{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Notes\n\nEvaluation:\n* Linear Regression (using randomly selected 500,000 examples from *train.parquet/partition_id=0*):\n    * Validation dataset r2 score = 0.0154\n    * Public leaderboard r2 score =\n* Linear Regression (using randomly selected 500,000 examples each of the 10 dataset partition id files):\n    * Validation dataset r2 score = 0.0072\n    * Public leaderboard r2 score = ","metadata":{}},{"cell_type":"code","source":"# Print date and time at the start of the run\n#from reformer_pytorch import LSHSelfAttention\nimport datetime\n\ncurTime = datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')\nprint('notebook running')\nprint(f'date: {curTime.split()[0]}')\nprint(f'time: {curTime.split()[1]}')\n\ndef print_time():\n    curTime = datetime.datetime.now().strftime('%H-%S-%M')\n    print(f'time (%H-%M-%S): {curTime}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:25:52.720595Z","iopub.execute_input":"2025-01-07T12:25:52.720807Z","iopub.status.idle":"2025-01-07T12:25:52.729648Z","shell.execute_reply.started":"2025-01-07T12:25:52.720783Z","shell.execute_reply":"2025-01-07T12:25:52.728885Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import packages\n\nimport time\nimport datetime\nimport os\nimport polars as pl\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n#import kaggle_evaluation.jane_street_inference_server\n#import kaggle_evaluation.jane_street_inference_server  \n\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.linear_model import LinearRegression\nfrom xgboost import XGBRegressor\n\nimport joblib\n\nfrom keras.optimizers import RMSprop\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras.callbacks import LearningRateScheduler\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\nfrom tensorflow.keras.callbacks import EarlyStopping\n\nprint('log: package imports successful')\nprint_time()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:25:52.823266Z","iopub.execute_input":"2025-01-07T12:25:52.824012Z","iopub.status.idle":"2025-01-07T12:26:05.091973Z","shell.execute_reply.started":"2025-01-07T12:25:52.823972Z","shell.execute_reply":"2025-01-07T12:26:05.091104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Import dataset as subset of training dataset\n\ndef import_unordered_data(num_samples=int(5e5), verbose=False):\n    \n    base_path = \"/kaggle/input/jane-street-real-time-market-data-forecasting/\"\n    random_state = 42\n    samples = []\n    total_entries_count = 0\n    \n    for i in range(1):\n\n        file_path = f\"train.parquet/partition_id={i}/part-0.parquet\"\n        \n        if verbose:\n            print(f\"importing file: {file_path}\")\n\n        try:\n            sample = pd.read_parquet(os.path.join(base_path, file_path))\n            total_entries_count += len(sample)\n            \n            if verbose:\n                print(f\"number of entries in '{file_path}': {len(sample):,}\")\n    \n            if num_samples < len(sample):\n                sample = sample.sample(n=num_samples, random_state=random_state)\n            samples.append(sample)\n    \n            if verbose:\n                print(' ')\n                \n        except Exception as e:\n            print(f\"error: {e}\")\n\n    sample_df = pd.concat(samples, ignore_index=True)\n    \n    if verbose:\n        print(f\"importing files complete\")\n        print(f'number of entries in full dataset: {total_entries_count:,}')\n        print(f'number of entries in dataframe: {len(sample_df):,}')\n\n    return sample_df\n\nt0 = datetime.datetime.now()\n\nsample_df = import_unordered_data(verbose=True)\n\nt1 =datetime.datetime.now()\ndif = (t1 - t0).total_seconds()\n\nprint(f'time taken to import data: {dif:.0f}s')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:05.093731Z","iopub.execute_input":"2025-01-07T12:26:05.094623Z","iopub.status.idle":"2025-01-07T12:26:09.617555Z","shell.execute_reply.started":"2025-01-07T12:26:05.094575Z","shell.execute_reply":"2025-01-07T12:26:09.616624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualise first 10 rows in dataset\n\nsample_df.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:09.618676Z","iopub.execute_input":"2025-01-07T12:26:09.618945Z","iopub.status.idle":"2025-01-07T12:26:09.650185Z","shell.execute_reply.started":"2025-01-07T12:26:09.618918Z","shell.execute_reply":"2025-01-07T12:26:09.649435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Split data into features and targets, and fill in missing values using SimpleImputer\n\n# Separate features, responders and weights\nfeatures = sample_df.filter(regex='^feature_')\nresponders = sample_df.filter(regex='^responder_')\nweights = sample_df['weight']\n\n# Convert to numpy arrays for further processing\nX = features.values\ny = responders[['responder_6']].values\nw = weights.values\n\n# Set nan values to 0.0\nX = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)\ny = np.nan_to_num(y, nan=0.0, posinf=0.0, neginf=0.0)\n\n# X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2)\n\n# Split features and targets into training and valudation sets\ntrain_test_split_output = train_test_split(X, y, w, test_size=0.2, random_state=42)\nX_train, X_val, y_train, y_val, w_train, w_val = train_test_split_output\n\n# Print feature, target and weight matrices shapes for training and validation dataset\nprint(f\"X_train shape: {X_train.shape}, X_val shape: {X_val.shape}\")\nprint(f\"y_train shape: {y_train.shape}, y_val shape: {y_val.shape}\")\nprint(f\"w_train shape: {w_train.shape}, w_val shape: {w_val.shape}\")\n\n# Train SimpleImputer to compute missing values and transform X_val to X_val_imputed\n# (making sure not to leak data from X_train to X_val by training the imputer on\n# X_train only)\nimputer = SimpleImputer(strategy='mean')\n\nX_train_imputed = imputer.fit_transform(X_train)\nX_val_imputed = imputer.transform(X_val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:09.651714Z","iopub.execute_input":"2025-01-07T12:26:09.651969Z","iopub.status.idle":"2025-01-07T12:26:10.937917Z","shell.execute_reply.started":"2025-01-07T12:26:09.651945Z","shell.execute_reply":"2025-01-07T12:26:10.936967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train LinearRegression model\n\nmodel = LinearRegression()\nmodel.fit(X_train_imputed, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:10.939097Z","iopub.execute_input":"2025-01-07T12:26:10.939465Z","iopub.status.idle":"2025-01-07T12:26:12.198059Z","shell.execute_reply.started":"2025-01-07T12:26:10.939412Z","shell.execute_reply":"2025-01-07T12:26:12.197336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Make predictions and evaluate\n\ny_val_pred = model.predict(X_val_imputed)\n\nscore = r2_score(y_val, y_val_pred)\nprint(f'LinearRegression model coefficient of determination (R^2): {score:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.198947Z","iopub.execute_input":"2025-01-07T12:26:12.199250Z","iopub.status.idle":"2025-01-07T12:26:12.219039Z","shell.execute_reply.started":"2025-01-07T12:26:12.199216Z","shell.execute_reply":"2025-01-07T12:26:12.217166Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    global lags_\n    if lags is not None:\n        lags_ = lags\n    # Extract the features for the model input\n    feature_columns = [col for col in test.columns if col.startswith(\"feature_\")]\n    features = test.select(feature_columns).to_numpy()  # Convert to numpy array for model input\n    print(feature_columns)\n    features = np.nan_to_num(features, nan=0.0, posinf=0.0, neginf=0.0)\n    # Generate predictions using the model\n    model_predictions = model.predict(features)\n    if Is_keras:\n        responder_6_predictions = model.predict(features)[:,0]\n    else:\n        responder_6_predictions = model_xgb.predict(features)\n    print(responder_6_predictions)    \n    #responder_6_predictions = model_predictions[:, 6]  # Assuming responder_6 is at index 6\n    # Create a new Polars DataFrame with row_id and responder_6 predictions\n    predictions = test.select(\"row_id\").with_columns(\n        pl.Series(\"responder_6\", responder_6_predictions)\n    )\n    print(predictions)\n    # Ensure the output format and length requirements\n    if isinstance(predictions, pl.DataFrame):\n        assert predictions.columns == ['row_id', 'responder_6']\n    elif isinstance(predictions, pd.DataFrame):\n        assert (predictions.columns == ['row_id', 'responder_6']).all()\n    else:\n        raise TypeError('The predict function must return a DataFrame')\n    \n    assert len(predictions) == len(test)\n    return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.221081Z","iopub.execute_input":"2025-01-07T12:26:12.221386Z","iopub.status.idle":"2025-01-07T12:26:12.247397Z","shell.execute_reply.started":"2025-01-07T12:26:12.221352Z","shell.execute_reply":"2025-01-07T12:26:12.245901Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Is_keras = True\nimport kaggle_evaluation.jane_street_inference_server  \n\ninference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.248757Z","iopub.execute_input":"2025-01-07T12:26:12.249047Z","iopub.status.idle":"2025-01-07T12:26:12.709189Z","shell.execute_reply.started":"2025-01-07T12:26:12.249015Z","shell.execute_reply":"2025-01-07T12:26:12.708409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# joblib.dump(model, \"linearRegression.pkl\")\n# model = joblib.load(\"linearRegression.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.710201Z","iopub.execute_input":"2025-01-07T12:26:12.710559Z","iopub.status.idle":"2025-01-07T12:26:12.714618Z","shell.execute_reply.started":"2025-01-07T12:26:12.710520Z","shell.execute_reply":"2025-01-07T12:26:12.713738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# output_folder = '/kaggle/working'\n# files = os.listdir(output_folder)\n# files","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.717182Z","iopub.execute_input":"2025-01-07T12:26:12.717498Z","iopub.status.idle":"2025-01-07T12:26:12.726952Z","shell.execute_reply.started":"2025-01-07T12:26:12.717471Z","shell.execute_reply":"2025-01-07T12:26:12.726273Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Is_keras = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.727711Z","iopub.execute_input":"2025-01-07T12:26:12.727939Z","iopub.status.idle":"2025-01-07T12:26:12.734402Z","shell.execute_reply.started":"2025-01-07T12:26:12.727916Z","shell.execute_reply":"2025-01-07T12:26:12.733612Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# X_train, X_val, y_train, y_val, weights_train, weights_val = train_test_split(\n#     X, y, weights, test_size=0.2, random_state=42\n# # ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.735313Z","iopub.execute_input":"2025-01-07T12:26:12.735570Z","iopub.status.idle":"2025-01-07T12:26:12.743800Z","shell.execute_reply.started":"2025-01-07T12:26:12.735546Z","shell.execute_reply":"2025-01-07T12:26:12.743027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def learning_rate_scheduler_xgb(epoch):\n#     initial_rate = 0.3\n#     decay_rate = 0.999\n#     return initial_rate * (decay_rate ** (np.log(epoch)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.744693Z","iopub.execute_input":"2025-01-07T12:26:12.744920Z","iopub.status.idle":"2025-01-07T12:26:12.752258Z","shell.execute_reply.started":"2025-01-07T12:26:12.744896Z","shell.execute_reply":"2025-01-07T12:26:12.751553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# model_xgb = XGBRegressor(\n#     n_estimators=200,\n#     learning_rate=0.1,\n#     tree_method='hist',\n#     max_depth=6,\n#     random_state=42\n# )\n\n# model_xgb.fit(\n#     X_train,\n#     y_train,\n#     eval_set=[(X_train, y_train), (X_val, y_val)],\n#     eval_metric='rmse',\n#     verbose=False\n# )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.753041Z","iopub.execute_input":"2025-01-07T12:26:12.753292Z","iopub.status.idle":"2025-01-07T12:26:12.763096Z","shell.execute_reply.started":"2025-01-07T12:26:12.753268Z","shell.execute_reply":"2025-01-07T12:26:12.762286Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# y_pred = model_xgb.predict(X_val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.763975Z","iopub.execute_input":"2025-01-07T12:26:12.764215Z","iopub.status.idle":"2025-01-07T12:26:12.770351Z","shell.execute_reply.started":"2025-01-07T12:26:12.764176Z","shell.execute_reply":"2025-01-07T12:26:12.769617Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# mse = mean_squared_error(y_val, y_pred, squared=False)\n# r2 = r2_score(y_val, y_pred)\n# print(f'RMSE: {mse:.4f}')\n# print(f'R²: {r2:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.771187Z","iopub.execute_input":"2025-01-07T12:26:12.771449Z","iopub.status.idle":"2025-01-07T12:26:12.780764Z","shell.execute_reply.started":"2025-01-07T12:26:12.771402Z","shell.execute_reply":"2025-01-07T12:26:12.780009Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# joblib.dump(model_xgb, 'xgboost_sklean.pkl')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.781564Z","iopub.execute_input":"2025-01-07T12:26:12.781786Z","iopub.status.idle":"2025-01-07T12:26:12.788231Z","shell.execute_reply.started":"2025-01-07T12:26:12.781763Z","shell.execute_reply":"2025-01-07T12:26:12.787481Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class GCRMSprop(RMSprop):\n#     def get_gradients(self, loss, params):\n#         grads = []\n#         gradients = super().get_gradients()\n#         for grad in gradients:\n#             grad_len = len(grad.shape)\n#             if grad_len > 1:\n#                 axis = list(range(grad_len) - 1)\n#                 grad -= ops.mean(grad, axis=axis, keep_dims=True)\n#             grads.append(grad)\n#         return grads\n\n# optimizer = GCRMSprop(learning_rate=1e4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.789214Z","iopub.execute_input":"2025-01-07T12:26:12.789483Z","iopub.status.idle":"2025-01-07T12:26:12.797889Z","shell.execute_reply.started":"2025-01-07T12:26:12.789455Z","shell.execute_reply":"2025-01-07T12:26:12.797116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# input_dim = X.shape[1]\n# output_dim = y.shape[1]\n\n# model = models.Sequential([\n#     layers.Input(shape=(input_dim,)),\n#     layers.Dense(64, activation='relu'),\n#     layers.Dense(32, activation='relu'),\n#     layers.Dense(64, activation='relu'),\n#     layers.Dense(output_dim, activation='linear')\n# ])\n\n# model.compile(optimizer='adam', loss='mse')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.798953Z","iopub.execute_input":"2025-01-07T12:26:12.799282Z","iopub.status.idle":"2025-01-07T12:26:12.809100Z","shell.execute_reply.started":"2025-01-07T12:26:12.799246Z","shell.execute_reply":"2025-01-07T12:26:12.808464Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def step_decay(epoch):\n#     initial_lr = 0.01\n#     drop = 0.5\n#     epochs_drop = 5\n    \n#     lr = initial_lr * (drop ** (epoch//epochs_drop))\n\n#     return lr\n\n# lr_scheduler = LearningRateScheduler(step_decay)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.810079Z","iopub.execute_input":"2025-01-07T12:26:12.810689Z","iopub.status.idle":"2025-01-07T12:26:12.816796Z","shell.execute_reply.started":"2025-01-07T12:26:12.810652Z","shell.execute_reply":"2025-01-07T12:26:12.816120Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# reduce_lr = ReduceLROnPlateau(\n#     monitor='val_loss',\n#     factor=0.5,\n#     patience=2,\n#     min_lr=1e-6\n# )\n\n# early_stopping = EarlyStopping(\n#     monitor='val_los',\n#     patience=10,\n#     min_delta=0.00001,\n#     restore_best_weights=True\n# )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.817631Z","iopub.execute_input":"2025-01-07T12:26:12.817834Z","iopub.status.idle":"2025-01-07T12:26:12.828538Z","shell.execute_reply.started":"2025-01-07T12:26:12.817813Z","shell.execute_reply":"2025-01-07T12:26:12.827886Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# if Is_keras:\n#     history = model.fit(\n#         X_train, y_train,\n#         sample_weight=weights_train,\n#         epochs=50,\n#         batch_size=32,\n#         validation_data=(X_val, y_val, weights_val),\n#         callbacks=[early_stopping, reduce_rl]\n#     )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.829365Z","iopub.execute_input":"2025-01-07T12:26:12.829663Z","iopub.status.idle":"2025-01-07T12:26:12.836550Z","shell.execute_reply.started":"2025-01-07T12:26:12.829639Z","shell.execute_reply":"2025-01-07T12:26:12.835815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# if Is_keras:\n#     model.save(\"/kaggle/working/model.keras\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.837405Z","iopub.execute_input":"2025-01-07T12:26:12.837740Z","iopub.status.idle":"2025-01-07T12:26:12.844491Z","shell.execute_reply.started":"2025-01-07T12:26:12.837703Z","shell.execute_reply":"2025-01-07T12:26:12.843777Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# t1 = datetime.datetime.now()\n# dif = (t1-t0).total_seconds()\n# print(f'time taken: {dif:.2f} s')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T12:26:12.845346Z","iopub.execute_input":"2025-01-07T12:26:12.845691Z","iopub.status.idle":"2025-01-07T12:26:12.852201Z","shell.execute_reply.started":"2025-01-07T12:26:12.845654Z","shell.execute_reply":"2025-01-07T12:26:12.851332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}