{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":9801075,"sourceType":"datasetVersion","datasetId":6006872},{"sourceId":9806342,"sourceType":"datasetVersion","datasetId":6010899},{"sourceId":203900450,"sourceType":"kernelVersion"},{"sourceId":171905,"sourceType":"modelInstanceVersion","modelInstanceId":146319,"modelId":168862}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":80.344101,"end_time":"2024-10-26T03:27:42.952247","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-10-26T03:26:22.608146","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"252dd2de87de42f4becd877fbbafc26b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_ff55584ae0ab4f39ae471f314eaad988","placeholder":"​","style":"IPY_MODEL_b8d338c473aa4dc3ba25f37a997a9037","value":" 1/1 [00:00&lt;00:00, 33.79it/s]"}},"48a2731fb59b4ce8ace5b53d6f0e3337":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a8dbc79c7c5a48318466a102c55801bb","placeholder":"​","style":"IPY_MODEL_ebd06aaaa7024a1684ba1b9fe89358cf","value":"100%"}},"56da652f3eeb42aca986e6fd815629dc":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5bb4e0df01c44716afebab25aafe9f5f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_f4e41234b0cc4f3e9313d971f5aadc9f","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_6408698650f74dd699bcc914b850e396","value":1}},"6408698650f74dd699bcc914b850e396":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"9cb493ec04fc4ed391f5ac28cc84500e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_48a2731fb59b4ce8ace5b53d6f0e3337","IPY_MODEL_5bb4e0df01c44716afebab25aafe9f5f","IPY_MODEL_252dd2de87de42f4becd877fbbafc26b"],"layout":"IPY_MODEL_56da652f3eeb42aca986e6fd815629dc"}},"a8dbc79c7c5a48318466a102c55801bb":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b8d338c473aa4dc3ba25f37a997a9037":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ebd06aaaa7024a1684ba1b9fe89358cf":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"f4e41234b0cc4f3e9313d971f5aadc9f":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ff55584ae0ab4f39ae471f314eaad988":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<div class=\"list-group\" id=\"list-tab\" role=\"tablist\">\n  <h3 class=\"list-group-item list-group-item-action active\" data-toggle=\"list\"  role=\"tab\" aria-controls=\"home\"  >Table of Contents</h3>\n  <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#data\" role=\"tab\" aria-controls=\"profile\">DATA LOADING AND PREPROCESSING<span class=\"badge badge-primary badge-pill \"></span></a>\n    <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#eda\" role=\"tab\" aria-controls=\"messages\">TIME SERIES ANALYSIS AND EDA<span class=\"badge badge-primary badge-pill\"></span></a>\n    <a class=\"list-group-item list-group-item-action\"  data-toggle=\"list\" href=\"#model\" role=\"tab\" aria-controls=\"settings\">MODELLING<span class=\"badge badge-primary badge-pill\"></span></a>\n    <a class=\"list-group-item list-group-item-action\" data-toggle=\"list\" href=\"#sub\" role=\"tab\" aria-controls=\"settings\">SUB TO SERVER<span class=\"badge badge-primary badge-pill\"></span></a> \n","metadata":{}},{"cell_type":"markdown","source":"# <div id='data' style=\"color:white;   font-weight:bold; font-size:120%; text-align:center;padding:12.0px; background:black\"> DATA LOADING AND PREPROCESSING</div>","metadata":{}},{"cell_type":"markdown","source":"<a href=\"#list-tab\" class=\"btn btn-success btn-lg active\" role=\"button\" aria-pressed=\"true\" style=\"color:Blue; font-size:140%; background:lightgrey;  font-weight:bold; \" data-toggle=\"popover\" title=\"go to Colors\">GO BACK</a>","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv, pd.read_parquet )\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\n\nimport os, gc\nfrom tqdm.auto import tqdm\nimport pickle # module to serialize and deserialize objects\nimport re # for Regular expression operations \n\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.optimizers import Adam\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data  import Dataset, DataLoader\nfrom pytorch_lightning import (LightningDataModule, LightningModule, Trainer)\nfrom pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, Timer\n\nfrom sklearn.metrics import r2_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import VotingRegressor\n\nimport lightgbm as lgb\nfrom lightgbm import LGBMRegressor\n\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-10T15:40:41.998321Z","iopub.execute_input":"2025-01-10T15:40:41.998649Z","iopub.status.idle":"2025-01-10T15:40:59.437906Z","shell.execute_reply.started":"2025-01-10T15:40:41.998624Z","shell.execute_reply":"2025-01-10T15:40:59.437032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gridColor = 'lightgrey'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-10T15:42:15.274405Z","iopub.execute_input":"2025-01-10T15:42:15.275361Z","iopub.status.idle":"2025-01-10T15:42:15.279384Z","shell.execute_reply.started":"2025-01-10T15:42:15.275323Z","shell.execute_reply":"2025-01-10T15:42:15.278413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\nsample_df = pd.read_parquet(f\"{path}/train.parquet/partition_id=9/part-0.parquet\")\nsample_df = sample_df.dropna()\nsample_df.round(1)\n\nprint(sample_df.describe())\n\nprint(sample_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-10T15:42:16.041967Z","iopub.execute_input":"2025-01-10T15:42:16.042337Z","iopub.status.idle":"2025-01-10T15:42:44.537891Z","shell.execute_reply.started":"2025-01-10T15:42:16.042294Z","shell.execute_reply":"2025-01-10T15:42:44.536702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_dfs = {}\n\nfor sequence_value in range(10):\n    \n    sample_dfs[sequence_value] = sample_df.iloc[int(sequence_value*len(sample_df)*.1):int((sequence_value+1) * len(sample_df)*.1):1]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-10T15:44:20.840486Z","iopub.execute_input":"2025-01-10T15:44:20.840849Z","iopub.status.idle":"2025-01-10T15:44:20.845908Z","shell.execute_reply.started":"2025-01-10T15:44:20.840824Z","shell.execute_reply":"2025-01-10T15:44:20.844896Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_dfs[0].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-10T15:44:22.180730Z","iopub.execute_input":"2025-01-10T15:44:22.181051Z","iopub.status.idle":"2025-01-10T15:44:22.242596Z","shell.execute_reply.started":"2025-01-10T15:44:22.181014Z","shell.execute_reply":"2025-01-10T15:44:22.241556Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(len(sample_dfs[0]))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-10T00:56:21.192193Z","iopub.execute_input":"2025-01-10T00:56:21.192587Z","iopub.status.idle":"2025-01-10T00:56:21.197652Z","shell.execute_reply.started":"2025-01-10T00:56:21.192555Z","shell.execute_reply":"2025-01-10T00:56:21.196496Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sum_length = sum(len(sample_dfs[i]) for i in sample_dfs)\nprint(sum_length)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-10T00:56:21.928874Z","iopub.execute_input":"2025-01-10T00:56:21.929183Z","iopub.status.idle":"2025-01-10T00:56:21.934095Z","shell.execute_reply.started":"2025-01-10T00:56:21.929159Z","shell.execute_reply":"2025-01-10T00:56:21.933028Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(10):\n    sample_dfs[i] = sample_dfs[i].dropna()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-10T15:44:26.606535Z","iopub.execute_input":"2025-01-10T15:44:26.606833Z","iopub.status.idle":"2025-01-10T15:44:27.637030Z","shell.execute_reply.started":"2025-01-10T15:44:26.606811Z","shell.execute_reply":"2025-01-10T15:44:27.636256Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_dfs[0].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-10T00:56:27.452963Z","iopub.execute_input":"2025-01-10T00:56:27.453314Z","iopub.status.idle":"2025-01-10T00:56:29.350685Z","shell.execute_reply.started":"2025-01-10T00:56:27.453284Z","shell.execute_reply":"2025-01-10T00:56:29.349683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample = sample_dfs[0].iloc[500:750]\n\nax = sample['responder_6'].plot.box()\nax.set_title('Box and Whiskers')\n\nax.set_ylabel('Responder 6')\nplt.figure(figsize = (12,6))\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T01:58:51.503432Z","iopub.execute_input":"2025-01-08T01:58:51.503761Z","iopub.status.idle":"2025-01-08T01:58:51.650066Z","shell.execute_reply.started":"2025-01-08T01:58:51.503737Z","shell.execute_reply":"2025-01-08T01:58:51.648961Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# <div id='eda'  style=\"color:white;   font-weight:bold; font-size:120%; text-align:center;padding:12.0px; background:black\"> TIME SERIES ANALYSIS AND EDA</div>","metadata":{}},{"cell_type":"markdown","source":"<a href=\"#list-tab\" class=\"btn btn-success btn-lg active\" role=\"button\" aria-pressed=\"true\" style=\"color:Blue; font-size:140%; background:lightgrey;  font-weight:bold; \" data-toggle=\"popover\" title=\"go to Colors\">GO BACK</a>","metadata":{}},{"cell_type":"markdown","source":"Let us take a look at the target values over time (for  `symbol_id`=1)","metadata":{}},{"cell_type":"code","source":"def train(i):\n    train = sample_dfs[i]\n\n    # Select rows where symbol_id == 1\n    filtered_data = train[train.symbol_id == 1]\n    xx = filtered_data['time_id']\n    yy = filtered_data['responder_6']\n\n    # Create the plot\n    plt.figure(figsize=(16, 5))\n    plt.plot(xx, yy, color='black', linewidth=0.05)\n    plt.suptitle('Returns, responder_6', weight='bold', fontsize=16)\n    plt.xlabel(\"Time\", fontsize=12)\n    plt.ylabel(\"Returns\", fontsize=12)\n    plt.grid(color='lightgray', linewidth=0.8)\n    plt.axhline(0, color='red', linestyle='-', linewidth=1.2)\n    plt.show()\n\ntrain(0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T00:29:51.172572Z","iopub.execute_input":"2025-01-07T00:29:51.172929Z","iopub.status.idle":"2025-01-07T00:29:51.476412Z","shell.execute_reply.started":"2025-01-07T00:29:51.172899Z","shell.execute_reply":"2025-01-07T00:29:51.475535Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let us take a look at the cumulative values of response over time","metadata":{}},{"cell_type":"code","source":"def train(i):\n    train = sample_dfs[i]\n\n    # Select rows where symbol_id == 1\n    filtered_data = train[train.symbol_id == 1]\n    xx = filtered_data['time_id']\n    yy = filtered_data['responder_6']\n\n    return xx, yy\n\n# Create subplots for i = 0 to 10\nfig, axes = plt.subplots(5, 2, figsize=(20, 25))  # Adjust grid size (5 rows, 2 columns)\n\nfor i, ax in enumerate(axes.flatten()[:10]):  # Loop over 10 plots\n    xx, yy = train(i)\n    ax.plot(xx, yy, color='black', linewidth=0.05)\n    ax.set_title(f'Returns, responder_6 (i={i})', fontsize=14, weight='bold')\n    ax.set_xlabel(\"Time\", fontsize=12)\n    ax.set_ylabel(\"Returns\", fontsize=12)\n    ax.grid(color='lightgray', linewidth=0.8)\n    ax.axhline(0, color='red', linestyle='-', linewidth=1.2)\n\nplt.tight_layout()  # Adjust layout for better spacing\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T00:31:49.173711Z","iopub.execute_input":"2025-01-07T00:31:49.174217Z","iopub.status.idle":"2025-01-07T00:31:51.653242Z","shell.execute_reply.started":"2025-01-07T00:31:49.174171Z","shell.execute_reply":"2025-01-07T00:31:51.652053Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now let's compare this responder (6) with other responders","metadata":{}},{"cell_type":"code","source":"# for symbol_id == 0\nplt.figure(figsize=(18, 7))\npredictor_cols = [col for col in sample_df.columns if 'responder' in col]\nfor i in predictor_cols: \n    if i == 'responder_6': \n        c='red'\n        lw=2.5\n        plt.plot((sample_df.groupby(['date_id'])[i].mean()).cumsum(), linewidth = lw, color = c)\n    else: \n        lw=1\n        plt.plot((sample_df.groupby(['date_id'])[i].mean()).cumsum(), linewidth = lw)\n\nplt.xlabel('Trade days')\nplt.ylabel('Cumulative response')\nplt.title('Response time series over trade days  \\n Responder 6 (red) and other responders', weight='bold')\nplt.grid(visible=True, color = gridColor, linewidth = 0.7)\nplt.axhline(0, color='blue', linestyle='-', linewidth=1)\nplt.legend(predictor_cols)\nsns.despine()\n#plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T00:33:22.313146Z","iopub.execute_input":"2025-01-07T00:33:22.313479Z","iopub.status.idle":"2025-01-07T00:33:23.741575Z","shell.execute_reply.started":"2025-01-07T00:33:22.313455Z","shell.execute_reply":"2025-01-07T00:33:23.740776Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- We can see that `resp6` (red) most closely follows `resp0` and `resp3`\n\nLet's build a correlation matrix and see it numerically.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(6, 6))\nresponders = pd.read_csv(f\"{path}/responders.csv\")\nmatrix = responders[[ f\"tag_{no}\" for no in range(0,5,1) ] ].T.corr()\nsns.heatmap(matrix, square=True, cmap=\"coolwarm\", alpha =0.9, vmin=-1, vmax=1, center= 0, linewidths=0.5, \n            linecolor='white', annot=True, fmt='.2f')\nplt.xlabel(\"Responder_0 - Responder_8\")\nplt.ylabel(\"Responder_0 - Responder_8\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T00:33:47.750985Z","iopub.execute_input":"2025-01-07T00:33:47.751300Z","iopub.status.idle":"2025-01-07T00:33:48.108562Z","shell.execute_reply.started":"2025-01-07T00:33:47.751278Z","shell.execute_reply":"2025-01-07T00:33:48.107713Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Seaborn Correlation Heatmap for Responders\n\nresponder_sample = []\n\nfor col in sample.columns:\n    if col.startswith('responder'):\n        responder_sample.append(col)\n\nresponder_df = sample[responder_sample]\n\nresponder_correlation = responder_df.corr(method = 'spearman').round(1)\n\n\nsns.heatmap(responder_correlation, square = True, cmap = 'coolwarm', vmin=-1, vmax = 1, linewidth = 0.5, annot = True)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:33:56.838548Z","iopub.execute_input":"2025-01-07T01:33:56.838866Z","iopub.status.idle":"2025-01-07T01:33:57.571952Z","shell.execute_reply.started":"2025-01-07T01:33:56.838841Z","shell.execute_reply":"2025-01-07T01:33:57.571083Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let us take a look at the returns and cumulative daily returns, and disribution of returns for all responders","metadata":{}},{"cell_type":"code","source":"df_train=sample_df\ns_id = 0                        # Change params to take a look at other symbols\nres_columns = [col for col in df_train.columns if re.match(\"responder_\", col)]\nrow = 9\nj = 0\n\nfig, axs = plt.subplots(figsize=(18, 4*row))\nfor i in range(1, 3 * len(res_columns) + 1, 3):\n    xx= sample_dfs[0][(sample_df.symbol_id==s_id)] ['time_id']\n    yy=sample_dfs[0][ (sample_df.symbol_id==s_id)][f'responder_{j}']\n    c='black'\n    if j == 6: c='red'\n        \n    ax1 = plt.subplot(9, 3, i)\n    ax1.plot(   xx,yy.cumsum()   , color = c, linewidth =0.8 )\n    plt.axhline(0, color='blue', linestyle='-', linewidth=0.9)\n    plt.grid(color =gridColor )\n    \n    ax2 = plt.subplot(9, 3, i+1)\n    #by_date = df_symbolX.groupby([\"date_id\"])\n    ax2.plot(xx,yy   , color = c, linewidth =0.05)\n    plt.axhline(0, color='blue', linestyle='-', linewidth=1.2)\n    ax2.set_title(f\"responder_{j}\", fontsize = 14)\n    plt.grid(color = gridColor)\n    \n    ax3 = plt.subplot(9, 3, i+2)\n    b=1000\n    ax3.hist(yy, bins=b, color = c,density=True, histtype=\"step\" )\n    ax3.hist(yy, bins=b, color = 'lightgrey',density=True)\n    plt.grid(color = gridColor)\n    ax3.set_ylim([0, 3.5])\n    ax3.set_xlim([-2.5, 2.5])\n    \n    j = j + 1\n    \nfig.patch.set_linewidth(3)\nfig.patch.set_edgecolor('#000000')\nfig.patch.set_facecolor('#eeeeee') \nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:34:49.952258Z","iopub.execute_input":"2025-01-07T01:34:49.952546Z","iopub.status.idle":"2025-01-07T01:35:05.957098Z","shell.execute_reply.started":"2025-01-07T01:34:49.952524Z","shell.execute_reply":"2025-01-07T01:35:05.955899Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We can see that responders have different behavior and distributions.\n\nLet us now study the behavior of `responder 6`  for different `symbol_id`","metadata":{}},{"cell_type":"code","source":"res_columns = [col for col in df_train.columns if re.match(\"responder_\", col)]\nrow=10\nfig, axs = plt.subplots(figsize=(18, 5*row))\nb=300\nj = 0\nfor i in range(1, 3 * row + 1, 3):\n    xx= sample_df[(sample_df.symbol_id==j)] ['time_id']\n    yy= sample_df[(sample_df.symbol_id==j)]['responder_6']\n    c='black'\n        \n    ax1 = plt.subplot(row, 3, i)\n    ax1.plot(   xx,yy.cumsum()   , color = c, linewidth =0.8 )\n    plt.axhline(0, color='red', linestyle='-', linewidth=0.7)\n    plt.grid(color = gridColor)\n    plt.xlabel('Time')\n    \n    ax2 = plt.subplot(row, 3, i+1)\n    ax2.plot(xx,yy   , color = c, linewidth =0.05)\n    plt.axhline(0, color='red', linestyle='-', linewidth=0.7)\n    ax2.set_title(f\"symbol_id={j}\", fontsize = '14')\n    plt.grid(color = gridColor)\n    plt.xlabel('Time')\n    \n    ax3 = plt.subplot(row, 3, i+2)\n    ax3.hist(yy, bins=b, color = c, density=True, histtype=\"step\" )\n    ax3.hist(yy, bins=b, color = 'lightgrey',density=True)\n    plt.grid(color = gridColor)\n    ax3.set_xlim([-2.5, 2.5])\n    ax3.set_ylim([0, 1.5])\n    plt.xlabel('Time')\n    \n    j = j + 1\n    \nfig.patch.set_linewidth(3)\nfig.patch.set_edgecolor('#000000')\nfig.patch.set_facecolor('#eeeeee') \nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:35:18.258449Z","iopub.execute_input":"2025-01-07T01:35:18.258817Z","iopub.status.idle":"2025-01-07T01:35:31.943783Z","shell.execute_reply.started":"2025-01-07T01:35:18.258785Z","shell.execute_reply":"2025-01-07T01:35:31.942750Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- We see that the behavior and distribution of one `responder 6` is  different for different `symbol_id`\n\nNow let's study the data in more detail and then continue diving into time series analysis\n","metadata":{}},{"cell_type":"markdown","source":"## Files and variables overview","metadata":{}},{"cell_type":"markdown","source":"### Features.csv\nfeatures.csv - metadata pertaining to the anonymized features\n\n#### Features have many missing values.","metadata":{}},{"cell_type":"code","source":"df_train = sample_df\nplt.figure(figsize=(20, 3))    # Plot missing values\nplt.bar(x=df_train.isna().sum().index, height=df_train.isna().sum().values, color=\"red\", label='missing')   # analog: using missingno\nplt.xticks(rotation=90)\nplt.title(f'Missing values over the {len(df_train)} samples which have a target')\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:35:50.536580Z","iopub.execute_input":"2025-01-07T01:35:50.536926Z","iopub.status.idle":"2025-01-07T01:35:52.358837Z","shell.execute_reply.started":"2025-01-07T01:35:50.536900Z","shell.execute_reply":"2025-01-07T01:35:52.357982Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Some columns are not very useful in our sample (either Null or show the partition number).","metadata":{}},{"cell_type":"markdown","source":"#### Structure of features:","metadata":{}},{"cell_type":"code","source":"features = pd.read_csv(f\"{path}/features.csv\")\nfeatures","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:36:44.127273Z","iopub.execute_input":"2025-01-07T01:36:44.127575Z","iopub.status.idle":"2025-01-07T01:36:44.155872Z","shell.execute_reply.started":"2025-01-07T01:36:44.127552Z","shell.execute_reply":"2025-01-07T01:36:44.155093Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Tags visualizing:","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(18, 6))\nplt.imshow(features.iloc[:, 1:].T.values, cmap=\"gray_r\")\nplt.xlabel(\"feature_00 - feature_78\")\nplt.ylabel(\"tag_0 - tag_16\")\nplt.yticks(np.arange(17))\nplt.xticks(np.arange(79))\nplt.grid(color = 'lightgrey')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:36:49.214734Z","iopub.execute_input":"2025-01-07T01:36:49.215019Z","iopub.status.idle":"2025-01-07T01:36:49.775631Z","shell.execute_reply.started":"2025-01-07T01:36:49.214992Z","shell.execute_reply":"2025-01-07T01:36:49.774614Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Correlation matrix between feature_XX and feature_YY","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(11, 11))\nmatrix = features[[ f\"tag_{no}\" for no in range(0,17,1) ] ].T.corr()\nsns.heatmap(matrix, square=True, cmap=\"coolwarm\", alpha =0.9, vmin=-1, vmax=1, center= 0, linewidths=0.5, linecolor='white')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:36:56.316592Z","iopub.execute_input":"2025-01-07T01:36:56.316931Z","iopub.status.idle":"2025-01-07T01:36:56.968496Z","shell.execute_reply.started":"2025-01-07T01:36:56.316906Z","shell.execute_reply":"2025-01-07T01:36:56.967584Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Responders.csv\nresponders.csv - metadata pertaining to the anonymized responders\n#### Structure of responders:","metadata":{}},{"cell_type":"code","source":"responders = pd.read_csv(f\"{path}/responders.csv\")\nresponders","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:37:11.912776Z","iopub.execute_input":"2025-01-07T01:37:11.913088Z","iopub.status.idle":"2025-01-07T01:37:11.926607Z","shell.execute_reply.started":"2025-01-07T01:37:11.913061Z","shell.execute_reply":"2025-01-07T01:37:11.925834Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Weights\n#### Basic stats:","metadata":{}},{"cell_type":"code","source":"sample_df['weight'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:37:13.381801Z","iopub.execute_input":"2025-01-07T01:37:13.382119Z","iopub.status.idle":"2025-01-07T01:37:13.536515Z","shell.execute_reply.started":"2025-01-07T01:37:13.382093Z","shell.execute_reply":"2025-01-07T01:37:13.535583Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8,3))\nplt.hist(sample_df['weight'], bins=30, color='grey', edgecolor = 'white',density=True )\nplt.title('Distribution of weights')\nplt.grid(color = 'lightgrey', linewidth=0.5)\nplt.axvline( 2.37399, color='red', linestyle='-', linewidth=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:37:18.013799Z","iopub.execute_input":"2025-01-07T01:37:18.014105Z","iopub.status.idle":"2025-01-07T01:37:18.325411Z","shell.execute_reply.started":"2025-01-07T01:37:18.014082Z","shell.execute_reply":"2025-01-07T01:37:18.324500Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Sample submission.csv\nsample_submission.csv - This file illustrates the format of the predictions your model should make.","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv(f\"{path}/sample_submission.csv\")\nprint( f\"shape = {sub.shape}\" )\nsub.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:37:22.676053Z","iopub.execute_input":"2025-01-07T01:37:22.676351Z","iopub.status.idle":"2025-01-07T01:37:22.695823Z","shell.execute_reply.started":"2025-01-07T01:37:22.676329Z","shell.execute_reply":"2025-01-07T01:37:22.694985Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Train.parquet\n\n- **train.parquet** - The training set, contains historical data and returns. For convenience, the training set has been partitioned into ten parts.\n  - `date_id` and `time_id` - Integer values that are ordinally sorted, providing a chronological structure to the data, although the actual time intervals between `time_id` values may vary.\n  - `symbol_id` - Identifies a unique financial instrument.\n  - `weight` - The weighting used for calculating the scoring function.\n  - `feature_{00...78}` - Anonymized market data.\n  - `responder_{0...8}` - Anonymized responders clipped between -5 and 5. The `responder_6` field is what you are trying to predict.\n  \n  \nEach row in the `{train/test}.parquet` dataset corresponds to a unique combination of a symbol (identified by `symbol_id`) and a timestamp (represented by `date_id` and `time_id`). You will be provided with multiple responders, with `responder_6` being the only responder used for scoring. The date_id column is an integer which represents the day of the event, while `time_id` represents a time ordering. It's important to note that the real time differences between each time_id are not guaranteed to be consistent.\n\n- The `symbol_id` column contains encrypted identifiers. Each `symbol_id` is not guaranteed to appear in all `time_id` and `date_id` combinations.\n- Additionally, new `symbol_id` values **may appear in future** test sets.est sets.","metadata":{}},{"cell_type":"markdown","source":"## Responders: analysis, statistics and distributions","metadata":{}},{"cell_type":"code","source":"col =[]\nfor i in range(9):\n    col.append(f\"responder_{i}\") \n\nsample_df[col].describe().round(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:37:29.543805Z","iopub.execute_input":"2025-01-07T01:37:29.544228Z","iopub.status.idle":"2025-01-07T01:37:31.216251Z","shell.execute_reply.started":"2025-01-07T01:37:29.544183Z","shell.execute_reply":"2025-01-07T01:37:31.215534Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Interesting fact:\n- The values ​​of all variables are strictly within the range of `[-5, 5]`","metadata":{}},{"cell_type":"markdown","source":"#### Responders-Responder distributions\nLet's dive deeper into the relationships between respondents and plot mutual distributions between 'reps 6' and other responders","metadata":{}},{"cell_type":"code","source":"numerical_features=[]\nnumerical_features=sample_df.filter(regex='^responder_').columns.tolist() # Separate responders\nnumerical_features.remove('responder_6')\n\ngs=600\nk=1;\ncol = 3\nrow = 3\nfig, axs = plt.subplots(row, col, figsize=(5*col, 5*row))\n\nfor i in numerical_features:\n    \n    plt.subplot(col,row, k)\n    plt.hexbin(sample_df[i], sample_df['responder_6'], gridsize=gs, cmap='CMRmap', bins='log', alpha = 0.2)\n    plt.xlabel(f'{i}', fontsize = 12)\n    plt.ylabel('responder_6', fontsize = 12)\n    plt.tick_params(axis='x', labelsize=6)\n    plt.tick_params(axis='y', labelsize=6)\n    k=k+1\nfig.patch.set_linewidth(3)\nfig.patch.set_edgecolor('#000000')\nfig.patch.set_facecolor('#eeeeee')   \n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:37:39.858772Z","iopub.execute_input":"2025-01-07T01:37:39.859072Z","iopub.status.idle":"2025-01-07T01:37:58.996332Z","shell.execute_reply.started":"2025-01-07T01:37:39.859049Z","shell.execute_reply":"2025-01-07T01:37:58.995396Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Responder6-Features distributions\nNow let's plot mutual distributions between 'reps 6' and some features","metadata":{}},{"cell_type":"code","source":"numerical_features=[]\nfor i in ['05', '06', '07', '08', '12', '15', '19', '32', '38', '39', '50', '51', '65', '66', '67']:\n    numerical_features.append(f'feature_{i}') \n\ngs=600\nk=1;\ncol = 3\nrow = int(np.ceil(len(numerical_features) /3 ))\nsz=5\nw=sz*col\nh = w/col *row\nplt.figure(figsize=(w, h))\n\nfig, axs = plt.subplots(figsize=(w, h))\n\nfor i in numerical_features:\n    \n    plt.subplot(row, col, k)\n    plt.hexbin(sample_df['responder_6'], sample_df[i], gridsize=gs, cmap='CMRmap', bins='log', alpha = 0.3)\n    \n    plt.xlabel(f'{i}')\n    plt.ylabel('responder_6')\n    plt.tick_params(axis='x', labelsize=6)\n    plt.tick_params(axis='y', labelsize=6)\n    k=k+1\n\nfig.patch.set_linewidth(3)\nfig.patch.set_edgecolor('#000000')\nfig.patch.set_facecolor('#eeeeee')   \nplt.show()   ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:37:58.997476Z","iopub.execute_input":"2025-01-07T01:37:58.997757Z","iopub.status.idle":"2025-01-07T01:38:35.249247Z","shell.execute_reply.started":"2025-01-07T01:37:58.997735Z","shell.execute_reply":"2025-01-07T01:38:35.248042Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numerical_features=[]\n\nfor i in range(5,9):\n    numerical_features.append(f'feature_0{i}') \nfor i in range(15,20):\n    numerical_features.append(f'feature_{i}') \n    \na=0; k=1;\nn=3; \n\nfig, axs = plt.subplots(figsize=(15, 4))\nfor i in numerical_features[:-1]:\n    a=a+1\n    for j in numerical_features[a:]:\n        plt.subplot(1,n, k)\n        plt.hexbin(sample_df[i], sample_df[j], gridsize=200, cmap='CMRmap', bins='log', alpha = 1)\n        plt.grid()\n        plt.xlabel(f'{i}', fontsize = 14)\n        plt.ylabel(f'{j}', fontsize = 14)\n        plt.tick_params(axis='x', labelsize=6)\n        plt.tick_params(axis='y', labelsize=6)\n        \n        k=k+1\n        if k == (n+1):    \n            k=1\n            plt.show()\n            plt.figure(figsize=(15, 4)) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-07T01:38:35.251076Z","iopub.execute_input":"2025-01-07T01:38:35.251482Z","iopub.status.idle":"2025-01-07T01:39:04.362118Z","shell.execute_reply.started":"2025-01-07T01:38:35.251444Z","shell.execute_reply":"2025-01-07T01:39:04.361215Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- There are many nonlinear and non-trivial distributions","metadata":{}},{"cell_type":"markdown","source":"# <div id='sub'  style=\"color:white;   font-weight:bold; font-size:120%; text-align:center;padding:12.0px; background:black\">SUB TO SERVER</div>","metadata":{}},{"cell_type":"markdown","source":"<a href=\"#list-tab\" class=\"btn btn-success btn-lg active\" role=\"button\" aria-pressed=\"true\" style=\"color:Blue; font-size:140%; background:lightgrey;  font-weight:bold; \" data-toggle=\"popover\" title=\"go to Colors\">GO BACK</a>","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import MinMaxScaler\nimport polars as pl\nimport kaggle_evaluation\nimport kaggle_evaluation.jane_street_inference_server as inference_server_module\n\n# Define global variables\nlags_: pd.DataFrame | None = None\nlstm_model = None\nscaler = None\n\n# Initialize the LSTM model\ndef initialize_lstm(input_shape):\n    \"\"\"Initializes the LSTM model.\"\"\"\n    global lstm_model\n    if lstm_model is None:\n        lstm_model = tf.keras.Sequential([\n            tf.keras.layers.LSTM(32, return_sequences=True, input_shape=input_shape),\n            tf.keras.layers.LSTM(16, return_sequences=False),\n            tf.keras.layers.Dense(1)\n        ])\n        lstm_model.compile(optimizer='adam', loss='mse', metrics=['mae'])\n\ndef train_lstm_model(df, epochs=1, batch_size=32):\n    \"\"\"Prepares data and trains the LSTM model.\"\"\"\n    global scaler, lstm_model\n\n    # Drop NaN values\n    df = df.dropna()\n\n    # Extract features and target\n    features = df.filter(like=\"feature_\", axis=1)  # Keep only feature columns\n    target = df['responder_6']\n\n    # Normalize features\n    scaler = MinMaxScaler()\n    features_scaled = scaler.fit_transform(features)\n\n    # Split data\n    X_train, X_test, y_train, y_test = train_test_split(\n        features_scaled, target, test_size=0.2, random_state=42\n    )\n\n    # Further split for validation\n    X_train, X_val, y_train, y_val = train_test_split(\n        X_train, y_train, test_size=0.2, random_state=42\n    )\n\n    # Reshape for LSTM input\n    X_train = np.expand_dims(X_train, axis=-1)\n    X_val = np.expand_dims(X_val, axis=-1)\n\n    # Initialize the model\n    initialize_lstm(input_shape=X_train.shape[1:])\n\n    # Train the model\n    lstm_model.fit(\n        X_train, y_train,\n        validation_data=(X_val, y_val),\n        epochs=epochs,\n        batch_size=batch_size,\n        verbose=1\n    )\n\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    global lags_, lstm_model, scaler\n\n    # Convert test and lag data to Pandas\n    if isinstance(test, pl.DataFrame):\n        test = test.to_pandas()\n    if lags is not None and isinstance(lags, pl.DataFrame):\n        lags = lags.to_pandas()\n\n    # Update lags for subsequent batches\n    if lags is not None:\n        lags_ = lags\n\n    # Ensure the model is trained\n    if lstm_model is None:\n        print(\"Error: LSTM model is not initialized or trained.\")\n        return pd.DataFrame({'row_id': test['row_id'], 'responder_6': [np.nan] * len(test)})\n\n    # Prepare test features\n    try:\n        test_features = test.filter(like=\"feature_\", axis=1)  # Keep only feature columns\n        if scaler is not None:\n            test_features = scaler.transform(test_features)\n        test_features = np.expand_dims(test_features, axis=-1)  # Reshape for LSTM\n    except Exception as e:\n        print(\"Error during test feature preparation:\", e)\n        return pd.DataFrame({'row_id': test['row_id'], 'responder_6': [np.nan] * len(test)})\n\n    # Debugging: Check input shape and NaN\n    print(\"Input shape for LSTM:\", test_features.shape)\n    if np.isnan(test_features).any():\n        print(\"Error: NaN values found in test_features\")\n        print(test_features)\n        test_features = np.nan_to_num(test_features)  # Replace NaN with zeros\n\n    # Make predictions\n    try:\n        predictions = lstm_model.predict(test_features)\n        print(\"Predictions:\", predictions)\n    except Exception as e:\n        print(\"Error during prediction:\", e)\n        return pd.DataFrame({'row_id': test['row_id'], 'responder_6': [np.nan] * len(test)})\n\n    # Format predictions for Kaggle submission\n    result = pd.DataFrame({\n        'row_id': test['row_id'],\n        'responder_6': predictions.flatten()\n    })\n\n    return result\n\n\n\n# Main function for Kaggle submission\ndef main():\n    # Load training data and train the LSTM model\n    train_data_path = '/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=9/part-0.parquet'\n    df = pd.read_parquet(train_data_path)\n    train_lstm_model(df, epochs=3, batch_size=32)\n\n    # Set up Kaggle inference server\n    inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\n    if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n        inference_server.serve()\n    else:\n        inference_server.run_local_gateway(\n            [\n                '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n                '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet'\n            ]\n        )\n\n# Run the main function\nif __name__ == \"__main__\":\n    main()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}