{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84493,"databundleVersionId":11037875,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Jane Street Market data forecasting notebook \n\n ### Plan:\n\n* Import libraries\n* Data cleaning\n* Vizualization\n* Building model\n* Training\n* Validation\n* Testing\n* Submission ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"###  Note : \n1. include time-series\n2. use only GPU\n3. use Regression algorithms","metadata":{}},{"cell_type":"markdown","source":"## Import libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\n\nfrom torch import nn as NN\nimport keras\nimport tensorflow as tf\n\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nimport lightgbm as lgb\nfrom lightgbm import LGBMRegressor , Booster\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nrandom_seed=42","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-09T15:45:28.93859Z","iopub.execute_input":"2025-02-09T15:45:28.938922Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data intruduction and cleaning","metadata":{}},{"cell_type":"code","source":"basedir= \"/kaggle/input/jane-street-real-time-market-data-forecasting/\"\ntrain_dir = basedir+'train'\ntest_dir = basedir+'test'\nfeatures_dir = basedir+'features.csv'\nresp_dir = basedir+'responders.csv'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T14:30:02.766327Z","iopub.execute_input":"2025-01-05T14:30:02.766929Z","iopub.status.idle":"2025-01-05T14:30:02.770811Z","shell.execute_reply.started":"2025-01-05T14:30:02.766895Z","shell.execute_reply":"2025-01-05T14:30:02.769954Z"}},"outputs":[],"execution_count":2},{"cell_type":"code","source":"tf.config.list_physical_devices()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T14:30:02.773051Z","iopub.execute_input":"2025-01-05T14:30:02.773379Z","iopub.status.idle":"2025-01-05T14:30:02.855705Z","shell.execute_reply.started":"2025-01-05T14:30:02.773352Z","shell.execute_reply":"2025-01-05T14:30:02.854933Z"}},"outputs":[{"execution_count":3,"output_type":"execute_result","data":{"text/plain":"[PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'),\n PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]"},"metadata":{}}],"execution_count":3},{"cell_type":"code","source":"def load_csv(df_dir):\n    df = pd.read_csv(df_dir)\n    return df\ndf_train = load_csv(features_dir)\ndf_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T14:30:02.85655Z","iopub.execute_input":"2025-01-05T14:30:02.856766Z","iopub.status.idle":"2025-01-05T14:30:02.90101Z","shell.execute_reply.started":"2025-01-05T14:30:02.856748Z","shell.execute_reply":"2025-01-05T14:30:02.900081Z"}},"outputs":[{"execution_count":4,"output_type":"execute_result","data":{"text/plain":"       feature  tag_0  tag_1  tag_2  tag_3  tag_4  tag_5  tag_6  tag_7  tag_8  \\\n0   feature_00  False  False   True  False  False  False  False  False  False   \n1   feature_01  False  False   True  False  False  False  False  False  False   \n2   feature_02  False  False   True  False  False  False  False  False  False   \n3   feature_03  False  False   True  False  False  False  False  False  False   \n4   feature_04  False  False   True  False  False  False  False  False  False   \n..         ...    ...    ...    ...    ...    ...    ...    ...    ...    ...   \n74  feature_74  False  False  False  False  False  False  False  False   True   \n75  feature_75  False  False  False  False  False  False  False  False   True   \n76  feature_76  False  False  False  False  False  False  False  False   True   \n77  feature_77  False  False  False  False  False  False  False  False   True   \n78  feature_78  False  False  False  False  False  False  False  False   True   \n\n    tag_9  tag_10  tag_11  tag_12  tag_13  tag_14  tag_15  tag_16  \n0   False   False   False   False   False    True   False    True  \n1   False   False   False   False    True    True   False    True  \n2   False   False   False    True   False   False   False    True  \n3   False   False   False   False    True   False   False    True  \n4   False   False   False    True    True   False   False    True  \n..    ...     ...     ...     ...     ...     ...     ...     ...  \n74  False   False   False   False   False    True   False   False  \n75  False   False   False    True   False   False   False   False  \n76  False   False   False    True   False   False   False   False  \n77  False   False   False   False    True   False   False   False  \n78  False   False   False   False    True   False   False   False  \n\n[79 rows x 18 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>feature</th>\n      <th>tag_0</th>\n      <th>tag_1</th>\n      <th>tag_2</th>\n      <th>tag_3</th>\n      <th>tag_4</th>\n      <th>tag_5</th>\n      <th>tag_6</th>\n      <th>tag_7</th>\n      <th>tag_8</th>\n      <th>tag_9</th>\n      <th>tag_10</th>\n      <th>tag_11</th>\n      <th>tag_12</th>\n      <th>tag_13</th>\n      <th>tag_14</th>\n      <th>tag_15</th>\n      <th>tag_16</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>feature_00</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>feature_01</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>feature_02</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>feature_03</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>feature_04</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>74</th>\n      <td>feature_74</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>75</th>\n      <td>feature_75</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>76</th>\n      <td>feature_76</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>77</th>\n      <td>feature_77</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>78</th>\n      <td>feature_78</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n  </tbody>\n</table>\n<p>79 rows × 18 columns</p>\n</div>"},"metadata":{}}],"execution_count":4},{"cell_type":"code","source":"df_train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T14:30:02.902007Z","iopub.execute_input":"2025-01-05T14:30:02.902326Z","iopub.status.idle":"2025-01-05T14:30:02.929126Z","shell.execute_reply.started":"2025-01-05T14:30:02.902287Z","shell.execute_reply":"2025-01-05T14:30:02.928139Z"}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 79 entries, 0 to 78\nData columns (total 18 columns):\n #   Column   Non-Null Count  Dtype \n---  ------   --------------  ----- \n 0   feature  79 non-null     object\n 1   tag_0    79 non-null     bool  \n 2   tag_1    79 non-null     bool  \n 3   tag_2    79 non-null     bool  \n 4   tag_3    79 non-null     bool  \n 5   tag_4    79 non-null     bool  \n 6   tag_5    79 non-null     bool  \n 7   tag_6    79 non-null     bool  \n 8   tag_7    79 non-null     bool  \n 9   tag_8    79 non-null     bool  \n 10  tag_9    79 non-null     bool  \n 11  tag_10   79 non-null     bool  \n 12  tag_11   79 non-null     bool  \n 13  tag_12   79 non-null     bool  \n 14  tag_13   79 non-null     bool  \n 15  tag_14   79 non-null     bool  \n 16  tag_15   79 non-null     bool  \n 17  tag_16   79 non-null     bool  \ndtypes: bool(17), object(1)\nmemory usage: 2.1+ KB\n","output_type":"stream"}],"execution_count":5},{"cell_type":"code","source":"df_resp = load_csv(resp_dir)\ndf_resp","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T14:30:02.929944Z","iopub.execute_input":"2025-01-05T14:30:02.930242Z","iopub.status.idle":"2025-01-05T14:30:02.943509Z","shell.execute_reply.started":"2025-01-05T14:30:02.930221Z","shell.execute_reply":"2025-01-05T14:30:02.942829Z"}},"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"     responder  tag_0  tag_1  tag_2  tag_3  tag_4\n0  responder_0   True  False   True  False  False\n1  responder_1   True  False  False   True  False\n2  responder_2   True   True  False  False  False\n3  responder_3  False  False   True  False   True\n4  responder_4  False  False  False   True   True\n5  responder_5  False   True  False  False   True\n6  responder_6  False  False   True  False  False\n7  responder_7  False  False  False   True  False\n8  responder_8  False   True  False  False  False","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>responder</th>\n      <th>tag_0</th>\n      <th>tag_1</th>\n      <th>tag_2</th>\n      <th>tag_3</th>\n      <th>tag_4</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>responder_0</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>responder_1</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>responder_2</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>responder_3</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>responder_4</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>responder_5</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>responder_6</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>responder_7</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>responder_8</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}],"execution_count":6},{"cell_type":"code","source":"df_resp.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T14:36:17.956258Z","iopub.execute_input":"2025-01-05T14:36:17.95658Z","iopub.status.idle":"2025-01-05T14:36:17.966136Z","shell.execute_reply.started":"2025-01-05T14:36:17.956555Z","shell.execute_reply":"2025-01-05T14:36:17.965297Z"}},"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 9 entries, 0 to 8\nData columns (total 6 columns):\n #   Column     Non-Null Count  Dtype \n---  ------     --------------  ----- \n 0   responder  9 non-null      object\n 1   tag_0      9 non-null      bool  \n 2   tag_1      9 non-null      bool  \n 3   tag_2      9 non-null      bool  \n 4   tag_3      9 non-null      bool  \n 5   tag_4      9 non-null      bool  \ndtypes: bool(5), object(1)\nmemory usage: 245.0+ bytes\n","output_type":"stream"}],"execution_count":7},{"cell_type":"code","source":"\nclass Partition:\n    def __init__(self, nums):\n        self.nums = nums\n        sorted = [_Num(nums[i], i) for i in range(len(nums))]\n        sorted.sort()\n        self.sorted = sorted\n\n    def run(self):\n        sorted = self.sorted[:]\n        N = len(sorted)\n        connections = [[] for i in range(N)]\n\n        while len(sorted) > 1:\n            bigger  = sorted.pop()\n            smaller = sorted.pop()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-09T15:45:02.23915Z","iopub.execute_input":"2025-02-09T15:45:02.239478Z","iopub.status.idle":"2025-02-09T15:45:02.24605Z","shell.execute_reply.started":"2025-02-09T15:45:02.23941Z","shell.execute_reply":"2025-02-09T15:45:02.245082Z"}},"outputs":[],"execution_count":1}]}