{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-20T03:07:16.589732Z","iopub.execute_input":"2024-10-20T03:07:16.590612Z","iopub.status.idle":"2024-10-20T03:07:17.70737Z","shell.execute_reply.started":"2024-10-20T03:07:16.590568Z","shell.execute_reply":"2024-10-20T03:07:17.706067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\nimport pandas as pd\nimport numpy as np\nfrom sklearn.linear_model import Ridge\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport kaggle_evaluation.jane_street_inference_server\n\nimport random\n\ndef seed_everything(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n\nseed_everything(seed=2024)\n\ntarget = \"responder_6\"\nop_path = f\"/kaggle/working\"\nip_path = f\"/kaggle/input/janestreet2024-dataload-v1\"\nstate = 42\nmethod = \"LGBM1R\"","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:17.70954Z","iopub.execute_input":"2024-10-20T03:07:17.71013Z","iopub.status.idle":"2024-10-20T03:07:20.176724Z","shell.execute_reply.started":"2024-10-20T03:07:17.71008Z","shell.execute_reply":"2024-10-20T03:07:20.175587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pl.read_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=9/part-0.parquet\")\ntrain=train.to_pandas()\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:20.178076Z","iopub.execute_input":"2024-10-20T03:07:20.178536Z","iopub.status.idle":"2024-10-20T03:07:29.95707Z","shell.execute_reply.started":"2024-10-20T03:07:20.178502Z","shell.execute_reply":"2024-10-20T03:07:29.956064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"symbol_counts = train['symbol_id'].value_counts()\nplt.bar(symbol_counts.index, symbol_counts.values)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:29.959734Z","iopub.execute_input":"2024-10-20T03:07:29.960113Z","iopub.status.idle":"2024-10-20T03:07:30.306661Z","shell.execute_reply.started":"2024-10-20T03:07:29.960075Z","shell.execute_reply":"2024-10-20T03:07:30.305484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weight analysis","metadata":{}},{"cell_type":"markdown","source":"Portfolio weight refers to the proportion of an individual asset’s value relative to the total value of a portfolio. It represents how much of the total investment is allocated to a specific asset. Portfolio weights are typically expressed as percentages and help in determining the level of exposure to each asset in a portfolio.\n\n\nFormula:\nThe weight of an asset in a portfolio is calculated using the following formula:\n\nPortfolio Weight of Asset\n=\nValue of the Asset\nTotal Value of the Portfolio\nPortfolio Weight of Asset= \nTotal Value of the Portfolio\nValue of the Asset\n​\t\n ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 6))\nplt.plot(train['weight'][:500])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:30.307904Z","iopub.execute_input":"2024-10-20T03:07:30.308259Z","iopub.status.idle":"2024-10-20T03:07:30.596567Z","shell.execute_reply.started":"2024-10-20T03:07:30.308223Z","shell.execute_reply":"2024-10-20T03:07:30.59525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n\nsns.histplot(train['weight'], bins=50)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:30.598043Z","iopub.execute_input":"2024-10-20T03:07:30.598479Z","iopub.status.idle":"2024-10-20T03:07:35.276713Z","shell.execute_reply.started":"2024-10-20T03:07:30.598431Z","shell.execute_reply":"2024-10-20T03:07:35.275608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pairs with highest wieght overtime","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 6))\naggregated = train.groupby('symbol_id')[['weight']].agg(['sum'])\nplt.bar(aggregated.index, aggregated['weight']['sum'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:35.278073Z","iopub.execute_input":"2024-10-20T03:07:35.278417Z","iopub.status.idle":"2024-10-20T03:07:35.796358Z","shell.execute_reply.started":"2024-10-20T03:07:35.278381Z","shell.execute_reply":"2024-10-20T03:07:35.79506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group by 'symbol_id' and count unique 'weight' values\nunique_weights_per_symbol = train.groupby('symbol_id')['weight'].nunique()\n\n# Display the result\nprint(unique_weights_per_symbol)","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:13:28.395086Z","iopub.execute_input":"2024-10-20T03:13:28.396422Z","iopub.status.idle":"2024-10-20T03:13:28.687724Z","shell.execute_reply.started":"2024-10-20T03:13:28.396318Z","shell.execute_reply":"2024-10-20T03:13:28.686599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Distribution of weights over time","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 6))\naggregated = train.groupby('date_id')[['weight']].agg(['sum'])\nplt.bar(aggregated.index, aggregated['weight']['sum'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:35.797933Z","iopub.execute_input":"2024-10-20T03:07:35.798427Z","iopub.status.idle":"2024-10-20T03:07:36.353388Z","shell.execute_reply.started":"2024-10-20T03:07:35.798376Z","shell.execute_reply":"2024-10-20T03:07:36.352376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 6))\naggregated = train.groupby('time_id')[['weight']].agg(['sum'])\nplt.bar(aggregated.index, aggregated['weight']['sum'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:16:59.545068Z","iopub.execute_input":"2024-10-20T03:16:59.54586Z","iopub.status.idle":"2024-10-20T03:17:01.202901Z","shell.execute_reply.started":"2024-10-20T03:16:59.545813Z","shell.execute_reply":"2024-10-20T03:17:01.201721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min(train['time_id'])","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:17:56.677356Z","iopub.execute_input":"2024-10-20T03:17:56.677784Z","iopub.status.idle":"2024-10-20T03:17:57.263681Z","shell.execute_reply.started":"2024-10-20T03:17:56.677742Z","shell.execute_reply":"2024-10-20T03:17:57.262551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max(train['time_id'])","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:18:13.929181Z","iopub.execute_input":"2024-10-20T03:18:13.929607Z","iopub.status.idle":"2024-10-20T03:18:14.513819Z","shell.execute_reply.started":"2024-10-20T03:18:13.929566Z","shell.execute_reply":"2024-10-20T03:18:14.512692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max(train['date_id'])","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:23:01.000288Z","iopub.execute_input":"2024-10-20T03:23:01.001388Z","iopub.status.idle":"2024-10-20T03:23:01.625879Z","shell.execute_reply.started":"2024-10-20T03:23:01.001298Z","shell.execute_reply":"2024-10-20T03:23:01.624598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min(train['date_id'])","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:23:30.519135Z","iopub.execute_input":"2024-10-20T03:23:30.519921Z","iopub.status.idle":"2024-10-20T03:23:31.12431Z","shell.execute_reply.started":"2024-10-20T03:23:30.519877Z","shell.execute_reply":"2024-10-20T03:23:31.123069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a pivot table\npivot_table = train.pivot_table(index='date_id', columns='time_id', values='symbol_id', aggfunc='count', fill_value=0)\n\nplt.figure(figsize=(12, 10))\nsns.heatmap(pivot_table, cmap='viridis')\nplt.title('Heatmap of Record Counts per date_id and time_id')\nplt.xlabel('time_id')\nplt.ylabel('date_id')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:20:28.159618Z","iopub.execute_input":"2024-10-20T03:20:28.160646Z","iopub.status.idle":"2024-10-20T03:20:29.670891Z","shell.execute_reply.started":"2024-10-20T03:20:28.160594Z","shell.execute_reply":"2024-10-20T03:20:29.669833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.kdeplot(train['time_id'], shade=True, color='blue', label='time_id')\nsns.kdeplot(train['date_id'], shade=True, color='red', label='date_id')\nplt.title('Overlayed Density Plots of time_id and date_id')\nplt.xlabel('Value')\nplt.ylabel('Density')\nplt.legend()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:21:23.154742Z","iopub.execute_input":"2024-10-20T03:21:23.155164Z","iopub.status.idle":"2024-10-20T03:22:18.562144Z","shell.execute_reply.started":"2024-10-20T03:21:23.155126Z","shell.execute_reply":"2024-10-20T03:22:18.561001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(20, 6))\n\n# Histogram of time_id\naxes[0].hist(train['time_id'], bins=50, color='skyblue', edgecolor='black')\naxes[0].set_title('Distribution of time_id')\naxes[0].set_xlabel('time_id')\naxes[0].set_ylabel('Frequency')\n\n# Histogram of date_id\naxes[1].hist(train['date_id'], bins=50, color='salmon', edgecolor='black')\naxes[1].set_title('Distribution of date_id')\naxes[1].set_xlabel('date_id')\naxes[1].set_ylabel('Frequency')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:27:13.025675Z","iopub.execute_input":"2024-10-20T03:27:13.026719Z","iopub.status.idle":"2024-10-20T03:27:13.9322Z","shell.execute_reply.started":"2024-10-20T03:27:13.026665Z","shell.execute_reply":"2024-10-20T03:27:13.931159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter the DataFrame\nsubset = train[(train['date_id'] == 1636) & (train['symbol_id'] == 4)]\n\n# Check the number of records in the subset\nprint(f\"Number of records for date_id=1600 and symbol_id=1: {len(subset)}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:37:52.042148Z","iopub.execute_input":"2024-10-20T03:37:52.043005Z","iopub.status.idle":"2024-10-20T03:37:52.059957Z","shell.execute_reply.started":"2024-10-20T03:37:52.042942Z","shell.execute_reply":"2024-10-20T03:37:52.058708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:37:54.210303Z","iopub.execute_input":"2024-10-20T03:37:54.210715Z","iopub.status.idle":"2024-10-20T03:37:54.239598Z","shell.execute_reply.started":"2024-10-20T03:37:54.210674Z","shell.execute_reply":"2024-10-20T03:37:54.238315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.figure(figsize=(12, 6))\nplt.hist(subset['time_id'], bins=30, color='skyblue', edgecolor='black')\nplt.title(f'Distribution of time_id for date_id {date_id_value}')\nplt.xlabel('time_id')\nplt.ylabel('Frequency')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:38:29.687772Z","iopub.execute_input":"2024-10-20T03:38:29.68866Z","iopub.status.idle":"2024-10-20T03:38:30.00534Z","shell.execute_reply.started":"2024-10-20T03:38:29.688608Z","shell.execute_reply":"2024-10-20T03:38:30.004039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"STD of weight is\", train['weight'].std())","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:36.354876Z","iopub.execute_input":"2024-10-20T03:07:36.355281Z","iopub.status.idle":"2024-10-20T03:07:36.407783Z","shell.execute_reply.started":"2024-10-20T03:07:36.355238Z","shell.execute_reply":"2024-10-20T03:07:36.406619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 6))\nplt.scatter(train['weight'][:15000], train['responder_6'][:15000])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:36.411493Z","iopub.execute_input":"2024-10-20T03:07:36.411828Z","iopub.status.idle":"2024-10-20T03:07:36.657013Z","shell.execute_reply.started":"2024-10-20T03:07:36.411796Z","shell.execute_reply":"2024-10-20T03:07:36.655913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Responder analysis","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 6))\nplt.plot(train['responder_6'][:5000])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:36.658153Z","iopub.execute_input":"2024-10-20T03:07:36.658479Z","iopub.status.idle":"2024-10-20T03:07:36.955462Z","shell.execute_reply.started":"2024-10-20T03:07:36.658445Z","shell.execute_reply":"2024-10-20T03:07:36.95431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['responder_6'].describe()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:36.956847Z","iopub.execute_input":"2024-10-20T03:07:36.95725Z","iopub.status.idle":"2024-10-20T03:07:37.201953Z","shell.execute_reply.started":"2024-10-20T03:07:36.957205Z","shell.execute_reply":"2024-10-20T03:07:37.200704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Responder has scaled between [-5, 5] with a mean score of -3.7","metadata":{}},{"cell_type":"code","source":"sns.histplot(train['responder_6'], bins=50)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:37.203298Z","iopub.execute_input":"2024-10-20T03:07:37.203739Z","iopub.status.idle":"2024-10-20T03:07:41.793945Z","shell.execute_reply.started":"2024-10-20T03:07:37.203692Z","shell.execute_reply":"2024-10-20T03:07:41.792841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the cumulative sum\nplt.figure(figsize=(16, 6))\ntrain['cumulative_sum'] = train['responder_6'].cumsum()\nplt.plot(train['cumulative_sum'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:41.795159Z","iopub.execute_input":"2024-10-20T03:07:41.795465Z","iopub.status.idle":"2024-10-20T03:07:42.720358Z","shell.execute_reply.started":"2024-10-20T03:07:41.795433Z","shell.execute_reply":"2024-10-20T03:07:42.719225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the cumulative sum\ntrain['cumulative_sum'] = train['weight'].cumsum()\nplt.plot(train['cumulative_sum'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:42.721641Z","iopub.execute_input":"2024-10-20T03:07:42.722007Z","iopub.status.idle":"2024-10-20T03:07:43.552464Z","shell.execute_reply.started":"2024-10-20T03:07:42.721951Z","shell.execute_reply":"2024-10-20T03:07:43.55137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Features analysis","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 6))\nplt.plot(train['feature_00'][:500])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:43.553827Z","iopub.execute_input":"2024-10-20T03:07:43.554177Z","iopub.status.idle":"2024-10-20T03:07:43.794923Z","shell.execute_reply.started":"2024-10-20T03:07:43.554142Z","shell.execute_reply":"2024-10-20T03:07:43.793655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(train['feature_00'], bins=50)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:43.79633Z","iopub.execute_input":"2024-10-20T03:07:43.796703Z","iopub.status.idle":"2024-10-20T03:07:48.369996Z","shell.execute_reply.started":"2024-10-20T03:07:43.796667Z","shell.execute_reply":"2024-10-20T03:07:48.36875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 6))\nsns.histplot(train['date_id'], bins=50)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:48.371471Z","iopub.execute_input":"2024-10-20T03:07:48.371899Z","iopub.status.idle":"2024-10-20T03:07:52.918256Z","shell.execute_reply.started":"2024-10-20T03:07:48.371852Z","shell.execute_reply":"2024-10-20T03:07:52.917202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['feature_00_cumulative_sum'] = train['feature_00'].cumsum()\nplt.plot(train['feature_00_cumulative_sum'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:52.920086Z","iopub.execute_input":"2024-10-20T03:07:52.920581Z","iopub.status.idle":"2024-10-20T03:07:53.760619Z","shell.execute_reply.started":"2024-10-20T03:07:52.920525Z","shell.execute_reply":"2024-10-20T03:07:53.759557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(train['feature_01'], bins=50)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:53.761938Z","iopub.execute_input":"2024-10-20T03:07:53.762313Z","iopub.status.idle":"2024-10-20T03:07:58.383678Z","shell.execute_reply.started":"2024-10-20T03:07:53.762277Z","shell.execute_reply":"2024-10-20T03:07:58.382653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 6))\ntrain['feature_01_cumulative_sum'] = train['feature_01'].cumsum()\nplt.plot(train['feature_01_cumulative_sum'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:58.385244Z","iopub.execute_input":"2024-10-20T03:07:58.386131Z","iopub.status.idle":"2024-10-20T03:07:59.302655Z","shell.execute_reply.started":"2024-10-20T03:07:58.386082Z","shell.execute_reply":"2024-10-20T03:07:59.301548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Explanetory Data Analysis","metadata":{}},{"cell_type":"code","source":"features_agg_sum = pd.DataFrame(train.iloc[:, 4:70].sum(), columns=['Value'])\nfeatures_agg_sum.plot(figsize=(24, 6))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:07:59.304451Z","iopub.execute_input":"2024-10-20T03:07:59.30492Z","iopub.status.idle":"2024-10-20T03:08:01.153449Z","shell.execute_reply.started":"2024-10-20T03:07:59.304868Z","shell.execute_reply":"2024-10-20T03:08:01.152418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['time_id'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:08:01.154835Z","iopub.execute_input":"2024-10-20T03:08:01.155271Z","iopub.status.idle":"2024-10-20T03:08:01.215031Z","shell.execute_reply.started":"2024-10-20T03:08:01.155227Z","shell.execute_reply":"2024-10-20T03:08:01.214122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=train[train['time_id']==0]\ndisplay(df)","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:08:01.216427Z","iopub.execute_input":"2024-10-20T03:08:01.216857Z","iopub.status.idle":"2024-10-20T03:08:01.256947Z","shell.execute_reply.started":"2024-10-20T03:08:01.216812Z","shell.execute_reply":"2024-10-20T03:08:01.255946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols=train.columns.tolist()\n\nfor col in cols[3:]:\n    plt.figure(figsize=(12, 6)) \n    for symbol in df['symbol_id'].unique():\n        subset = df[df['symbol_id'] == symbol]\n        plt.plot(subset['date_id'], subset[col], label=symbol)\n\n    plt.xlabel('Date')\n    plt.ylabel(f'{col} Value')\n    plt.title(f'Symbol-wise {col}')\n    plt.legend(title='Symbol ID', bbox_to_anchor=(1.05, 1), loc='upper left')\n\n    plt.xticks(rotation=45)\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:08:01.258161Z","iopub.execute_input":"2024-10-20T03:08:01.25848Z","iopub.status.idle":"2024-10-20T03:09:09.632996Z","shell.execute_reply.started":"2024-10-20T03:08:01.258447Z","shell.execute_reply":"2024-10-20T03:09:09.631891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Runing the model","metadata":{}},{"cell_type":"code","source":"def custom_metric(y_true,y_pred,weight):\n    weighted_r2=1-(np.sum(weight*(y_true-y_pred)**2)/np.sum(weight*y_true**2))\n    return weighted_r2\nprint(\"read data\")\ntrain=pl.read_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=9/part-0.parquet\")\ntrain=train.to_pandas()\nprint(\"get X,y\")\n#相关性绝对值大于0.01的feature列\ncols=['feature_04', 'feature_06', 'feature_07', 'feature_08', 'feature_15', 'feature_16', 'feature_17', 'feature_19', 'feature_25', 'feature_36', 'feature_45', 'feature_56', 'feature_60', 'feature_66']\nX=train[cols].fillna(3).values\ny=train['responder_6'].values\nprint(\"train test split\")\nsplit=1300000#大约是8:2\nweights=train['weight'].values\ntrain_X,train_y,test_X,test_y,train_weight,test_weight=X[:-split],y[:-split],X[-split:],y[-split:],weights[:-split],weights[-split:]\nprint(f\"train_X.shape:{train_X.shape},test_X.shape:{test_X.shape}\")\nprint(\"fit and predict\")\nmodel=Ridge()\nmodel.fit(train_X,train_y)\ntrain_pred=model.predict(train_X)\ntest_pred=model.predict(test_X)\nprint(f\"train weighted_r2:{custom_metric(train_y,train_pred,weight=train_weight)}\")\nprint(f\"test weighted_r2:{custom_metric(test_y,test_pred,weight=test_weight)}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:09:09.634348Z","iopub.execute_input":"2024-10-20T03:09:09.635403Z","iopub.status.idle":"2024-10-20T03:09:15.734925Z","shell.execute_reply.started":"2024-10-20T03:09:09.635355Z","shell.execute_reply":"2024-10-20T03:09:15.733825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(test,lags):\n    cols=['feature_04', 'feature_06', 'feature_07', 'feature_08', 'feature_15', 'feature_16', 'feature_17', 'feature_19', 'feature_25', 'feature_36', 'feature_45', 'feature_56', 'feature_60', 'feature_66']\n    predictions = test.select(\n        'row_id',\n        pl.lit(0.0).alias('responder_6'),\n    )\n    test_preds=model.predict(test[cols].to_pandas().fillna(3).values)\n    predictions = predictions.with_columns(pl.Series('responder_6', test_preds.ravel()))\n    return predictions\n\ninference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"execution":{"iopub.status.busy":"2024-10-20T03:09:15.736361Z","iopub.execute_input":"2024-10-20T03:09:15.73732Z","iopub.status.idle":"2024-10-20T03:09:15.924795Z","shell.execute_reply.started":"2024-10-20T03:09:15.737267Z","shell.execute_reply":"2024-10-20T03:09:15.923733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}