{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":9801075,"sourceType":"datasetVersion","datasetId":6006872},{"sourceId":9806342,"sourceType":"datasetVersion","datasetId":6010899},{"sourceId":9835899,"sourceType":"datasetVersion","datasetId":6033335},{"sourceId":10139918,"sourceType":"datasetVersion","datasetId":6258261},{"sourceId":10139922,"sourceType":"datasetVersion","datasetId":6258265},{"sourceId":10253875,"sourceType":"datasetVersion","datasetId":6297065},{"sourceId":10304887,"sourceType":"datasetVersion","datasetId":6378806},{"sourceId":10351700,"sourceType":"datasetVersion","datasetId":6410107},{"sourceId":10452398,"sourceType":"datasetVersion","datasetId":6470240},{"sourceId":203900450,"sourceType":"kernelVersion"},{"sourceId":214889877,"sourceType":"kernelVersion"},{"sourceId":216017958,"sourceType":"kernelVersion"},{"sourceId":216577393,"sourceType":"kernelVersion"},{"sourceId":217425505,"sourceType":"kernelVersion"},{"sourceId":219922,"sourceType":"modelInstanceVersion","modelInstanceId":187554,"modelId":209619}],"dockerImageVersionId":30822,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install rtdl_num_embeddings -q --no-index --find-links=/kaggle/input/jane-street-import/rtdl_num_embeddings","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:30:58.634415Z","iopub.execute_input":"2025-01-13T11:30:58.634808Z","iopub.status.idle":"2025-01-13T11:31:03.410042Z","shell.execute_reply.started":"2025-01-13T11:30:58.634776Z","shell.execute_reply":"2025-01-13T11:31:03.408454Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os , gc\nimport pandas as pd\nimport numpy as np\nimport polars as pl\nfrom matplotlib import pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom pytorch_lightning import (LightningDataModule, LightningModule, Trainer)\n\nfrom sklearn.metrics import r2_score\n\nimport torch.optim\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\nfrom sklearn.model_selection import train_test_split\nimport math\nfrom tqdm import tqdm\nfrom collections import OrderedDict\nfrom my_utility import Model, make_parameter_groups\n\nimport warnings\nimport joblib\nfrom pytorch_lightning.callbacks import Callback\n\nimport lightgbm as lgb\nfrom lightgbm import LGBMRegressor, Booster\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:31:05.884899Z","iopub.execute_input":"2025-01-13T11:31:05.885338Z","iopub.status.idle":"2025-01-13T11:31:12.861847Z","shell.execute_reply.started":"2025-01-13T11:31:05.885298Z","shell.execute_reply":"2025-01-13T11:31:12.860673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ROOT_DIR = \"/kaggle/input/jane-street-real-time-market-data-forecasting/\"","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:31:17.524816Z","iopub.execute_input":"2025-01-13T11:31:17.525591Z","iopub.status.idle":"2025-01-13T11:31:17.529721Z","shell.execute_reply.started":"2025-01-13T11:31:17.525552Z","shell.execute_reply":"2025-01-13T11:31:17.528726Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#listall files and directories\nfor root, dirs, files in os.walk(ROOT_DIR):\n    print(f\"Directory: {root}\")\n    for file in files:\n        print(f\"  File: {file}\")","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:31:18.050914Z","iopub.execute_input":"2025-01-13T11:31:18.051325Z","iopub.status.idle":"2025-01-13T11:31:18.108666Z","shell.execute_reply.started":"2025-01-13T11:31:18.051287Z","shell.execute_reply":"2025-01-13T11:31:18.107615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features = pd.read_csv(f\"{ROOT_DIR}/features.csv\")\nfeatures","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:31:18.654129Z","iopub.execute_input":"2025-01-13T11:31:18.654567Z","iopub.status.idle":"2025-01-13T11:31:18.705844Z","shell.execute_reply.started":"2025-01-13T11:31:18.654503Z","shell.execute_reply":"2025-01-13T11:31:18.704313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(20, 10))\nplt.imshow(features.iloc[:, 1:].T.values, cmap=\"gray\")\nplt.xlabel(\"feature_00  ~  feature_78\")\nplt.ylabel(\"tag_0  ~  tag_16\")\nplt.yticks(np.arange(17))\nplt.xticks(np.arange(79))\nplt.grid()\nplt.show()","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:31:20.653085Z","iopub.execute_input":"2025-01-13T11:31:20.653420Z","iopub.status.idle":"2025-01-13T11:31:21.378654Z","shell.execute_reply.started":"2025-01-13T11:31:20.653394Z","shell.execute_reply":"2025-01-13T11:31:21.377350Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nsns.heatmap(features[[ f\"tag_{no}\" for no in range(0,17,1) ] ].T.corr(), square=True, cmap=\"jet\")\nplt.xlabel(\"feature_0  ~  feature_78\")\nplt.ylabel(\"feature_0  ~  feature_78\")\nplt.show()","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:31:22.053097Z","iopub.execute_input":"2025-01-13T11:31:22.053440Z","iopub.status.idle":"2025-01-13T11:31:22.750687Z","shell.execute_reply.started":"2025-01-13T11:31:22.053413Z","shell.execute_reply":"2025-01-13T11:31:22.749415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"responders = pd.read_csv(f\"{ROOT_DIR}/responders.csv\")\nresponders","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:31:23.123686Z","iopub.execute_input":"2025-01-13T11:31:23.124047Z","iopub.status.idle":"2025-01-13T11:31:23.141131Z","shell.execute_reply.started":"2025-01-13T11:31:23.124015Z","shell.execute_reply":"2025-01-13T11:31:23.139822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.heatmap(responders[[ f\"tag_{no}\" for no in range(0,5,1) ] ].T.corr(), annot=True, square=True, cmap=\"jet\")\nplt.xlabel(\"responder_0  ~  responder_8\")\nplt.ylabel(\"responder_0  ~  responder_8\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:31:23.835497Z","iopub.execute_input":"2025-01-13T11:31:23.836040Z","iopub.status.idle":"2025-01-13T11:31:24.288753Z","shell.execute_reply.started":"2025-01-13T11:31:23.836002Z","shell.execute_reply":"2025-01-13T11:31:24.287298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub = pd.read_csv(f\"{ROOT_DIR}/sample_submission.csv\")\nprint( f\"sub.shape = {sub.shape}\")\nsub","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:31:24.352117Z","iopub.execute_input":"2025-01-13T11:31:24.352480Z","iopub.status.idle":"2025-01-13T11:31:24.370586Z","shell.execute_reply.started":"2025-01-13T11:31:24.352449Z","shell.execute_reply":"2025-01-13T11:31:24.369578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = (\n    pl.read_parquet(f\"{ROOT_DIR}/train.parquet/partition_id=0/part-0.parquet\")\n)\ntrain.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:31:24.405917Z","iopub.execute_input":"2025-01-13T11:31:24.406286Z","iopub.status.idle":"2025-01-13T11:31:26.603061Z","shell.execute_reply.started":"2025-01-13T11:31:24.406255Z","shell.execute_reply":"2025-01-13T11:31:26.602007Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train.head())\nprint(str(train.columns))","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:31:26.604335Z","iopub.execute_input":"2025-01-13T11:31:26.604637Z","iopub.status.idle":"2025-01-13T11:31:26.620223Z","shell.execute_reply.started":"2025-01-13T11:31:26.604613Z","shell.execute_reply":"2025-01-13T11:31:26.618950Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(15, 15))\nsns.heatmap(train[[ f\"feature_{target:02d}\" for target in range(79)]].corr(), square=True, cmap=\"jet\")\nplt.xlabel(\"feature_00  ~  feature_78\")\nplt.ylabel(\"feature_00  ~  feature_78\")\nplt.grid()\nplt.show()\nplt.figure(figsize=(15, 15))\nsns.heatmap(train[[ f\"responder_{target}\" for target in range(9)]].corr(), square=True, cmap=\"jet\")\nplt.xlabel(\"responder_0  ~  responder_8\")\nplt.ylabel(\"responder_0  ~  responder_8\")\nplt.grid()\nplt.show()","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:31:26.621864Z","iopub.execute_input":"2025-01-13T11:31:26.622152Z","iopub.status.idle":"2025-01-13T11:31:29.715857Z","shell.execute_reply.started":"2025-01-13T11:31:26.622127Z","shell.execute_reply":"2025-01-13T11:31:29.714756Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for target in range(9):\n    col = f\"responder_{target}\"\n    mean_, sgm_ = train[col].mean(), np.sqrt(train[col].var())\n    min_, max_ = train[col].min(), train[col].max()\n    print(\"-\" * 30)\n    print( f\"column = {col}\" )\n    print( f\" - mean  : {mean_:.4f}\",  )\n    print( f\" - sigma : {sgm_:.4f}\",  )\n    print( f\" - min  : {min_:.4f}\",  )\n    print( f\" - max  : {max_:.4f}\",  )\n    \n    plt.hist(train[col], bins=20)\n    plt.xlabel(col)\n    plt.ylabel(\"frequency / records\")\n    #plt.yscale(\"log\")\n    plt.grid()\n    plt.show()","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:31:29.717137Z","iopub.execute_input":"2025-01-13T11:31:29.717406Z","iopub.status.idle":"2025-01-13T11:31:32.059803Z","shell.execute_reply.started":"2025-01-13T11:31:29.717384Z","shell.execute_reply":"2025-01-13T11:31:32.058402Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for partition_id in range(10):\n    print(f\"> train.parquet/partition_id={partition_id}/part-0.parquet\")\n    train_data = pl.read_parquet(f\"{ROOT_DIR}/train.parquet/partition_id={partition_id}/part-0.parquet\")\n\n    print( f\"symbol_id: \", train_data[\"symbol_id\"].min(), \"-\", train_data[\"symbol_id\"].max())\n    bins = train_data[\"symbol_id\"].max() - train_data[\"symbol_id\"].min() + 1\n    plt.hist(train_data[\"symbol_id\"], bins=bins)\n    plt.xlabel(\"symbol_id\")\n    plt.ylabel(\"frequency / records\")\n    plt.grid()\n    plt.show()","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:31:32.061005Z","iopub.execute_input":"2025-01-13T11:31:32.061685Z","iopub.status.idle":"2025-01-13T11:32:20.768295Z","shell.execute_reply.started":"2025-01-13T11:31:32.061645Z","shell.execute_reply":"2025-01-13T11:32:20.767188Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for partition_id in range(10):\n    print(f\"> train.parquet/partition_id={partition_id}/part-0.parquet\")\n    train_data = pl.read_parquet(f\"{ROOT_DIR}/train.parquet/partition_id={partition_id}/part-0.parquet\")\n\n\n    print( f\"date_id: \", train_data[\"date_id\"].min(), \"-\", train_data[\"date_id\"].max())\n    bins = train_data[\"date_id\"].max() - train_data[\"date_id\"].min() + 1\n    plt.hist(train_data[\"date_id\"], bins=bins)\n    plt.xlabel(\"date_id\")\n    plt.ylabel(\"frequency / records\")\n    plt.grid()\n    plt.show()","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:32:20.769261Z","iopub.execute_input":"2025-01-13T11:32:20.769580Z","iopub.status.idle":"2025-01-13T11:32:39.502650Z","shell.execute_reply.started":"2025-01-13T11:32:20.769544Z","shell.execute_reply":"2025-01-13T11:32:39.501243Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = (\n    pl.read_parquet(f\"{ROOT_DIR}/test.parquet/date_id=0/part-0.parquet\")\n)\nprint(test.shape)\ntest","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:32:39.503881Z","iopub.execute_input":"2025-01-13T11:32:39.504234Z","iopub.status.idle":"2025-01-13T11:32:39.535358Z","shell.execute_reply.started":"2025-01-13T11:32:39.504195Z","shell.execute_reply":"2025-01-13T11:32:39.534228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#missin val\nsupervised_usable = (\n    test\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\n\nplt.figure(figsize=(6, 20))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:32:39.538063Z","iopub.execute_input":"2025-01-13T11:32:39.538352Z","iopub.status.idle":"2025-01-13T11:32:40.560009Z","shell.execute_reply.started":"2025-01-13T11:32:39.538327Z","shell.execute_reply":"2025-01-13T11:32:40.558958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags = (\n    pl.read_parquet(f\"{ROOT_DIR}/lags.parquet/date_id=0/part-0.parquet\")\n)\nprint(lags.shape)\nprint(lags.columns)\nlags","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:32:40.562757Z","iopub.execute_input":"2025-01-13T11:32:40.563215Z","iopub.status.idle":"2025-01-13T11:32:40.585244Z","shell.execute_reply.started":"2025-01-13T11:32:40.563171Z","shell.execute_reply":"2025-01-13T11:32:40.584104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(lags[\"responder_6_lag_1\"])\nplt.grid()\nplt.xlabel(\"symbol_id\")\nplt.ylabel(\"responder_6_lag_1\")\nplt.show()","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:32:40.586296Z","iopub.execute_input":"2025-01-13T11:32:40.586713Z","iopub.status.idle":"2025-01-13T11:32:40.799401Z","shell.execute_reply.started":"2025-01-13T11:32:40.586674Z","shell.execute_reply":"2025-01-13T11:32:40.798256Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"subset_features = [f\"feature_{i:02d}\" for i in range(10)]  \n\ntrain = train.with_columns([pl.col(f).cast(pl.Float32) for f in subset_features])\n\ncorrelation_matrix = (\n    train.select(subset_features)\n    .to_pandas()\n    .corr()\n)\n\nplt.figure(figsize=(10, 8))\nsns.heatmap(\n    correlation_matrix,\n    square=True,\n    cmap=\"coolwarm\", \n    annot=False, \n    cbar_kws={\"label\": \"Correlation Coefficient\"}\n)\nplt.title(\"Feature Correlation Heatmap (Subset)\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:32:40.800448Z","iopub.execute_input":"2025-01-13T11:32:40.800877Z","iopub.status.idle":"2025-01-13T11:32:42.017012Z","shell.execute_reply.started":"2025-01-13T11:32:40.800839Z","shell.execute_reply":"2025-01-13T11:32:42.015648Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numeric_cols = [f\"feature_{i:02d}\" for i in range(79)] \nnumeric_data = train.select(numeric_cols).to_pandas()\n\nnumeric_data.hist(\n    figsize=(20, 15), bins=30, color='darkblue', alpha=0.7, grid=False\n)\n\nplt.suptitle(\"Distribution of Features\", fontsize=16)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:32:42.018170Z","iopub.execute_input":"2025-01-13T11:32:42.018577Z","iopub.status.idle":"2025-01-13T11:32:58.196719Z","shell.execute_reply.started":"2025-01-13T11:32:42.018538Z","shell.execute_reply":"2025-01-13T11:32:58.195693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.express as px\n\npairplot_data_sample = train.sample(n=2000, seed=42).select([f\"feature_{i:02d}\" for i in range(5)] + [\"responder_6\"])\n\nfig = px.scatter_matrix(\n    pairplot_data_sample,\n    dimensions=[f\"feature_{i:02d}\" for i in range(5)],\n    color=\"responder_6\",\n    title=\"Scatter Matrix (Pairplot Alternative with Plotly)\"\n)\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:32:58.197798Z","iopub.execute_input":"2025-01-13T11:32:58.198211Z","iopub.status.idle":"2025-01-13T11:32:59.764723Z","shell.execute_reply.started":"2025-01-13T11:32:58.198175Z","shell.execute_reply":"2025-01-13T11:32:59.763347Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"correlation_matrix = train.corr()\nprint(correlation_matrix)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:32:59.765829Z","iopub.execute_input":"2025-01-13T11:32:59.766096Z","iopub.status.idle":"2025-01-13T11:33:03.524113Z","shell.execute_reply.started":"2025-01-13T11:32:59.766074Z","shell.execute_reply":"2025-01-13T11:33:03.518400Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\n\ntrain_pandas = train.to_pandas()\n\ncorrelations = train_pandas.corr()[\"responder_6\"].sort_values(ascending=False)\n\nprint(\"Top positively correlated features:\")\nprint(correlations.head(10))  \n\nprint(\"\\nTop negatively correlated features:\")\nprint(correlations.tail(10))  \n\nthreshold = 0.1 \ntop_features = correlations[correlations.abs() > threshold].index.tolist()\n\ntop_features.remove(\"responder_6\")  \n\nprint(f\"\\nTop features selected based on correlation: {top_features}\")\n\ntest_pandas = test.to_pandas()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:33:03.531225Z","iopub.execute_input":"2025-01-13T11:33:03.532402Z","iopub.status.idle":"2025-01-13T11:33:51.010037Z","shell.execute_reply.started":"2025-01-13T11:33:03.532203Z","shell.execute_reply":"2025-01-13T11:33:51.008634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import r2_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:33:51.011791Z","iopub.execute_input":"2025-01-13T11:33:51.012138Z","iopub.status.idle":"2025-01-13T11:33:51.017624Z","shell.execute_reply.started":"2025-01-13T11:33:51.012099Z","shell.execute_reply":"2025-01-13T11:33:51.016329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"correlation_matrix = train_pandas.corr()\n\nplt.figure(figsize=(15, 12))\nsns.heatmap(\n    correlation_matrix, \n    annot=False,  \n    cmap=\"coolwarm\",\n    vmin=-1, \n    vmax=1\n)\nplt.title(\"Correlation Matrix Heatmap\")\nplt.show()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:33:51.018934Z","iopub.execute_input":"2025-01-13T11:33:51.019294Z","iopub.status.idle":"2025-01-13T11:34:30.524371Z","shell.execute_reply.started":"2025-01-13T11:33:51.019263Z","shell.execute_reply":"2025-01-13T11:34:30.523315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#print(train.collect_schema())  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:30.525642Z","iopub.execute_input":"2025-01-13T11:34:30.525943Z","iopub.status.idle":"2025-01-13T11:34:30.530297Z","shell.execute_reply.started":"2025-01-13T11:34:30.525917Z","shell.execute_reply":"2025-01-13T11:34:30.529340Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import train_test_split\n\nclass CONFIG:\n    target_col = \"responder_6\"\n    lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_rename = {f\"responder_{idx}\": f\"responder_{idx}_lag_1\" for idx in range(9)}\n    valid_ratio = 0.05\n    start_dt = 1100","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:30.531531Z","iopub.execute_input":"2025-01-13T11:34:30.531877Z","iopub.status.idle":"2025-01-13T11:34:30.553779Z","shell.execute_reply.started":"2025-01-13T11:34:30.531846Z","shell.execute_reply":"2025-01-13T11:34:30.552736Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pl.scan_parquet(f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\") \\\n    .select(\n        pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n        pl.all(),\n    ) \\\n    .with_columns(\n        (pl.col(CONFIG.target_col) * 2).cast(pl.Int32).alias(\"label\"),\n    ) \\\n    .filter(\n        pl.col(\"date_id\").gt(CONFIG.start_dt)\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:30.554907Z","iopub.execute_input":"2025-01-13T11:34:30.555289Z","iopub.status.idle":"2025-01-13T11:34:30.581647Z","shell.execute_reply.started":"2025-01-13T11:34:30.555192Z","shell.execute_reply":"2025-01-13T11:34:30.580457Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"''''lags = train.select(pl.col(CONFIG.lag_cols_original))\nlags = lags.rename(CONFIG.lag_cols_rename)\nlags = lags.with_columns(\n    date_id = pl.col('date_id') + 1,  # lag by 1day\n    )\nlags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()  \nlags'''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:30.583028Z","iopub.execute_input":"2025-01-13T11:34:30.583549Z","iopub.status.idle":"2025-01-13T11:34:30.601674Z","shell.execute_reply.started":"2025-01-13T11:34:30.583399Z","shell.execute_reply":"2025-01-13T11:34:30.600363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags = train.select(pl.col(CONFIG.lag_cols_original))\nlags = lags.rename(CONFIG.lag_cols_rename)\nlags = lags.with_columns(\n    date_id=pl.col('date_id') + 1,  # lag by 1 day\n)\n\nfor col in lags.collect_schema().names():\n        if col not in [\"date_id\", \"symbol_id\"]:\n            lags = lags.with_columns(\n                pl.col(col).shift(1).alias(f\"{col}_lag_1\")\n            )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:30.602902Z","iopub.execute_input":"2025-01-13T11:34:30.603375Z","iopub.status.idle":"2025-01-13T11:34:30.667698Z","shell.execute_reply.started":"2025-01-13T11:34:30.603334Z","shell.execute_reply":"2025-01-13T11:34:30.666775Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#train/validationsplit\nlen_train = train.select(pl.col(\"date_id\")).collect().shape[0]\nvalid_records = int(len_train * CONFIG.valid_ratio)\nlen_ofl_mdl = len_train - valid_records\nlast_tr_dt = train.select(pl.col(\"date_id\")).collect().row(len_ofl_mdl)[0]\n\nprint(f\"\\nlen_train = {len_train}\")\nprint(f\"len_ofl_mdl = {len_ofl_mdl}\")\nprint(f\"---> Last offline train date = {last_tr_dt}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:30.673743Z","iopub.execute_input":"2025-01-13T11:34:30.674091Z","iopub.status.idle":"2025-01-13T11:34:30.808987Z","shell.execute_reply.started":"2025-01-13T11:34:30.674063Z","shell.execute_reply":"2025-01-13T11:34:30.808010Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"training_data = train.filter(pl.col(\"date_id\").le(last_tr_dt))\nvalidation_data = train.filter(pl.col(\"date_id\").gt(last_tr_dt))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T13:44:35.966875Z","iopub.status.idle":"2025-01-13T13:44:35.967299Z","shell.execute_reply":"2025-01-13T13:44:35.967151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.collect()  \ntrain = train.join(lags, on=[\"date_id\", \"symbol_id\"], how=\"left\")\nprint(\"Schema after join:\", train.schema())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T13:44:35.862896Z","iopub.execute_input":"2025-01-13T13:44:35.863293Z","iopub.status.idle":"2025-01-13T13:44:35.964139Z","shell.execute_reply.started":"2025-01-13T13:44:35.863252Z","shell.execute_reply":"2025-01-13T13:44:35.962567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler = MinMaxScaler()\nfeature_cols = [col for col in train.collect_schema().names() if col not in [\"date_id\", \"symbol_id\", \"label\"]]\n\ntrain_df = train.collect()\ntrain_features = train_df.select(fearure_cols[:91]).to_numpy()\n\ntrain_features_scaled = scaler.fit_transform(train_features())\nfor idx, col in enumerate(feature_cols):\n    train_df = train_df.with_columns(\n        pl.Series(name=f\"{col}_scaled\", values=train_features_scaled[:, idx])\n    )\n\nvalidation_df = validation_data.collect()\nvalidation_features = validation_df[feature_cols].to_numpy()\nvalidation_features_scaled = scaler.transform(validation_features)\nfor idx, col in enumerate(feature_cols):\n    validation_df= validation_df.with_columns(\n        pl.Series(name=f\"{col}_scaled\", values=validation_features_scaled[:, idx])\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T13:44:35.964880Z","iopub.status.idle":"2025-01-13T13:44:35.965288Z","shell.execute_reply":"2025-01-13T13:44:35.965142Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_scaled_df.write_parquet(\"train_scaled.parquet\", partition_by=\"date_id\")\nvalidation_scaled_df.write_parquet(\"validation_scaled.parquet\", partition_by=\"date_id\")\n\nprint(\"\\nPreprocessing completed. Processed data saved as:\")\nprint(\"- train_scaled.parquet\")\nprint(\"- validation_scaled.parquet\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:31.028660Z","iopub.status.idle":"2025-01-13T11:34:31.028994Z","shell.execute_reply":"2025-01-13T11:34:31.028859Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.concat([train, valid]).reset_index(drop=True)\ntrain.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:31.029785Z","iopub.status.idle":"2025-01-13T11:34:31.030090Z","shell.execute_reply":"2025-01-13T11:34:31.029968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(training_data.explain())\nprint(validation_data.explain())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:31.031019Z","iopub.status.idle":"2025-01-13T11:34:31.031477Z","shell.execute_reply":"2025-01-13T11:34:31.031231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#--------------\ntrain = pl.scan_parquet(\"/kaggle/input/js24-preprocessing-create-lags/training.parquet\").collect().to_pandas()\nvalid = pl.scan_parquet(\"/kaggle/input/js24-preprocessing-create-lags/validation.parquet\").collect().to_pandas()\ntrain.shape, valid.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:31.032107Z","iopub.status.idle":"2025-01-13T11:34:31.032394Z","shell.execute_reply":"2025-01-13T11:34:31.032278Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_model(seed):\n    # XGBoost parameters\n    XGB_Params = {\n        'learning_rate': 0.05,\n        'max_depth': 6,\n        'n_estimators': 200,\n        'subsample': 0.8,\n        'colsample_bytree': 0.8,\n        'reg_alpha': 1,\n        'reg_lambda': 5,\n        'random_state': seed,\n        'tree_method': 'gpu_hist',\n        'device' : 'cuda',\n        'n_gpus' : 2,\n    }\n    \n    XGB_Model = XGBRegressor(**XGB_Params)\n    return XGB_Model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:31.033367Z","iopub.status.idle":"2025-01-13T11:34:31.033766Z","shell.execute_reply":"2025-01-13T11:34:31.033611Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = train[ CONFIG.feature_cols ]\ny_train = train[ CONFIG.target_col ]\nw_train = train[ \"weight\" ]\nX_valid = valid[ CONFIG.feature_cols ]\ny_valid = valid[ CONFIG.target_col ]\nw_valid = valid[ \"weight\" ]\n\nX_train.shape, y_train.shape, w_train.shape, X_valid.shape, y_valid.shape, w_valid.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:31.035149Z","iopub.status.idle":"2025-01-13T11:34:31.035655Z","shell.execute_reply":"2025-01-13T11:34:31.035431Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nmodel = get_model(CONFIG.seed)\nmodel.fit( X_train, y_train, sample_weight=w_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:31.036714Z","iopub.status.idle":"2025-01-13T11:34:31.037151Z","shell.execute_reply":"2025-01-13T11:34:31.036958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred_train1 = model.predict(X_train.iloc[:X_train.shape[0]//2])\ny_pred_train2 = model.predict(X_train.iloc[X_train.shape[0]//2:])\ntrain_score = r2_score(y_train, np.concatenate([y_pred_train1, y_pred_train2], axis=0), sample_weight=w_train )\ntrain_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:31.038056Z","iopub.status.idle":"2025-01-13T11:34:31.038493Z","shell.execute_reply":"2025-01-13T11:34:31.038302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred_valid = model.predict(X_valid)\nvalid_score = r2_score(y_valid, y_pred_valid, sample_weight=w_valid )\nvalid_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:31.039745Z","iopub.status.idle":"2025-01-13T11:34:31.040186Z","shell.execute_reply":"2025-01-13T11:34:31.039983Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_means = { symbol_id : -1 for symbol_id in range(39) }\nfor symbol_id, gdf in train[[\"symbol_id\", CONFIG.target_col]].groupby(\"symbol_id\"):\n    y_mean = gdf[ CONFIG.target_col ].mean()\n    y_means[symbol_id] = y_mean\n    print(f\"symbol_id = {symbol_id}, y_means = {y_mean:.5f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:31.041120Z","iopub.status.idle":"2025-01-13T11:34:31.041586Z","shell.execute_reply":"2025-01-13T11:34:31.041369Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cv_detail = { symbol_id : 0 for symbol_id in range(39) }\nfor symbol_id, gdf in valid.groupby(\"symbol_id\"):\n    X_valid = gdf[ CONFIG.feature_cols ]\n    y_valid = gdf[ CONFIG.target_col ]\n    w_valid = gdf[ \"weight\" ]\n    y_pred_valid = model.predict(X_valid)\n    score = r2_score(y_valid, y_pred_valid, sample_weight=w_valid )\n    cv_detail[symbol_id] = score\n    \n    print(f\"symbol_id = {symbol_id}, score = {score:.5f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:31.042553Z","iopub.status.idle":"2025-01-13T11:34:31.042978Z","shell.execute_reply":"2025-01-13T11:34:31.042791Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sids = list(cv_detail.keys())\nplt.bar(sids, [cv_detail[sid] for sid in sids])\nplt.grid()\nplt.xlabel(\"symbol_id\")\nplt.ylabel(\"CV score\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:31.043988Z","iopub.status.idle":"2025-01-13T11:34:31.044440Z","shell.execute_reply":"2025-01-13T11:34:31.044247Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"result = {\n    \"model\" : model,\n    \"cv\" : valid_score,\n    \"cv_detail\" : cv_detail,\n    \"y_mean\" : y_means,\n}\nwith open(\"result.pkl\", \"wb\") as fp:\n    pickle.dump(result, fp)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T11:34:31.046112Z","iopub.status.idle":"2025-01-13T11:34:31.046474Z","shell.execute_reply":"2025-01-13T11:34:31.046321Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#correlation w responder_6\nif \"responder_6\" in correlation_matrix.columns:\n    correlation_with_responder_6 = correlation_matrix[\"responder_6\"].drop(\"responder_6\").sort_values(ascending=False)\n    top_features = correlation_with_responder_6.head(10).index \n    valid_top_features = [feature for feature in top_features if feature in test.columns]\n    print(f\"Valid top features: {valid_top_features}\")\n    print(f\"Top features correlated with 'responder_6': {list(top_features)}\")\n\n    plt.figure(figsize=(12, 6))\n    sns.barplot(\n        x=correlation_with_responder_6.loc[valid_top_features],\n        y=valid_top_features,\n        palette=\"coolwarm\",\n    )\n    plt.title(\"Top Features Correlated with Responder_6\", fontsize=14)\n    plt.xlabel(\"Correlation Coefficient\")\n    plt.ylabel(\"Features\")\n    plt.show()\n\n    train_data = train_pandas[list(valid_top_features) + [\"responder_6\"]]\n\n    X = train_data.drop(columns=[\"responder_6\"]) \n    y = train_data[\"responder_6\"] \n    from sklearn.impute import SimpleImputer\n\n    imputer = SimpleImputer(strategy='mean')  # Use 'median' if preferred\n    X = pd.DataFrame(imputer.fit_transform(X), columns=X.columns)\n    X = X.fillna(0)\n\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n    scaler = StandardScaler()\n    X_train_scaled = scaler.fit_transform(X_train)\n    X_test_scaled = scaler.transform(X_test)\n\n    model = LinearRegression()\n    model.fit(X_train_scaled, y_train)\n\n    y_pred_train = model.predict(X_train_scaled)\n    y_pred_test = model.predict(X_test_scaled)\n\n    r2_train = r2_score(y_train, y_pred_train)\n    r2_test = r2_score(y_test, y_pred_test)\n\n\n    print(f\"R2 Score (Train): {r2_train:.4f}\")\n    print(f\"R2 Score (Test): {r2_test:.4f}\")\n\n    if hasattr(model, 'coef_'):\n        feature_importance = pd.Series(model.coef_, index=X.columns)\n        feature_importance.sort_values(ascending=False).plot(kind='bar', figsize=(10, 6), title='Feature Importance')\n        plt.show()\n    else:\n        print(\"Model does not support feature importance.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T16:37:58.747847Z","iopub.status.idle":"2025-01-12T16:37:58.748286Z","shell.execute_reply":"2025-01-12T16:37:58.748085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data = test.select(valid_top_features).to_pandas()\n\nprint(test_data.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T16:37:58.749276Z","iopub.status.idle":"2025-01-12T16:37:58.749670Z","shell.execute_reply":"2025-01-12T16:37:58.749520Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntest_data_imputed = imputer.transform(test_data)\ntest_data_imputed = pd.DataFrame(test_data_imputed, columns=test_data.columns)\n\ntest_data_scaled = scaler.transform(test_data_imputed)\n\ntest_predictions = model.predict(test_data_scaled)\n\nsubmission = pd.DataFrame({\n    'row_id': test_pandas['row_id'], \n    'responder_6': test_predictions\n})\nsubmission.to_parquet('submission.parquet', index=False)\nprint(\"Submission file created: submission.parquet\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T16:37:58.750523Z","iopub.status.idle":"2025-01-12T16:37:58.750829Z","shell.execute_reply":"2025-01-12T16:37:58.750706Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}