{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport gc\nfrom matplotlib import pyplot as plt\nimport matplotlib.cm as cm\nfrom sklearn.model_selection import StratifiedGroupKFold","metadata":{"execution":{"iopub.status.busy":"2024-12-27T08:10:47.001588Z","iopub.execute_input":"2024-12-27T08:10:47.002228Z","iopub.status.idle":"2024-12-27T08:10:48.465514Z","shell.execute_reply.started":"2024-12-27T08:10:47.002187Z","shell.execute_reply":"2024-12-27T08:10:48.464308Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Configurations","metadata":{}},{"cell_type":"code","source":"class CONFIG:\n    target_col = \"responder_6\"\n    lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_rename = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1\" for idx in range(9)}\n    valid_ratio = 0.1\n    start_dt = 1500","metadata":{"execution":{"iopub.status.busy":"2024-12-27T08:10:48.467204Z","iopub.execute_input":"2024-12-27T08:10:48.467710Z","iopub.status.idle":"2024-12-27T08:10:48.473891Z","shell.execute_reply.started":"2024-12-27T08:10:48.467670Z","shell.execute_reply":"2024-12-27T08:10:48.472783Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load training data","metadata":{}},{"cell_type":"code","source":"# Use last 2 parquets\ntrain = pl.scan_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n).select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),\n).with_columns(\n    (pl.col(CONFIG.target_col)*2).cast(pl.Int32).alias(\"label\"),\n).filter(\n    pl.col(\"date_id\").gt(CONFIG.start_dt)\n)","metadata":{"execution":{"iopub.status.busy":"2024-12-27T08:10:48.475022Z","iopub.execute_input":"2024-12-27T08:10:48.475373Z","iopub.status.idle":"2024-12-27T08:10:48.518215Z","shell.execute_reply.started":"2024-12-27T08:10:48.475327Z","shell.execute_reply":"2024-12-27T08:10:48.516952Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create Lags data from training data","metadata":{}},{"cell_type":"code","source":"lags = train.select(pl.col(CONFIG.lag_cols_original))\nlags = lags.rename(CONFIG.lag_cols_rename)\nlags = lags.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )\nlags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()  # pick up last record of previous date\nlags","metadata":{"execution":{"iopub.status.busy":"2024-12-27T08:10:48.520886Z","iopub.execute_input":"2024-12-27T08:10:48.521305Z","iopub.status.idle":"2024-12-27T08:10:48.845460Z","shell.execute_reply.started":"2024-12-27T08:10:48.521268Z","shell.execute_reply":"2024-12-27T08:10:48.844152Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Merge training data and lags data","metadata":{}},{"cell_type":"code","source":"train = train.join(lags, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\ntrain","metadata":{"execution":{"iopub.status.busy":"2024-12-27T08:10:48.847063Z","iopub.execute_input":"2024-12-27T08:10:48.848195Z","iopub.status.idle":"2024-12-27T08:10:48.885816Z","shell.execute_reply.started":"2024-12-27T08:10:48.848139Z","shell.execute_reply":"2024-12-27T08:10:48.884566Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Split training data and validation data","metadata":{}},{"cell_type":"code","source":"len_train   = train.select(pl.col(\"date_id\")).collect().shape[0]\nvalid_records = int(len_train * CONFIG.valid_ratio)\nlen_ofl_mdl = len_train - valid_records\nlast_tr_dt  = train.select(pl.col(\"date_id\")).collect().row(len_ofl_mdl)[0]\n\nprint(f\"\\n len_train = {len_train}\")\nprint(f\"\\n len_ofl_mdl = {len_ofl_mdl}\")\nprint(f\"\\n---> Last offline train date = {last_tr_dt}\\n\")\n\ntraining_data = train.filter(pl.col(\"date_id\").le(last_tr_dt))\nvalidation_data   = train.filter(pl.col(\"date_id\").gt(last_tr_dt))","metadata":{"execution":{"iopub.status.busy":"2024-12-27T08:10:48.887284Z","iopub.execute_input":"2024-12-27T08:10:48.887708Z","iopub.status.idle":"2024-12-27T08:10:50.290776Z","shell.execute_reply.started":"2024-12-27T08:10:48.887661Z","shell.execute_reply":"2024-12-27T08:10:50.288911Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data","metadata":{"execution":{"iopub.status.busy":"2024-12-27T08:10:50.292515Z","iopub.execute_input":"2024-12-27T08:10:50.293002Z","iopub.status.idle":"2024-12-27T08:10:50.326917Z","shell.execute_reply.started":"2024-12-27T08:10:50.292951Z","shell.execute_reply":"2024-12-27T08:10:50.325461Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Save data as parquets","metadata":{}},{"cell_type":"code","source":"training_data.collect().\\\nwrite_parquet(\n    f\"training.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:10:50.328624Z","iopub.execute_input":"2024-12-27T08:10:50.329094Z","iopub.status.idle":"2024-12-27T08:11:44.950835Z","shell.execute_reply.started":"2024-12-27T08:10:50.329043Z","shell.execute_reply":"2024-12-27T08:11:44.949736Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data.collect().\\\nwrite_parquet(\n    \"validation.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:11:44.952043Z","iopub.execute_input":"2024-12-27T08:11:44.952396Z","iopub.status.idle":"2024-12-27T08:12:26.521695Z","shell.execute_reply.started":"2024-12-27T08:11:44.952361Z","shell.execute_reply":"2024-12-27T08:12:26.520436Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pl.scan_parquet(\"/kaggle/working/training.parquet\").collect().to_pandas()\nvalid = pl.scan_parquet(\"/kaggle/working/validation.parquet\").collect().to_pandas()\ntrain.shape, valid.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:12:26.526080Z","iopub.execute_input":"2024-12-27T08:12:26.526417Z","iopub.status.idle":"2024-12-27T08:12:38.866023Z","shell.execute_reply.started":"2024-12-27T08:12:26.526384Z","shell.execute_reply":"2024-12-27T08:12:38.864683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:12:38.867438Z","iopub.execute_input":"2024-12-27T08:12:38.867899Z","iopub.status.idle":"2024-12-27T08:12:40.208349Z","shell.execute_reply.started":"2024-12-27T08:12:38.867861Z","shell.execute_reply":"2024-12-27T08:12:40.207318Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"valid","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:12:40.209517Z","iopub.execute_input":"2024-12-27T08:12:40.209864Z","iopub.status.idle":"2024-12-27T08:12:40.363149Z","shell.execute_reply.started":"2024-12-27T08:12:40.209829Z","shell.execute_reply":"2024-12-27T08:12:40.361767Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:12:40.364375Z","iopub.execute_input":"2024-12-27T08:12:40.364749Z","iopub.status.idle":"2024-12-27T08:12:40.732568Z","shell.execute_reply.started":"2024-12-27T08:12:40.364713Z","shell.execute_reply":"2024-12-27T08:12:40.731394Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.scatter(train['feature_01'], train['responder_6'],alpha=0.3)\nplt.title('Scatter Plot: feature_01 vs responder_6')\nplt.xlabel('feature_01')\nplt.ylabel('responder_6')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:12:40.734018Z","iopub.execute_input":"2024-12-27T08:12:40.734531Z","iopub.status.idle":"2024-12-27T08:12:58.626258Z","shell.execute_reply.started":"2024-12-27T08:12:40.734463Z","shell.execute_reply":"2024-12-27T08:12:58.625144Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.scatter(train['feature_01'], train['responder_6'],alpha=0.3)\nplt.title('Scatter Plot: feature_01 vs responder_6')\nplt.xscale('log')\nplt.xlabel('feature_01')\nplt.ylabel('responder_6')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:12:58.627774Z","iopub.execute_input":"2024-12-27T08:12:58.628226Z","iopub.status.idle":"2024-12-27T08:13:08.837532Z","shell.execute_reply.started":"2024-12-27T08:12:58.628175Z","shell.execute_reply":"2024-12-27T08:13:08.836344Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"将特征feature_01与responder_6做散点分析，可以看出数据是比较均匀分布的，但是数据量太大，不能发现其他的","metadata":{}},{"cell_type":"code","source":"train_date_id_mean=train.groupby(\"date_id\",as_index=False)[['feature_01','responder_6']].mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:13:08.838782Z","iopub.execute_input":"2024-12-27T08:13:08.839162Z","iopub.status.idle":"2024-12-27T08:13:09.014252Z","shell.execute_reply.started":"2024-12-27T08:13:08.839127Z","shell.execute_reply":"2024-12-27T08:13:09.013001Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(train_date_id_mean['date_id'], train_date_id_mean['feature_01'],color='blue', linestyle='-')\nplt.plot(train_date_id_mean['date_id'], train_date_id_mean['responder_6'],color='red', linestyle='--')\nplt.title('Line Plot: date_id vs responder_6 or feature_01')\nplt.xlabel('date_id')\nplt.ylabel('responder_6 or feature_01')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:13:09.015651Z","iopub.execute_input":"2024-12-27T08:13:09.015991Z","iopub.status.idle":"2024-12-27T08:13:09.279720Z","shell.execute_reply.started":"2024-12-27T08:13:09.015956Z","shell.execute_reply":"2024-12-27T08:13:09.278485Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"由于其与时间有关，将特征feature_01与responder_6关于天数进行分类，并得到均值，可看出随天数波动，有较大改变，似乎在特征feature_01下降，responder_6也下降，特征feature_01上升，responder_6也上升，有线性关系，成正比","metadata":{}},{"cell_type":"code","source":"train_time_id_mean=train.groupby(\"time_id\",as_index=False)[['feature_01','responder_6']].mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:13:09.280964Z","iopub.execute_input":"2024-12-27T08:13:09.281286Z","iopub.status.idle":"2024-12-27T08:13:09.454309Z","shell.execute_reply.started":"2024-12-27T08:13:09.281255Z","shell.execute_reply":"2024-12-27T08:13:09.453079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(train_time_id_mean['time_id'], train_time_id_mean['feature_01'],color='blue', linestyle='-')\nplt.plot(train_time_id_mean['time_id'], train_time_id_mean['responder_6'],color='red', linestyle='--')\nplt.title('Line Plot: time_id vs responder_6 or feature_01')\nplt.xlabel('time_id')\nplt.ylabel('responder_6 or feature_01')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:13:09.455748Z","iopub.execute_input":"2024-12-27T08:13:09.456114Z","iopub.status.idle":"2024-12-27T08:13:09.768532Z","shell.execute_reply.started":"2024-12-27T08:13:09.456076Z","shell.execute_reply":"2024-12-27T08:13:09.767440Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"在关于一天内的不同时间段做将特征feature_01与responder_6分类，发现在一天中，有一时刻feature_01的值达到了最高，随后一直下降，但是responder_6基本保持不变，但是略微有线性正比关系，在最开始feature_01很大时，responder_6也有略微上升，反之，在最后也有下降。","metadata":{}},{"cell_type":"code","source":"sns.pairplot(train[['feature_01', 'responder_6']])\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:13:09.769811Z","iopub.execute_input":"2024-12-27T08:13:09.770149Z","iopub.status.idle":"2024-12-27T08:13:44.180443Z","shell.execute_reply.started":"2024-12-27T08:13:09.770114Z","shell.execute_reply":"2024-12-27T08:13:44.178819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"correlation_matrix = train[['date_id', 'time_id','symbol_id','feature_01','responder_6']].corr()\nsns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Heatmap')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:13:44.182389Z","iopub.execute_input":"2024-12-27T08:13:44.182777Z","iopub.status.idle":"2024-12-27T08:13:45.236403Z","shell.execute_reply.started":"2024-12-27T08:13:44.182742Z","shell.execute_reply":"2024-12-27T08:13:45.235178Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"将不同特征放一起来看相关性，可以看出特征feature_01与responder_6之间的关系值为0.03，说明之前的正比是对的，但是几乎接近于0，所有没有相关性，基本是独立的。","metadata":{}},{"cell_type":"code","source":"train_symbol_id_mean=train.groupby(\"symbol_id\",as_index=False)[['feature_01','responder_6']].mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:30:13.697136Z","iopub.execute_input":"2024-12-27T08:30:13.697594Z","iopub.status.idle":"2024-12-27T08:30:13.887836Z","shell.execute_reply.started":"2024-12-27T08:30:13.697555Z","shell.execute_reply":"2024-12-27T08:30:13.886145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(train_symbol_id_mean['symbol_id'], train_symbol_id_mean['feature_01'],color='blue', linestyle='-')\nplt.plot(train_symbol_id_mean['symbol_id'], train_symbol_id_mean['responder_6'],color='red', linestyle='--')\nplt.title('Line Plot: symbol_id vs responder_6 or feature_01')\nplt.xlabel('symbol_id')\nplt.ylabel('responder_6 or feature_01')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:31:07.860853Z","iopub.execute_input":"2024-12-27T08:31:07.861317Z","iopub.status.idle":"2024-12-27T08:31:08.160285Z","shell.execute_reply.started":"2024-12-27T08:31:07.861279Z","shell.execute_reply":"2024-12-27T08:31:08.158832Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"又取了symbol_id为分组特征feature_01与responder_6，看其在不同类别下的规律","metadata":{}},{"cell_type":"code","source":"train_loss=train[train['date_id']>=1677]\ntrain_loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:44:22.720729Z","iopub.execute_input":"2024-12-27T08:44:22.721097Z","iopub.status.idle":"2024-12-27T08:44:22.783798Z","shell.execute_reply.started":"2024-12-27T08:44:22.721065Z","shell.execute_reply":"2024-12-27T08:44:22.782580Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.regplot(x='feature_01', y='responder_6', data=train_loss, scatter_kws={'s':10}, line_kws={'color':'red'})\nplt.title('Scatter Plot with Regression Line')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:44:33.669062Z","iopub.execute_input":"2024-12-27T08:44:33.669590Z","iopub.status.idle":"2024-12-27T08:44:44.835193Z","shell.execute_reply.started":"2024-12-27T08:44:33.669543Z","shell.execute_reply":"2024-12-27T08:44:44.834112Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"将数据集缩小，通过此图可以看出基本独立，验证了之前的想法","metadata":{}},{"cell_type":"code","source":"train[\"feature_01_bancut\"] = pd.cut(train['feature_01'],bins=10,labels=[1,2,3,4,5,6,7,8,9,10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T09:12:19.024401Z","iopub.execute_input":"2024-12-27T09:12:19.025325Z","iopub.status.idle":"2024-12-27T09:12:20.206954Z","shell.execute_reply.started":"2024-12-27T09:12:19.025272Z","shell.execute_reply":"2024-12-27T09:12:20.205728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.boxplot(x='feature_01_bancut', y='responder_6', data=train)\nplt.title('Box Plot: feature_01_bancut vs Target')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T09:13:17.242506Z","iopub.execute_input":"2024-12-27T09:13:17.243713Z","iopub.status.idle":"2024-12-27T09:13:18.943672Z","shell.execute_reply.started":"2024-12-27T09:13:17.243666Z","shell.execute_reply":"2024-12-27T09:13:18.942553Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"将特征feature_01通过大小给分为10组，看其在不同大小下对于responder_6的关系","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}