{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport gc\nfrom matplotlib import pyplot as plt\nimport matplotlib.cm as cm\nfrom sklearn.model_selection import StratifiedGroupKFold","metadata":{"execution":{"iopub.status.busy":"2024-12-29T05:42:35.947803Z","iopub.execute_input":"2024-12-29T05:42:35.948275Z","iopub.status.idle":"2024-12-29T05:42:38.675886Z","shell.execute_reply.started":"2024-12-29T05:42:35.948215Z","shell.execute_reply":"2024-12-29T05:42:38.674542Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Configurations","metadata":{}},{"cell_type":"code","source":"class CONFIG:\n    target_col = \"responder_6\"\n    lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_rename = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1\" for idx in range(9)}\n    valid_ratio = 0.1\n    start_dt = 0","metadata":{"execution":{"iopub.status.busy":"2024-12-29T05:42:38.679178Z","iopub.execute_input":"2024-12-29T05:42:38.680198Z","iopub.status.idle":"2024-12-29T05:42:38.686764Z","shell.execute_reply.started":"2024-12-29T05:42:38.680140Z","shell.execute_reply":"2024-12-29T05:42:38.685363Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load training data","metadata":{}},{"cell_type":"code","source":"train = pl.read_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n).select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),\n).with_columns(\n    (pl.col(CONFIG.target_col)*2).cast(pl.Int32).alias(\"label\"),\n).filter(\n    pl.col(\"date_id\").gt(CONFIG.start_dt)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:42:38.688564Z","iopub.execute_input":"2024-12-29T05:42:38.689000Z","iopub.status.idle":"2024-12-29T05:43:40.094850Z","shell.execute_reply.started":"2024-12-29T05:42:38.688960Z","shell.execute_reply":"2024-12-29T05:43:40.092293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:43:40.100859Z","iopub.execute_input":"2024-12-29T05:43:40.102698Z","iopub.status.idle":"2024-12-29T05:43:40.165933Z","shell.execute_reply.started":"2024-12-29T05:43:40.102588Z","shell.execute_reply":"2024-12-29T05:43:40.164285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 数据分析与探索\nfeature_12 = train.select(\"feature_12\").to_pandas()[\"feature_12\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:43:40.168029Z","iopub.execute_input":"2024-12-29T05:43:40.168632Z","iopub.status.idle":"2024-12-29T05:43:41.240817Z","shell.execute_reply.started":"2024-12-29T05:43:40.168546Z","shell.execute_reply":"2024-12-29T05:43:41.239263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 基本统计描述\nprint(feature_12.describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:43:41.242681Z","iopub.execute_input":"2024-12-29T05:43:41.243250Z","iopub.status.idle":"2024-12-29T05:43:43.172017Z","shell.execute_reply.started":"2024-12-29T05:43:41.243193Z","shell.execute_reply":"2024-12-29T05:43:43.170657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 分布可视化\nplt.figure(figsize=(12, 6))\nplt.hist(feature_12, bins=50, color='blue', alpha=0.7)\nplt.title('Distribution of feature_12')\nplt.xlabel('feature_12')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:43:43.176374Z","iopub.execute_input":"2024-12-29T05:43:43.176770Z","iopub.status.idle":"2024-12-29T05:43:44.301459Z","shell.execute_reply.started":"2024-12-29T05:43:43.176735Z","shell.execute_reply":"2024-12-29T05:43:44.299922Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 分析`feature_12`的基本统计描述和分布情况。从直方图可以看出，`feature_12`的分布较为集中，但存在一些异常值。","metadata":{}},{"cell_type":"code","source":"# 特征变换\n# 对数变换\nfeature_12_log = np.log1p(feature_12)\nplt.figure(figsize=(12, 6))\nplt.hist(feature_12_log, bins=50, color='green', alpha=0.7)\nplt.title('Distribution of log1p(feature_12)')\nplt.xlabel('log1p(feature_12)')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:43:44.303126Z","iopub.execute_input":"2024-12-29T05:43:44.303663Z","iopub.status.idle":"2024-12-29T05:43:46.191047Z","shell.execute_reply.started":"2024-12-29T05:43:44.303609Z","shell.execute_reply":"2024-12-29T05:43:46.189835Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 对`feature_12`进行对数变换后，分布变得更加对称，异常值的影响减小。","metadata":{}},{"cell_type":"code","source":"# 平方根变换\nfeature_12_sqrt = np.sqrt(feature_12)\nplt.figure(figsize=(12, 6))\nplt.hist(feature_12_sqrt, bins=50, color='red', alpha=0.7)\nplt.title('Distribution of sqrt(feature_12)')\nplt.xlabel('sqrt(feature_12)')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:43:46.192840Z","iopub.execute_input":"2024-12-29T05:43:46.193325Z","iopub.status.idle":"2024-12-29T05:43:46.966115Z","shell.execute_reply.started":"2024-12-29T05:43:46.193274Z","shell.execute_reply":"2024-12-29T05:43:46.964773Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 对`feature_12`进行平方根变换后，分布也变得更加对称，但不如对数变换效果明显。","metadata":{}},{"cell_type":"code","source":"# 与 responder_0 进行交互\nresponder_0 = train.select(\"responder_0\").to_pandas()[\"responder_0\"]\nfeature_12_responder_0_interaction = feature_12 * responder_0\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:43:46.968462Z","iopub.execute_input":"2024-12-29T05:43:46.968974Z","iopub.status.idle":"2024-12-29T05:43:47.921738Z","shell.execute_reply.started":"2024-12-29T05:43:46.968920Z","shell.execute_reply":"2024-12-29T05:43:47.920456Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 特征交互\n# 添加新特征到 DataFrame\ntrain = train.with_columns(\n    pl.Series(\"feature_12_log\", feature_12_log),\n    pl.Series(\"feature_12_sqrt\", feature_12_sqrt),\n    pl.Series(\"feature_12_responder_0_interaction\", feature_12_responder_0_interaction)\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:43:47.923423Z","iopub.execute_input":"2024-12-29T05:43:47.923835Z","iopub.status.idle":"2024-12-29T05:43:49.456703Z","shell.execute_reply.started":"2024-12-29T05:43:47.923794Z","shell.execute_reply":"2024-12-29T05:43:49.455367Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 通过与`responder_0`进行交互，生成新的特征`feature_12_responder_0_interaction`，并将其添加到原始数据集中。","metadata":{}},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:43:49.458672Z","iopub.execute_input":"2024-12-29T05:43:49.459113Z","iopub.status.idle":"2024-12-29T05:43:49.486252Z","shell.execute_reply.started":"2024-12-29T05:43:49.459074Z","shell.execute_reply":"2024-12-29T05:43:49.484660Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 提取 feature_12_responder_0_interaction 列\nfeature_12_responder_0_interaction = train.select(\"feature_12_responder_0_interaction\").to_pandas()[\"feature_12_responder_0_interaction\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:51:00.719050Z","iopub.execute_input":"2024-12-29T05:51:00.719611Z","iopub.status.idle":"2024-12-29T05:51:01.224105Z","shell.execute_reply.started":"2024-12-29T05:51:00.719569Z","shell.execute_reply":"2024-12-29T05:51:01.222720Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 基本统计描述\nfeature_12_responder_0_interaction_stats = feature_12_responder_0_interaction.describe()\nprint(feature_12_responder_0_interaction_stats)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:51:11.770450Z","iopub.execute_input":"2024-12-29T05:51:11.770872Z","iopub.status.idle":"2024-12-29T05:51:13.641846Z","shell.execute_reply.started":"2024-12-29T05:51:11.770835Z","shell.execute_reply":"2024-12-29T05:51:13.640374Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 分布可视化\nplt.figure(figsize=(12, 6))\nplt.hist(feature_12_responder_0_interaction, bins=50, color='blue', alpha=0.7)\nplt.title('Distribution of feature_12_responder_0_interaction')\nplt.xlabel('feature_12_responder_0_interaction')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:51:21.434119Z","iopub.execute_input":"2024-12-29T05:51:21.434599Z","iopub.status.idle":"2024-12-29T05:51:22.523370Z","shell.execute_reply.started":"2024-12-29T05:51:21.434558Z","shell.execute_reply":"2024-12-29T05:51:22.522191Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 从原始分布图中可以看出，`feature_12_responder_0_interaction` 特征的分布较为偏斜，存在一些极端值。","metadata":{}},{"cell_type":"code","source":"# 特征变换\n# 对数变换\nfeature_12_responder_0_interaction_log = np.log1p(feature_12_responder_0_interaction)\nplt.figure(figsize=(12, 6))\nplt.hist(feature_12_responder_0_interaction_log, bins=50, color='green', alpha=0.7)\nplt.title('Distribution of log1p(feature_12_responder_0_interaction)')\nplt.xlabel('log1p(feature_12_responder_0_interaction)')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:51:32.823130Z","iopub.execute_input":"2024-12-29T05:51:32.823595Z","iopub.status.idle":"2024-12-29T05:51:34.497626Z","shell.execute_reply.started":"2024-12-29T05:51:32.823555Z","shell.execute_reply":"2024-12-29T05:51:34.496382Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 通过对数变换后的分布更加接近正态分布，减少了极端值的影响。","metadata":{}},{"cell_type":"code","source":"# 平方根变换\nfeature_12_responder_0_interaction_sqrt = np.sqrt(feature_12_responder_0_interaction)\nplt.figure(figsize=(12, 6))\nplt.hist(feature_12_responder_0_interaction_sqrt, bins=50, color='red', alpha=0.7)\nplt.title('Distribution of sqrt(feature_12_responder_0_interaction)')\nplt.xlabel('sqrt(feature_12_responder_0_interaction)')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T05:51:45.003470Z","iopub.execute_input":"2024-12-29T05:51:45.003906Z","iopub.status.idle":"2024-12-29T05:51:46.245059Z","shell.execute_reply.started":"2024-12-29T05:51:45.003866Z","shell.execute_reply":"2024-12-29T05:51:46.243726Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 通过平方根变换后的分布也有所改善，但不如对数变换的效果明显。","metadata":{}}]}