{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 这一个cell copy from EDA and time series any......\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv, pd.read_parquet )\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\n\nimport os, gc\nfrom tqdm.auto import tqdm\nimport pickle # module to serialize and deserialize objects\nimport re # for Regular expression operations \n\n# import tensorflow as tf\n# from tensorflow.keras import layers, models\n# from tensorflow.keras.optimizers import Adam\n\n# import torch\n# import torch.nn as nn\n# import torch.nn.functional as F\n# from torch.utils.data  import Dataset, DataLoader\n# from pytorch_lightning import (LightningDataModule, LightningModule, Trainer)\n# from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, Timer\n\n# from sklearn.metrics import r2_score\n# from sklearn.model_selection import train_test_split\n# from sklearn.ensemble import VotingRegressor\n\n# import lightgbm as lgb\n# from lightgbm import LGBMRegressor\n\n# from xgboost import XGBRegressor\n# from catboost import CatBoostRegressor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-18T01:59:05.799904Z","iopub.execute_input":"2024-12-18T01:59:05.800236Z","iopub.status.idle":"2024-12-18T01:59:06.106673Z","shell.execute_reply.started":"2024-12-18T01:59:05.800202Z","shell.execute_reply":"2024-12-18T01:59:06.10529Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"筛选180天的，这个本地能跑","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# 目标路径\npath = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\nsamples = []\n\n# 逐步加载数据并筛选最近 360 天的数据\nr = range(10)  # 遍历所有分区\nrecent_days = []  # 用于存储最近 360 天的数据\n\n# 先扫描所有分区，找到最大 date_id\nmax_date_id = 0\nfor i in r:\n    file_path = f\"{path}/train.parquet/partition_id={i}/part-0.parquet\"\n    part = pd.read_parquet(file_path, columns=['date_id'])  # 只读取 date_id 列\n    max_date_id = max(max_date_id, part['date_id'].max())\n\n# 确定最近 360 天的起始 date_id\nstart_date_id = max_date_id - 180\n\n# 再次遍历，筛选出符合条件的数据\nfor i in r:\n    file_path = f\"{path}/train.parquet/partition_id={i}/part-0.parquet\"\n    \n    # 读取数据\n    part = pd.read_parquet(file_path)\n    \n    # 筛选出最近 360 天的数据\n    part_recent = part[part['date_id'] > start_date_id]\n    recent_days.append(part_recent)\n\n# 合并所有筛选出的数据\nrecent_df = pd.concat(recent_days, ignore_index=True)\n\n# # 输出结果\n# print(\"最近 180 天的数据：\")\n# print(recent_df.info())\n# print(recent_df.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T02:00:00.407634Z","iopub.execute_input":"2024-12-18T02:00:00.407962Z","iopub.status.idle":"2024-12-18T02:01:30.517753Z","shell.execute_reply.started":"2024-12-18T02:00:00.407933Z","shell.execute_reply":"2024-12-18T02:01:30.517012Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"普通读取方式，即使优化也读不全","metadata":{}},{"cell_type":"code","source":"# import pandas as pd\n\n# # 目标路径\n# path = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\n# samples = []\n\n# # 数据类型转换映射，可以降低读取压力\n# dtype_map = {\n#     'feature1': 'float32',\n#     'feature2': 'int16',\n#     'responder_6': 'float32'\n# }\n\n# # 加载数据并转换列的数据类型\n# r = range(10)  # 读取前两个分区文件\n# for i in r:\n#     file_path = f\"{path}/train.parquet/partition_id={i}/part-0.parquet\"\n    \n#     # 加载数据\n#     part = pd.read_parquet(file_path)\n    \n#     # 转换指定列的数据类型\n#     for col, dtype in dtype_map.items():\n#         if col in part.columns:  # 确保列存在于数据中\n#             part[col] = part[col].astype(dtype)\n# # pandas.read_parquet() 不支持直接传递 dtype 参数，这与 pandas.read_csv() 不同\n# # 在读取 parquet 文件时，数据类型需要单独转换，而不能通过 dtype 参数直接指定。\n\n#     samples.append(part)\n\n# # 合并所有样本\n# sample_df = pd.concat(samples, ignore_index=True)\n\n# # 四舍五入到1位小数\n# sample_df = sample_df.round(1)\n\n# # 显示结果\n# # print(sample_df.info())  # 查看数据类型是否已正确转换\n# # print(sample_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T11:33:32.64012Z","iopub.execute_input":"2024-12-17T11:33:32.640744Z","iopub.status.idle":"2024-12-17T11:33:33.262436Z","shell.execute_reply.started":"2024-12-17T11:33:32.640676Z","shell.execute_reply":"2024-12-17T11:33:33.260614Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"惰性加载，后续会用到","metadata":{}},{"cell_type":"code","source":"# # 惰性加载所有数据\n\n# path = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\n# samples = [] \n\n# # 直接从trainparquet里面读取数据\n# r = range(9)\n# for i in r:\n#     file_path = f\"{path}/train.parquet/partition_id={i}/part-0.parquet\"\n#     # part = pd.read_parquet(file_path)\n#     # samples.append(part)\n\n#     # 使用 polars 的 scan_parquet 进行惰性加载\n#     part = pl.scan_parquet(file_path)  # 使用 scan_parquet 来进行惰性加载\n#     samples.append(part)\n\n# # 合并所有惰性加载的 DataFrame\n# # 使用 collect() 来将所有的惰性加载 DataFrame 合并成一个真正的 DataFrame\n# sample_df = pl.concat(samples)\n# df_result = sample_df.collect()\n# # sample_df = pd.concat(samples, ignore_index=True) #将所有的sample放入一个dataframe里面\n\n# # 获取所有浮动类型的列名\n# float_columns = [col for col in df_result.columns if df_result[col].dtype in [pl.Float32, pl.Float64]]\n\n# # 对浮动类型列进行四舍五入\n# df_result = df_result.with_columns([\n#     pl.col(col).round(1) if col in float_columns else pl.col(col)\n#     for col in df_result.columns\n# ])\n# # 打印结果\n# print(df_result)\n\n\n# # # 进行实际的数据读取（惰性加载，只有在这里才会真正读取数据）\n# # df_result = df_optimized.collect()\n\n# # 另外两种处理\n# # 1.只提取 \"weight\", \"feature1\", \"feature2\" 列\n# # df_filtered = df.select([\"weight\", \"feature1\", \"feature2\"])\n\n# # 2.将 Int32 优化为 Int16，较为常用，有金牌选手用过\n# # df_optimized = df_filtered.with_columns([\n# #     pl.col(col).cast(pl.Int16) for col in df_filtered.columns if \"int\" in str(df_filtered[col].dtype)\n# # ])\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T11:26:51.818387Z","iopub.execute_input":"2024-12-17T11:26:51.81878Z","iopub.status.idle":"2024-12-17T11:27:42.026778Z","shell.execute_reply.started":"2024-12-17T11:26:51.818749Z","shell.execute_reply":"2024-12-17T11:27:42.025516Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"可以发现一共有1529天（可能不是天，但这里理解成天）  \n那么不妨就选出来1529-360天，只看后面的","metadata":{}},{"cell_type":"code","source":"gridColor = 'lightgrey'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T01:16:09.315034Z","iopub.execute_input":"2024-12-18T01:16:09.315883Z","iopub.status.idle":"2024-12-18T01:16:09.319766Z","shell.execute_reply.started":"2024-12-18T01:16:09.315848Z","shell.execute_reply":"2024-12-18T01:16:09.318815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df\n\n# 跑第一个，能在这里看","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T11:29:40.954391Z","iopub.execute_input":"2024-12-17T11:29:40.955548Z","iopub.status.idle":"2024-12-17T11:29:41.216726Z","shell.execute_reply.started":"2024-12-17T11:29:40.955504Z","shell.execute_reply":"2024-12-17T11:29:41.215508Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 数据加载总结  \n9个parquet实际上有接近五千万份数据，根据金融市场的相关要求，这些数据不需要全部都用  \n具体有三个解决办法：1.更改数据类型（上面已经做了），这样的话会损失一些精度  \n2.用Polar的惰性加载，后面进行EDA时，尤其绘图时分批加载  \n3.最重要的方法。越接近预测日，数据其实越重要，所以可以直接用date_id>1500的数据，大概在两千万条左右  ","metadata":{}},{"cell_type":"markdown","source":"# 缺失值处理","metadata":{}},{"cell_type":"code","source":"# 计算缺失值比例并显示为表格\nmissing_values = recent_df.isnull().sum() / len(recent_df) * 100\nmissing_values = missing_values[missing_values > 0]  # 只显示有缺失值的列\nprint(\"\\n缺失值比例：\")\nprint(missing_values.sort_values(ascending=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T12:07:18.142126Z","iopub.execute_input":"2024-12-17T12:07:18.142481Z","iopub.status.idle":"2024-12-17T12:07:18.785626Z","shell.execute_reply.started":"2024-12-17T12:07:18.142453Z","shell.execute_reply":"2024-12-17T12:07:18.784745Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"缺失值条形可视化（热力图画不出来）","metadata":{}},{"cell_type":"code","source":"# 绘制条形图\nplt.figure(figsize=(12, 8))\nmissing_values.plot(kind='bar', color='skyblue')\nplt.title(\"Missing Values Percentage per Column\", fontsize=14)\nplt.xlabel(\"Columns\", fontsize=12)\nplt.ylabel(\"Missing Values (%)\", fontsize=12)\nplt.xticks(rotation=90)  # 旋转 x 轴标签，避免重叠\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T12:08:38.7531Z","iopub.execute_input":"2024-12-17T12:08:38.753444Z","iopub.status.idle":"2024-12-17T12:08:39.141013Z","shell.execute_reply.started":"2024-12-17T12:08:38.753415Z","shell.execute_reply":"2024-12-17T12:08:39.140035Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"比较明显划分为三层，着重看一下39,42,50,53,看看有没有必要删除掉  \n2%左右的考虑赋中位数  \n剩下的直接填充","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T12:17:14.880881Z","iopub.execute_input":"2024-12-17T12:17:14.881769Z","iopub.status.idle":"2024-12-17T12:17:17.426023Z","shell.execute_reply.started":"2024-12-17T12:17:14.881734Z","shell.execute_reply":"2024-12-17T12:17:17.425118Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# responder6的分布分析  \n极端值需要分析吗","metadata":{}},{"cell_type":"code","source":"# 绘制目标变量的分布图\nplt.figure(figsize=(12, 6))\nsns.histplot(recent_df['responder_6'].dropna(), bins=50, kde=True)\nplt.title(\"Distribution of Responder_6\")\nplt.xlabel(\"Responder_6\")\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T12:20:32.412742Z","iopub.execute_input":"2024-12-17T12:20:32.413858Z","iopub.status.idle":"2024-12-17T12:20:58.248751Z","shell.execute_reply.started":"2024-12-17T12:20:32.41382Z","shell.execute_reply":"2024-12-17T12:20:58.247855Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(recent_df['responder_6'].describe())\nprint(\"偏度 (Skewness):\", recent_df['responder_6'].skew())\nprint(\"峰度 (Kurtosis):\", recent_df['responder_6'].kurt())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T12:23:23.817553Z","iopub.execute_input":"2024-12-17T12:23:23.818239Z","iopub.status.idle":"2024-12-17T12:23:24.129163Z","shell.execute_reply.started":"2024-12-17T12:23:23.818202Z","shell.execute_reply":"2024-12-17T12:23:24.128157Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"与responder6相关性最高的几个特性","metadata":{}},{"cell_type":"code","source":"corr_with_responder6 = recent_df.corr()['responder_6'].sort_values(ascending=False)\nprint(\"与 Responder_6 相关性最高的特征：\")\nprint(corr_with_responder6.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T12:23:39.852532Z","iopub.execute_input":"2024-12-17T12:23:39.853407Z","iopub.status.idle":"2024-12-17T12:25:50.014801Z","shell.execute_reply.started":"2024-12-17T12:23:39.853368Z","shell.execute_reply":"2024-12-17T12:25:50.013956Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"这个图的作用，还得想想  \n下一步不成熟的想法：1.responder3*responder6  \n2.分析一下极端值","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 7))\nplt.hexbin(recent_df['responder_3'], recent_df['responder_6'], gridsize=50, cmap=\"viridis\", mincnt=1)\nplt.colorbar(label=\"Count in Hexagon\")\nplt.title(\"Responder_6 vs Responder_3 (Hexbin Plot)\", fontsize=16)\nplt.xlabel(\"Responder_3\", fontsize=12)\nplt.ylabel(\"Responder_6\", fontsize=12)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T12:38:38.0319Z","iopub.execute_input":"2024-12-17T12:38:38.032283Z","iopub.status.idle":"2024-12-17T12:38:38.91775Z","shell.execute_reply.started":"2024-12-17T12:38:38.032251Z","shell.execute_reply":"2024-12-17T12:38:38.916825Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**feature之间相关性的热力图*","metadata":{}},{"cell_type":"code","source":"# 计算特征之间的相关性矩阵\nfeatures = recent_df.loc[:, 'feature_00':'feature_78']\nfeature_corr = features.corr()\n\n# 绘制特征相关性热力图\nplt.figure(figsize=(15, 12))\nsns.heatmap(feature_corr, cmap='coolwarm', center=0, annot=False)\nplt.title(\"Feature Correlation Heatmap\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T15:47:59.672032Z","iopub.execute_input":"2024-12-17T15:47:59.672452Z","iopub.status.idle":"2024-12-17T15:50:04.440943Z","shell.execute_reply.started":"2024-12-17T15:47:59.672414Z","shell.execute_reply":"2024-12-17T15:50:04.439653Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"responder6的时间趋势","metadata":{}},{"cell_type":"code","source":"# 计算每日平均 responder_6\ndaily_avg_responder = recent_df.groupby('date_id')['responder_6'].mean()\n\n# 绘制时间序列图\nplt.figure(figsize=(14, 6))\nplt.plot(daily_avg_responder.index, daily_avg_responder.values, color='black', linewidth=0.6)\nplt.title(\"Responder_6 Trend over Time\")\nplt.xlabel(\"Date_ID\")\nplt.ylabel(\"Average Responder_6\")\nplt.axhline(0, color='red', linestyle='--', linewidth=1)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T16:05:21.867302Z","iopub.execute_input":"2024-12-17T16:05:21.868219Z","iopub.status.idle":"2024-12-17T16:05:22.36855Z","shell.execute_reply.started":"2024-12-17T16:05:21.868173Z","shell.execute_reply":"2024-12-17T16:05:22.367235Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 分析不同symbol_id对时间的波动情况  \n# 这里还是需要思考date_id和time_id结合的可行性，为什么图会这么奇怪  \n","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nsymbol_ids = [1, 5]  # 选择几个示例的 symbol_id\n\nplt.figure(figsize=(14, 6))\nfor symbol in symbol_ids:\n    # 筛选出当前 symbol_id 的数据\n    symbol_data = recent_df[recent_df['symbol_id'] == symbol]\n    \n    # 按照时间顺序（date_id 和 time_id）排序\n    symbol_data = symbol_data.sort_values(by=['date_id', 'time_id'])\n    \n    # 累积求和计算\n    cumulative_responder = symbol_data['responder_6'].cumsum()\n    \n    # 使用 date_id 和 time_id 组合来表示时间\n    time_index = symbol_data['date_id'] * 1000 + symbol_data['time_id']\n    \n    # 绘图\n    plt.plot(time_index, cumulative_responder, label=f\"Symbol_ID: {symbol}\")\n\nplt.title(\"Cumulative Responder_6 for Different Symbol_IDs\")\nplt.xlabel(\"Time (Date_ID * 1000 + Time_ID)\")\nplt.ylabel(\"Cumulative Responder_6\")\nplt.axhline(0, color='red', linestyle='--', linewidth=1)\nplt.legend()\nplt.grid()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T16:10:39.369972Z","iopub.execute_input":"2024-12-17T16:10:39.370454Z","iopub.status.idle":"2024-12-17T16:10:40.679329Z","shell.execute_reply.started":"2024-12-17T16:10:39.370419Z","shell.execute_reply":"2024-12-17T16:10:40.678272Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 补充：对时间信息进行EDA# ","metadata":{}},{"cell_type":"code","source":"# 统计每个 date_id 下的样本量\ndate_counts = recent_df['date_id'].value_counts().sort_index()\n\n# 可视化分布\nplt.figure(figsize=(12, 6))\nplt.plot(date_counts.index, date_counts.values, marker='o')\nplt.title(\"Sample Count per date_id\")\nplt.xlabel(\"date_id\")\nplt.ylabel(\"Count\")\nplt.grid()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T03:24:31.465166Z","iopub.execute_input":"2024-12-18T03:24:31.466065Z","iopub.status.idle":"2024-12-18T03:24:31.763989Z","shell.execute_reply.started":"2024-12-18T03:24:31.466005Z","shell.execute_reply":"2024-12-18T03:24:31.763008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 检查 date_id 和 time_id 组合的唯一性\nunique_combinations = recent_df[['date_id', 'time_id']].duplicated().sum()\nprint(f\"重复的 date_id 和 time_id 组合数量: {unique_combinations}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T03:41:16.390573Z","iopub.execute_input":"2024-12-18T03:41:16.391709Z","iopub.status.idle":"2024-12-18T03:41:16.567133Z","shell.execute_reply.started":"2024-12-18T03:41:16.39166Z","shell.execute_reply":"2024-12-18T03:41:16.566091Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 按 date_id 和 time_id 排序\nrecent_df = recent_df.sort_values(by=['date_id', 'time_id']).reset_index(drop=True)\n\n# 检查时间间隔\nrecent_df['time_diff'] = recent_df['time_id'].diff()\nprint(recent_df[['date_id', 'time_id', 'time_diff']].head(100))\n\n# 可视化时间间隔分布\nimport matplotlib.pyplot as plt\nplt.figure(figsize=(12, 6))\nplt.plot(recent_df['time_diff'], label='Time Difference')\nplt.title(\"Time Difference Between Consecutive Rows\")\nplt.xlabel(\"Index\")\nplt.ylabel(\"Time Difference\")\nplt.legend()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T03:47:26.826102Z","iopub.execute_input":"2024-12-18T03:47:26.827031Z","iopub.status.idle":"2024-12-18T03:47:51.859825Z","shell.execute_reply.started":"2024-12-18T03:47:26.826995Z","shell.execute_reply":"2024-12-18T03:47:51.858925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 筛选某个 symbol_id\nsymbol_id_example = 0\nsymbol_df = recent_df[recent_df['symbol_id'] == symbol_id_example]\n\n# 重新进行时间特征EDA\nsymbol_date_counts = symbol_df['date_id'].value_counts().sort_index()\n\n# 可视化单个 symbol_id 的样本量分布\nplt.figure(figsize=(12, 6))\nplt.plot(symbol_date_counts.index, symbol_date_counts.values, marker='o')\nplt.title(f\"Sample Count per date_id for symbol_id {symbol_id_example}\")\nplt.xlabel(\"date_id\")\nplt.ylabel(\"Count\")\nplt.grid()\nplt.show()\n\n# 检查时间间隔\nsymbol_df = symbol_df.sort_values(by=['date_id', 'time_id'])\nsymbol_df['time_diff'] = symbol_df['time_id'].diff()\n\nplt.figure(figsize=(12, 6))\nplt.plot(symbol_df['time_diff'], label='Time Difference')\nplt.title(f\"Time Difference Between Consecutive Rows for symbol_id {symbol_id_example}\")\nplt.xlabel(\"Index\")\nplt.ylabel(\"Time Difference\")\nplt.legend()\nplt.grid()\nplt.show()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T03:44:45.76649Z","iopub.execute_input":"2024-12-18T03:44:45.767142Z","iopub.status.idle":"2024-12-18T03:44:46.981811Z","shell.execute_reply.started":"2024-12-18T03:44:45.76711Z","shell.execute_reply":"2024-12-18T03:44:46.980911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train =recent_df\n# train['N']=train.index.values \n# train['id']=train.index.values \n\n# xx= recent_df[(recent_df.symbol_id==3)] ['id']\n# yy=recent_df[ (recent_df.symbol_id==3)]['responder_6']\n\n# plt.figure(figsize=(16, 5))\n# plt.plot(xx,yy, color = 'black', linewidth =0.05)\n# plt.suptitle('Returns, responder_6', weight='bold', fontsize=16)\n# plt.xlabel(\"Time\", fontsize=12)\n# plt.ylabel(\"Returns\", fontsize=12)\n# plt.grid(color = gridColor , linewidth=0.8)\n# plt.axhline(0, color='red', linestyle='-', linewidth=1.2)\n# plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T12:28:34.34122Z","iopub.execute_input":"2024-12-17T12:28:34.341564Z","iopub.status.idle":"2024-12-17T12:28:35.029246Z","shell.execute_reply.started":"2024-12-17T12:28:34.341534Z","shell.execute_reply":"2024-12-17T12:28:35.028342Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"不太理解，为什么会出现突变的情况  \n详情可以见notebook2，并且最好还得根据date_id选一些前面的天数  \n选择前357天的不会有这样的情况  \n选择后300天的会有比较明显的这个情况","metadata":{}},{"cell_type":"code","source":"# for symbol_id == 0\nplt.figure(figsize=(18, 7))\npredictor_cols = [col for col in recent_df.columns if 'responder' in col]\nfor i in predictor_cols: \n    if i == 'responder_6': \n        c='red'\n        lw=2.5\n        plt.plot((recent_df[recent_df.symbol_id == 10].groupby(['date_id'])[i].mean()).cumsum(), linewidth = lw, color = c)\n    else: \n        lw=1\n        plt.plot((recent_df[recent_df.symbol_id == 10].groupby(['date_id'])[i].mean()).cumsum(), linewidth = lw)\n\nplt.xlabel('Trade days')\nplt.ylabel('Cumulative response')\nplt.title('Response time series over trade days  \\n Responder 6 (red) and other responders', weight='bold')\nplt.grid(visible=True, color = gridColor, linewidth = 0.7)\nplt.axhline(0, color='blue', linestyle='-', linewidth=1)\nplt.legend(predictor_cols)\nsns.despine()\n#plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T16:17:01.946029Z","iopub.execute_input":"2024-12-17T16:17:01.946729Z","iopub.status.idle":"2024-12-17T16:17:04.02599Z","shell.execute_reply.started":"2024-12-17T16:17:01.946681Z","shell.execute_reply":"2024-12-17T16:17:04.024802Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Tag与Feature之间的可视化","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfeatures = pd.read_csv(f\"{path}/features.csv\")\n\nplt.figure(figsize=(18, 6))\nplt.imshow(features.iloc[:, 1:].T.values, cmap=\"cool\")\nplt.xlabel(\"feature_00 - feature_78\")\nplt.ylabel(\"tag_0 - tag_16\")\nplt.yticks(np.arange(17))\nplt.xticks(np.arange(79))\nplt.grid(color = 'lightblue')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T13:01:31.178144Z","iopub.execute_input":"2024-12-17T13:01:31.179263Z","iopub.status.idle":"2024-12-17T13:01:31.834456Z","shell.execute_reply.started":"2024-12-17T13:01:31.179187Z","shell.execute_reply":"2024-12-17T13:01:31.833321Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"feature9-11 21-31 删除或者合并  \n我个人感觉，tag和feature之间的作用，应该是不同的tag构成了同一个feature  \n那么相同tag构成的feature完全可以做某种处理","metadata":{}},{"cell_type":"markdown","source":"# 滑动窗口平滑\n备注：滑动窗口平滑的前提是时间信息其实是连续的，这里生成的图是假设时间信息是连续的，但是等前面时间进一步处理之后  \n需要重新进行处理，然后看结果","metadata":{}},{"cell_type":"code","source":"# 滑动窗口大小\nwindow_size = 100  \n\n# 计算滑动平均值\nrolling_mean = recent_df['responder_6'].rolling(window=window_size).mean()\n\n# 原始数据与滑动平均的可视化\nplt.figure(figsize=(12, 6))\nplt.plot(recent_df['responder_6'], label='Original responder_6', alpha=0.5)\nplt.plot(rolling_mean, label=f'{window_size}-Step Rolling Mean', color='red')\nplt.title(\"Sliding Window Smoothing for responder_6\")\nplt.xlabel(\"Index\")\nplt.ylabel(\"responder_6\")\nplt.legend()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T16:37:21.896023Z","iopub.execute_input":"2024-12-17T16:37:21.896477Z","iopub.status.idle":"2024-12-17T16:38:08.498632Z","shell.execute_reply.started":"2024-12-17T16:37:21.896439Z","shell.execute_reply":"2024-12-17T16:38:08.49747Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 指数平滑","metadata":{}},{"cell_type":"code","source":"# 指数移动平均\nema = recent_df['responder_6'].ewm(span=window_size).mean()\n\n# 可视化原始数据与EMA平滑\nplt.figure(figsize=(12, 6))\nplt.plot(recent_df['responder_6'], label='Original responder_6', alpha=0.5)\nplt.plot(ema, label=f'{window_size}-Step EMA', color='green')\nplt.title(\"Exponential Moving Average Smoothing for responder_6\")\nplt.xlabel(\"Index\")\nplt.ylabel(\"responder_6\")\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T16:41:20.349907Z","iopub.execute_input":"2024-12-17T16:41:20.350728Z","iopub.status.idle":"2024-12-17T16:42:06.489791Z","shell.execute_reply.started":"2024-12-17T16:41:20.350679Z","shell.execute_reply":"2024-12-17T16:42:06.488726Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 两个图放在一起进行对比","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nplt.plot(recent_df['responder_6'], label='Original responder_6', alpha=0.5)\nplt.plot(rolling_mean, label=f'{window_size}-Step Rolling Mean', color='red')\nplt.plot(ema, label=f'{window_size}-Step EMA', color='green')\nplt.title(\"Comparison of Rolling Mean and EMA Smoothing\")\nplt.xlabel(\"Index\")\nplt.ylabel(\"responder_6\")\nplt.legend()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T16:42:13.472034Z","iopub.execute_input":"2024-12-17T16:42:13.47245Z","iopub.status.idle":"2024-12-17T16:43:23.871847Z","shell.execute_reply.started":"2024-12-17T16:42:13.472414Z","shell.execute_reply":"2024-12-17T16:43:23.870549Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**从上述结果来看，还是更适合用EMA**","metadata":{}},{"cell_type":"code","source":"from statsmodels.graphics.tsaplots import plot_acf, plot_pacf\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T02:02:08.861115Z","iopub.execute_input":"2024-12-18T02:02:08.861486Z","iopub.status.idle":"2024-12-18T02:02:11.452798Z","shell.execute_reply.started":"2024-12-18T02:02:08.861455Z","shell.execute_reply":"2024-12-18T02:02:11.452027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 提取特征\nrecent_df['responder_6_lag40'] = recent_df['responder_6'].shift(40)\nrecent_df['responder_6_lag80'] = recent_df['responder_6'].shift(80)\nrecent_df['responder_6_lag120'] = recent_df['responder_6'].shift(120)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T03:04:11.071447Z","iopub.execute_input":"2024-12-18T03:04:11.071801Z","iopub.status.idle":"2024-12-18T03:04:11.107054Z","shell.execute_reply.started":"2024-12-18T03:04:11.071771Z","shell.execute_reply":"2024-12-18T03:04:11.106351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"recent_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T03:04:21.733392Z","iopub.execute_input":"2024-12-18T03:04:21.733718Z","iopub.status.idle":"2024-12-18T03:04:22.872029Z","shell.execute_reply.started":"2024-12-18T03:04:21.733688Z","shell.execute_reply":"2024-12-18T03:04:22.870898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_squared_error\n\n# 删除包含缺失值的行\nrecent_df = recent_df.dropna()\n\n# 准备输入特征和目标变量\nfeatures = ['responder_6_lag40', 'responder_6_lag80', 'responder_6_lag120']\nX = recent_df[features]\ny = recent_df['responder_6']\n\n# 划分训练集和测试集\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# 训练线性回归模型\nmodel = LinearRegression()\nmodel.fit(X_train, y_train)\n\n# 预测与评估\ny_pred = model.predict(X_test)\nmse = mean_squared_error(y_test, y_pred)\nprint(f\"Baseline Model (Linear Regression) MSE: {mse:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T03:06:41.165343Z","iopub.execute_input":"2024-12-18T03:06:41.165702Z","iopub.status.idle":"2024-12-18T03:06:45.128806Z","shell.execute_reply.started":"2024-12-18T03:06:41.165672Z","shell.execute_reply":"2024-12-18T03:06:45.125571Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import lightgbm as lgb\n\n# 对特征进行平滑，window看看需不需要更多，可变\nrecent_df['responder_6_rolling_mean'] = recent_df['responder_6'].rolling(window=50).mean()\nrecent_df['responder_6_ema'] = recent_df['responder_6'].ewm(span=50).mean()\n\n# 删除缺失值\nrecent_df = recent_df.dropna()\n\n# 输入特征与目标变量\nfeatures = ['responder_6_lag40', 'responder_6_lag80', 'responder_6_lag120', \n            'responder_6_rolling_mean', 'responder_6_ema']\nX = recent_df[features]\ny = recent_df['responder_6']\n\n# 划分训练集和测试集\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# LightGBM 模型训练\nlgb_model = lgb.LGBMRegressor()\nlgb_model.fit(X_train, y_train)\n\n# 预测与评估\ny_pred_lgb = lgb_model.predict(X_test)\nmse_lgb = mean_squared_error(y_test, y_pred_lgb)\nprint(f\"LightGBM Model MSE: {mse_lgb:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T03:08:07.400544Z","iopub.execute_input":"2024-12-18T03:08:07.40088Z","iopub.status.idle":"2024-12-18T03:08:38.310239Z","shell.execute_reply.started":"2024-12-18T03:08:07.400853Z","shell.execute_reply":"2024-12-18T03:08:38.309197Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"相比线性回归的 0.5578，有了明显的提升  \nLightGBM 能够更好地捕捉非线性关系，尤其是在使用了滞后特征和平滑特征的情况下  \n所以考虑进一步进行优化","metadata":{}},{"cell_type":"markdown","source":"---------------------以下忽略-----------------------------------------  \n\n优化方法1：差分  \n计算 responder_6 与滞后特征之间的差分  \n其他方式待尝试：  \n1.对date_id 和 time_id 的周期性编码（如日、周、月）  \n但是这个方法之前遇到了问题，有乱码的情况  还得再探索  \n\n2.结合多个滞后特征与平滑特征的组合关系\n\n有没有可能，在这个题目中，不能用差分，效果太好了","metadata":{}},{"cell_type":"code","source":"# # 构建差分特征\n# recent_df['responder_6_diff40'] = recent_df['responder_6'] - recent_df['responder_6_lag40']\n# recent_df['responder_6_diff80'] = recent_df['responder_6'] - recent_df['responder_6_lag80']\n# recent_df['responder_6_diff120'] = recent_df['responder_6'] - recent_df['responder_6_lag120']\n\n# # 删除因新特征产生的缺失值\n# recent_df = recent_df.dropna()\n\n# # 检查新特征\n# print(recent_df[['responder_6', 'responder_6_lag40', 'responder_6_diff40', \n#                  'responder_6_lag80', 'responder_6_diff80']].head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T03:29:24.780651Z","iopub.execute_input":"2024-12-18T03:29:24.781018Z","iopub.status.idle":"2024-12-18T03:29:32.365047Z","shell.execute_reply.started":"2024-12-18T03:29:24.780968Z","shell.execute_reply":"2024-12-18T03:29:32.364059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import lightgbm as lgb\n# from sklearn.model_selection import train_test_split\n# from sklearn.metrics import mean_squared_error\n\n# # 更新特征列表\n# features = ['responder_6_lag40', 'responder_6_lag80', 'responder_6_lag120', \n#             'responder_6_diff40', 'responder_6_diff80', 'responder_6_diff120',\n#             'responder_6_rolling_mean', 'responder_6_ema']\n\n# # 输入与目标变量\n# X = recent_df[features]\n# y = recent_df['responder_6']\n\n# # 数据集划分\n# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# # 训练 LightGBM 模型\n# lgb_model = lgb.LGBMRegressor()\n# lgb_model.fit(X_train, y_train)\n\n# # 预测与评估\n# y_pred_lgb = lgb_model.predict(X_test)\n# mse_lgb = mean_squared_error(y_test, y_pred_lgb)\n# print(f\"Updated LightGBM Model MSE: {mse_lgb:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T03:29:34.892794Z","iopub.execute_input":"2024-12-18T03:29:34.893136Z","iopub.status.idle":"2024-12-18T03:29:57.789442Z","shell.execute_reply.started":"2024-12-18T03:29:34.893106Z","shell.execute_reply":"2024-12-18T03:29:57.788469Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"优化方法2:","metadata":{}},{"cell_type":"markdown","source":"调参结果发现，并没有太大的提升，可能已经到了模型的上限","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\n\nparams = {\n    'num_leaves': [31, 50],\n    'learning_rate': [0.05, 0.1],\n    'n_estimators': [100, 200]\n}\n\nmodel = lgb.LGBMRegressor()\ngrid = GridSearchCV(model, params, cv=3, scoring='neg_mean_squared_error', verbose=2)\ngrid.fit(X_train, y_train)\n\nprint(f\"Best Parameters: {grid.best_params_}\")\nprint(f\"Best MSE: {-grid.best_score_}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T03:14:38.457981Z","iopub.execute_input":"2024-12-18T03:14:38.459256Z","iopub.status.idle":"2024-12-18T03:24:31.463255Z","shell.execute_reply.started":"2024-12-18T03:14:38.459217Z","shell.execute_reply":"2024-12-18T03:24:31.462232Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# 绘制特征重要性\nlgb.plot_importance(lgb_model, max_num_features=10, importance_type='gain')\nplt.title(\"Feature Importance\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T03:26:59.751001Z","iopub.execute_input":"2024-12-18T03:26:59.752072Z","iopub.status.idle":"2024-12-18T03:27:00.024362Z","shell.execute_reply.started":"2024-12-18T03:26:59.752017Z","shell.execute_reply":"2024-12-18T03:27:00.023261Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"下面会将数据分段，会影响lag特征","metadata":{}},{"cell_type":"code","source":"num_segments = 10  # 将数据分成10段，方便绘图观察\nsegment_size = len(recent_df) // num_segments\n\nfor i in range(num_segments):\n    start = i * segment_size\n    end = (i + 1) * segment_size\n    segment_data = recent_df['responder_6'].iloc[start:end].dropna()\n    \n    print(f\"Segment {i+1}:\")\n    plt.figure(figsize=(12, 4))\n    plot_acf(segment_data, lags=50, title=f\"ACF for Segment {i+1}\")\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T02:02:14.914572Z","iopub.execute_input":"2024-12-18T02:02:14.915566Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom statsmodels.graphics.tsaplots import plot_acf\n\n# 分段数量\nnum_segments = 10\nsegment_size = len(recent_df) // num_segments\n\n# 创建子图布局（2行5列，共10个子图）\nfig, axes = plt.subplots(nrows=2, ncols=5, figsize=(20, 8))\nfig.suptitle(\"ACF for All Segments\", fontsize=16, y=1.02)\n\n# 遍历每个段，绘制ACF\nfor i, ax in enumerate(axes.flat):\n    start = i * segment_size\n    end = (i + 1) * segment_size\n    segment_data = recent_df['responder_6'].iloc[start:end].dropna()\n    \n    # 绘制 ACF 到指定的子图中\n    plot_acf(segment_data, lags=50, ax=ax, title=f\"Segment {i+1}\")\n\n# 调整子图间距\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T02:25:33.285998Z","iopub.execute_input":"2024-12-18T02:25:33.286737Z","iopub.status.idle":"2024-12-18T02:42:41.718665Z","shell.execute_reply.started":"2024-12-18T02:25:33.286688Z","shell.execute_reply":"2024-12-18T02:42:41.717686Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"很明显，在lag40步长的时候会出现  \n那么PACF就不需要了  \n下一步很明确就是添加滞后，跑baseline","metadata":{}},{"cell_type":"code","source":"# 按 symbol_id 分组统计 responder_6 均值\nsymbol_trend = recent_df.groupby('symbol_id')['responder_6'].mean().sort_values(ascending=False)\n\n# 可视化\nplt.figure(figsize=(14, 6))\nsymbol_trend.plot(kind='bar', color='skyblue')\nplt.title(\"Average responder_6 by symbol_id\")\nplt.xlabel(\"symbol_id\")\nplt.ylabel(\"Average responder_6\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T02:53:40.296235Z","iopub.execute_input":"2024-12-18T02:53:40.296576Z","iopub.status.idle":"2024-12-18T02:53:40.940522Z","shell.execute_reply.started":"2024-12-18T02:53:40.296539Z","shell.execute_reply":"2024-12-18T02:53:40.939468Z"}},"outputs":[],"execution_count":null}]}