{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":10442185,"sourceType":"datasetVersion","datasetId":6463253},{"sourceId":224299,"sourceType":"modelInstanceVersion","modelInstanceId":191359,"modelId":213311},{"sourceId":225162,"sourceType":"modelInstanceVersion","modelInstanceId":192059,"modelId":214013},{"sourceId":227362,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":193855,"modelId":215772},{"sourceId":228787,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":195090,"modelId":216987}],"dockerImageVersionId":30823,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # nn lgb cat\n# import os\n# import gc\n# import numpy as np\n# import pandas as pd\n# import polars as pls\n# import torch\n# import torch.nn as nn\n# import torch.nn.functional as F\n# import pytorch_lightning as pl\n# import time\n\n# import lightgbm as lgb\n# from catboost import CatBoostRegressor\n\n# # 若需要官方评测接口\n# import kaggle_evaluation.jane_street_inference_server\n\n# from torch.optim.lr_scheduler import ExponentialLR\n\n# ##########################################################\n# # 0) 全局配置\n# ##########################################################\n# ONLINE_LEARNING_PARAMS = {\n#     \"lr\":          1e-5,\n#     \"weight_decay\":5e-6,\n#     \"batch_size\":  512,\n#     \"epochs\":      1,\n#     \"lr_gamma\":    0.99\n# }\n\n# # NN ckpt\n# MODEL_CKPT_PATH = \"/kaggle/input/nnlgbcat/other/default/1/offline_best.model (1).ckpt\"\n\n# # LGB & Cat\n# LGB_MODEL_PATH = \"/kaggle/input/nnlgbcat/other/default/1/lgb_model_single_0.txt\"\n# CAT_MODEL_PATH = \"/kaggle/input/nnlgbcat/other/default/1/cat_symmetric_best.cbm\"\n\n# # NN特征(仅79列)\n# NN_FEATURES_79 = [f\"feature_{i:02d}\" for i in range(79)]\n# # LGB/Cat特征(79 + 9 lag + symbol_id => 89列)\n# LGB_CAT_FEATS_89 = (\n#     [f\"feature_{i:02d}\" for i in range(79)]\n#     + [f\"responder_{i}_lag_1\" for i in range(9)]\n#     + [\"symbol_id\"]\n# )\n\n# # 融合系数(仅示例)\n# ALPHA_NN   = 0.60\n# ALPHA_LGB  = 0.60\n\n# # 是否假设行对齐(在合并NN和lags数据时用)\n# ASSUME_ROW_ALIGNMENT = False\n\n# ##########################################################\n# # 1) 定义 NN(与离线训练一致), 用于在线学习\n# ##########################################################\n# class NN(pl.LightningModule):\n#     def __init__(self, input_dim, hidden_dims, dropouts, lr, weight_decay):\n#         super().__init__()\n#         self.save_hyperparameters()\n#         layers = []\n#         in_dim = input_dim\n#         for i, hidden_dim in enumerate(hidden_dims):\n#             layers.append(nn.BatchNorm1d(in_dim))\n#             if i > 0:\n#                 layers.append(nn.SiLU())\n#             if i < len(dropouts):\n#                 layers.append(nn.Dropout(dropouts[i]))\n#             layers.append(nn.Linear(in_dim, hidden_dim))\n#             in_dim = hidden_dim\n\n#         layers.append(nn.Linear(in_dim, 1))\n#         layers.append(nn.Tanh())\n#         self.model = nn.Sequential(*layers)\n\n#         self.lr = lr\n#         self.weight_decay = weight_decay\n\n#     def forward(self, x: torch.Tensor) -> torch.Tensor:\n#         return 5.0 * self.model(x).squeeze(-1)\n\n# ##########################################################\n# # 2) forward_fill_and_zero\n# ##########################################################\n# def forward_fill_and_zero(\n#     df: pd.DataFrame,\n#     sort_cols=[\"time_id\"],\n#     group_cols=[\"symbol_id\"],\n#     fill_cols=None\n# ):\n#     \"\"\"\n#     groupby(...) => ffill => fillna(0)\n#     \"\"\"\n#     if fill_cols is None:\n#         fill_cols = NN_FEATURES_79[:]\n#         if \"weight\" in df.columns:\n#             fill_cols.append(\"weight\")\n\n#     df = df.sort_values(group_cols + sort_cols, ignore_index=True)\n#     df[fill_cols] = df.groupby(group_cols)[fill_cols].ffill()\n#     df[fill_cols] = df[fill_cols].fillna(0)\n#     return df\n\n# ##########################################################\n# # 3) 全局变量 => 3个模型(已加载) & 在线学习cache\n# ##########################################################\n# # （A）NN\n# trained_nn      = None\n# test_cache_nn   = {}\n# first_date_nn   = None   # 用于跳过首日在线学习\n\n# # （B）LGB, Cat\n# lgb_model_single = None\n# cat_model_single = None\n\n# # （C）全局 lags_ 用于拼接 9 lag 特征 (Polars方式)\n# lags_ = None\n\n# ##########################################################\n# # 4) 在脚本加载阶段，就把 3 个模型初始化好\n# ##########################################################\n# def load_lgb_model_txt(model_file: str) -> lgb.Booster:\n#     \"\"\"加载 LightGBM txt 模型\"\"\"\n#     with open(model_file, 'r') as f:\n#         model_str = f.read()\n#     return lgb.Booster(model_str=model_str)\n\n# def load_cat_model(model_file: str) -> CatBoostRegressor:\n#     model = CatBoostRegressor()\n#     model.load_model(model_file)\n#     return model\n\n# # ============ A) 先加载 NN ============\n# print(f\"[Init] Loading offline NN => {MODEL_CKPT_PATH}\")\n# _nn_tmp = NN.load_from_checkpoint(\n#     MODEL_CKPT_PATH,\n#     input_dim=79,\n#     hidden_dims=[512,512,256],\n#     dropouts=[0.1,0.1],\n#     lr=1e-3,\n#     weight_decay=5e-4\n# )\n# dev = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# _nn_tmp.to(dev)\n# _nn_tmp.eval()\n# trained_nn = _nn_tmp\n\n# # 约定: 第一次出现 date_id => 记录到 first_date_nn => 跳过在线学习\n# # 但要直到 predict() 被调用时，才知道 date_id\n# # 因此这里的 first_date_nn 先不赋值\n\n# # ============ B) 加载 LGB & Cat ============\n# print(f\"[Init] Loading LGB => {LGB_MODEL_PATH}\")\n# lgb_model_single = load_lgb_model_txt(LGB_MODEL_PATH)\n# print(f\"[Init] => lgb_model_single => #trees={lgb_model_single.num_trees()}\")\n\n# print(f\"[Init] Loading Cat => {CAT_MODEL_PATH}\")\n# cat_model_single = load_cat_model(CAT_MODEL_PATH)\n# print(\"[Init] => cat_model_single => loaded done\")\n\n# ##########################################################\n# # 5) predict(test, lags)\n# ##########################################################\n# def predict(test: pls.DataFrame, lags: pls.DataFrame | None) -> pd.DataFrame:\n#     \"\"\"\n#     1) NN => 在线学习 => nn_pred\n#     2) LGB+Cat => 拼接 lag 特征 => lgb_pred & cat_pred\n#     3) final_pred = ALPHA_NN*nn_pred + (1-ALPHA_NN)*( ALPHA_LGB*lgb + (1-ALPHA_LGB)*cat )\n#        => clip(-5,5)\n#     \"\"\"\n#     global trained_nn, test_cache_nn, first_date_nn\n#     global lgb_model_single, cat_model_single\n#     global lags_  # 新增\n\n#     # A) polars => pandas  (for NN part)\n#     test_pd = test.to_pandas()\n#     date_now = test_pd[\"date_id\"].iloc[0]\n#     row_ids  = test_pd[\"row_id\"].values if \"row_id\" in test_pd.columns else np.arange(len(test_pd))\n\n#     # B) NN => 缓存 => 在线学习\n#     if first_date_nn is None:\n#         # 第一次 => 记录\n#         first_date_nn = date_now\n#     if date_now not in test_cache_nn:\n#         test_cache_nn[date_now] = []\n#     test_cache_nn[date_now].append(test_pd)\n\n#     # 当有新一天时, 先对“上一天”做一次在线学习\n#     if lags is not None:\n#         # 说明这是新的一天(竞赛环境下, 通常会先给你当天 lags)\n#         if date_now == first_date_nn:\n#             print(\"[NN] skip online train => first day\")\n#         else:\n#             # 训练上一天的数据\n#             prev_day = date_now - 1\n#             if prev_day in test_cache_nn:\n#                 t0 = time.time()\n#                 dev = next(trained_nn.parameters()).device\n#                 prev_pd = pd.concat(test_cache_nn[prev_day], ignore_index=True)\n                \n#                 # 这里假设 lags 也是当前新的, 不一定要用\n#                 # 所以我们不再做 merges 依赖 lags, 直接用 test里的第6个responder_6?\n#                 # 但原逻辑还是要 merges \"responder_6_lag_1\" 当作 y\n#                 # 你可以保持原逻辑(只要 data alignment OK).\n#                 lags_pd = lags.to_pandas()\n\n#                 if ASSUME_ROW_ALIGNMENT:\n#                     if len(prev_pd)!=len(lags_pd):\n#                         print(\"[WARN] mismatch row => naive alignment\")\n#                     merged = prev_pd.copy()\n#                     merged[\"responder_6_lag_1\"] = lags_pd[\"responder_6_lag_1\"].values\n#                 else:\n#                     merged = pd.merge(\n#                         prev_pd, lags_pd,\n#                         on=[\"time_id\",\"symbol_id\"],\n#                         how=\"inner\"\n#                     )\n\n#                 merged = forward_fill_and_zero(merged)\n#                 X_tr = torch.FloatTensor(merged[NN_FEATURES_79].values).to(dev)\n#                 y_tr = torch.FloatTensor(merged[\"responder_6_lag_1\"].values).to(dev)\n#                 w_tr = torch.FloatTensor(merged[\"weight\"].values).to(dev)\n\n#                 # freeze BN\n#                 trained_nn.train()\n#                 for m in trained_nn.modules():\n#                     if isinstance(m, nn.BatchNorm1d):\n#                         m.eval()\n#                         for p in m.parameters():\n#                             p.requires_grad=False\n\n#                 opt = torch.optim.Adam(\n#                     filter(lambda p: p.requires_grad, trained_nn.parameters()),\n#                     lr=ONLINE_LEARNING_PARAMS[\"lr\"],\n#                     weight_decay=ONLINE_LEARNING_PARAMS[\"weight_decay\"]\n#                 )\n#                 sch = ExponentialLR(opt, gamma=ONLINE_LEARNING_PARAMS[\"lr_gamma\"])\n\n#                 for _ep in range(ONLINE_LEARNING_PARAMS[\"epochs\"]):\n#                     perm = torch.randperm(X_tr.size(0))\n#                     bs_  = ONLINE_LEARNING_PARAMS[\"batch_size\"]\n#                     for st in range(0, X_tr.size(0), bs_):\n#                         idx = perm[st:st+bs_]\n#                         x_b, y_b, w_b = X_tr[idx], y_tr[idx], w_tr[idx]\n#                         opt.zero_grad()\n#                         p_b = trained_nn(x_b)\n#                         loss_b = (w_b*(p_b - y_b)**2).mean()\n#                         loss_b.backward()\n#                         opt.step()\n#                     sch.step()\n\n#                 trained_nn.eval()\n#                 # 清理上一天的缓存\n#                 del test_cache_nn[prev_day]\n#                 print(f\"[NN] day {prev_day}->{date_now} => online train => cost={time.time()-t0:.2f}s\")\n\n#     # NN推理\n#     dev = next(trained_nn.parameters()).device\n#     test_ff = forward_fill_and_zero(test_pd)\n#     X_nn    = torch.FloatTensor(test_ff[NN_FEATURES_79].values).to(dev)\n#     with torch.no_grad():\n#         nn_pred = trained_nn(X_nn).cpu().numpy().ravel()\n\n#     # C) 处理 LGB+Cat => 9lag (Polars 的简单写法)\n#     #    如果当日开始时 lags 不为 None, 更新全局 lags_\n#     if lags is not None:\n#         lags_ = lags  # 存储下来, 当天多次 predict 时复用\n\n#     # 在这里拼接最后时刻(上一时刻)的 9 lag 特征 => group_by + last => join\n#     # 如果没有 lags_, 就直接给 9 个 lag 列补 0\n#     if lags_ is not None:\n#         lag_tail = (\n#             lags_\n#             .group_by([\"date_id\", \"symbol_id\"], maintain_order=True)\n#             .last()\n#             .drop([\"time_id\"])  # 不需要 time_id\n#         )\n#         lgbcat = test.join(lag_tail, on=[\"date_id\",\"symbol_id\"], how=\"left\")\n#     else:\n#         # 补 0\n#         for idx in range(9):\n#             test = test.with_columns(pl.lit(0.0).alias(f\"responder_{idx}_lag_1\"))\n#         lgbcat = test\n\n#     # 转成 pandas => 送入 lgb/cat\n#     lgbcat_pd = lgbcat.to_pandas()\n#     X_lgbcat  = lgbcat_pd[LGB_CAT_FEATS_89]\n#     lgb_pred  = lgb_model_single.predict(X_lgbcat)\n#     cat_pred  = cat_model_single.predict(X_lgbcat)\n\n#     # D) 最终融合 => alpha\n#     lgbcat_ens = ALPHA_LGB*lgb_pred + (1.0 - ALPHA_LGB)*cat_pred\n#     final_pred = ALPHA_NN*nn_pred + (1.0 - ALPHA_NN)*lgbcat_ens\n\n#     # E) clip => [-5,5]\n#     final_pred = np.clip(final_pred, a_min=-5.0, a_max=5.0)\n\n#     # 输出\n#     out_df = pd.DataFrame({\n#         \"row_id\":   row_ids,\n#         \"responder_6\": final_pred\n#     })\n#     assert (out_df.columns == [\"row_id\",\"responder_6\"]).all()\n#     assert len(out_df) == len(test_pd)\n\n#     return out_df","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-09T18:37:56.040946Z","iopub.execute_input":"2025-01-09T18:37:56.041223Z","iopub.status.idle":"2025-01-09T18:38:11.439610Z","shell.execute_reply.started":"2025-01-09T18:37:56.041200Z","shell.execute_reply":"2025-01-09T18:38:11.438789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# nn lgb cat\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport polars as pls\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport pytorch_lightning as pl\nimport time\n\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor\n\n# 若需要官方评测接口\nimport kaggle_evaluation.jane_street_inference_server\n\nfrom torch.optim.lr_scheduler import ExponentialLR\n\n##########################################################\n# 0) 全局配置\n##########################################################\nONLINE_LEARNING_PARAMS = {\n    \"lr\":          2.05943e-05,     # 在线训练的学习率\n    \"weight_decay\":0.00096991,\n    \"batch_size\":  21260,\n    \"epochs\":      1,\n    \"lr_gamma\":    0.99\n}\n\n# NN ckpt (NN1)\nMODEL_CKPT_PATH = \"/kaggle/input/trial/pytorch/default/1/trialx.model.ckpt\"\n# 新增: NN2 ckpt (请改为你的实际文件路径)\nMODEL2_CKPT_PATH = \"/kaggle/input/trial/pytorch/default/1/trialA.model.ckpt\"\n\nMODEL3_CKPT_PATH = \"/kaggle/input/trial/pytorch/default/1/trialAA.model.ckpt\"\n\nMODEL4_CKPT_PATH = \"/kaggle/input/trial/pytorch/default/1/trialAAA.model.ckpt\"\n\n# LGB & Cat\nLGB_MODEL_PATH = \"/kaggle/input/nnlgbcat/other/default/1/lgb_model_single_0.txt\"\nLGB2_MODEL_PATH = \"/kaggle/input/lgb_2/other/default/1/lgb_model_single_0_0.txt\"\nCAT_MODEL_PATH = \"/kaggle/input/nnlgbcat/other/default/1/cat_symmetric_best.cbm\"\n\n# NN特征(仅79列)\nNN_FEATURES_79 = [\n    f\"feature_{i:02d}\" for i in range(79)\n    if i not in (9, 10, 11)\n]\n# LGB/Cat特征(79 + 9 lag + symbol_id => 89列)\nLGB_CAT_FEATS_89 = (\n    [f\"feature_{i:02d}\" for i in range(79)]\n    + [f\"responder_{i}_lag_1\" for i in range(9)]\n    + [\"symbol_id\"]\n)\n\n# 融合系数(仅示例)\nALPHA_NN   = 0.60\nALPHA_LGB  = 0.60\n\n# 是否假设行对齐(在合并NN和lags数据时用)\nASSUME_ROW_ALIGNMENT = False\n\n##########################################################\n# 1) 定义 NN(与离线训练一致), 用于在线学习\n##########################################################\nclass NN(pl.LightningModule):\n    def __init__(self, input_dim, hidden_dims, dropouts, lr, weight_decay):\n        super().__init__()\n        self.save_hyperparameters()\n\n        # 与你训练时相同的网络结构\n        layers = []\n        in_dim = input_dim\n\n        # 第1层: dropout -> Linear -> SiLU\n        layers.append(nn.Dropout(dropouts[0]))\n        layers.append(nn.Linear(in_dim, hidden_dims[0]))\n        layers.append(nn.SiLU())\n\n        # 后续几层\n        for i in range(1, len(hidden_dims)):\n            layers.append(nn.Dropout(dropouts[i]))\n            layers.append(nn.Linear(hidden_dims[i-1], hidden_dims[i]))\n            layers.append(nn.SiLU())\n\n        # 最后 dropout\n        layers.append(nn.Dropout(dropouts[-1]))\n        # 输出 => 1 => Tanh => *5\n        layers.append(nn.Linear(hidden_dims[-1], 1))\n        layers.append(nn.Tanh())\n\n        self.model = nn.Sequential(*layers)\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        return 5.0 * self.model(x).squeeze(-1)\n\n##########################################################\n# 2) forward_fill_and_zero\n##########################################################\ndef forward_fill_and_zero(\n    df: pd.DataFrame,\n    sort_cols=[\"time_id\"],\n    group_cols=[\"symbol_id\"],\n    fill_cols=None\n):\n    \"\"\"\n    groupby(...) => ffill => fillna(0)\n    \"\"\"\n    if fill_cols is None:\n        fill_cols = NN_FEATURES_79[:]\n        if \"weight\" in df.columns:\n            fill_cols.append(\"weight\")\n\n    df = df.sort_values(group_cols + sort_cols, ignore_index=True)\n    df[fill_cols] = df.groupby(group_cols)[fill_cols].ffill()\n    df[fill_cols] = df[fill_cols].fillna(0)\n    return df\n\n##########################################################\n# 3) 全局变量 => 3个模型(已加载) & 在线学习cache\n##########################################################\n# （A）NN1 & NN2，共用 test_cache_nn & first_date_nn\ntrained_nn      = None   # NN1\ntrained_nn2     = None   # NN2\ntrained_nn3     = None   # NN3\ntrained_nn4     = None   # NN4\ntest_cache_nn   = {}\nfirst_date_nn   = None   # 用于跳过首日在线学习\n\n# （B）LGB, Cat\nlgb_model_single = None\ncat_model_single = None\n\n# （C）全局 lags_ 用于拼接 9 lag 特征 (Polars方式)\nlags_ = None\n\n##########################################################\n# 4) 在脚本加载阶段，就把 3 个模型初始化好\n##########################################################\ndef load_lgb_model_txt(model_file: str) -> lgb.Booster:\n    \"\"\"加载 LightGBM txt 模型\"\"\"\n    with open(model_file, 'r') as f:\n        model_str = f.read()\n    return lgb.Booster(model_str=model_str)\n\ndef load_cat_model(model_file: str) -> CatBoostRegressor:\n    model = CatBoostRegressor()\n    model.load_model(model_file)\n    return model\n\n# ============ A) 先加载 NN1 ============\nprint(f\"[Init] Loading offline NN => {MODEL_CKPT_PATH}\")\n_nn_tmp = NN.load_from_checkpoint(\n    MODEL_CKPT_PATH,\n    input_dim=76,\n    hidden_dims=[384,896,896,394],\n    dropouts=[0.1014378698,0.1972033905,0.1123435323,0.2314834093,0.2157768967],\n    lr=1e-3,\n    weight_decay=5e-4\n)\ndev = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n_nn_tmp.to(dev)\n_nn_tmp.eval()\ntrained_nn = _nn_tmp\n\n# ============ A2) 再加载 NN2 ============\nprint(f\"[Init] Loading offline NN2 => {MODEL2_CKPT_PATH}\")\n_nn_tmp2 = NN.load_from_checkpoint(\n    MODEL2_CKPT_PATH,\n    input_dim=76,\n    hidden_dims=[384,896,896,394],\n    dropouts=[0.1014378698,0.1972033905,0.1123435323,0.2314834093,0.2157768967],\n    lr=1e-3,\n    weight_decay=5e-4\n)\n_nn_tmp2.to(dev)\n_nn_tmp2.eval()\ntrained_nn2 = _nn_tmp2\n\nprint(f\"[Init] Loading offline NN3 => {MODEL3_CKPT_PATH}\")\n_nn_tmp3 = NN.load_from_checkpoint(\n    MODEL3_CKPT_PATH,\n    input_dim=76,\n    hidden_dims=[384,896,896,394],\n    dropouts=[0.1014378698,0.1972033905,0.1123435323,0.2314834093,0.2157768967],\n    lr=1e-3,\n    weight_decay=5e-4\n)\n_nn_tmp3.to(dev)\n_nn_tmp3.eval()\ntrained_nn3 = _nn_tmp3\n\nprint(f\"[Init] Loading offline NN4 => {MODEL4_CKPT_PATH}\")\n_nn_tmp4 = NN.load_from_checkpoint(\n    MODEL4_CKPT_PATH,\n    input_dim=76,\n    hidden_dims=[384,896,896,394],\n    dropouts=[0.1014378698,0.1972033905,0.1123435323,0.2314834093,0.2157768967],\n    lr=1e-3,\n    weight_decay=5e-4\n)\n_nn_tmp4.to(dev)\n_nn_tmp4.eval()\ntrained_nn4 = _nn_tmp4\n\n# 约定: 第一次出现 date_id => 记录到 first_date_nn => 跳过在线学习\n# 但要直到 predict() 被调用时，才知道 date_id\n# 因此这里的 first_date_nn 先不赋值\n\n# ============ B) 加载 LGB & Cat ============\nprint(f\"[Init] Loading LGB => {LGB_MODEL_PATH}\")\nlgb_model_single = load_lgb_model_txt(LGB_MODEL_PATH)\nprint(f\"[Init] => lgb_model_single => #trees={lgb_model_single.num_trees()}\")\n\nprint(f\"[Init] Loading second LGB => {LGB2_MODEL_PATH}\")\nlgb_model_single2 = load_lgb_model_txt(LGB2_MODEL_PATH)\nprint(f\"[Init] => lgb_model_single2 => #trees={lgb_model_single2.num_trees()}\")\n\nprint(f\"[Init] Loading Cat => {CAT_MODEL_PATH}\")\ncat_model_single = load_cat_model(CAT_MODEL_PATH)\nprint(\"[Init] => cat_model_single => loaded done\")\n\n##########################################################\n# 5) predict(test, lags)\n##########################################################\ndef predict(test: pls.DataFrame, lags: pls.DataFrame | None) -> pd.DataFrame:\n    \"\"\"\n    1) NN1 & NN2 => 在线学习 => nn_pred1, nn_pred2\n    2) 取 nn_pred_avg = 0.5*(nn_pred1 + nn_pred2)\n    3) LGB+Cat => 同样处理 => lgb_pred & cat_pred\n    4) final_pred = ALPHA_NN*nn_pred_avg + (1-ALPHA_NN)*( ALPHA_LGB*lgb + (1-ALPHA_LGB)*cat )\n       => clip(-5,5)\n    \"\"\"\n    global trained_nn, trained_nn2, trained_nn3, trained_nn4, test_cache_nn, first_date_nn\n    global lgb_model_single, cat_model_single\n    global lags_  # 新增\n\n    # A) polars => pandas  (for NN part)\n    test_pd = test.to_pandas()\n    date_now = test_pd[\"date_id\"].iloc[0]\n    row_ids  = test_pd[\"row_id\"].values if \"row_id\" in test_pd.columns else np.arange(len(test_pd))\n\n    # B) NN => 缓存 => 在线学习 (同一份)\n    if first_date_nn is None:\n        first_date_nn = date_now\n    if date_now not in test_cache_nn:\n        test_cache_nn[date_now] = []\n    test_cache_nn[date_now].append(test_pd)\n\n    if lags is not None:\n        # 说明这是新的一天(竞赛环境下, 通常会先给你当天 lags)\n        if date_now == first_date_nn:\n            print(\"[NN] skip online train => first day\")\n        else:\n            # 训练上一天的数据 => 对 NN1 & NN2 都要做\n            prev_day = date_now - 1\n            if prev_day in test_cache_nn:\n                t0 = time.time()\n                dev_cur = next(trained_nn.parameters()).device\n                prev_pd = pd.concat(test_cache_nn[prev_day], ignore_index=True)\n\n                lags_pd = lags.to_pandas()\n\n                if ASSUME_ROW_ALIGNMENT:\n                    if len(prev_pd)!=len(lags_pd):\n                        print(\"[WARN] mismatch row => naive alignment\")\n                    merged = prev_pd.copy()\n                    merged[\"responder_6_lag_1\"] = lags_pd[\"responder_6_lag_1\"].values\n                else:\n                    merged = pd.merge(\n                        prev_pd, lags_pd,\n                        on=[\"time_id\",\"symbol_id\"],\n                        how=\"inner\"\n                    )\n\n                merged = forward_fill_and_zero(merged)\n                X_tr = torch.FloatTensor(merged[NN_FEATURES_79].values).to(dev_cur)\n                y_tr = torch.FloatTensor(merged[\"responder_6_lag_1\"].values).to(dev_cur)\n                w_tr = torch.FloatTensor(merged[\"weight\"].values).to(dev_cur)\n\n                # == 1) 在线学习 NN1 ==\n                trained_nn.train()\n                opt = torch.optim.Adam(\n                    trained_nn.parameters(),\n                    lr=ONLINE_LEARNING_PARAMS[\"lr\"],\n                    weight_decay=ONLINE_LEARNING_PARAMS[\"weight_decay\"]\n                )\n                sch = ExponentialLR(opt, gamma=ONLINE_LEARNING_PARAMS[\"lr_gamma\"])\n\n                for _ep in range(ONLINE_LEARNING_PARAMS[\"epochs\"]):\n                    perm = torch.randperm(X_tr.size(0))\n                    bs_  = ONLINE_LEARNING_PARAMS[\"batch_size\"]\n                    for st in range(0, X_tr.size(0), bs_):\n                        idx = perm[st:st+bs_]\n                        x_b, y_b, w_b = X_tr[idx], y_tr[idx], w_tr[idx]\n                        opt.zero_grad()\n                        p_b = trained_nn(x_b)\n                        loss_b = (w_b*(p_b - y_b)**2).mean()\n                        loss_b.backward()\n                        opt.step()\n                    sch.step()\n                trained_nn.eval()\n\n                # == 2) 在线学习 NN2 ==\n                trained_nn2.train()\n                opt = torch.optim.Adam(\n                    trained_nn2.parameters(),\n                    lr=ONLINE_LEARNING_PARAMS[\"lr\"],\n                    weight_decay=ONLINE_LEARNING_PARAMS[\"weight_decay\"]\n                )\n                sch = ExponentialLR(opt, gamma=ONLINE_LEARNING_PARAMS[\"lr_gamma\"])\n\n                for _ep in range(ONLINE_LEARNING_PARAMS[\"epochs\"]):\n                    perm = torch.randperm(X_tr.size(0))\n                    bs_  = ONLINE_LEARNING_PARAMS[\"batch_size\"]\n                    for st in range(0, X_tr.size(0), bs_):\n                        idx = perm[st:st+bs_]\n                        x_b, y_b, w_b = X_tr[idx], y_tr[idx], w_tr[idx]\n                        opt.zero_grad()\n                        p_b = trained_nn2(x_b)\n                        loss_b = (w_b*(p_b - y_b)**2).mean()\n                        loss_b.backward()\n                        opt.step()\n                    sch.step()\n                trained_nn2.eval()\n\n                # == 3) 在线学习 NN3 ==\n                trained_nn3.train()\n                opt = torch.optim.Adam(\n                    trained_nn3.parameters(),\n                    lr=ONLINE_LEARNING_PARAMS[\"lr\"],\n                    weight_decay=ONLINE_LEARNING_PARAMS[\"weight_decay\"]\n                )\n                sch = ExponentialLR(opt, gamma=ONLINE_LEARNING_PARAMS[\"lr_gamma\"])\n\n                for _ep in range(ONLINE_LEARNING_PARAMS[\"epochs\"]):\n                    perm = torch.randperm(X_tr.size(0))\n                    bs_  = ONLINE_LEARNING_PARAMS[\"batch_size\"]\n                    for st in range(0, X_tr.size(0), bs_):\n                        idx = perm[st:st+bs_]\n                        x_b, y_b, w_b = X_tr[idx], y_tr[idx], w_tr[idx]\n                        opt.zero_grad()\n                        p_b = trained_nn3(x_b)\n                        loss_b = (w_b*(p_b - y_b)**2).mean()\n                        loss_b.backward()\n                        opt.step()\n                    sch.step()\n                trained_nn3.eval()\n\n                # == 4) 在线学习 NN4 ==\n                trained_nn4.train()\n                opt = torch.optim.Adam(\n                    trained_nn4.parameters(),\n                    lr=ONLINE_LEARNING_PARAMS[\"lr\"],\n                    weight_decay=ONLINE_LEARNING_PARAMS[\"weight_decay\"]\n                )\n                sch = ExponentialLR(opt, gamma=ONLINE_LEARNING_PARAMS[\"lr_gamma\"])\n\n                for _ep in range(ONLINE_LEARNING_PARAMS[\"epochs\"]):\n                    perm = torch.randperm(X_tr.size(0))\n                    bs_  = ONLINE_LEARNING_PARAMS[\"batch_size\"]\n                    for st in range(0, X_tr.size(0), bs_):\n                        idx = perm[st:st+bs_]\n                        x_b, y_b, w_b = X_tr[idx], y_tr[idx], w_tr[idx]\n                        opt.zero_grad()\n                        p_b = trained_nn4(x_b)\n                        loss_b = (w_b*(p_b - y_b)**2).mean()\n                        loss_b.backward()\n                        opt.step()\n                    sch.step()\n                trained_nn4.eval()\n\n                # 清理上一天的缓存\n                del test_cache_nn[prev_day]\n                print(f\"[NN] day {prev_day}->{date_now} => online train => cost={time.time()-t0:.2f}s\")\n\n    # NN推理 => NN1\n    dev_cur = next(trained_nn.parameters()).device\n    test_ff = forward_fill_and_zero(test_pd)\n    X_nn    = torch.FloatTensor(test_ff[NN_FEATURES_79].values).to(dev_cur)\n    with torch.no_grad():\n        nn_pred1 = trained_nn(X_nn).cpu().numpy().ravel()\n\n    # NN推理 => NN2\n    with torch.no_grad():\n        nn_pred2 = trained_nn2(X_nn).cpu().numpy().ravel()\n\n    # NN推理 => NN3\n    with torch.no_grad():\n        nn_pred3 = trained_nn3(X_nn).cpu().numpy().ravel()\n\n    # NN推理 => NN4\n    with torch.no_grad():\n        nn_pred4 = trained_nn4(X_nn).cpu().numpy().ravel()\n\n    # NN avg\n    nn_pred_avg = 0.25*(nn_pred1 + nn_pred2 + nn_pred3 + nn_pred4)\n\n    # C) 处理 LGB+Cat => 9lag (Polars 的简单写法)\n    if lags is not None:\n        lags_ = lags  # 存储下来, 当天多次 predict 时复用\n\n    if lags_ is not None:\n        lag_tail = (\n            lags_\n            .group_by([\"date_id\", \"symbol_id\"], maintain_order=True)\n            .last()\n            .drop([\"time_id\"])  # 不需要 time_id\n        )\n        lgbcat = test.join(lag_tail, on=[\"date_id\",\"symbol_id\"], how=\"left\")\n    else:\n        # 补 0\n        for idx in range(9):\n            test = test.with_columns(pl.lit(0.0).alias(f\"responder_{idx}_lag_1\"))\n        lgbcat = test\n\n    # 转成 pandas => 送入 lgb/cat\n    lgbcat_pd = lgbcat.to_pandas()\n    X_lgbcat  = lgbcat_pd[LGB_CAT_FEATS_89]\n    lgb_pred1 = lgb_model_single.predict(X_lgbcat)\n    lgb_pred2 = lgb_model_single2.predict(X_lgbcat)\n    lgb_pred  = 0.5*lgb_pred1 + 0.5*lgb_pred2\n    cat_pred  = cat_model_single.predict(X_lgbcat)\n\n    # ----------------------------------------------------\n    # 关键：针对 symbol_id <39 和 >=39 做不同融合\n    # ----------------------------------------------------\n    symbol_arr = test_pd[\"symbol_id\"].values  # numpy array\n    mask_new   = (symbol_arr >= 39)\n\n    # 先生成容器\n    final_pred = np.zeros(len(test_pd), dtype=np.float32)\n\n    # 1) 对老symbol =>  NN avg + LGB + Cat\n    #    融合 => final_pred_old = ALPHA_NN*nn_avg + (1-ALPHA_NN)* [ ALPHA_LGB*lgb + (1-ALPHA_LGB)*cat ]\n    #    其中 ALPHA_NN, ALPHA_LGB 全局定义\n    lgbcat_fuse = ALPHA_LGB*lgb_pred + (1.0 - ALPHA_LGB)*cat_pred\n    final_pred[~mask_new] = ( ALPHA_NN*nn_pred_avg[~mask_new]\n                              + (1-ALPHA_NN)*lgbcat_fuse[~mask_new] )\n\n    # 2) 对新symbol => 仅 NN avg + Cat\n    #    final_pred_new = ALPHA_NN*nn_avg + (1-ALPHA_NN)*cat\n    final_pred[mask_new] = ( ALPHA_NN*nn_pred_avg[mask_new]\n                             + (1-ALPHA_NN)*cat_pred[mask_new] )\n\n    # clip\n    final_pred = np.clip(final_pred, -5, 5)\n\n    out_df = pd.DataFrame({\n        \"row_id\": row_ids,\n        \"responder_6\": final_pred\n    })\n    assert (out_df.columns == [\"row_id\",\"responder_6\"]).all()\n    assert len(out_df) == len(test_pd)\n\n    return out_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T16:03:26.751981Z","iopub.execute_input":"2025-01-13T16:03:26.752284Z","iopub.status.idle":"2025-01-13T16:03:39.825538Z","shell.execute_reply.started":"2025-01-13T16:03:26.752258Z","shell.execute_reply":"2025-01-13T16:03:39.824566Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T16:05:09.956206Z","iopub.execute_input":"2025-01-13T16:05:09.956535Z","iopub.status.idle":"2025-01-13T16:05:10.584275Z","shell.execute_reply.started":"2025-01-13T16:05:09.956508Z","shell.execute_reply":"2025-01-13T16:05:10.583423Z"}},"outputs":[],"execution_count":null}]}