{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":10421155,"sourceType":"datasetVersion","datasetId":6459010},{"sourceId":214081095,"sourceType":"kernelVersion"},{"sourceId":217350000,"sourceType":"kernelVersion"},{"sourceId":226170,"sourceType":"modelInstanceVersion","modelInstanceId":192905,"modelId":214855},{"sourceId":226171,"sourceType":"modelInstanceVersion","modelInstanceId":192906,"modelId":214856},{"sourceId":228020,"sourceType":"modelInstanceVersion","modelInstanceId":194419,"modelId":216328},{"sourceId":228624,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":194952,"modelId":216849}],"dockerImageVersionId":30823,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from typing import List, Any\n\nimport polars as pl\nimport pandas as pd\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom datetime import datetime\nfrom tqdm.notebook import tqdm\nimport joblib\nimport gc\n\nimport sys\nimport os\nimport kaggle_evaluation.jane_street_inference_server\nsys.path.append(r\"/kaggle/input/dnn-online-config\")\nfrom eval_metric import *\nfrom config import Config","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-13T13:56:57.846994Z","iopub.execute_input":"2025-01-13T13:56:57.847270Z","iopub.status.idle":"2025-01-13T13:57:01.678566Z","shell.execute_reply.started":"2025-01-13T13:56:57.847247Z","shell.execute_reply":"2025-01-13T13:57:01.677874Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def same_seed(seed: int) -> None:\n    \"\"\"设置随机数种子（便于复现）\n\n    Parameters\n    ----------\n    seed : int\n        要设置的随机数种子的值\n    \"\"\"\n    # 设置 CUDNN 为确定性操作，确保每次运行结果相同（有助于复现）\n    torch.backends.cudnn.deterministic = True\n    \n    # 禁用 CUDNN 的自动优化功能，以避免影响确定性行为\n    torch.backends.cudnn.benchmark = False\n    \n    # 设置 numpy 的随机种子，确保 numpy 的随机数生成器是可复现的\n    np.random.seed(seed)\n    \n    # 设置 Pytorch 的 CPU 随机种子，确保 CPU 上的操作是可复现的\n    torch.manual_seed(seed)\n    \n    # 如果有 GPU 可用，设置所有 GPU 设备的随机数种子，确保 GPU 上的操作也是可复现的\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        \n    pl.set_random_seed(seed)\n        \n    # 打印随机数种子值，便于追踪\n    print(f\"Set Seed = {seed}\")\n\ndef r2_score(y_true: torch.Tensor, \n             y_pred: torch.Tensor, \n             sample_weight: torch.Tensor):\n    numerator = torch.sum(sample_weight * (y_pred - y_true) ** 2) / torch.sum(sample_weight)\n    denominator = torch.sum(sample_weight * (y_true ** 2)) / torch.sum(sample_weight) + 1e-38\n    return 1 - numerator / denominator\n\nclass WeightedMSELoss(nn.Module):\n    def __init__(self):\n        super(WeightedMSELoss, self).__init__()\n    \n    def forward(self, y_true, y_pred, sample_weight):\n        numerator = torch.mean(sample_weight*((y_pred - y_true) ** 2))\n        denominator = torch.mean(sample_weight*(y_true ** 2)) + 1e-38\n        return numerator / denominator\n\nclass JsDNNDataset(Dataset):\n    def __init__(self, config: Config, df: pl.DataFrame):\n        self.X: torch.Tensor = torch.tensor(df.select(config.features).to_numpy(), dtype=torch.float32)\n        self.target: torch.Tensor = torch.tensor(df.select(config.target).to_numpy().squeeze(), dtype=torch.float32)\n        self.weight: torch.Tensor = torch.tensor(df.select(config.sample_weight).to_numpy().squeeze(), dtype=torch.float32)\n        \n        # 打印形状以检查数据是否正确\n        # print(f\"X shape: {self.X.shape}\")\n        # print(f\"target shape: {self.target.shape}\")\n        # print(f\"weight shape: {self.weight.shape}\")\n        \n    def __len__(self):\n        return len(self.target)\n    \n    def __getitem__(self, idx):\n        # print(f\"Accessing index {idx}\")  # 打印索引调试信息\n        return self.X[idx], self.target[idx], self.weight[idx]\n    \nclass DNN(nn.Module):\n    \n    def __init__(self, input_dim: int,\n                       hidden_dims: List[int],\n                       dropouts: List[float],):\n        super(DNN, self).__init__()\n        \n        assert len(hidden_dims) == len(dropouts)\n        \n        layers: List[Any] = []\n        self.input_dim = input_dim\n        \n        # 设置隐藏层\n        for i, hidden_dim in enumerate(hidden_dims):\n            if i == 0:\n                layers.append(nn.Linear(input_dim, hidden_dim))\n                layers.append(nn.BatchNorm1d(hidden_dim))\n                layers.append(nn.SiLU())\n                # layers.append(nn.BatchNorm1d(hidden_dim))\n                layers.append(nn.Dropout(dropouts[i]))\n            else:\n                layers.append(nn.Linear(hidden_dims[i-1], hidden_dim))\n                layers.append(nn.BatchNorm1d(hidden_dim))\n                layers.append(nn.SiLU())\n                # layers.append(nn.BatchNorm1d(hidden_dim))\n                layers.append(nn.Dropout(dropouts[i]))\n              \n        # 设置输出层  \n        layers.append(nn.Linear(hidden_dims[-1], 1))\n        self.model = nn.Sequential(*layers)\n        \n    def forward(self, x):\n        return self.model(x).squeeze(-1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T13:57:01.679575Z","iopub.execute_input":"2025-01-13T13:57:01.679985Z","iopub.status.idle":"2025-01-13T13:57:01.691100Z","shell.execute_reply.started":"2025-01-13T13:57:01.679961Z","shell.execute_reply":"2025-01-13T13:57:01.690264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_agg_list(day, columns, agg_type: str, last=True):\n    agg_mean_list = [pl.col(c).mean().name.suffix(f\"_mean_{agg_type}_{day}d\") for c in columns]\n    agg_std_list = [pl.col(c).std().name.suffix(f\"_std_{agg_type}_{day}d\") for c in columns]\n    agg_max_list = [pl.col(c).max().name.suffix(f\"_max_{agg_type}_{day}d\") for c in columns]\n    agg_min_list = [pl.col(c).min().name.suffix(f\"_min_{agg_type}_{day}d\") for c in columns]\n    \n    agg_list = agg_mean_list + agg_std_list + agg_max_list + agg_min_list\n    \n    if last:\n        agg_last_list = [pl.col(c).last().name.suffix(f\"_last_{agg_type}_{day}d\") for c in columns]\n        agg_list += agg_last_list\n        \n    return agg_list","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T13:57:01.692575Z","iopub.execute_input":"2025-01-13T13:57:01.692836Z","iopub.status.idle":"2025-01-13T13:57:01.707450Z","shell.execute_reply.started":"2025-01-13T13:57:01.692814Z","shell.execute_reply":"2025-01-13T13:57:01.706523Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class JaneStreetOnlinePredictor:\n    \n    def __init__(self, \n                 config: Config,\n                 dnn_model,\n                 gbdt_models, \n                 train_data: pl.DataFrame,\n                 history_data: pl.DataFrame,\n                 retrain_interval: int):\n        self.config: Config = config\n        \n        # 配置 nn model 训练器的参数\n        self.dnn_model = dnn_model\n        self.criterion = WeightedMSELoss()\n        self.optimizer = optim.Adam(self.dnn_model.parameters(), lr=5e-6, weight_decay=1e-5)\n        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        self.dnn_model.to(self.device)\n        \n        self.train_data: pl.DataFrame = train_data\n        self.retrain_data: pl.DataFrame = None\n        self.scored_data: pl.DataFrame = None # 用于计算测试 score 的数据\n        self.lags: pl.DataFrame = None\n        self.new_day_test: List[pl.DataFrame] = []\n        \n        self.retrain_interval: int = retrain_interval\n        self.count_date: int =  0\n        self.max_date_id: int = (\n            self.train_data\n            .select(\"date_id\")\n            .max()\n            .to_series()\n            .cast(pl.Int32)\n            .to_numpy()[0]\n        )\n        self.min_date_id: int = (\n            self.train_data\n            .select(\"date_id\")\n            .min()\n            .to_series()\n            .cast(pl.Int32)\n            .to_numpy()[0]\n        )\n    \n        self.is_zero_day = True \n        self.retrain = False\n\n        # 配置 gbdt model 需要的数据\n        self.gbdt_models = gbdt_models\n        self.history_data = history_data\n        \n        self.lags_infer: pl.DataFrame = None\n    \n    def retrain_dnn(self):\n\n        self.dnn_model.train()\n        retrain_dataloader = DataLoader(JsDNNDataset(self.config, self.retrain_data), \n                                        batch_size=8196, shuffle=True)\n        epochs = 5\n        for epoch in range(epochs):\n            print(f\"epoch: {epoch+1}\")\n            \n            if self.config.debug:\n                y_train_true_list = []\n                y_train_pred_list = []\n                weight_train_list = []\n                \n            retrain_pbar= tqdm(retrain_dataloader, \n                               total=len(retrain_dataloader), \n                               desc=f\"Epoch [{epoch + 1} / {epochs}]\")\n            for X, y, weight in retrain_pbar:\n                self.optimizer.zero_grad()\n                X, y, w = X.to(self.device), y.to(self.device), weight.to(self.device)\n                y_train_pred = self.dnn_model(X)\n                loss = self.criterion(y, y_train_pred, w)\n                loss.backward()\n                self.optimizer.step()\n\n                if self.config.debug: \n                    y_train_true_list.append(y.cpu())\n                    y_train_pred_list.append(y_train_pred.detach().cpu())\n                    weight_train_list.append(w.cpu())\n            if self.config.debug:\n                train_score = r2_score(y_true=torch.cat(y_train_true_list),\n                                        y_pred=torch.cat(y_train_pred_list),\n                                        sample_weight=torch.cat(weight_train_list))\n                print(f\"目前的训练分数：{train_score}\")\n    \n    def update_data(self, lags: pl.DataFrame):\n        \n        # 将前一天的 test 数据和 lags(label) 合并，并统一数据类型\n        test = self.join_lags_with_test(lags)\n        \n        # 从历史的 train_data 中采样数据，然后和前一天的 test data 合并, 并填充缺失值\n        sampled_data = (\n            self.train_data\n            .sample(fraction=1/48, with_replacement=False)\n            .cast(self.config.row_column_tpyes)\n        )\n        self.retrain_data = pl.concat([sampled_data, test]).fill_null(value=0)\n        \n        # 新旧数据的 push 和 pop\n        self.train_data = (\n            pl.concat([self.train_data, test]) # push \n            .filter(pl.col(\"date_id\") > self.min_date_id) # pop\n        )\n\n        # 在 debug 时实时计算每一天累计的测试数据的 score\n        if self.config.debug:\n            if self.scored_data is None:\n                self.scored_data = test.select([\"responder_6\", \"pred\", \"weight\"])\n            else:\n                self.scored_data = pl.concat([self.scored_data,test.select([\"responder_6\", \"pred\", \"weight\"])])\n            \n        # 数据更新完成后，将 new_day_test 重新弄设置为空列表，接收下一个 date 的数据\n        self.new_day_test = []\n        \n        gc.collect()\n        return\n    \n    def cal_current_test_score(self):\n        self.current_score = self.config.score(\n            y_true=self.scored_data.select(self.config.target).to_numpy().squeeze(),\n            y_pred=self.scored_data.select(\"pred\").to_numpy().squeeze(),\n            sample_weight = self.scored_data.select(self.config.sample_weight).to_numpy().squeeze())\n    \n    def join_lags_with_test(self, lags: pl.DataFrame) -> pl.DataFrame:\n        \n        test: pl.DataFrame = pl.concat(self.new_day_test)\n        test = (\n            test\n            .with_columns(\n                pl.lit(self.max_date_id)\n                .alias(\"date_id\")\n            )\n        )\n        \n        # lags 2 label\n        lags2label = (\n            lags\n            .with_columns(\n                pl.lit(self.max_date_id)\n                .alias(\"date_id\")\n            )\n            .rename({f\"{resp}_lag_1\": resp for resp in [f\"responder_{i}\" for i in range(9)]})\n        )\n        \n        # join test with label\n        test = test.join(lags2label, on=[\"symbol_id\", \"date_id\", \"time_id\"], how=\"left\")\n        test = test.cast(self.config.row_column_tpyes)\n\n        # join test with lag features\n        self.lags_infer = (\n            self.lags_infer\n            .with_columns(\n                pl.lit(self.max_date_id)\n                .alias(\"date_id\")\n            )\n            .cast({\"date_id\": pl.Int16})\n        )\n        test = test.join(self.lags_infer, on=[\"symbol_id\", \"date_id\"], how=\"left\")\n        \n        return test.select(self.train_data.columns)\n\n    def feature_engineering(self, lags: pl.DataFrame, current_date: int) :\n        \n        # 原始lags先存储到history更新历史数据\n        lags = lags.rename(self.config.lag_cols_rename)\n        lags = lags.cast(self.config.id_column_types)\n        lags = lags.cast(self.config.responder_column_types)\n\n        self.history_data = pl.concat([self.history_data, lags])\n        \n        # 只储存最近N天的历史数据\n        self.history_data = (\n            self.history_data\n            .filter(pl.col(\"date_id\") > (current_date - self.config.lag_n_days))\n            .sort([\"symbol_id\", \"date_id\", \"time_id\"])\n        )\n\n        # shift 1天的统计值\n        shift_1d_agg_list = create_agg_list(1, self.config.responder_cols, agg_type=\"shift\", last=True)\n        shift_1d_data = self.history_data.filter(pl.col(\"date_id\") == current_date)\n        shift_1d_lags_infer = (\n            shift_1d_data\n            .group_by([\"symbol_id\", \"date_id\"], maintain_order=True)\n            .agg(shift_1d_agg_list)\n        )\n        \n        self.lags_infer = shift_1d_lags_infer\n    \n    def predict(self, \n                test: pl.DataFrame,\n                lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n        current_date = test.select(\"date_id\").to_numpy()[:, 0][0]\n        if lags is not None:\n            \n            self.count_date += 1\n            \n            if not self.is_zero_day:\n                print(self.min_date_id)\n                # ============================================== 更新数据 ====================================\n                start1 = datetime.now()\n                \n                # lags 和 lags2label 都在第 1 天才开始处理（date 从第 0 天开始）\n                # lags 要在前一天保存，后一天统一处理，处理完之后，新的 lags 覆盖旧的 lags\n                self.update_data(lags)\n\n                end1 = datetime.now()\n                print(f\"更新数据用时：{end1 - start1}秒！\")\n                # ===========================================================================================\n                \n                # ============================================== 计算score ====================================\n                # 仅在 debug 时计算\n                if self.config.debug:\n                    self.cal_current_test_score()\n                    print(f\"current_test_score: {self.current_score}\")\n                # ===========================================================================================\n                \n                # ============================================== 再训练 =======================================\n                if self.count_date % self.retrain_interval == 0:\n                    start2 = datetime.now()\n                    \n                    self.retrain = True\n                    self.retrain_dnn()\n\n                    end2 = datetime.now()\n                    print(f\"再训练用时：{end2 - start2}秒！\")\n                # ============================================================================================\n                \n            self.is_zero_day = False # 他在第 0 天之后都是 False\n            self.max_date_id += 1   # 用来替换新数据的日期相当于 push\n            self.min_date_id += 1   # 用来过滤旧数据：增加一天的新数据，删除一天的旧数据, 相当于 pop\n            \n            # 前一天的数据更新完之后，保存 lags，用于下次更新\n            self.lags = None # 为了释放旧的 lags 内存\n            gc.collect() \n            self.lags = lags\n            \n            # ====================================== feature engineering  ====================================\n            self.feature_engineering(lags, current_date)\n            # ================================================================================================\n        else:\n            self.retrain = False\n        \n        # =========================================== gbdt offline预测 ===============================\n        gbdt_test = test.cast(self.config.id_column_types)\n        gbdt_test = gbdt_test.cast(self.config.feature_column_types)\n\n        # 将新来的 test 数据和当天的 lags 数据合并\n        my_X_test = (\n            gbdt_test\n            .join(self.lags_infer, on=[\"symbol_id\", \"date_id\"], how=\"left\")\n        )\n        # gbdt_X_test = my_X_test.select(self.config.features).to_numpy()\n        # gbdt_y_test_pred = np.mean([model.predict(gbdt_X_test) for model in self.gbdt_models], axis=0)\n        # ===========================================================================================\n        \n        # ========================================== nn online 预测 ==================================\n        self.dnn_model.eval()\n        dnn_X_test = torch.tensor( # 将数据填充缺失值后转换为 tensor\n            my_X_test\n            .select(self.config.features)\n            .fill_null(0)\n            .to_numpy(),\n            dtype=torch.float32\n        ).to(self.device)\n        with torch.no_grad():\n            self.optimizer.zero_grad()\n            dnn_y_test_pred = self.dnn_model(dnn_X_test)\n        \n        dnn_y_test_pred = dnn_y_test_pred.cpu().numpy()\n        # ===========================================================================================\n\n        # =============================================== 组合预测 ===================================\n        y_test_pred = dnn_y_test_pred # * 0.5 + gbdt_y_test_pred * 0.5\n        y_test_pred = np.clip(y_test_pred, a_min=-5, a_max=5)\n        # print(y_test_pred)\n        # ===========================================================================================\n        \n        # ========================================= 保存 test data ===================================\n        test = test.with_columns(pl.Series(y_test_pred).alias(\"pred\"))\n        # 将 test 数据添加到 new_day_test 列表中，并在 next_date 的 time_0 进行合并，然后将被重置为空列表\n        selected_test = test.select(pl.all().exclude(\"row_id\", \"is_scored\"))\n        self.new_day_test.append(selected_test)\n        # ===========================================================================================\n        \n        # ============================================== 交卷 ========================================\n        predictions = (\n            test\n            .select(\n                \"row_id\",\n                pl.lit(0.0).alias(\"responder_6\")\n            )\n            .with_columns(pl.Series(y_test_pred).alias(\"responder_6\"))\n        )\n        assert isinstance(predictions, pl.DataFrame | pd.DataFrame)\n        assert list(predictions.columns) == ['row_id', 'responder_6']\n        assert len(predictions) == len(test)\n\n        if self.retrain:\n            print(f\"总用时：{datetime.now() - start1}秒！\")\n\n        return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T13:57:01.709252Z","iopub.execute_input":"2025-01-13T13:57:01.709549Z","iopub.status.idle":"2025-01-13T13:57:01.744696Z","shell.execute_reply.started":"2025-01-13T13:57:01.709521Z","shell.execute_reply":"2025-01-13T13:57:01.743813Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"config = Config","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T13:57:01.745593Z","iopub.execute_input":"2025-01-13T13:57:01.746370Z","iopub.status.idle":"2025-01-13T13:57:01.759341Z","shell.execute_reply.started":"2025-01-13T13:57:01.746335Z","shell.execute_reply":"2025-01-13T13:57:01.758678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"config.lag_n_days = 2\nhistory_data = (\n    pl.scan_parquet(r\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\")\n    .select(['date_id','time_id','symbol_id'] + config.responder_cols)\n    .filter(\n        (pl.col(\"date_id\")>=(1698 - config.lag_n_days))&(pl.col(\"date_id\")<1698)\n    )\n)\nhistory_data = history_data.with_columns(\n    date_id = (pl.col(\"date_id\") - pl.lit(1698))\n              .cast(pl.Int16)\n)\nhistory_data = history_data.collect()\n\nhistory_data = history_data.cast(config.id_column_types)\nhistory_data = history_data.cast(config.responder_column_types)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T13:57:01.760116Z","iopub.execute_input":"2025-01-13T13:57:01.760402Z","iopub.status.idle":"2025-01-13T13:57:02.114835Z","shell.execute_reply.started":"2025-01-13T13:57:01.760382Z","shell.execute_reply":"2025-01-13T13:57:02.113860Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = (\n    pl.scan_parquet(r\"/kaggle/input/js-generate-retrain-data/data.parquet\")\n    .filter(pl.col(\"date_id\").is_between(1219, 1698))\n    .fill_null(value=0)\n    .with_columns([\n        pl.lit(0, dtype=pl.Float32).alias(\"pred\")\n    ])\n    .cast(config.row_column_tpyes)\n    .collect()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T13:57:02.115658Z","iopub.execute_input":"2025-01-13T13:57:02.115967Z","iopub.status.idle":"2025-01-13T13:57:19.552137Z","shell.execute_reply.started":"2025-01-13T13:57:02.115945Z","shell.execute_reply":"2025-01-13T13:57:19.551426Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"same_seed(42)\ndnn_model_params_path = r\"/kaggle/input/50_0.039666_0.022241/pytorch/default/1/50_0.039666_0.022241.pkl\"\ndnn_model_params_dict = joblib.load(dnn_model_params_path)\nprint(\"epoch: \", dnn_model_params_dict[\"epoch\"])\ndnn_model = DNN(\n    input_dim=dnn_model_params_dict[\"input_dim\"],\n    hidden_dims=dnn_model_params_dict[\"hidden_dims\"],\n    dropouts=dnn_model_params_dict[\"dropouts\"]\n)\ndnn_model.load_state_dict(dnn_model_params_dict[\"model_state_dict\"])\n\ngbdt_models = []\n# xgb_models = joblib.load(\n#     r\"/kaggle/input/xgb_0.015296_800_1698/scikitlearn/default/1/2025-01-01_19_13_XGB_0.015296_800_1698.pkl\"\n# )\n# gbdt_models += xgb_models\n\n# lgb_models = joblib.load(\n#     r\"/kaggle/input/lgb_0.017605_800_1698/scikitlearn/default/1/2025-01-01_19_14_LGB_0.017605_800_1698.pkl\"\n# )\n# gbdt_models += lgb_models","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T13:57:19.553953Z","iopub.execute_input":"2025-01-13T13:57:19.554192Z","iopub.status.idle":"2025-01-13T13:57:20.206252Z","shell.execute_reply.started":"2025-01-13T13:57:19.554170Z","shell.execute_reply":"2025-01-13T13:57:20.205525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 生成 lags 特征列表\nnew_features = []\nstat_names = [\"mean\", \"std\", \"max\", \"min\",]\nlast = True\nif last:\n    stat_names.append(\"last\")\nfor col in list(config.responder_cols):\n    for stat_name in stat_names:\n        new_features.append(f\"{col}_{stat_name}_shift_{1}d\")\n \nconfig.features += new_features\n\nprint(len(config.features))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T13:57:20.207140Z","iopub.execute_input":"2025-01-13T13:57:20.207384Z","iopub.status.idle":"2025-01-13T13:57:20.212334Z","shell.execute_reply.started":"2025-01-13T13:57:20.207351Z","shell.execute_reply":"2025-01-13T13:57:20.211516Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"same_seed(42)\nconfig.debug = False\n\njs_predict = JaneStreetOnlinePredictor(config, \n                                       dnn_model, \n                                       gbdt_models, \n                                       train_data.drop(\"partition_id\"), \n                                       history_data, \n                                       1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T13:57:20.213127Z","iopub.execute_input":"2025-01-13T13:57:20.213558Z","iopub.status.idle":"2025-01-13T13:57:21.114502Z","shell.execute_reply.started":"2025-01-13T13:57:20.213528Z","shell.execute_reply":"2025-01-13T13:57:21.113563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ntest = False\n\ninference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(js_predict.predict)\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    if test:\n         inference_server.run_local_gateway(\n            (\n                \"/kaggle/input/js24-create-simulate-data/test.parquet\",\n                \"/kaggle/input/js24-create-simulate-data/lags.parquet\",\n            )\n        )\n    else:\n        inference_server.run_local_gateway(\n            (\n                \"/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet\",\n                \"/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet\",\n            )\n        )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T13:57:21.115430Z","iopub.execute_input":"2025-01-13T13:57:21.115804Z","iopub.status.idle":"2025-01-13T13:57:21.655799Z","shell.execute_reply.started":"2025-01-13T13:57:21.115777Z","shell.execute_reply":"2025-01-13T13:57:21.654927Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}