{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":9801075,"sourceType":"datasetVersion","datasetId":6006872},{"sourceId":9806342,"sourceType":"datasetVersion","datasetId":6010899},{"sourceId":203900450,"sourceType":"kernelVersion"}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":7.594014,"end_time":"2024-10-10T11:58:36.355301","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-10-10T11:58:28.761287","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport os, gc\nfrom tqdm.auto import tqdm\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nimport pickle\n\nfrom sklearn.ensemble import RandomForestRegressor\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom pytorch_lightning import (LightningDataModule, LightningModule, Trainer)\nfrom pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, Timer\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.metrics import r2_score\nfrom sklearn.model_selection import train_test_split\nfrom torch.utils.data import Dataset, DataLoader\n\n\nfrom sklearn.metrics import r2_score\nfrom lightgbm import LGBMRegressor\nimport lightgbm as lgb\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T23:22:46.282958Z","iopub.execute_input":"2024-12-05T23:22:46.283630Z","iopub.status.idle":"2024-12-05T23:22:56.323518Z","shell.execute_reply.started":"2024-12-05T23:22:46.283593Z","shell.execute_reply":"2024-12-05T23:22:56.322818Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"gridColor = 'lightgrey'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T23:22:56.324910Z","iopub.execute_input":"2024-12-05T23:22:56.325467Z","iopub.status.idle":"2024-12-05T23:22:56.329177Z","shell.execute_reply.started":"2024-12-05T23:22:56.325437Z","shell.execute_reply":"2024-12-05T23:22:56.328401Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Analysis of Dataset","metadata":{}},{"cell_type":"code","source":"%%time\npath = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\nsamples = [] \n\n# Load a data from each file:\nr = range(2)\nfor i in r:\n    file_path = f\"{path}/train.parquet/partition_id={i}/part-0.parquet\"\n    part = pd.read_parquet(file_path)\n    samples.append(part)\n    \nsample_df = pd.concat(samples, ignore_index=True) # Concatenate all samples into one DataFrame if needed\n\nsample_df.round(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T23:22:56.330351Z","iopub.execute_input":"2024-12-05T23:22:56.330749Z","iopub.status.idle":"2024-12-05T23:23:06.657519Z","shell.execute_reply.started":"2024-12-05T23:22:56.330712Z","shell.execute_reply":"2024-12-05T23:23:06.656460Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Plots","metadata":{}},{"cell_type":"code","source":"train =sample_df\ntrain['N']=train.index.values \ntrain['id']=train.index.values \n\nxx= sample_df[(sample_df.symbol_id==1)] ['id']\nyy=sample_df[ (sample_df.symbol_id==1)]['responder_6']\n\nplt.figure(figsize=(16, 5))\nplt.plot(xx,yy, color = 'black', linewidth =0.05)\nplt.suptitle('Returns, responder_6', weight='bold', fontsize=16)\nplt.xlabel(\"Time\", fontsize=12)\nplt.ylabel(\"Returns\", fontsize=12)\nplt.grid(color = gridColor , linewidth=0.8)\nplt.axhline(0, color='red', linestyle='-', linewidth=1.2)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T23:23:06.659283Z","iopub.execute_input":"2024-12-05T23:23:06.659617Z","iopub.status.idle":"2024-12-05T23:23:07.543097Z","shell.execute_reply.started":"2024-12-05T23:23:06.659589Z","shell.execute_reply":"2024-12-05T23:23:07.542258Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(9):\n    plt.hist(sample_df[(sample_df.symbol_id == 1)][f\"responder_{i}\"], bins=100);","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T00:15:22.842363Z","iopub.execute_input":"2024-12-06T00:15:22.842644Z","iopub.status.idle":"2024-12-06T00:15:25.877557Z","shell.execute_reply.started":"2024-12-06T00:15:22.842618Z","shell.execute_reply":"2024-12-06T00:15:25.876587Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#for symbol_id=1\nplt.figure(figsize=(14, 4))\nplt.plot(xx,yy.cumsum(), color = 'black', linewidth =0.6)\nplt.suptitle('Cumulative responder_6', weight='bold', fontsize=16)\nplt.xlabel(\"Time\", fontsize=12)\nplt.ylabel(\"Cumulative res\", fontsize=12)\nplt.yticks(np.arange(-500,1000,250))\n#plt.xticks(np.arange(0,170,10))\nplt.grid(color = gridColor)\n#plt.grid(color = 'lightblue')\nplt.axhline(0, color='red', linestyle='-', linewidth=0.7)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T23:23:08.049650Z","iopub.execute_input":"2024-12-05T23:23:08.050211Z","iopub.status.idle":"2024-12-05T23:23:08.295641Z","shell.execute_reply.started":"2024-12-05T23:23:08.050168Z","shell.execute_reply":"2024-12-05T23:23:08.294832Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for symbol_id == 0\nplt.figure(figsize=(18, 7))\npredictor_cols = [col for col in sample_df.columns if 'responder' in col]\nfor i in predictor_cols: \n    if i == 'responder_6': \n        c='red'\n        lw=2.5\n        plt.plot((sample_df[sample_df.symbol_id == 0].groupby(['date_id'])[i].mean()).cumsum(), linewidth = lw, color = c)\n    else: \n        lw=1\n        plt.plot((sample_df[sample_df.symbol_id == 0].groupby(['date_id'])[i].mean()).cumsum(), linewidth = lw)\n\nplt.xlabel('Trade days')\nplt.ylabel('Cumulative response')\nplt.title('Response time series over trade days  \\n Responder 6 (red) and other responders', weight='bold')\nplt.grid(visible=True, color = gridColor, linewidth = 0.7)\nplt.axhline(0, color='blue', linestyle='-', linewidth=1)\nplt.legend(predictor_cols)\nsns.despine()\n#plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T23:31:10.756448Z","iopub.execute_input":"2024-12-05T23:31:10.757210Z","iopub.status.idle":"2024-12-05T23:31:12.747093Z","shell.execute_reply.started":"2024-12-05T23:31:10.757178Z","shell.execute_reply":"2024-12-05T23:31:12.746213Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(6, 6))\nresponders = pd.read_csv(f\"{path}/responders.csv\")\nmatrix = responders[[ f\"tag_{no}\" for no in range(0,5,1) ] ].T.corr()\nsns.heatmap(matrix, square=True, cmap=\"coolwarm\", alpha =0.9, vmin=-1, vmax=1, center= 0, linewidths=0.5, \n            linecolor='white', annot=True, fmt='.2f')\nplt.xlabel(\"Responder_0 - Responder_8\")\nplt.ylabel(\"Responder_0 - Responder_8\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T23:31:14.389405Z","iopub.execute_input":"2024-12-05T23:31:14.389721Z","iopub.status.idle":"2024-12-05T23:31:14.974343Z","shell.execute_reply.started":"2024-12-05T23:31:14.389694Z","shell.execute_reply":"2024-12-05T23:31:14.973466Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.apply(lambda x: x.fillna(x.mean()), axis=0)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"markdown","source":"## Configurations Fore Prediction","metadata":{}},{"cell_type":"code","source":"class CONFIG:\n    seed = 42\n    target_col = \"responder_6\"\n    # feature_cols = [\"symbol_id\", \"time_id\"] + [f\"feature_{idx:02d}\" for idx in range(79)]+ [f\"responder_{idx}_lag_1\" for idx in range(9)]\n    feature_cols_small = [f\"feature_{idx:02d}\" for idx in range(79)]\n    feature_cols = [f\"feature_{idx:02d}\" for idx in range(79)]+ [f\"responder_{idx}_lag_1\" for idx in range(9)]\n    responders = [f\"responder_{i}\" for i in range(9)]\n    \n    model_paths = [\n        #\"/kaggle/input/js24-train-gbdt-model-with-lags-singlemodel/result.pkl\",\n        #\"/kaggle/input/js24-trained-gbdt-model/result.pkl\",\n        \"/kaggle/input/js-xs-nn-trained-model\",\n        \"/kaggle/input/js-with-lags-trained-xgb/result.pkl\",\n    ]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T00:14:01.021379Z","iopub.execute_input":"2024-12-06T00:14:01.022045Z","iopub.status.idle":"2024-12-06T00:14:01.027219Z","shell.execute_reply.started":"2024-12-06T00:14:01.022009Z","shell.execute_reply":"2024-12-06T00:14:01.026371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_plot_matrix(matrix):\n    matrix = matrix - np.min(matrix)\n    matrix /= np.max(matrix)\n    return matrix","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T00:14:01.299015Z","iopub.execute_input":"2024-12-06T00:14:01.299651Z","iopub.status.idle":"2024-12-06T00:14:01.303488Z","shell.execute_reply.started":"2024-12-06T00:14:01.299618Z","shell.execute_reply":"2024-12-06T00:14:01.302663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"col_list = CONFIG.feature_cols_small[:] + CONFIG.responders\ncorr_matrix = train[col_list].corr()\n# print(np.max(np.abs(corr_matrix.iloc[:-1,-1])))\nindex = col_list.index(\"responder_6\")\nprint(np.max(np.abs(corr_matrix.drop(index=\"responder_6\").iloc[:,index])))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T00:14:01.368609Z","iopub.execute_input":"2024-12-06T00:14:01.368840Z","iopub.status.idle":"2024-12-06T00:15:22.388779Z","shell.execute_reply.started":"2024-12-06T00:14:01.368817Z","shell.execute_reply":"2024-12-06T00:15:22.387869Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.hist(corr_matrix.iloc[:-1,-1]);","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T00:22:00.651483Z","iopub.execute_input":"2024-12-06T00:22:00.651810Z","iopub.status.idle":"2024-12-06T00:22:00.872536Z","shell.execute_reply.started":"2024-12-06T00:22:00.651784Z","shell.execute_reply":"2024-12-06T00:22:00.871616Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(get_plot_matrix(corr_matrix))\nplt.colorbar()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T00:22:03.168136Z","iopub.execute_input":"2024-12-06T00:22:03.168511Z","iopub.status.idle":"2024-12-06T00:22:03.411752Z","shell.execute_reply.started":"2024-12-06T00:22:03.168479Z","shell.execute_reply":"2024-12-06T00:22:03.410914Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_matrix","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T00:15:22.774213Z","iopub.execute_input":"2024-12-06T00:15:22.774591Z","iopub.status.idle":"2024-12-06T00:15:22.841277Z","shell.execute_reply.started":"2024-12-06T00:15:22.774551Z","shell.execute_reply":"2024-12-06T00:15:22.840377Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T00:11:44.536666Z","iopub.execute_input":"2024-12-06T00:11:44.537486Z","iopub.status.idle":"2024-12-06T00:11:44.540981Z","shell.execute_reply.started":"2024-12-06T00:11:44.537450Z","shell.execute_reply":"2024-12-06T00:11:44.539994Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# col_list","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T00:08:42.693876Z","iopub.execute_input":"2024-12-06T00:08:42.694711Z","iopub.status.idle":"2024-12-06T00:08:42.698247Z","shell.execute_reply.started":"2024-12-06T00:08:42.694677Z","shell.execute_reply":"2024-12-06T00:08:42.697268Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train[\"feature_21\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T00:08:43.984808Z","iopub.execute_input":"2024-12-06T00:08:43.985469Z","iopub.status.idle":"2024-12-06T00:08:43.989523Z","shell.execute_reply.started":"2024-12-06T00:08:43.985432Z","shell.execute_reply":"2024-12-06T00:08:43.988404Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train.iloc[:,25]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T00:08:45.014020Z","iopub.execute_input":"2024-12-06T00:08:45.014778Z","iopub.status.idle":"2024-12-06T00:08:45.018390Z","shell.execute_reply.started":"2024-12-06T00:08:45.014741Z","shell.execute_reply":"2024-12-06T00:08:45.017403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# np.corrcoef(train.iloc[:,21],train.iloc[:,21])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T00:08:50.562541Z","iopub.execute_input":"2024-12-06T00:08:50.562871Z","iopub.status.idle":"2024-12-06T00:08:50.566923Z","shell.execute_reply.started":"2024-12-06T00:08:50.562842Z","shell.execute_reply":"2024-12-06T00:08:50.566047Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"count = 0\nfor feature in CONFIG.feature_cols:\n    train[train.symbol_id == 1][feature].rolling(100).mean().plot()\n    count += 1\n    if count == 10:\n        break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T23:38:13.373520Z","iopub.execute_input":"2024-12-05T23:38:13.373859Z","iopub.status.idle":"2024-12-05T23:38:15.560614Z","shell.execute_reply.started":"2024-12-05T23:38:13.373830Z","shell.execute_reply":"2024-12-05T23:38:15.559646Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.iloc[:,20]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T23:56:32.519651Z","iopub.execute_input":"2024-12-05T23:56:32.520019Z","iopub.status.idle":"2024-12-05T23:56:32.527718Z","shell.execute_reply.started":"2024-12-05T23:56:32.519980Z","shell.execute_reply":"2024-12-05T23:56:32.526606Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load preprocessed data (to calculate CV)","metadata":{}},{"cell_type":"code","source":"valid = pl.scan_parquet(\n    f\"/kaggle/input/js24-preprocessing-create-lags/validation.parquet/\"\n).collect().to_pandas()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:35:16.339180Z","iopub.execute_input":"2024-12-05T19:35:16.339543Z","iopub.status.idle":"2024-12-05T19:35:18.862148Z","shell.execute_reply.started":"2024-12-05T19:35:16.339511Z","shell.execute_reply":"2024-12-05T19:35:18.861417Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load model","metadata":{}},{"cell_type":"code","source":"xgb_model = None\nmodel_path = CONFIG.model_paths[1]\nwith open( model_path, \"rb\") as fp:\n    result = pickle.load(fp)\n    xgb_model = result[\"model\"]\n\nxgb_feature_cols = [\"symbol_id\", \"time_id\"] + CONFIG.feature_cols\n\n# Show model\ndisplay(xgb_model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:35:18.864019Z","iopub.execute_input":"2024-12-05T19:35:18.864389Z","iopub.status.idle":"2024-12-05T19:35:18.945283Z","shell.execute_reply.started":"2024-12-05T19:35:18.864350Z","shell.execute_reply":"2024-12-05T19:35:18.944441Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load Randomforest model","metadata":{}},{"cell_type":"code","source":"# Initialize and train the Random Forest model\nrf_model = RandomForestRegressor(n_estimators=100, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:35:18.946333Z","iopub.execute_input":"2024-12-05T19:35:18.946597Z","iopub.status.idle":"2024-12-05T19:35:18.950356Z","shell.execute_reply.started":"2024-12-05T19:35:18.946571Z","shell.execute_reply":"2024-12-05T19:35:18.949562Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Custom R2 metric and PyTorch Lightning model implementation with validation metrics logging, incorporating sample-weighted R2 calculation, learning rate scheduler, and progress bar visualization.","metadata":{}},{"cell_type":"code","source":"# Custom R2 metric for validation\ndef r2_val(y_true, y_pred, sample_weight):\n    r2 = 1 - np.average((y_pred - y_true) ** 2, weights=sample_weight) / (np.average((y_true) ** 2, weights=sample_weight) + 1e-38)\n    return r2\n\n\nclass NN(LightningModule):\n    def __init__(self, input_dim, hidden_dims, dropouts, lr, weight_decay):\n        super().__init__()\n        self.save_hyperparameters()\n        layers = []\n        in_dim = input_dim\n        for i, hidden_dim in enumerate(hidden_dims):\n            layers.append(nn.BatchNorm1d(in_dim))\n            if i > 0:\n                layers.append(nn.SiLU())\n            if i < len(dropouts):\n                layers.append(nn.Dropout(dropouts[i]))\n            layers.append(nn.Linear(in_dim, hidden_dim))\n            # layers.append(nn.ReLU())\n            in_dim = hidden_dim\n        layers.append(nn.Linear(in_dim, 1))  # 输出层\n        layers.append(nn.Tanh())\n        self.model = nn.Sequential(*layers)\n        self.lr = lr\n        self.weight_decay = weight_decay\n        self.validation_step_outputs = []\n\n    def forward(self, x):\n        return 5 * self.model(x).squeeze(-1)  # 输出为一维张量\n\n    def training_step(self, batch):\n        x, y, w = batch\n        y_hat = self(x)\n        loss = F.mse_loss(y_hat, y, reduction='none') * w  # 考虑样本权重\n        loss = loss.mean()\n        self.log('train_loss', loss, on_step=False, on_epoch=True, batch_size=x.size(0))\n        return loss\n\n    def validation_step(self, batch):\n        x, y, w = batch\n        y_hat = self(x)\n        loss = F.mse_loss(y_hat, y, reduction='none') * w\n        loss = loss.mean()\n        self.log('val_loss', loss, on_step=False, on_epoch=True, batch_size=x.size(0))\n        self.validation_step_outputs.append((y_hat, y, w))\n        return loss\n\n    def on_validation_epoch_end(self):\n        \"\"\"Calculate validation WRMSE at the end of the epoch.\"\"\"\n        y = torch.cat([x[1] for x in self.validation_step_outputs]).cpu().numpy()\n        if self.trainer.sanity_checking:\n            prob = torch.cat([x[0] for x in self.validation_step_outputs]).cpu().numpy()\n        else:\n            prob = torch.cat([x[0] for x in self.validation_step_outputs]).cpu().numpy()\n            weights = torch.cat([x[2] for x in self.validation_step_outputs]).cpu().numpy()\n            # r2_val\n            val_r_square = r2_val(y, prob, weights)\n            self.log(\"val_r_square\", val_r_square, prog_bar=True, on_step=False, on_epoch=True)\n        self.validation_step_outputs.clear()\n\n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.parameters(), lr=self.lr, weight_decay=self.weight_decay)\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5,\n                                                               verbose=True)\n        return {\n            'optimizer': optimizer,\n            'lr_scheduler': {\n                'scheduler': scheduler,\n                'monitor': 'val_loss',\n            }\n        }\n\n    def on_train_epoch_end(self):\n        if self.trainer.sanity_checking:\n            return\n        epoch = self.trainer.current_epoch\n        metrics = {k: v.item() if isinstance(v, torch.Tensor) else v for k, v in self.trainer.logged_metrics.items()}\n        formatted_metrics = {k: f\"{v:.5f}\" for k, v in metrics.items()}\n        print(f\"Epoch {epoch}: {formatted_metrics}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:35:18.952129Z","iopub.execute_input":"2024-12-05T19:35:18.952390Z","iopub.status.idle":"2024-12-05T19:35:18.967642Z","shell.execute_reply.started":"2024-12-05T19:35:18.952365Z","shell.execute_reply":"2024-12-05T19:35:18.966758Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load and initialize the best models for `N_folds` cross-validation using PyTorch Lightning checkpoints on GPU.\n","metadata":{}},{"cell_type":"code","source":"N_folds = 5\n# 加载最佳模型\nmodels = []\nfor fold in range(N_folds):\n    checkpoint_path = f\"{CONFIG.model_paths[0]}/nn_{fold}.model\"\n    model = NN.load_from_checkpoint(checkpoint_path)\n    models.append(model.to(\"cuda:0\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:35:22.227549Z","iopub.execute_input":"2024-12-05T19:35:22.228381Z","iopub.status.idle":"2024-12-05T19:35:23.115742Z","shell.execute_reply.started":"2024-12-05T19:35:22.228344Z","shell.execute_reply":"2024-12-05T19:35:23.115009Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CV Score","metadata":{}},{"cell_type":"code","source":"X_valid = valid[ xgb_feature_cols ]\ny_valid = valid[ CONFIG.target_col ]\nw_valid = valid[ \"weight\" ]\ny_pred_valid_xgb = xgb_model.predict(X_valid)\nvalid_score = r2_score( y_valid, y_pred_valid_xgb, sample_weight=w_valid )\nvalid_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:35:23.990350Z","iopub.execute_input":"2024-12-05T19:35:23.991009Z","iopub.status.idle":"2024-12-05T19:35:26.743965Z","shell.execute_reply.started":"2024-12-05T19:35:23.990975Z","shell.execute_reply":"2024-12-05T19:35:26.743002Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Prepare validation data by filling missing values in features with forward fill and zeros, then output shapes of `X_valid`, `y_valid`, and `w_valid`.\n","metadata":{}},{"cell_type":"code","source":"X_valid = valid[ CONFIG.feature_cols ]\ny_valid = valid[ CONFIG.target_col ]\nw_valid = valid[ \"weight\" ]\nX_valid = X_valid.fillna(method = 'ffill').fillna(0)\nX_valid.shape, y_valid.shape, w_valid.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:35:26.745789Z","iopub.execute_input":"2024-12-05T19:35:26.746802Z","iopub.status.idle":"2024-12-05T19:35:28.807866Z","shell.execute_reply.started":"2024-12-05T19:35:26.746758Z","shell.execute_reply":"2024-12-05T19:35:28.806965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import ElasticNet\nfrom sklearn.metrics import r2_score\n\n# Train ElasticNet model\nen_model = ElasticNet(alpha=0.1, l1_ratio=0.5)  # You can adjust alpha and l1_ratio as needed\nen_model.fit(X_valid, y_valid, sample_weight=w_valid)\n\n# Predict with the trained ElasticNet model\ny_pred_valid_en = en_model.predict(X_valid)\n\n# Evaluate the model's performance using R2 score\nvalid_score_en = r2_score(y_valid, y_pred_valid_en, sample_weight=w_valid)\nprint(f\"Validation R2 score for ElasticNet: {valid_score_en}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:35:28.809063Z","iopub.execute_input":"2024-12-05T19:35:28.809847Z","iopub.status.idle":"2024-12-05T19:35:34.562216Z","shell.execute_reply.started":"2024-12-05T19:35:28.809805Z","shell.execute_reply":"2024-12-05T19:35:34.559146Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train Gradient Boosting Model (LightGBM)\nlgb_model = LGBMRegressor(n_estimators=1000, learning_rate=0.05, max_depth=7)\nlgb_model.fit(X_valid, y_valid, sample_weight=w_valid)\n\n# Predict with the trained LightGBM model\ny_pred_valid_lgb = lgb_model.predict(X_valid)\nvalid_score_lgb = r2_score(y_valid, y_pred_valid_lgb, sample_weight=w_valid)\nprint(f\"Validation R2 score for LightGBM: {valid_score_lgb}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:36:59.570189Z","iopub.execute_input":"2024-12-05T19:36:59.570845Z","iopub.status.idle":"2024-12-05T19:39:47.525853Z","shell.execute_reply.started":"2024-12-05T19:36:59.570809Z","shell.execute_reply":"2024-12-05T19:39:47.524539Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Aggregate predictions from all models for validation data using PyTorch, calculate the weighted R2 score, and store the validation score.","metadata":{}},{"cell_type":"code","source":"y_pred_valid_nn = np.zeros(y_valid.shape)\nwith torch.no_grad():\n    for model in models:\n        model.eval()\n        y_pred_valid_nn += model(torch.FloatTensor(X_valid.values).to(\"cuda:0\")).cpu().numpy() / len(models)\nvalid_score = r2_score( y_valid, y_pred_valid_nn, sample_weight=w_valid )\nvalid_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:40:00.236748Z","iopub.execute_input":"2024-12-05T19:40:00.237538Z","iopub.status.idle":"2024-12-05T19:40:07.014518Z","shell.execute_reply.started":"2024-12-05T19:40:00.237507Z","shell.execute_reply":"2024-12-05T19:40:07.013591Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Combine predictions from XGBoost and neural network models in a weighted ensemble, compute the weighted R2 score, and store the validation score.","metadata":{}},{"cell_type":"code","source":"y_pred_valid_ensemble = 0.5 * (y_pred_valid_xgb + y_pred_valid_nn)\nvalid_score = r2_score( y_valid, y_pred_valid_ensemble, sample_weight=w_valid )\nvalid_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:40:22.552711Z","iopub.execute_input":"2024-12-05T19:40:22.553023Z","iopub.status.idle":"2024-12-05T19:40:22.572262Z","shell.execute_reply.started":"2024-12-05T19:40:22.552998Z","shell.execute_reply":"2024-12-05T19:40:22.571393Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Release validation data from memory and trigger garbage collection to optimize memory usage.","metadata":{}},{"cell_type":"code","source":"del valid, X_valid, y_valid, w_valid\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:40:31.315928Z","iopub.execute_input":"2024-12-05T19:40:31.316248Z","iopub.status.idle":"2024-12-05T19:40:31.562341Z","shell.execute_reply.started":"2024-12-05T19:40:31.316220Z","shell.execute_reply":"2024-12-05T19:40:31.561542Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Define a prediction function that processes test data, integrates lag features, combines predictions from XGBoost, neural network, and LightGBM models, and outputs clipped predictions as a DataFrame.\n","metadata":{}},{"cell_type":"code","source":"lags_ : pl.DataFrame | None = None\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    global lags_\n    if lags is not None:\n        lags_ = lags\n\n    predictions = test.select(\n        'row_id',\n        pl.lit(0.0).alias('responder_6'),\n    )\n    \n    symbol_ids = test.select('symbol_id').to_numpy()[:, 0]\n\n    if lags is not None:\n        lags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last() # pick up last record of previous date\n        test = test.join(lags, on=[\"date_id\", \"symbol_id\"], how=\"left\")\n    else:\n        test = test.with_columns(\n            (pl.lit(0.0).alias(f'responder_{idx}_lag_1') for idx in range(9))\n        )\n\n    preds = np.zeros((test.shape[0],))\n    \n    # Adding XGBoost predictions\n    preds += xgb_model.predict(test[xgb_feature_cols].to_pandas()) / 3\n    \n    # Adding Neural Network predictions\n    test_input = test[CONFIG.feature_cols].to_pandas()\n    test_input = test_input.fillna(method='ffill').fillna(0)\n    test_input = torch.FloatTensor(test_input.values).to(\"cuda:0\")\n    with torch.no_grad():\n        for nn_model in models:\n            nn_model.eval()\n            preds += nn_model(test_input).cpu().numpy() / len(models)\n    \n    # Adding LightGBM predictions\n    preds += lgb_model.predict(test[CONFIG.feature_cols].to_pandas()) / 3\n    \n    print(f\"predict> preds.shape =\", preds.shape)\n    \n    predictions = test.select('row_id').with_columns(\n        pl.Series(\n            name='responder_6',\n            values=np.clip(preds, a_min=-5, a_max=5),\n            dtype=pl.Float64,\n        )\n    )\n\n    assert isinstance(predictions, pl.DataFrame | pd.DataFrame)\n    assert list(predictions.columns) == ['row_id', 'responder_6']\n    assert len(predictions) == len(test)\n\n    return predictions\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.018344,"end_time":"2024-10-10T11:58:33.59684","exception":false,"start_time":"2024-10-10T11:58:33.578496","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:40:33.847125Z","iopub.execute_input":"2024-12-05T19:40:33.847474Z","iopub.status.idle":"2024-12-05T19:40:33.856836Z","shell.execute_reply.started":"2024-12-05T19:40:33.847440Z","shell.execute_reply":"2024-12-05T19:40:33.855880Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":2.225871,"end_time":"2024-10-10T11:58:35.830964","exception":false,"start_time":"2024-10-10T11:58:33.605093","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:40:37.222406Z","iopub.execute_input":"2024-12-05T19:40:37.223045Z","iopub.status.idle":"2024-12-05T19:40:37.458369Z","shell.execute_reply.started":"2024-12-05T19:40:37.223009Z","shell.execute_reply":"2024-12-05T19:40:37.456884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}