{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":9801075,"sourceType":"datasetVersion","datasetId":6006872},{"sourceId":9806342,"sourceType":"datasetVersion","datasetId":6010899},{"sourceId":10139918,"sourceType":"datasetVersion","datasetId":6258261},{"sourceId":10139922,"sourceType":"datasetVersion","datasetId":6258265},{"sourceId":10253875,"sourceType":"datasetVersion","datasetId":6297065},{"sourceId":10304887,"sourceType":"datasetVersion","datasetId":6378806},{"sourceId":10351700,"sourceType":"datasetVersion","datasetId":6410107},{"sourceId":203900450,"sourceType":"kernelVersion"},{"sourceId":215616115,"sourceType":"kernelVersion"},{"sourceId":216017958,"sourceType":"kernelVersion"},{"sourceId":216577393,"sourceType":"kernelVersion"}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":80.344101,"end_time":"2024-10-26T03:27:42.952247","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-10-26T03:26:22.608146","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"252dd2de87de42f4becd877fbbafc26b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_ff55584ae0ab4f39ae471f314eaad988","placeholder":"​","style":"IPY_MODEL_b8d338c473aa4dc3ba25f37a997a9037","value":" 1/1 [00:00&lt;00:00, 33.79it/s]"}},"48a2731fb59b4ce8ace5b53d6f0e3337":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a8dbc79c7c5a48318466a102c55801bb","placeholder":"​","style":"IPY_MODEL_ebd06aaaa7024a1684ba1b9fe89358cf","value":"100%"}},"56da652f3eeb42aca986e6fd815629dc":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5bb4e0df01c44716afebab25aafe9f5f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_f4e41234b0cc4f3e9313d971f5aadc9f","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_6408698650f74dd699bcc914b850e396","value":1}},"6408698650f74dd699bcc914b850e396":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"9cb493ec04fc4ed391f5ac28cc84500e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_48a2731fb59b4ce8ace5b53d6f0e3337","IPY_MODEL_5bb4e0df01c44716afebab25aafe9f5f","IPY_MODEL_252dd2de87de42f4becd877fbbafc26b"],"layout":"IPY_MODEL_56da652f3eeb42aca986e6fd815629dc"}},"a8dbc79c7c5a48318466a102c55801bb":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b8d338c473aa4dc3ba25f37a997a9037":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ebd06aaaa7024a1684ba1b9fe89358cf":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"f4e41234b0cc4f3e9313d971f5aadc9f":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ff55584ae0ab4f39ae471f314eaad988":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Additional\n\n- 2025/01/03 : add weights ensemble of public notebook\n    - https://www.kaggle.com/code/yunsuxiaozi/js-ridge-baseline?scriptVersionId=202739388\n    - https://www.kaggle.com/code/hideyukizushi/js-nnx5-xgbx5-weighted-blend-lb-0-0078/notebook?scriptVersionId=215419804\n    - https://www.kaggle.com/code/i2nfinit3y/jane-street-tabm-ft-transformer-inference?scriptVersionId=213715783","metadata":{}},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:#3371ff;text-align:center;letter-spacing:0.1px;overflow:hidden;padding:20px;color:white;overflow:hidden;margin:0;font-size:100%\">Imports and installation</div>","metadata":{}},{"cell_type":"code","source":"!pip install rtdl_num_embeddings -q --no-index --find-links=/kaggle/input/jane-street-import/rtdl_num_embeddings","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T14:33:20.838600Z","iopub.execute_input":"2025-01-13T14:33:20.839236Z","iopub.status.idle":"2025-01-13T14:33:29.399336Z","shell.execute_reply.started":"2025-01-13T14:33:20.839199Z","shell.execute_reply":"2025-01-13T14:33:29.398171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os, sys, gc\n# import pickle\n# import dill\n# import numpy as np\n# import pandas as pd\n# import polars as pl\n\n# import torch\n# import torch.nn as nn\n# import torch.nn.functional as F\n# from pytorch_lightning import (LightningDataModule, LightningModule, Trainer)\n# from tensorflow.keras.layers import Dense\n\n# from sklearn.metrics import r2_score\n\n# import torch.optim\n# from torch.utils.data import Dataset, DataLoader, TensorDataset\n# from sklearn.model_selection import train_test_split\n# import math\n# from tqdm import tqdm\n# from collections import OrderedDict\n# from tabm_reference import Model, make_parameter_groups\n\n# import warnings\n# import joblib\n# from pytorch_lightning.callbacks import Callback\n# import gc\n\n# import lightgbm as lgb\n# from lightgbm import LGBMRegressor, Booster\n# from xgboost import XGBRegressor\n# from catboost import CatBoostRegressor\n\n# import warnings\n# warnings.filterwarnings('ignore')\n# pd.options.display.max_columns = None\n\n# sys.path.append(\"/kaggle/input/jane-street-real-time-market-data-forecasting\")\n# import cupy as cp\n\n\n# # デバック用のコード\n# import torch\n# from torch.utils.data import DataLoader\n# import numpy as np\n# from sklearn.metrics import r2_score\n\n# # データローダーから1バッチを取得して入力形状を確認\n# def debug_model_input_and_r2(model, dataloader, device='cuda'):\n#     \"\"\"\n#     モデルの入力形状とR²スコアを確認するためのデバッグ関数\n\n#     Args:\n#         model: PyTorchモデル\n#         dataloader: DataLoaderオブジェクト\n#         device: デバイス名 ('cuda' or 'cpu')\n#     \"\"\"\n#     # モデルを適切なデバイスに送る\n#     model.to(device)\n#     model.eval()  # 評価モードに設定\n\n#     # データローダーから1バッチを取得\n#     batch = next(iter(dataloader))\n#     x, y, w = batch[0].to(device), batch[1].to(device), batch[2].to(device)\n\n#     # 入力形状の確認\n#     print(f\"Input shape: {x.shape}\")\n\n#     # モデルの予測\n#     with torch.no_grad():\n#         y_pred = model(x)\n\n#     # R²スコアの計算\n#     y_pred_np = y_pred.cpu().numpy()\n#     y_true_np = y.cpu().numpy()\n#     weights_np = w.cpu().numpy()\n#     r2 = 1 - np.average((y_pred_np - y_true_np) ** 2, weights=weights_np) / (\n#         np.average((y_true_np) ** 2, weights=weights_np) + 1e-38\n#     )\n#     print(f\"R² Score: {r2:.4f}\")\n\n\n# class CONFIG:\n#     \"\"\"Configuration class for model parameters\"\"\"\n#     seed = 42  # Random seed for reproducibility\n#     target_col = \"responder_6\"  # Target variable name\n#     # Features: 79 base features + 9 lagged features\n#     feature_cols = [f\"feature_{idx:02d}\" for idx in range(79)] + [f\"responder_{idx}_lag_1\" for idx in range(9)]\n#     # Paths to pre-trained models\n#     model_paths = [\n#         \"/kaggle/input/js-xs-nn-trained-model\",  # Neural Network models\n#         \"/kaggle/input/js-with-lags-trained-xgb/result.pkl\", # XGBoost model\n#         \"/kaggle/input/als-e-106-pp0-40-xgb-5fold/result0.pkl\",\n#         \"/kaggle/input/als-e-106-pp0-40-xgb-5fold/result1.pkl\",\n#         \"/kaggle/input/als-e-106-pp0-40-xgb-5fold/result2.pkl\",\n#         \"/kaggle/input/als-e-106-pp0-40-xgb-5fold/result3.pkl\",\n#         \"/kaggle/input/als-e-106-pp0-40-xgb-5fold/result4.pkl\"  \n#     ]\n\n# # Load validation data\n# valid = pl.scan_parquet(f\"/kaggle/input/js24-preprocessing-create-lags/validation.parquet/\").collect().to_pandas()\n\n# # Load XGBoost model\n# xgb_model = None\n# with open(CONFIG.model_paths[2], \"rb\") as fp:\n#     result = pickle.load(fp)\n#     xgb_model = result[\"model\"]\n# xgb_feature_cols = [\"symbol_id\", \"time_id\"] + CONFIG.feature_cols\n\n# xgb_model2 = None\n# with open(CONFIG.model_paths[4], \"rb\") as fp:\n#     result = pickle.load(fp)\n#     xgb_model2 = result[\"model\"]\n\n# xgb_model4 = None\n# with open(CONFIG.model_paths[6], \"rb\") as fp:\n#     result = pickle.load(fp)\n#     xgb_model4 = result[\"model\"]\n\n# xgb_model5 = None\n# with open(CONFIG.model_paths[1], \"rb\") as fp:\n#     result = pickle.load(fp)\n#     xgb_model5 = result[\"model\"]\n\n# def r2_val(y_true, y_pred, sample_weight):\n#     \"\"\"\n#     Calculate weighted R² score\n#     Args:\n#         y_true: True values\n#         y_pred: Predicted values\n#         sample_weight: Weights for each sample\n#     Returns:\n#         Weighted R² score\n#     \"\"\"\n#     r2 = 1 - np.average((y_pred - y_true) ** 2, weights=sample_weight) / (np.average((y_true) ** 2, weights=sample_weight) + 1e-38)\n#     return r2\n\n# def r2_loss(y_pred, y_true):\n#     \"\"\"\n#     R² Loss\n#     Args:\n#         y_pred: 予測値 (torch.Tensor)\n#         y_true: 実際の値 (torch.Tensor)\n#     Returns:\n#         R² Loss\n#     \"\"\"\n#     ss_total = torch.sum((y_true - torch.mean(y_true)) ** 2)\n#     ss_residual = torch.sum((y_true - y_pred) ** 2)\n#     r2 = 1 - ss_residual / (ss_total + 1e-38)  # 過剰な小数でのゼロ除算を防ぐ\n#     return 1 - r2  # R² の補完としての Loss\n\n# class NN(LightningModule):\n#     \"\"\"Neural Network model using PyTorch Lightning\"\"\"\n    \n#     def __init__(self, input_dim, hidden_dims, dropouts, lr, weight_decay):\n#         \"\"\"\n#         Initialize the neural network\n#         Args:\n#             input_dim: Input feature dimension\n#             hidden_dims: List of hidden layer dimensions\n#             dropouts: List of dropout rates\n#             lr: Learning rate\n#             weight_decay: Weight decay for regularization\n#         \"\"\"\n#         super().__init__()\n#         self.save_hyperparameters()\n        \n#         # Build network architecture\n#         layers = []\n#         in_dim = input_dim\n#         for i, hidden_dim in enumerate(hidden_dims):\n#             layers.append(nn.BatchNorm1d(in_dim))  # Batch normalization\n#             if i > 0:\n#                 layers.append(nn.SiLU())  # SiLU activation (except first layer)\n#             if i < len(dropouts):\n#                 layers.append(nn.Dropout(dropouts[i]))  # Dropout for regularization\n#             layers.append(nn.Linear(in_dim, hidden_dim))  # Linear layer\n#             in_dim = hidden_dim\n            \n#         # Output layer\n#         layers.append(nn.Linear(in_dim, 1))\n        \n#         self.model = nn.Sequential(*layers)\n#         self.lr = lr\n#         self.weight_decay = weight_decay\n#         self.validation_step_outputs = []\n\n#     def forward(self, x):\n#         \"\"\"Forward pass with scaling\"\"\"\n#         return 5 * self.model(x).squeeze(-1)  # Scale output to [-5, 5] range\n\n#     def training_step(self, batch):\n#         \"\"\"Single training step\"\"\"\n#         x, y, w = batch\n#         y_hat = self(x)\n#         mse_loss = F.mse_loss(y_hat, y, reduction='none') * w  # Weighted MSE loss\n#         r2_loss_value = r2_loss(y_hat, y)  # R² Loss\n#         loss = mse_loss.mean() + r2_loss_value  # 合算した損失\n#         self.log('train_loss', loss, on_step=False, on_epoch=True, batch_size=x.size(0))\n#         return loss\n\n#     def validation_step(self, batch):\n#         \"\"\"Single validation step\"\"\"\n#         x, y, w = batch\n#         y_hat = self(x)\n#         mse_loss = F.mse_loss(y_hat, y, reduction='none') * w\n#         r2_loss_value = r2_loss(y_hat, y)  # R² Loss\n#         loss = mse_loss.mean() + r2_loss_value\n#         self.log('val_loss', loss, on_step=False, on_epoch=True, batch_size=x.size(0))\n#         self.validation_step_outputs.append((y_hat, y, w))\n#         return loss\n\n#     def on_validation_epoch_end(self):\n#         \"\"\"Compute validation metrics at epoch end\"\"\"\n#         if not self.trainer.sanity_checking:\n#             y = torch.cat([x[1] for x in self.validation_step_outputs]).cpu().numpy()\n#             prob = torch.cat([x[0] for x in self.validation_step_outputs]).cpu().numpy()\n#             weights = torch.cat([x[2] for x in self.validation_step_outputs]).cpu().numpy()\n#             val_r_square = r2_val(y, prob, weights)\n#             self.log(\"val_r_square\", val_r_square, prog_bar=True, on_step=False, on_epoch=True)\n#         self.validation_step_outputs.clear()\n\n#     def configure_optimizers(self):\n#         \"\"\"Configure optimizer and learning rate scheduler\"\"\"\n#         optimizer = torch.optim.Adam(self.parameters(), lr=self.lr, weight_decay=self.weight_decay)\n#         scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n#             optimizer, \n#             mode='min', \n#             factor=0.5, \n#             patience=5, \n#             verbose=True\n#         )\n#         return {\n#             'optimizer': optimizer,\n#             'lr_scheduler': {\n#                 'scheduler': scheduler,\n#                 'monitor': 'val_loss',\n#             }\n#         }\n\n#     def on_train_epoch_end(self):\n#         \"\"\"Log metrics at end of training epoch\"\"\"\n#         if not self.trainer.sanity_checking:\n#             epoch = self.trainer.current_epoch\n#             metrics = {k: v.item() if isinstance(v, torch.Tensor) else v \n#                       for k, v in self.trainer.logged_metrics.items()}\n#             formatted_metrics = {k: f\"{v:.5f}\" for k, v in metrics.items()}\n#             print(f\"Epoch {epoch}: {formatted_metrics}\")\n\n# # Load ensemble of models (5-fold cross-validation)\n# N_folds = 5\n# models = []\n# for fold in range(N_folds):\n#     checkpoint_path = f\"{CONFIG.model_paths[0]}/nn_{fold}.model\"\n#     model = NN.load_from_checkpoint(checkpoint_path)\n#     models.append(model.to(\"cuda:0\"))\n\n\n# # Clear validation data from memory to free up space\n# del valid\n# gc.collect()\n\n# # Global variable to store lagged features\n# lags_: pl.DataFrame | None = None\n\n# def predict_nn_xgb(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n#     \"\"\"\n#     Make predictions using ensemble of XGBoost and Neural Network models\n    \n#     Args:\n#         test: DataFrame containing test data\n#         lags: DataFrame containing lagged features (optional)\n        \n#     Returns:\n#         DataFrame with predictions\n#     \"\"\"\n#     global lags_\n    \n#     # Store lags in global variable if provided\n#     if lags is not None:\n#         lags_ = lags\n\n#     # Initialize predictions DataFrame with row_id and placeholder predictions\n#     predictions_nn = test.select('row_id', pl.lit(0.0).alias('responder_6',))\n\n#     # Process lagged features\n#     # Get last record for each date_id and symbol_id combination\n#     lags = lags_.clone().group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()\n    \n#     # Join test data with lagged features\n#     test = test.join(lags, on=[\"date_id\", \"symbol_id\"], how=\"left\")\n\n#     # Initialize arrays for model predictions\n#     preds_xgb = np.zeros((test.shape[0],))  # XGBoost predictions\n#     preds_nn = np.zeros((test.shape[0],))   # Neural Network predictions\n\n#     # Generate XGBoost predictions\n#     test_pd = test[xgb_feature_cols].to_pandas()\n\n#     # cupy にデータを移行\n#     test_cp = cp.array(test_pd.values)\n\n#     for model in xgb_models:\n#         preds_xgb += model.predict(test_cp) * (1 / len(xgb_models))\n\n#     # Generate Neural Network predictions\n#     # Prepare input data\n#     test_input = test[CONFIG.feature_cols].to_pandas()\n#     # Handle missing values: forward fill then fill remaining with zeros\n#     test_input = test_input.fillna(method='ffill').fillna(0)\n#     # Convert to PyTorch tensor and move to GPU\n#     test_input = torch.FloatTensor(test_input.values).to(\"cuda:0\")\n\n#     # Generate predictions from Neural Network ensemble\n#     with torch.no_grad():  # Disable gradient calculation for inference\n#         for i, nn_model in enumerate(models):\n#             nn_model.eval()  # Set model to evaluation mode\n#             # Average predictions from all models\n#             preds_nn += nn_model(test_input).cpu().numpy() / len(models)\n\n#     # Combine predictions with equal weights (50% XGBoost, 50% Neural Network)\n#     preds = 0.55 * preds_xgb + 0.45 * preds_nn\n#     preds = np.clip(preds, a_min=-5, a_max=5)\n\n#     # Create final predictions DataFrame\n#     predictions_nn = test.select('row_id').\\\n#         with_columns(\n#             pl.Series(\n#                 name='responder_6',\n#                 values=preds,\n#                 dtype=pl.Float64,\n#             )\n#         )\n\n#     return predictions_nn\n\n\n# # Create list of feature names from 0-78, excluding feature_61\n# feature_list = [f\"feature_{idx:02d}\" for idx in range(79) if idx != 61]\n\n# # Define target column name\n# target_col = \"responder_6\" \n\n# # Create list of features for testing, combining feature_list with lagged responder features\n# feature_test = feature_list + [f\"responder_{idx}_lag_1\" for idx in range(9)] \n\n# # Define categorical features\n# feature_cat = [\"feature_09\", \"feature_10\", \"feature_11\"]\n\n# # Define continuous features by excluding categorical ones from feature_test\n# feature_cont = [item for item in feature_test if item not in feature_cat]\n\n# # Set batch size for model training\n# batch_size = 8192\n\n# # Create list of features to standardize (continuous features + lagged responder features)\n# std_feature = [i for i in feature_list if i not in feature_cat] + [f\"responder_{idx}_lag_1\" for idx in range(9)]\n\n# # Load pre-computed statistics for standardization\n# data_stats = joblib.load(\"/kaggle/input/my-own-js/data_stats.pkl\")\n# means = data_stats['mean']\n# stds = data_stats['std']\n\n# def standardize(df, feature_cols, means, stds):\n#     \"\"\"\n#     Standardize features using pre-computed means and standard deviations\n#     Args:\n#         df: Input dataframe\n#         feature_cols: List of columns to standardize\n#         means: Dictionary of mean values\n#         stds: Dictionary of standard deviation values\n#     Returns:\n#         Standardized dataframe\n#     \"\"\"\n#     return df.with_columns([\n#         ((pl.col(col) - means[col]) / stds[col]).alias(col) for col in feature_cols\n#     ])\n\n# # Dictionary mappings for categorical variables encoding\n# category_mappings = {'feature_09': {2: 0, 4: 1, 9: 2, 11: 3, 12: 4, 14: 5, 15: 6, 25: 7, 26: 8, 30: 9, 34: 10, 42: 11, 44: 12, 46: 13, 49: 14, 50: 15, 57: 16, 64: 17, 68: 18, 70: 19, 81: 20, 82: 21},\n#  'feature_10': {1: 0, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 10: 7, 12: 8},\n#  'feature_11': {9: 0, 11: 1, 13: 2, 16: 3, 24: 4, 25: 5, 34: 6, 40: 7, 48: 8, 50: 9, 59: 10, 62: 11, 63: 12, 66: 13,\n#   76: 14, 150: 15, 158: 16, 159: 17, 171: 18, 195: 19, 214: 20, 230: 21, 261: 22, 297: 23, 336: 24, 376: 25, 388: 26, 410: 27, 522: 28, 534: 29, 539: 30},\n#  'symbol_id': {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8, 9: 9, 10: 10, 11: 11, 12: 12, 13: 13, 14: 14, 15: 15, 16: 16, 17: 17, 18: 18, 19: 19,\n#   20: 20, 21: 21, 22: 22, 23: 23, 24: 24, 25: 25, 26: 26, 27: 27, 28: 28, 29: 29, 30: 30, 31: 31, 32: 32, 33: 33, 34: 34, 35: 35, 36: 36, 37: 37, 38: 38},\n#  'time_id' : {i : i for i in range(968)}}\n\n# def encode_column(df, column, mapping):\n#     \"\"\"\n#     Encode categorical columns using provided mapping\n#     Args:\n#         df: Input dataframe\n#         column: Column name to encode\n#         mapping: Dictionary with encoding mappings\n#     Returns:\n#         Dataframe with encoded column\n#     \"\"\"\n#     max_value = max(mapping.values())  \n    \n#     def encode_category(category):\n#         return mapping.get(category, max_value + 1)  \n    \n#     return df.with_columns(\n#         pl.col(column).map_elements(encode_category).alias(column)\n#     )\n\n# class R2Loss(nn.Module):\n#     \"\"\"Custom R-squared loss function for PyTorch\"\"\"\n#     def __init__(self):\n#         super(R2Loss, self).__init__()\n\n#     def forward(self, y_pred, y_true):\n#         # Calculate MSE\n#         mse_loss = torch.sum((y_pred - y_true) ** 2)\n#         # Calculate variance of true values\n#         var_y = torch.sum(y_true ** 2)\n#         # Calculate R² loss (adding small epsilon to avoid division by zero)\n#         loss = mse_loss / (var_y + 1e-38)\n#         return loss\n\n# class TabmModel(LightningModule):\n#     \"\"\"Tabular Model using PyTorch Lightning\"\"\"\n\n#     def __init__(self, n_cont_features, cat_cardinalities, n_classes, lr, weight_decay):\n#         \"\"\"\n#         Initialize the Tabm model.\n\n#         Args:\n#             n_cont_features (int): Number of continuous features.\n#             cat_cardinalities (list): List of categorical feature cardinalities.\n#             n_classes (int | None): Number of classes (None for regression).\n#             lr (float): Learning rate.\n#             weight_decay (float): Weight decay for regularization.\n#         \"\"\"\n#         super().__init__()\n#         self.save_hyperparameters()\n#         self.k = 16\n#         self.model = Model(\n#             n_num_features=n_cont_features,\n#             cat_cardinalities=cat_cardinalities,\n#             n_classes=n_classes,\n#             backbone={\n#                 'type': 'MLP',\n#                 'n_blocks': 3,\n#                 'd_block': 512,\n#                 'dropout': 0.25,\n#             },\n#             bins=None,\n#             num_embeddings=None,\n#             arch_type='tabm',\n#             k=self.k,\n#         )\n#         self.lr = lr\n#         self.weight_decay = weight_decay\n#         self.training_step_outputs = []\n#         self.validation_step_outputs = []\n#         self.loss_fn = R2Loss()\n\n#     def forward(self, x_cont, x_cat):\n#         return self.model(x_cont, x_cat).squeeze(-1)\n\n#     def training_step(self, batch, batch_idx):\n#         \"\"\"Single training step.\"\"\"\n#         x_cont, x_cat, y, w, w_y = batch\n#         x_cont = x_cont + torch.randn_like(x_cont) * 0.02  # Data augmentation\n#         y_hat = self(x_cont, x_cat)\n#         loss = self.loss_fn(y_hat.flatten(0, 1), y.repeat_interleave(self.k))\n#         self.log('train_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True, batch_size=x_cont.size(0))\n#         self.training_step_outputs.append((y_hat.mean(1), y, w))\n#         return loss\n\n#     def validation_step(self, batch, batch_idx):\n#         \"\"\"Single validation step.\"\"\"\n#         x_cont, x_cat, y, w, w_y = batch\n#         x_cont = x_cont + torch.randn_like(x_cont) * 0.02  # Data augmentation\n#         y_hat = self(x_cont, x_cat)\n#         loss = self.loss_fn(y_hat.flatten(0, 1), y.repeat_interleave(self.k))\n#         self.log('val_loss', loss, on_step=False, on_epoch=True, prog_bar=True, logger=True, batch_size=x_cont.size(0))\n#         self.validation_step_outputs.append((y_hat.mean(1), y, w))\n#         return loss\n\n#     def on_validation_epoch_end(self):\n#         \"\"\"Calculate validation R² at the end of the epoch.\"\"\"\n#         if not self.trainer.sanity_checking:\n#             y = torch.cat([x[1] for x in self.validation_step_outputs]).cpu().numpy()\n#             prob = torch.cat([x[0] for x in self.validation_step_outputs]).cpu().numpy()\n#             weights = torch.cat([x[2] for x in self.validation_step_outputs]).cpu().numpy()\n#             val_r_square = r2_val(y, prob, weights)\n#             self.log(\"val_r_square\", val_r_square, prog_bar=True, on_step=False, on_epoch=True)\n#         self.validation_step_outputs.clear()\n\n#     def configure_optimizers(self):\n#         \"\"\"Configure optimizer.\"\"\"\n#         optimizer = torch.optim.AdamW(make_parameter_groups(self.model), lr=self.lr, weight_decay=self.weight_decay)\n#         return {'optimizer': optimizer}\n\n#     def on_train_epoch_end(self):\n#         \"\"\"Log training metrics at end of epoch.\"\"\"\n#         if self.trainer.sanity_checking:\n#             return\n\n#         y = torch.cat([x[1] for x in self.training_step_outputs]).cpu().numpy()\n#         prob = torch.cat([x[0] for x in self.training_step_outputs]).detach().cpu().numpy()\n#         weights = torch.cat([x[2] for x in self.training_step_outputs]).cpu().numpy()\n#         train_r_square = r2_val(y, prob, weights)\n#         self.log(\"train_r_square\", train_r_square, prog_bar=True, on_step=False, on_epoch=True)\n#         self.training_step_outputs.clear()\n\n#         epoch = self.trainer.current_epoch\n#         metrics = {k: v.item() if isinstance(v, torch.Tensor) else v for k, v in self.trainer.logged_metrics.items()}\n#         formatted_metrics = {k: f\"{v:.5f}\" for k, v in metrics.items()}\n#         print(f\"Epoch {epoch}: {formatted_metrics}\")\n\n# class CustomArgs:\n#     \"\"\"Custom arguments for model configuration.\"\"\"\n#     def __init__(self):\n#         self.usegpu = True\n#         self.gpuid = 0\n#         self.seed = 42\n#         self.model = 'nn'\n#         self.use_wandb = False\n#         self.project = 'js-tabm-with-lags'\n#         self.dname = \"./input_df/\"\n#         self.loader_workers = 10\n#         self.bs = 8192\n#         self.lr = 1e-3\n#         self.weight_decay = 8e-4\n#         self.n_cont_features = 84\n#         self.n_cat_features = 5\n#         self.n_classes = None\n#         self.cat_cardinalities = [23, 10, 32, 40, 969]\n#         self.patience = 7\n#         self.max_epochs = 10\n#         self.N_fold = 5\n\n# my_args = CustomArgs()\n# device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n\n# # Load Tabm model checkpoint\n# tabm_model = TabmModel.load_from_checkpoint('/kaggle/input/my-own-js/tabm_epochepoch03.ckpt').to(device)\n# tabm_model.eval()\n\n# lags_history = None\n\n# def predict_tabm(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame:\n#     \"\"\"\n#     Make predictions using Tabm model.\n\n#     Args:\n#         test (pl.DataFrame): DataFrame containing test data.\n#         lags (pl.DataFrame | None): DataFrame containing lagged features.\n\n#     Returns:\n#         pl.DataFrame: DataFrame with predictions.\n#     \"\"\"\n#     global lags_, lags_history\n\n#     if lags is not None:\n#         lags_ = lags\n\n#     for col in feature_cat + ['symbol_id', 'time_id']:\n#         test = encode_column(test, col, category_mappings[col])\n\n#     predictions = test.select('row_id').with_columns(pl.lit(0.0).alias('responder_6'))\n\n#     symbol_ids = test['symbol_id'].to_numpy()\n#     time_id_array = test['time_id'].to_numpy()\n\n#     if time_id_array[0] == 0:\n#         lags = lags_.with_columns([\n#             pl.col('time_id').cast(pl.Int64),\n#             pl.col('symbol_id').cast(pl.Int64)\n#         ])\n#         lags_history = lags\n#         lags = lags.filter(pl.col(\"time_id\") == 0)\n#         test = test.join(lags, on=[\"time_id\", \"symbol_id\"], how=\"left\")\n#     else:\n#         lags = lags_history.filter(pl.col(\"time_id\") == time_id_array[0])\n#         test = test.join(lags, on=[\"time_id\", \"symbol_id\"], how=\"left\")\n\n#     # Fill missing values\n#     test = test.with_columns([\n#         pl.col(col).fill_null(0) for col in feature_list + [f\"responder_{idx}_lag_1\" for idx in range(9)]\n#     ])\n\n#     # Standardize features\n#     test = standardize(test, std_feature, means, stds)\n\n#     # Prepare tensors\n#     X_test = test[feature_test].to_numpy()\n#     X_test_tensor = torch.tensor(X_test, dtype=torch.float32).to(device)\n\n#     symbol_tensor = torch.tensor(symbol_ids, dtype=torch.float32).to(device)\n#     time_tensor = torch.tensor(time_id_array, dtype=torch.float32).to(device)\n#     X_cat = X_test_tensor[:, [9, 10, 11]]\n#     X_cont = X_test_tensor[:, [i for i in range(X_test_tensor.shape[1]) if i not in [9, 10, 11]]]\n\n#     X_cat = torch.cat([X_cat, symbol_tensor.unsqueeze(-1), time_tensor.unsqueeze(-1)], axis=1).to(torch.int64)\n\n#     # Generate predictions\n#     with torch.no_grad():\n#         outputs = tabm_model(X_cont, X_cat)\n#         preds = outputs.squeeze(-1).cpu().numpy()\n#         preds = preds.mean(axis=1)\n\n#     preds = np.clip(preds, a_min=-5, a_max=5)\n\n#     predictions = test.select('row_id').with_columns(\n#         pl.Series('responder_6', preds, dtype=pl.Float64)\n#     )\n\n#     return predictions\n\n# def load_from_dill(model_name: str, model_path: str, file_ext: str = '.dill'):\n#     \"\"\"\n#     Load a model from a dill file.\n\n#     Args:\n#         model_name (str): Name of the model file (without extension).\n#         model_path (str): Directory path containing the model file.\n#         file_ext (str): File extension (default: '.dill').\n\n#     Returns:\n#         object: Loaded model object.\n#     \"\"\"\n#     with open(os.path.join(model_path, f\"{model_name}{file_ext}\"), \"rb\") as file_handle:\n#         model_object = dill.load(file_handle)\n#     return model\n#     # Load pre-trained Ridge Regression model\n# ridge_model = load_from_dill(\n#     model_name='Ridge',\n#     model_path=\"/kaggle/input/jsridgev01011635\"\n# )\n\n# def predict_ridge(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame:\n#     \"\"\"\n#     Make predictions using Ridge Regression model.\n\n#     Args:\n#         test (pl.DataFrame): DataFrame containing test data.\n#         lags (pl.DataFrame | None): DataFrame containing lagged features (unused).\n\n#     Returns:\n#         pl.DataFrame: DataFrame with predictions.\n#     \"\"\"\n#     cols = [f'feature_{i:02}' for i in range(79)]\n#     test_pd = test[cols].to_pandas().fillna(3)\n#     test_preds = ridge_model.predict(test_pd.values)\n\n#     predictions = test.select('row_id').with_columns(\n#         pl.Series('responder_6', test_preds.ravel(), dtype=pl.Float64)\n#     )\n\n#     return predictions\n\n# def predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame:\n#     \"\"\"\n#     Make ensemble predictions combining Neural Network + XGBoost, Tabm, and Ridge Regression.\n\n#     Args:\n#         test (pl.DataFrame): DataFrame containing test data.\n#         lags (pl.DataFrame | None): DataFrame containing lagged features.\n\n#     Returns:\n#         pl.DataFrame: DataFrame with final ensemble predictions.\n#     \"\"\"\n#     # Get predictions from each model/ensemble\n#     pd_nn_xgb = predict_nn_xgb(test, lags).to_pandas()\n#     pd_ridge = predict_ridge(test, lags).to_pandas()\n#     pd_tabm = predict_tabm(test, lags).to_pandas()\n\n#     # Rename prediction columns to avoid conflicts\n#     pd_nn_xgb.rename(columns={'responder_6': 'col_nn_xgb'}, inplace=True)\n#     pd_ridge.rename(columns={'responder_6': 'col_ridge'}, inplace=True)\n#     pd_tabm.rename(columns={'responder_6': 'col_tabm'}, inplace=True)\n\n#     # Merge all predictions based on row_id\n#     pds = pd.merge(pd_nn_xgb, pd_ridge, on='row_id')\n#     pds = pd.merge(pds, pd_tabm, on='row_id')\n\n#     # Weighted ensemble\n#     e_weights = [0.50, 0.10, 0.40]\n#     pds['responder_6'] = (\n#         pds['col_nn_xgb'] * e_weights[0] +\n#         pds['col_ridge'] * e_weights[1] +\n#         pds['col_tabm'] * e_weights[2]\n#     )\n\n#     # Create final predictions DataFrame\n#     predictions = test.select('row_id').with_columns(\n#         pl.Series('responder_6', pds['responder_6'].values, dtype=pl.Float64)\n#     )\n\n#     return predictions\n\n# # Initialize inference server\n# import kaggle_evaluation.jane_street_inference_server\n# inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\n# # Serve predictions based on environment\n# if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n#     inference_server.serve()\n# else:\n#     inference_server.run_local_gateway(\n#         (\n#             '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n#             '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n#         )\n#     )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T14:33:29.401884Z","iopub.execute_input":"2025-01-13T14:33:29.402207Z","iopub.status.idle":"2025-01-13T14:33:29.426533Z","shell.execute_reply.started":"2025-01-13T14:33:29.402177Z","shell.execute_reply":"2025-01-13T14:33:29.425588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, sys, gc\nimport pickle\nimport dill\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom pytorch_lightning import (LightningDataModule, LightningModule, Trainer)\nfrom tensorflow.keras.layers import Dense\n\nfrom sklearn.metrics import r2_score\n\nimport torch.optim\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\nfrom sklearn.model_selection import train_test_split\nimport math\nfrom tqdm import tqdm\nfrom collections import OrderedDict\nfrom tabm_reference import Model, make_parameter_groups\n\nimport warnings\nimport joblib\nfrom pytorch_lightning.callbacks import Callback\nimport gc\n\nimport lightgbm as lgb\nfrom lightgbm import LGBMRegressor, Booster\nfrom xgboost import XGBRegressor\nimport xgboost as xgb\nfrom catboost import CatBoostRegressor\nimport cupy as cp \n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nsys.path.append(\"/kaggle/input/jane-street-real-time-market-data-forecasting\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T14:33:29.427760Z","iopub.execute_input":"2025-01-13T14:33:29.428089Z","iopub.status.idle":"2025-01-13T14:33:29.443865Z","shell.execute_reply.started":"2025-01-13T14:33:29.428060Z","shell.execute_reply":"2025-01-13T14:33:29.442981Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# デバック用のコード\nimport torch\nfrom torch.utils.data import DataLoader\nimport numpy as np\nfrom sklearn.metrics import r2_score\n\n# データローダーから1バッチを取得して入力形状を確認\ndef debug_model_input_and_r2(model, dataloader, device='cuda'):\n    \"\"\"\n    モデルの入力形状とR²スコアを確認するためのデバッグ関数\n\n    Args:\n        model: PyTorchモデル\n        dataloader: DataLoaderオブジェクト\n        device: デバイス名 ('cuda' or 'cpu')\n    \"\"\"\n    # モデルを適切なデバイスに送る\n    model.to(device)\n    model.eval()  # 評価モードに設定\n\n    # データローダーから1バッチを取得\n    batch = next(iter(dataloader))\n    x, y, w = batch[0].to(device), batch[1].to(device), batch[2].to(device)\n\n    # 入力形状の確認\n    print(f\"Input shape: {x.shape}\")\n\n    # モデルの予測\n    with torch.no_grad():\n        y_pred = model(x)\n\n    # R²スコアの計算\n    y_pred_np = y_pred.cpu().numpy()\n    y_true_np = y.cpu().numpy()\n    weights_np = w.cpu().numpy()\n    r2 = 1 - np.average((y_pred_np - y_true_np) ** 2, weights=weights_np) / (\n        np.average((y_true_np) ** 2, weights=weights_np) + 1e-38\n    )\n    print(f\"R² Score: {r2:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T14:33:29.446355Z","iopub.execute_input":"2025-01-13T14:33:29.447142Z","iopub.status.idle":"2025-01-13T14:33:29.460105Z","shell.execute_reply.started":"2025-01-13T14:33:29.447113Z","shell.execute_reply":"2025-01-13T14:33:29.459379Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:#3371ff;text-align:center;letter-spacing:0.1px;overflow:hidden;padding:20px;color:white;overflow:hidden;margin:0;font-size:100%\">Top Public Notebook</div>","metadata":{}},{"cell_type":"markdown","source":"## JS|NNx5+XGBx5(MyTrain+Pub)|WeightBlend|LB.0.0078 (hideyukizushi)\n\n`LB: 0.0078` https://www.kaggle.com/code/yunsuxiaozi/js2024-starter?scriptVersionId=206770572","metadata":{"papermill":{"duration":0.022173,"end_time":"2024-10-26T03:27:39.285949","exception":false,"start_time":"2024-10-26T03:27:39.263776","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CONFIG:\n    \"\"\"Configuration class for model parameters\"\"\"\n    seed = 42  # Random seed for reproducibility\n    target_col = \"responder_6\", \"responder_3\"  # Target variable name\n    # Features: 79 base features + 9 lagged features\n    feature_cols = [f\"feature_{idx:02d}\" for idx in range(79)] + [f\"responder_{idx}_lag_1\" for idx in range(9)]\n    # Paths to pre-trained models\n    model_paths = [\n        \"/kaggle/input/js-xs-nn-trained-model\",  # Neural Network models\n        \"/kaggle/input/js-with-lags-trained-xgb/result.pkl\", # XGBoost model\n        \"/kaggle/input/als-e-106-pp0-40-xgb-5fold/result0.pkl\",\n        \"/kaggle/input/als-e-106-pp0-40-xgb-5fold/result1.pkl\",\n        \"/kaggle/input/als-e-106-pp0-40-xgb-5fold/result2.pkl\",\n        \"/kaggle/input/als-e-106-pp0-40-xgb-5fold/result3.pkl\",\n        \"/kaggle/input/als-e-106-pp0-40-xgb-5fold/result4.pkl\"  \n    ]\n\n# Load validation data\nvalid = pl.scan_parquet(f\"/kaggle/input/js24-preprocessing-create-lags/validation.parquet/\").collect().to_pandas()\n\n# Load XGBoost model\nxgb_model = None\nwith open(CONFIG.model_paths[2], \"rb\") as fp:\n    result = pickle.load(fp)\n    xgb_model = result[\"model\"]\nxgb_feature_cols = [\"symbol_id\", \"time_id\"] + CONFIG.feature_cols\n\nxgb_model2 = None\nwith open(CONFIG.model_paths[4], \"rb\") as fp:\n    result = pickle.load(fp)\n    xgb_model2 = result[\"model\"]\n\nxgb_model4 = None\nwith open(CONFIG.model_paths[6], \"rb\") as fp:\n    result = pickle.load(fp)\n    xgb_model4 = result[\"model\"]\n\nxgb_model5 = None\nwith open(CONFIG.model_paths[1], \"rb\") as fp:\n    result = pickle.load(fp)\n    xgb_model5 = result[\"model\"]\n\ndef r2_val(y_true, y_pred, sample_weight):\n    \"\"\"\n    Calculate weighted R² score\n    Args:\n        y_true: True values\n        y_pred: Predicted values\n        sample_weight: Weights for each sample\n    Returns:\n        Weighted R² score\n    \"\"\"\n    r2 = 1 - np.average((y_pred - y_true) ** 2, weights=sample_weight) / (np.average((y_true) ** 2, weights=sample_weight) + 1e-38)\n    return r2\n\ndef r2_loss(y_pred, y_true):\n    \"\"\"\n    R² Loss\n    Args:\n        y_pred: 予測値 (torch.Tensor)\n        y_true: 実際の値 (torch.Tensor)\n    Returns:\n        R² Loss\n    \"\"\"\n    ss_total = torch.sum((y_true - torch.mean(y_true)) ** 2)\n    ss_residual = torch.sum((y_true - y_pred) ** 2)\n    r2 = 1 - ss_residual / (ss_total + 1e-6)  # 過剰な小数でのゼロ除算を防ぐ\n    return 1 - r2  # R² の補完としての Loss\n\nclass NN(LightningModule):\n    \"\"\"Neural Network model using PyTorch Lightning\"\"\"\n    \n    def __init__(self, input_dim, hidden_dims, dropouts, lr, weight_decay):\n        \"\"\"\n        Initialize the neural network\n        Args:\n            input_dim: Input feature dimension\n            hidden_dims: List of hidden layer dimensions\n            dropouts: List of dropout rates\n            lr: Learning rate\n            weight_decay: Weight decay for regularization\n        \"\"\"\n        super().__init__()\n        self.save_hyperparameters()\n        \n        # Build network architecture\n        layers = []\n        in_dim = input_dim\n        for i, hidden_dim in enumerate(hidden_dims):\n            layers.append(nn.BatchNorm1d(in_dim))  # Batch normalization\n            if i > 0:\n                layers.append(nn.SiLU())  # SiLU activation (except first layer)\n            if i < len(dropouts):\n                layers.append(nn.Dropout(dropouts[i]))  # Dropout for regularization\n            layers.append(nn.Linear(in_dim, hidden_dim))  # Linear layer\n            in_dim = hidden_dim\n            \n        # Output layer\n        layers.append(nn.Linear(in_dim, 1))\n        #layers.append(nn.Tanh())  # Tanh activation for bounded output\n        \n        self.model = nn.Sequential(*layers)\n        self.lr = lr\n        self.weight_decay = weight_decay\n        self.validation_step_outputs = []\n\n    def forward(self, x):\n        \"\"\"Forward pass with scaling\"\"\"\n        return 5 * self.model(x).squeeze(-1)  # Scale output to [-5, 5] range\n\n    def training_step(self, batch):\n        \"\"\"Single training step\"\"\"\n        x, y, w = batch\n        y_hat = self(x)\n        mse_loss = F.mse_loss(y_hat, y, reduction='none') * w  # Weighted MSE loss\n        r2_loss_value = r2_loss(y_hat, y)  # R² Loss\n        loss = mse_loss.mean() + r2_loss_value  # 合算した損失\n        self.log('train_loss', loss, on_step=False, on_epoch=True, batch_size=x.size(0))\n        return loss\n\n    def validation_step(self, batch):\n        \"\"\"Single validation step\"\"\"\n        x, y, w = batch\n        y_hat = self(x)\n        mse_loss = F.mse_loss(y_hat, y, reduction='none') * w\n        r2_loss_value = r2_loss(y_hat, y)  # R² Loss\n        loss = mse_loss.mean() + r2_loss_value\n        self.log('val_loss', total_loss, on_step=False, on_epoch=True, batch_size=x.size(0))\n        self.validation_step_outputs.append((y_hat, y, w))\n        return loss\n\n    def on_validation_epoch_end(self):\n        \"\"\"Compute validation metrics at epoch end\"\"\"\n        if not self.trainer.sanity_checking:\n            y = torch.cat([x[1] for x in self.validation_step_outputs]).cpu().numpy()\n            prob = torch.cat([x[0] for x in self.validation_step_outputs]).cpu().numpy()\n            weights = torch.cat([x[2] for x in self.validation_step_outputs]).cpu().numpy()\n            val_r_square = r2_val(y, prob, weights)\n            self.log(\"val_r_square\", val_r_square, prog_bar=True, on_step=False, on_epoch=True)\n        self.validation_step_outputs.clear()\n\n    def configure_optimizers(self):\n        \"\"\"Configure optimizer and learning rate scheduler\"\"\"\n        optimizer = torch.optim.Adam(self.parameters(), lr=self.lr, weight_decay=self.weight_decay)\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            optimizer, \n            mode='min', \n            factor=0.5, \n            patience=5, \n            verbose=True\n        )\n        return {\n            'optimizer': optimizer,\n            'lr_scheduler': {\n                'scheduler': scheduler,\n                'monitor': 'val_loss',\n            }\n        }\n\n    def on_train_epoch_end(self):\n        \"\"\"Log metrics at end of training epoch\"\"\"\n        if not self.trainer.sanity_checking:\n            epoch = self.trainer.current_epoch\n            metrics = {k: v.item() if isinstance(v, torch.Tensor) else v \n                      for k, v in self.trainer.logged_metrics.items()}\n            formatted_metrics = {k: f\"{v:.5f}\" for k, v in metrics.items()}\n            print(f\"Epoch {epoch}: {formatted_metrics}\")\n\n# Load ensemble of models (5-fold cross-validation)\nN_folds = 5\nmodels = []\nfor fold in range(N_folds):\n    checkpoint_path = f\"{CONFIG.model_paths[0]}/nn_{fold}.model\"\n    model = NN.load_from_checkpoint(checkpoint_path)\n    models.append(model.to(\"cuda:0\"))","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-01-13T14:33:29.461314Z","iopub.execute_input":"2025-01-13T14:33:29.461576Z","iopub.status.idle":"2025-01-13T14:33:30.851430Z","shell.execute_reply.started":"2025-01-13T14:33:29.461551Z","shell.execute_reply":"2025-01-13T14:33:30.850421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Clear validation data from memory to free up space\ndel valid\ngc.collect()\n\n# Global variable to store lagged features\nlags_: pl.DataFrame | None = None\n\ndef predict_nn_xgb(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame:\n    global lags_\n\n    if lags is not None:\n        lags_ = lags\n\n    predictions_nn = test.select('row_id').with_columns(pl.lit(0.0).alias('responder_6'))\n\n    # 遅延特徴量の処理\n    lags = lags_.clone().group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()\n    test = test.join(lags, on=[\"date_id\", \"symbol_id\"], how=\"left\")\n\n    # XGBoostの予測\n    test_pd = test[xgb_feature_cols].to_pandas()\n\n    # モデルにGPU予測を設定\n    # XGBoostの予測の設定を修正\n    xgb_model.set_params(tree_method='hist', device='cuda')\n    xgb_model2.set_params(tree_method='hist', device='cuda')\n    xgb_model4.set_params(tree_method='hist', device='cuda')\n    xgb_model5.set_params(tree_method='hist', device='cuda')\n\n    # preds_xgbを初期化\n    preds_xgb = np.zeros(test_pd.shape[0])\n\n    # 各モデルからの予測を取得\n    preds_xgb += xgb_model.predict(test_pd.values) * 0.25\n    preds_xgb += xgb_model2.predict(test_pd.values) * 0.25\n    preds_xgb += xgb_model4.predict(test_pd.values) * 0.25\n    preds_xgb += xgb_model5.predict(test_pd.values) * 0.25\n\n    # ニューラルネットワークの予測\n    test_input = test[CONFIG.feature_cols].to_pandas()\n    test_input = test_input.fillna(method='ffill').fillna(0)\n    test_input = torch.FloatTensor(test_input.values).to(\"cuda:0\")\n\n    preds_nn = np.zeros(test_pd.shape[0])\n    with torch.no_grad():\n        for i, nn_model in enumerate(models):\n            nn_model.eval()\n            preds_nn += nn_model(test_input).cpu().numpy() / len(models)\n\n    # 予測値の組み合わせ\n    preds = 0.55 * preds_xgb + 0.45 * preds_nn\n\n    # 結果のDataFrame作成\n    predictions_nn = test.select('row_id').with_columns(\n        pl.Series(\n            name='responder_6',\n            values=np.clip(preds, a_min=-5, a_max=5),\n            dtype=pl.Float64,\n        )\n    )\n\n    return predictions_nn","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.031534,"end_time":"2024-10-26T03:27:39.453592","exception":false,"start_time":"2024-10-26T03:27:39.422058","status":"completed"},"tags":[],"_kg_hide-output":false,"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T14:33:30.853046Z","iopub.execute_input":"2025-01-13T14:33:30.853641Z","iopub.status.idle":"2025-01-13T14:33:31.268436Z","shell.execute_reply.started":"2025-01-13T14:33:30.853600Z","shell.execute_reply":"2025-01-13T14:33:31.267643Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Jane Street | TabM/FT-Transformer inference (i2nfinit3y)\n`LB: 0.0074` https://www.kaggle.com/code/i2nfinit3y/jane-street-tabm-ft-transformer-inference?scriptVersionId=213715783","metadata":{}},{"cell_type":"code","source":"# Create list of feature names from 0-78, excluding feature_61\nfeature_list = [f\"feature_{idx:02d}\" for idx in range(79) if idx != 61]\n\n# Define target column name\ntarget_col = \"responder_6\" \n\n# Create list of features for testing, combining feature_list with lagged responder features\nfeature_test = feature_list + [f\"responder_{idx}_lag_1\" for idx in range(9)] \n\n# Define categorical features\nfeature_cat = [\"feature_09\", \"feature_10\", \"feature_11\"]\n\n# Define continuous features by excluding categorical ones from feature_test\nfeature_cont = [item for item in feature_test if item not in feature_cat]\n\n# Set batch size for model training\nbatch_size = 8192\n\n# Create list of features to standardize (continuous features + lagged responder features)\nstd_feature = [i for i in feature_list if i not in feature_cat] + [f\"responder_{idx}_lag_1\" for idx in range(9)]\n\n# Load pre-computed statistics for standardization\ndata_stats = joblib.load(\"/kaggle/input/my-own-js/data_stats.pkl\")\nmeans = data_stats['mean']\nstds = data_stats['std']\n\ndef standardize(df, feature_cols, means, stds):\n    \"\"\"\n    Standardize features using pre-computed means and standard deviations\n    Args:\n        df: Input dataframe\n        feature_cols: List of columns to standardize\n        means: Dictionary of mean values\n        stds: Dictionary of standard deviation values\n    Returns:\n        Standardized dataframe\n    \"\"\"\n    return df.with_columns([\n        ((pl.col(col) - means[col]) / stds[col]).alias(col) for col in feature_cols\n    ])\n\n# Dictionary mappings for categorical variables encoding\ncategory_mappings = {'feature_09': {2: 0, 4: 1, 9: 2, 11: 3, 12: 4, 14: 5, 15: 6, 25: 7, 26: 8, 30: 9, 34: 10, 42: 11, 44: 12, 46: 13, 49: 14, 50: 15, 57: 16, 64: 17, 68: 18, 70: 19, 81: 20, 82: 21},\n 'feature_10': {1: 0, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 10: 7, 12: 8},\n 'feature_11': {9: 0, 11: 1, 13: 2, 16: 3, 24: 4, 25: 5, 34: 6, 40: 7, 48: 8, 50: 9, 59: 10, 62: 11, 63: 12, 66: 13,\n  76: 14, 150: 15, 158: 16, 159: 17, 171: 18, 195: 19, 214: 20, 230: 21, 261: 22, 297: 23, 336: 24, 376: 25, 388: 26, 410: 27, 522: 28, 534: 29, 539: 30},\n 'symbol_id': {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8, 9: 9, 10: 10, 11: 11, 12: 12, 13: 13, 14: 14, 15: 15, 16: 16, 17: 17, 18: 18, 19: 19,\n  20: 20, 21: 21, 22: 22, 23: 23, 24: 24, 25: 25, 26: 26, 27: 27, 28: 28, 29: 29, 30: 30, 31: 31, 32: 32, 33: 33, 34: 34, 35: 35, 36: 36, 37: 37, 38: 38},\n 'time_id' : {i : i for i in range(968)}}\n\ndef encode_column(df, column, mapping):\n    \"\"\"\n    Encode categorical columns using provided mapping\n    Args:\n        df: Input dataframe\n        column: Column name to encode\n        mapping: Dictionary with encoding mappings\n    Returns:\n        Dataframe with encoded column\n    \"\"\"\n    max_value = max(mapping.values())  \n    \n    def encode_category(category):\n        # Return max_value + 1 for any unseen categories\n        return mapping.get(category, max_value + 1)  \n    \n    return df.with_columns(\n        pl.col(column).map_elements(encode_category).alias(column)\n    )\n\nclass R2Loss(nn.Module):\n    \"\"\"\n    Custom R-squared loss function for PyTorch\n    R² = 1 - (MSE / variance of y)\n    \"\"\"\n    def __init__(self):\n        super(R2Loss, self).__init__()\n\n    def forward(self, y_pred, y_true):\n        # Calculate MSE\n        mse_loss = torch.sum((y_pred - y_true) ** 2)\n        # Calculate variance of true values\n        var_y = torch.sum(y_true ** 2)\n        # Calculate R² loss (adding small epsilon to avoid division by zero)\n        loss = mse_loss / (var_y + 1e-38)\n        return loss\n\nclass NN(LightningModule):\n    \"\"\"\n    Neural Network model using PyTorch Lightning\n    Implements a custom architecture with continuous and categorical inputs\n    \"\"\"\n    def __init__(self, n_cont_features, cat_cardinalities, n_classes, lr, weight_decay):\n        super().__init__()\n        self.save_hyperparameters()\n        self.k = 16  # Number of ensemble members\n\n        # Initialize the main model architecture\n        self.model = Model(\n                n_num_features=n_cont_features,\n                cat_cardinalities=cat_cardinalities,\n                n_classes=n_classes,\n                backbone={\n                    'type': 'MLP',\n                    'n_blocks': 3,\n                    'd_block': 512,\n                    'dropout': 0.25,\n                },\n                bins=None,\n                num_embeddings=None,\n                arch_type='tabm',\n                k=self.k,\n            )\n        \n        # Set learning parameters\n        self.lr = lr\n        self.weight_decay = weight_decay\n        \n        # Initialize lists to store outputs during training and validation\n        self.training_step_outputs = []\n        self.validation_step_outputs = []\n        \n        # Define loss function\n        self.loss_fn = R2Loss()\n\n    def forward(self, x_cont, x_cat):\n        \"\"\"Forward pass of the model\"\"\"\n        return self.model(x_cont, x_cat).squeeze(-1)\n\n    def training_step(self, batch):\n        \"\"\"\n        Perform a single training step\n        Args:\n            batch: Tuple containing (continuous features, categorical features, \n                   target values, weights, weighted targets)\n        \"\"\"\n        x_cont, x_cat, y, w, w_y = batch\n        \n        # Add random noise to continuous features for regularization\n        x_cont = x_cont + torch.randn_like(x_cont) * 0.02\n        \n        # Get model predictions\n        y_hat = self(x_cont, x_cat)\n        \n        # Calculate loss\n        loss = self.loss_fn(y_hat.flatten(0, 1), y.repeat_interleave(self.k))\n        \n        # Log training loss\n        self.log('train_loss', loss, on_step=True, on_epoch=True, \n                prog_bar=True, logger=True, batch_size=x_cont.size(0))\n        \n        # Store outputs for epoch-end calculations\n        self.training_step_outputs.append((y_hat.mean(1), y, w))\n        \n        return loss\n\n    def validation_step(self, batch):\n        \"\"\"\n        Perform a single validation step\n        Similar to training_step but used for validation\n        \"\"\"\n        x_cont, x_cat, y, w, w_y = batch\n        x_cont = x_cont + torch.randn_like(x_cont) * 0.02\n        y_hat = self(x_cont, x_cat)\n        \n        loss = self.loss_fn(y_hat.flatten(0, 1), y.repeat_interleave(self.k))\n        \n        self.log('val_loss', loss, on_step=False, on_epoch=True, \n                prog_bar=True, logger=True, batch_size=x_cont.size(0))\n        \n        self.validation_step_outputs.append((y_hat.mean(1), y, w))\n        return loss\n\n    def on_validation_epoch_end(self):\n        \"\"\"Calculate validation metrics at the end of each validation epoch\"\"\"\n        y = torch.cat([x[1] for x in self.validation_step_outputs]).cpu().numpy()\n        \n        if self.trainer.sanity_checking:\n            prob = torch.cat([x[0] for x in self.validation_step_outputs]).cpu().numpy()\n        else:\n            prob = torch.cat([x[0] for x in self.validation_step_outputs]).cpu().numpy()\n            weights = torch.cat([x[2] for x in self.validation_step_outputs]).cpu().numpy()\n            \n            # Calculate R² score for validation\n            val_r_square = r2_val(y, prob, weights)\n            self.log(\"val_r_square\", val_r_square, prog_bar=True, on_step=False, on_epoch=True)\n        \n        self.validation_step_outputs.clear()\n\n    def configure_optimizers(self):\n        \"\"\"Configure the optimizer for training\"\"\"\n        optimizer = torch.optim.AdamW(\n            make_parameter_groups(self.model), \n            lr=self.lr, \n            weight_decay=self.weight_decay\n        )\n        return {'optimizer': optimizer}\n\n    def on_train_epoch_end(self):\n        \"\"\"Calculate and log training metrics at the end of each training epoch\"\"\"\n        if self.trainer.sanity_checking:\n            return\n            \n        # Gather all outputs from training steps\n        y = torch.cat([x[1] for x in self.training_step_outputs]).cpu().numpy()\n        prob = torch.cat([x[0] for x in self.training_step_outputs]).detach().cpu().numpy()\n        weights = torch.cat([x[2] for x in self.training_step_outputs]).cpu().numpy()\n        \n        # Calculate R² score for training\n        train_r_square = r2_val(y, prob, weights)\n        self.log(\"train_r_square\", train_r_square, prog_bar=True, on_step=False, on_epoch=True)\n        \n        self.training_step_outputs.clear()\n        \n        # Print epoch metrics\n        epoch = self.trainer.current_epoch\n        metrics = {k: v.item() if isinstance(v, torch.Tensor) else v \n                  for k, v in self.trainer.logged_metrics.items()}\n        formatted_metrics = {k: f\"{v:.5f}\" for k, v in metrics.items()}\n        print(f\"Epoch {epoch}: {formatted_metrics}\")\n\n        \nclass custom_args:\n    \"\"\"\n    Custom arguments class to store model and training configuration\n    Acts as a configuration container similar to argparse.Namespace\n    \"\"\"\n    def __init__(self):\n        # GPU Configuration\n        self.usegpu = True\n        self.gpuid = 0\n        \n        # Random seed for reproducibility\n        self.seed = 42\n        \n        # Model Configuration\n        self.model = 'nn'  # Neural network model type\n        \n        # Wandb logging configuration\n        self.use_wandb = False\n        self.project = 'js-tabm-with-lags'\n        \n        # Data and loading configuration\n        self.dname = \"./input_df/\"  # Data directory\n        self.loader_workers = 10    # Number of workers for data loading\n        self.bs = 8192             # Batch size\n        \n        # Model hyperparameters\n        self.lr = 1e-4             # Learning rate\n        self.weight_decay = 1e-5    # Weight decay for regularization\n        \n        # Feature configuration\n        self.n_cont_features = 84   # Number of continuous features\n        self.n_cat_features = 5     # Number of categorical features\n        self.n_classes = None       # Number of classes (None for regression)\n        \n        # Categorical feature cardinalities\n        # [feature_09, feature_10, feature_11, symbol_id, time_id]\n        self.cat_cardinalities = [23, 10, 32, 40, 969]\n        \n        # Training configuration\n        self.patience = 7           # Early stopping patience\n        self.max_epochs = 10        # Maximum training epochs\n        self.N_fold = 5            # Number of cross-validation folds\n\n# Create instance of custom arguments\nmy_args = custom_args()\n\n# Set up device (GPU if available, else CPU)\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n\n# Load pre-trained model from checkpoint and move to appropriate device\nmodel = NN.load_from_checkpoint('/kaggle/input/my-own-js/tabm_epochepoch03.ckpt').to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T14:33:31.270013Z","iopub.execute_input":"2025-01-13T14:33:31.270294Z","iopub.status.idle":"2025-01-13T14:33:31.384189Z","shell.execute_reply.started":"2025-01-13T14:33:31.270266Z","shell.execute_reply":"2025-01-13T14:33:31.383133Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Global variables to store lag features\nlags_ : pl.DataFrame | None = None\n\nlags_history = None\n\ndef predict_tabm(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"\n    Make predictions using the TABM (Tabular Model)\n    \n    Args:\n        test: Input DataFrame containing test features\n        lags: DataFrame containing lagged features\n        \n    Returns:\n        DataFrame with predictions\n    \"\"\"\n    global lags_, lags_history\n    # Update global lags if new ones provided\n    if lags is not None:\n        lags_ = lags\n\n    # Encode categorical features\n    for col in feature_cat + ['symbol_id', 'time_id']:\n        test = encode_column(test, col, category_mappings[col])\n\n    # Initialize predictions DataFrame with row_ids\n    predictions = test.select(\n        'row_id',\n        pl.lit(0.0).alias('responder_6'),\n    )\n\n    # Extract symbol and time information\n    symbol_ids = test.select('symbol_id').to_numpy()[:, 0]\n    time_id = test.select(\"time_id\").to_numpy()[0]\n    timie_id_array = test.select(\"time_id\").to_numpy()[:, 0]\n    \n    # Handle time_id = 0 case (first prediction)\n    if time_id == 0:\n        # Convert time_id and symbol_id to integers\n        lags = lags.with_columns(pl.col('time_id').cast(pl.Int64))\n        lags = lags.with_columns(pl.col('symbol_id').cast(pl.Int64)) \n        # Store full lags history and filter for time_id 0\n        lags_history = lags\n        lags = lags.filter(pl.col(\"time_id\") == 0)  \n        test = test.join(lags, on=[\"time_id\", \"symbol_id\"],  how=\"left\")\n    else:\n        # Filter lags for current time_id\n        lags = lags_history.filter(pl.col(\"time_id\") == time_id)\n        test = test.join(lags, on=[\"time_id\", \"symbol_id\"],  how=\"left\")\n\n    # Fill missing values with 0\n    test = test.with_columns([\n        pl.col(col).fill_null(0) for col in feature_list + [f\"responder_{idx}_lag_1\" for idx in range(9)] \n    ])\n    \n    # Standardize features\n    test = standardize(test, std_feature, means, stds)\n\n    # Convert to numpy array and then to torch tensors\n    X_test = test[feature_test].to_numpy()\n    X_test_tensor = torch.tensor(X_test, dtype=torch.float32).to(device)\n    symbol_tensor = torch.tensor(symbol_ids, dtype=torch.float32).to(device)\n    time_tensor = torch.tensor(timie_id_array, dtype=torch.float32).to(device)\n    \n    # Separate categorical and continuous features\n    X_cat = X_test_tensor[:, [9, 10, 11]]\n    X_cont = X_test_tensor[:, [i for i in range(X_test_tensor.shape[1]) if i not in [9, 10, 11]]]\n\n    # Combine categorical features with symbol and time information\n    X_cat = (torch.concat([X_cat, symbol_tensor.unsqueeze(-1), time_tensor.unsqueeze(-1)], axis=1)).to(torch.int64)\n\n    # Make predictions\n    model.eval()\n    with torch.no_grad():\n        \n        outputs = model(X_cont, X_cat)\n        # Assuming the model outputs a tensor of shape (batch_size, 1)\n        preds = outputs.squeeze(-1).cpu().numpy()\n        preds = preds.mean(1)\n\n    # Create final predictions DataFrame\n    predictions = \\\n    test.select('row_id').\\\n    with_columns(\n        pl.Series(\n            name   = 'responder_6', \n            values = np.clip(preds, a_min = -5, a_max = 5),\n            dtype  = pl.Float64,\n        )\n    )\n\n    return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T14:33:31.385670Z","iopub.execute_input":"2025-01-13T14:33:31.386043Z","iopub.status.idle":"2025-01-13T14:33:31.400578Z","shell.execute_reply.started":"2025-01-13T14:33:31.386012Z","shell.execute_reply":"2025-01-13T14:33:31.399655Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## JS Ridge baseline (yunsuxiaozi)\n\n`LB: 0.0042` https://www.kaggle.com/code/yunsuxiaozi/js-ridge-baseline?scriptVersionId=202938352","metadata":{}},{"cell_type":"code","source":"def load_from_dill(model_name, model_path=None, file_ext='.dill'):\n    \"\"\"\n    Load a model from a dill file\n    \n    Args:\n        model_name: Name of the model file (without extension)\n        model_path: Directory path containing the model file\n        file_ext: File extension (default: '.dill')\n        \n    Returns:\n        Loaded model object\n    \"\"\"\n    model_object = None\n    # Open and load the model file using dill\n    with open(f\"{model_path}/{model_name}{file_ext}\", \"rb\") as file_handle:\n        model_object = dill.load(file_handle)\n    return model_object\n\n# Load pre-trained Ridge Regression model\nrdg = load_from_dill(\n    model_name='Ridge', \n    model_path=\"/kaggle/input/jsridgev01011635\"\n)\n\ndef predict_ridge(test, lags):\n    \"\"\"\n    Make predictions using Ridge Regression model\n    \n    Args:\n        test: DataFrame containing test data\n        lags: DataFrame containing lagged features (unused in this function)\n        \n    Returns:\n        DataFrame with predictions\n    \"\"\"\n    # Select the 79 numerical features\n    cols = [f'feature_{i:02}' for i in range(79)]\n\n    # Initialize predictions DataFrame with row_id and placeholder predictions\n    predictions = test.select(\n        'row_id',\n        pl.lit(0.0).alias('responder_6'),\n    )\n\n    # Generate predictions:\n    # 1. Select required features\n    # 2. Convert to pandas\n    # 3. Fill missing values with 3\n    # 4. Make predictions using Ridge model\n    test_preds = rdg.predict(test[cols].to_pandas().fillna(3).values)\n\n    # Add predictions to result DataFrame\n    predictions = predictions.with_columns(pl.Series('responder_6', test_preds.ravel()))\n\n    return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T14:33:31.401732Z","iopub.execute_input":"2025-01-13T14:33:31.402049Z","iopub.status.idle":"2025-01-13T14:33:31.416474Z","shell.execute_reply.started":"2025-01-13T14:33:31.402015Z","shell.execute_reply":"2025-01-13T14:33:31.415754Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:#3371ff;text-align:center;letter-spacing:0.1px;overflow:hidden;padding:20px;color:white;overflow:hidden;margin:0;font-size:100%\">Ensemble notebook</div>","metadata":{}},{"cell_type":"code","source":"def predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"\n    Make ensemble predictions combining three different models:\n    - Neural Network + XGBoost ensemble\n    - Ridge Regression\n    - Tabular Model (TABM)\n    \n    Args:\n        test: DataFrame containing test features\n        lags: DataFrame containing lagged features\n        \n    Returns:\n        DataFrame with weighted ensemble predictions\n    \"\"\"\n    # Get predictions from each model/ensemble\n    pd_nn_xgb = predict_nn_xgb(test, lags).to_pandas()  # Neural Network + XGBoost predictions\n    pd_ridge = predict_ridge(test, lags).to_pandas()     # Ridge Regression predictions \n    pd_tabm = predict_tabm(test, lags).to_pandas()       # Tabular Model predictions\n    \n    # Rename prediction columns to avoid naming conflicts during merge\n    pd_nn_xgb = pd_nn_xgb.rename(columns={'responder_6': 'col_nn_xgb'})\n    pd_ridge = pd_ridge.rename(columns={'responder_6': 'col_ridge'})\n    pd_tabm = pd_tabm.rename(columns={'responder_6': 'col_tabm'})\n    \n    # Merge predictions from all models based on row_id\n    pds = pd.merge(pd_nn_xgb, pd_ridge, on=['row_id'])\n    pds = pd.merge(pds, pd_tabm, on=['row_id'])\n    \n    # Define ensemble weights for each model\n    e_weights = [0.50,  # Weight for Neural Network + XGBoost\n                0.10,   # Weight for Ridge Regression\n                0.40]   # Weight for Tabular Model\n    \n    # Create weighted ensemble predictions\n    pds['responder_6'] = (\n        pds['col_nn_xgb'] * e_weights[0] + \n        pds['col_ridge'] * e_weights[1] +    \n        pds['col_tabm'] * e_weights[2]     \n    )\n    \n    # Format final predictions DataFrame\n    predictions = test.select('row_id', pl.lit(0.0).alias('responder_6'))\n    pred = pds['responder_6'].to_numpy()\n    predictions = predictions.with_columns(pl.Series('responder_6', pred.ravel()))\n    display(predictions)\n    display(pds)\n    \n    return predictions","metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"papermill":{"duration":0.351292,"end_time":"2024-10-26T03:27:42.101707","exception":false,"start_time":"2024-10-26T03:27:41.750415","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T14:33:31.418433Z","iopub.execute_input":"2025-01-13T14:33:31.418773Z","iopub.status.idle":"2025-01-13T14:33:31.431530Z","shell.execute_reply.started":"2025-01-13T14:33:31.418725Z","shell.execute_reply":"2025-01-13T14:33:31.430753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import kaggle_evaluation.jane_street_inference_server\n\n# Initialize the Jane Street inference server with our predict function\ninference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\n# Check if this is running in competition environment\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    # If in competition environment, serve predictions in real-time\n    inference_server.serve()\nelse:\n    # If running locally, use test data from provided parquet files\n    inference_server.run_local_gateway(\n        (\n            # Path to test data parquet file\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            # Path to lagged features parquet file\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T14:33:31.432670Z","iopub.execute_input":"2025-01-13T14:33:31.433052Z","iopub.status.idle":"2025-01-13T14:33:31.600465Z","shell.execute_reply.started":"2025-01-13T14:33:31.433004Z","shell.execute_reply":"2025-01-13T14:33:31.599211Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}