{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":210658417,"sourceType":"kernelVersion"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport os\nfrom tqdm.auto import tqdm\nfrom matplotlib import pyplot as plt\nimport pickle\n\nfrom sklearn.metrics import r2_score\nfrom lightgbm import LGBMRegressor\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-03T09:57:23.869423Z","iopub.execute_input":"2024-12-03T09:57:23.869785Z","iopub.status.idle":"2024-12-03T09:57:28.338225Z","shell.execute_reply.started":"2024-12-03T09:57:23.869757Z","shell.execute_reply":"2024-12-03T09:57:28.337571Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CONFIG:\n    seed = 42\n    target_col = \"responder_6\"\n    feature_cols = [f\"feature_{idx:02d}\" for idx in range(73)] \\\n        + ['feature_73_74', 'feature_75_76', 'feature_77_78']\\\n        + [f\"responder_{idx}_lag_1\" for idx in range(9)]\n    categorical_cols = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T09:57:28.339624Z","iopub.execute_input":"2024-12-03T09:57:28.340057Z","iopub.status.idle":"2024-12-03T09:57:28.344688Z","shell.execute_reply.started":"2024-12-03T09:57:28.340029Z","shell.execute_reply":"2024-12-03T09:57:28.343768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!rm -rf /kaggle/working/*","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T09:57:29.204401Z","iopub.execute_input":"2024-12-03T09:57:29.204885Z","iopub.status.idle":"2024-12-03T09:57:30.219217Z","shell.execute_reply.started":"2024-12-03T09:57:29.204851Z","shell.execute_reply":"2024-12-03T09:57:30.217792Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nos.listdir(\"../input/jane-street-preprocessing\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T09:57:30.221507Z","iopub.execute_input":"2024-12-03T09:57:30.221833Z","iopub.status.idle":"2024-12-03T09:57:30.233065Z","shell.execute_reply.started":"2024-12-03T09:57:30.221801Z","shell.execute_reply":"2024-12-03T09:57:30.232216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_idx = sorted([int(s.replace(\"date_id=\", \"\")) for s in os.listdir(\"/kaggle/input/jane-street-preprocessing/training.parquet\")])[-400:]\nvalid_idx = sorted([int(s.replace(\"date_id=\", \"\")) for s in os.listdir(\"/kaggle/input/jane-street-preprocessing/validation.parquet\")])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T09:57:30.234134Z","iopub.execute_input":"2024-12-03T09:57:30.234408Z","iopub.status.idle":"2024-12-03T09:57:30.259751Z","shell.execute_reply.started":"2024-12-03T09:57:30.234383Z","shell.execute_reply":"2024-12-03T09:57:30.259116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train = pl.scan_parquet(\"/kaggle/input/jane-street-preprocessing/training.parquet\").collect().to_pandas()\n# valid = pl.scan_parquet(\"/kaggle/input/jane-street-preprocessing/validation.parquet\").collect().to_pandas()\n# train.shape, valid.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T09:57:30.308033Z","iopub.execute_input":"2024-12-03T09:57:30.308889Z","iopub.status.idle":"2024-12-03T09:57:30.312817Z","shell.execute_reply.started":"2024-12-03T09:57:30.308847Z","shell.execute_reply":"2024-12-03T09:57:30.311513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from typing import List, Callable\n\nclass Iterator(xgb.DataIter):\n    data_dir = \"/kaggle/input/jane-street-preprocessing\"\n    def __init__(self, partitions: List[str], feature_cols: List[str], target_col: List[str], weight_col='weight', fill_null=None, file_name=\"training\"):\n        self._file_paths = partitions\n        self._it = 0\n        self.feature_cols = feature_cols\n        self.target_col = target_col\n        self.weight_col = weight_col\n        self.fill_null = fill_null\n        self.file_name = file_name\n        super().__init__(cache_prefix=os.path.join(\".\", \"cache\"))\n\n    def next(self, input_data: Callable):\n        # return 0 to let XGBoost know this is the end of iteration\n        if self._it == len(self._file_paths):\n            return 0\n    \n        # load formatted data\n        train = pl.read_parquet(\n            os.path.join(self.data_dir, f'{self.file_name}.parquet/date_id={self._file_paths[self._it]}/00000000.parquet')\n        ).fill_null(self.fill_null)\n        \n        X_train = train.select(pl.col(self.feature_cols)).to_pandas()\n        y_train = train.select(pl.col(self.target_col)).to_pandas()\n        w_train = train.select(pl.col(self.weight_col)).get_columns()[0].to_pandas()\n        \n        input_data(data=X_train, label=y_train, weight=w_train)\n        self._it += 1\n        # Return 1 to let XGBoost know we haven't seen all the files yet.\n        return 1\n\n    def reset(self):\n        \"\"\"Reset the iterator to its beginning\"\"\"\n        self._it = 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T09:57:30.974202Z","iopub.execute_input":"2024-12-03T09:57:30.974820Z","iopub.status.idle":"2024-12-03T09:57:30.982592Z","shell.execute_reply.started":"2024-12-03T09:57:30.974785Z","shell.execute_reply":"2024-12-03T09:57:30.981742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def r2_score_factory(eval=True):\n    def r2_score(y_pred, dtrain: xgb.DMatrix):\n        y_true = dtrain.get_label()\n        weight = dtrain.get_weight()\n        r2 = 1 - np.average((y_pred - y_true) ** 2, weights=weight) / (np.average((y_true) ** 2, weights=weight) + 1e-38)\n        if eval:\n            return 'r2', -r2\n        else:\n            return r2\n    \n    return r2_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T09:57:31.619602Z","iopub.execute_input":"2024-12-03T09:57:31.619905Z","iopub.status.idle":"2024-12-03T09:57:31.625140Z","shell.execute_reply.started":"2024-12-03T09:57:31.619876Z","shell.execute_reply":"2024-12-03T09:57:31.624192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Preparing XGB Data Loader...\")\ntrain_it = Iterator(train_idx, feature_cols=CONFIG.feature_cols, target_col=[CONFIG.target_col], fill_null=0, file_name='training')\ndtrain = xgb.DMatrix(train_it)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T09:57:32.211235Z","iopub.execute_input":"2024-12-03T09:57:32.211917Z","iopub.status.idle":"2024-12-03T09:58:25.982004Z","shell.execute_reply.started":"2024-12-03T09:57:32.211881Z","shell.execute_reply":"2024-12-03T09:58:25.981234Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"valid_it = Iterator(valid_idx, feature_cols=CONFIG.feature_cols, target_col=[CONFIG.target_col], fill_null=0, file_name='validation')\ndvalid = xgb.DMatrix(valid_it)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T09:58:25.983779Z","iopub.execute_input":"2024-12-03T09:58:25.984828Z","iopub.status.idle":"2024-12-03T09:58:33.392483Z","shell.execute_reply.started":"2024-12-03T09:58:25.984783Z","shell.execute_reply":"2024-12-03T09:58:33.391507Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"XGB_Params = {\n    'random_state': CONFIG.seed,\n    'disable_default_eval_metric': True,\n    'tree_method': 'hist',\n    'objective': 'reg:squarederror',\n    'learning_rate': 0.1,\n    'max_depth': 6,\n    'subsample': 0.7,\n    'colsample_bytree': 0.7,\n    'reg_alpha': 1, 'reg_lambda': 1,\n    'n_estimators': 200,\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T09:58:33.393626Z","iopub.execute_input":"2024-12-03T09:58:33.393919Z","iopub.status.idle":"2024-12-03T09:58:33.937219Z","shell.execute_reply.started":"2024-12-03T09:58:33.393892Z","shell.execute_reply":"2024-12-03T09:58:33.936106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"booster = xgb.train(\n    XGB_Params, dtrain, num_boost_round=50, early_stopping_rounds=5,\n    evals=[(dtrain, 'train'), (dvalid, 'valid')],\n    custom_metric=r2_score_factory(eval=True),\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T09:58:33.939522Z","iopub.execute_input":"2024-12-03T09:58:33.939900Z","iopub.status.idle":"2024-12-03T10:05:01.672166Z","shell.execute_reply.started":"2024-12-03T09:58:33.939857Z","shell.execute_reply":"2024-12-03T10:05:01.671441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred_train = booster.predict(dtrain)\nr2_score_factory(eval=False)(y_pred_train, dtrain)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T10:05:01.673387Z","iopub.execute_input":"2024-12-03T10:05:01.674106Z","iopub.status.idle":"2024-12-03T10:05:13.574127Z","shell.execute_reply.started":"2024-12-03T10:05:01.674065Z","shell.execute_reply":"2024-12-03T10:05:13.573234Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred_valid = booster.predict(dvalid)\nr2_score_factory(eval=False)(y_pred_valid, dvalid)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T10:05:13.575256Z","iopub.execute_input":"2024-12-03T10:05:13.575621Z","iopub.status.idle":"2024-12-03T10:05:15.319587Z","shell.execute_reply.started":"2024-12-03T10:05:13.575584Z","shell.execute_reply":"2024-12-03T10:05:15.318565Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"booster.save_model('xgboost_model.json')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T10:05:15.320538Z","iopub.execute_input":"2024-12-03T10:05:15.320786Z","iopub.status.idle":"2024-12-03T10:05:15.331645Z","shell.execute_reply.started":"2024-12-03T10:05:15.320762Z","shell.execute_reply":"2024-12-03T10:05:15.331053Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pickle\nfile_name = \"xgb_reg.pkl\"\n\n# save\npickle.dump(booster, open(file_name, \"wb\"))\n\n# load\nxgb_model_loaded = pickle.load(open(file_name, \"rb\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T10:05:15.333070Z","iopub.execute_input":"2024-12-03T10:05:15.333502Z","iopub.status.idle":"2024-12-03T10:05:15.347926Z","shell.execute_reply.started":"2024-12-03T10:05:15.333462Z","shell.execute_reply":"2024-12-03T10:05:15.347214Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!rm -rf /kaggle/working/*.page","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T10:05:15.348627Z","iopub.execute_input":"2024-12-03T10:05:15.348873Z","iopub.status.idle":"2024-12-03T10:05:18.215541Z","shell.execute_reply.started":"2024-12-03T10:05:15.348847Z","shell.execute_reply":"2024-12-03T10:05:18.214258Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}