{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":9838282,"sourceType":"datasetVersion","datasetId":6035149}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":7.594014,"end_time":"2024-10-10T11:58:36.355301","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-10-10T11:58:28.761287","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Thanks to these great works:\n  - [motono0223: js24-train-gbdt-model-with-lags-singlemodel](https://www.kaggle.com/code/motono0223/js24-train-gbdt-model-with-lags-singlemodel)\n  - [yuanzhezhou: jane-street-baseline-lgb-xgb-and-catboost](https://www.kaggle.com/code/yuanzhezhou/jane-street-baseline-lgb-xgb-and-catboost)\n\n这篇notebook主要包含如下内容:\n  - 如何建立滞后N个date_id的(shift, rolling)特征工程\n  - 如何动态存储最近N个date_id的数据以更新lags特征\n\n这里用到的模型是一个加入lag前一天特征的XGBoost单折模型, 在LB取得了0.0052得分.\n\n在每个新的date_id的time_id=0到来时会给一个当前date_id前一天的lags标签, 比如当date_id=100时，会给date_id=99的标签，但是这个lags数据里的date_id还是100(为了和当前主数据对齐)； 需要将这个lags和动态存储最近N天，以构造更多的shift和rolling特征。","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport os, gc\nfrom tqdm.auto import tqdm\nfrom matplotlib import pyplot as plt\nimport pickle\n\nfrom sklearn.metrics import r2_score\nfrom lightgbm import LGBMRegressor\nimport lightgbm as lgb\nfrom xgboost import XGBRegressor\nimport xgboost as xgb\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport kaggle_evaluation.jane_street_inference_server\n\npl.Config.set_tbl_rows(100)\npl.Config.set_tbl_cols(400)\npl.Config.set_fmt_table_cell_list_len(5)","metadata":{"execution":{"iopub.status.busy":"2024-12-21T23:42:46.148958Z","iopub.execute_input":"2024-12-21T23:42:46.149326Z","iopub.status.idle":"2024-12-21T23:42:50.555112Z","shell.execute_reply.started":"2024-12-21T23:42:46.149277Z","shell.execute_reply":"2024-12-21T23:42:50.554264Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Configurations","metadata":{}},{"cell_type":"code","source":"class CONFIG:\n    debug = False\n    seed = 42\n    target_col = \"responder_6\"\n    lag_cols_rename = { f\"responder_{idx}_lag_1\" : f\"responder_{idx}\" for idx in range(9)}\n    lag_target_cols_name = [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_original = [\"date_id\", \"time_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    model_path = \"/kaggle/input/janestreet-public-model/xgb_001.pkl\"\n    lag_ndays = 4","metadata":{"execution":{"iopub.status.busy":"2024-12-21T23:42:50.556831Z","iopub.execute_input":"2024-12-21T23:42:50.557284Z","iopub.status.idle":"2024-12-21T23:42:50.562443Z","shell.execute_reply.started":"2024-12-21T23:42:50.557254Z","shell.execute_reply":"2024-12-21T23:42:50.561607Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_agg_list(day, columns):\n    agg_mean_list = [pl.col(c).mean().name.suffix(f\"_mean_{day}d\") for c in columns]\n    agg_std_list = [pl.col(c).std().name.suffix(f\"_std_{day}d\") for c in columns]\n    agg_max_list = [pl.col(c).max().name.suffix(f\"_max_{day}d\") for c in columns]\n    agg_last_list = [pl.col(c).last().name.suffix(f\"_last_{day}d\") for c in columns]\n    agg_list = agg_mean_list + agg_std_list + agg_max_list + agg_last_list\n    return agg_list","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T23:42:50.563617Z","iopub.execute_input":"2024-12-21T23:42:50.564378Z","iopub.status.idle":"2024-12-21T23:42:50.573388Z","shell.execute_reply.started":"2024-12-21T23:42:50.564327Z","shell.execute_reply":"2024-12-21T23:42:50.572611Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load model","metadata":{}},{"cell_type":"code","source":"with open( CONFIG.model_path, \"rb\") as fp:\n    result = pickle.load(fp)\n    \nmodel = result[\"model\"]\nfeatures = result[\"features\"]\nprint(len(features))","metadata":{"execution":{"iopub.status.busy":"2024-12-21T23:42:50.574450Z","iopub.execute_input":"2024-12-21T23:42:50.575046Z","iopub.status.idle":"2024-12-21T23:42:50.674994Z","shell.execute_reply.started":"2024-12-21T23:42:50.575009Z","shell.execute_reply":"2024-12-21T23:42:50.673348Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"在对test进行推理前需要提前构造好历史数据，需要注意的地方如下：\n  - 假如用到了lags N天的特征，需要先存储train中最后N天的数据.\n  - test数据中的date_id是从0开始计数的，需要预先调整train与test的date_id一致.\n  - 目前不确定test数据是否是紧随train，这里先改为和test一致(-N, -N+1, ..., -2, -1).","metadata":{}},{"cell_type":"code","source":"history = pl.scan_parquet(\n    \"/kaggle/input/jane-street-realtime-marketdata-forecasting/train.parquet\"\n).select(['date_id','time_id','symbol_id'] + [f\"responder_{idx}\" for idx in range(9)]).filter(\n    (pl.col(\"date_id\")>=(1698 - CONFIG.lag_ndays))&(pl.col(\"date_id\")<1698)\n)\n# 这里将历史date_id变为从-N到-1, 假设test的date_id=0紧随train的date_id=1698,\n# 在第一个batch给出的lags应该是date_id=1698的responser(但date_id给的0),\n# 这样history中最后一个date_id=1697变为-1, 正好可以和推理时给的lags衔接上\nhistory = history.with_columns(\n    date_id = (pl.col(\"date_id\") - pl.lit(1698)).cast(pl.Int16)\n)\nhistory = history.collect()\n\n# 这里是为了统一特征的dtypes(polars在concat时如果dtype对不上会报错)\nhistory_column_types = {\n    'date_id': pl.Int16,\n    'time_id': pl.Int16,\n    'symbol_id': pl.Int16\n}\nfeature_column_types = {}\nfor f in [f\"feature_{idx:02d}\" for idx in range(79)]:\n    feature_column_types[f] = pl.Float32\n\nresponder_column_types = {}\nfor f in [f\"responder_{idx}\" for idx in range(9)]:\n    responder_column_types[f] = pl.Float32\n\nhistory = history.cast(history_column_types)\nhistory = history.cast(responder_column_types)\nhistory.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T23:42:50.676486Z","iopub.execute_input":"2024-12-21T23:42:50.680826Z","iopub.status.idle":"2024-12-21T23:42:51.055332Z","shell.execute_reply.started":"2024-12-21T23:42:50.680795Z","shell.execute_reply":"2024-12-21T23:42:51.054554Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \n    global history\n    global lags_infer\n    \n    symbol_ids = test.select(\"symbol_id\").to_numpy()[:, 0]\n    current_date = test.select(\"date_id\").to_numpy()[:, 0][0]\n\n    if lags is not None:\n        \n        # 原始lags先存储到history更新历史数据\n        lags = lags.rename(CONFIG.lag_cols_rename)\n        lags = lags.cast(history_column_types)\n        lags = lags.cast(responder_column_types)\n\n        history = pl.concat([history, lags])\n        \n        # 只储存最近N天的历史数据\n        history = history.filter(pl.col(\"date_id\") > (current_date - CONFIG.lag_ndays))\n        # 这里构造当前date_id下所有batch用的特征\n        # 如果想用shift N天, 可以在history这样取：\n        # -- shift_n_data = history.filter(pl.col(\"date_id\") == (current_date - N))\n        # 如果想用rolling平均 N天, 可以在history这样取：\n        # -- rolling_n_data = history.filter(pl.col(\"date_id\") >= (current_date - N)).agg(任何统计指标)\n\n        # 这里用的XGB模型只使用了shift 1天的统计值\n        agg_list = create_agg_list(1, CONFIG.lag_target_cols_name)\n        shift_n_data = history.filter(pl.col(\"date_id\") == current_date)\n        lags_infer = shift_n_data.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).agg(agg_list)\n  \n    \n    test = test.cast(history_column_types)\n    test = test.cast(feature_column_types)\n    # 在一个date_id下的所有batch用到的lags_infer是相同的\n    # 像lags_infer这样的统计特征在每个date_id的time_id=0时构造完成\n    X_test = test.join(lags_infer, on=[\"date_id\", \"symbol_id\"], how=\"left\")\n    \n    preds = np.zeros((X_test.shape[0],))\n    preds += model.predict(X_test[features].to_pandas().values)\n    preds = np.clip(preds, a_min=-5, a_max=5)\n    \n    predictions = (\n        test.select('row_id').with_columns(\n            pl.Series(name='responder_6', values=preds, dtype=pl.Float64)\n        )\n    )\n\n    assert isinstance(predictions, pl.DataFrame | pd.DataFrame)\n    assert list(predictions.columns) == ['row_id', 'responder_6']\n    assert len(predictions) == len(test)\n\n    return predictions","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.018344,"end_time":"2024-10-10T11:58:33.59684","exception":false,"start_time":"2024-10-10T11:58:33.578496","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-21T23:42:51.056729Z","iopub.execute_input":"2024-12-21T23:42:51.057320Z","iopub.status.idle":"2024-12-21T23:42:51.066727Z","shell.execute_reply.started":"2024-12-21T23:42:51.057280Z","shell.execute_reply":"2024-12-21T23:42:51.065876Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-realtime-marketdata-forecasting/test.parquet',\n            '/kaggle/input/jane-street-realtime-marketdata-forecasting/lags.parquet',\n        )\n    )","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":2.225871,"end_time":"2024-10-10T11:58:35.830964","exception":false,"start_time":"2024-10-10T11:58:33.605093","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-21T23:42:51.067861Z","iopub.execute_input":"2024-12-21T23:42:51.068214Z","iopub.status.idle":"2024-12-21T23:42:51.249580Z","shell.execute_reply.started":"2024-12-21T23:42:51.068175Z","shell.execute_reply":"2024-12-21T23:42:51.248885Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"下面是训练时可以参考的shift N与rolling N的构造方法：","metadata":{}},{"cell_type":"code","source":"if CONFIG.debug:\n\n    train = pl.read_parquet(\"/kaggle/input/jane-street-realtime-marketdata-forecasting/train.parquet\")\n    lags = train.select(pl.col(CONFIG.lag_cols_original))\n    lags = lags.with_columns(date_id = pl.col(\"date_id\") + 1)\n    \n    date_ids = lags.select(\"date_id\").unique().to_series()\n    agg_list = create_agg_list(1, CONFIG.lag_target_cols_name)\n    \n    result = []\n    for date_id in tqdm(date_ids, total=len(date_ids)):\n        try:\n            # rolling N天\n            lags_ = lags.filter((pl.col(\"date_id\") > date_id - CONFIG.lag_ndays) & (pl.col(\"date_id\") <= date_id))\n            # shift N天\n            lags_ = lags.filter((pl.col(\"date_id\") == date_id - CONFIG.lag_ndays))\n            # 为了merge，将date_id统一到对应的date_id\n            # 比如在统计第10天的rolling 3天的数据时, 数据中的date_id应该是8,9,10, 统一为10和主数据对应\n            lags_ = lags_.with_columns(date_id=date_id)\n            lags_ = lags_.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).agg(agg_list)\n            result.append(lags_)\n        except:\n            continue\n    \n    lag_Ndays = pl.concat(result).sort(\"date_id\")\n    lag_Ndays = lag_Ndays.cast({\"date_id\": pl.Int16})\n    \n    train = train.join(lag_Ndays, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T23:42:51.250487Z","iopub.execute_input":"2024-12-21T23:42:51.250710Z","iopub.status.idle":"2024-12-21T23:42:51.257166Z","shell.execute_reply.started":"2024-12-21T23:42:51.250687Z","shell.execute_reply":"2024-12-21T23:42:51.256392Z"}},"outputs":[],"execution_count":null}]}