{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":210944874,"sourceType":"kernelVersion"},{"sourceId":210977729,"sourceType":"kernelVersion"}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport os, gc\nfrom tqdm.auto import tqdm\nfrom matplotlib import pyplot as plt\nimport pickle\n\nfrom sklearn.metrics import r2_score\nfrom xgboost import XGBRegressor\nimport xgboost as xgb\n\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:09:21.653539Z","iopub.execute_input":"2024-12-03T11:09:21.653942Z","iopub.status.idle":"2024-12-03T11:09:23.492944Z","shell.execute_reply.started":"2024-12-03T11:09:21.653908Z","shell.execute_reply":"2024-12-03T11:09:23.492032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CONFIG:\n    seed = 42\n    target_col = \"responder_6\"\n    feature_cols = [f\"feature_{idx:02d}\" for idx in range(73)] \\\n        + ['feature_73_74', 'feature_75_76', 'feature_77_78']\\\n        + [f\"responder_{idx}_lag_1\" for idx in range(9)]\n    categorical_cols = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T10:06:39.706482Z","iopub.execute_input":"2024-12-03T10:06:39.707002Z","iopub.status.idle":"2024-12-03T10:06:39.712968Z","shell.execute_reply.started":"2024-12-03T10:06:39.706965Z","shell.execute_reply":"2024-12-03T10:06:39.711803Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nprint(os.listdir(\"../input/jane-street-training\"))\nprint(os.listdir(\"../input/jane-street-preprocessing\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T10:06:40.450619Z","iopub.execute_input":"2024-12-03T10:06:40.451028Z","iopub.status.idle":"2024-12-03T10:06:40.465885Z","shell.execute_reply.started":"2024-12-03T10:06:40.450990Z","shell.execute_reply":"2024-12-03T10:06:40.464877Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = xgb.Booster({'nthread': 4})  # init model\nmodel.load_model('../input/jane-street-training/xgboost_model.json')  # load data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T10:06:40.994866Z","iopub.execute_input":"2024-12-03T10:06:40.995262Z","iopub.status.idle":"2024-12-03T10:06:41.039672Z","shell.execute_reply.started":"2024-12-03T10:06:40.995226Z","shell.execute_reply":"2024-12-03T10:06:41.038774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_08_fill_dict = pickle.load(open(\"../input/jane-street-preprocessing/feature08_fill_dict.pkl\", \"rb\"))\n# null_fill_dict = {\n#     'feature_39': 0.7065854668617249,\n#      'feature_42': -0.7328284382820129,\n#      'feature_53': -0.6800054907798767,\n#      'feature_50': 0.6517677903175354,\n#      'feature_15': -0.3965674638748169,\n#      'feature_44': -0.8057575225830078,\n#      'feature_41': 0.8103188276290894,\n#      'feature_52': 0.8985169529914856,\n#      'feature_55': -0.8813456296920776\n# }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T10:06:42.116468Z","iopub.execute_input":"2024-12-03T10:06:42.116899Z","iopub.status.idle":"2024-12-03T10:06:42.125628Z","shell.execute_reply.started":"2024-12-03T10:06:42.116867Z","shell.execute_reply":"2024-12-03T10:06:42.124577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess(test: pl.DataFrame, feature_08_fill_dict):\n    print(\"test\")\n    # Combine Feature 73-74, 75-76, 77-78\n    test = test.with_columns(\n        feature_73_74= (pl.col('feature_73') + pl.col('feature_74')) * 0.5,\n        feature_75_76= (pl.col('feature_75') + pl.col('feature_76')) * 0.5,\n        feature_77_78= (pl.col('feature_77') + pl.col('feature_78')) * 0.5,\n    ).drop([f'feature_{i}' for i in range(73, 79)])\n\n    # Fill Null Feature_08 by time_id \n    test = test.with_columns(\n        pl.col(\"time_id\").map_elements(lambda x: feature_08_fill_dict.get(x, None)).alias(\"feature_08_null_fill\")\n    ).with_columns(\n        pl.when(pl.col(\"feature_08\").is_null())\n        .then(pl.col(\"feature_08_null_fill\"))\n        .otherwise(pl.col(\"feature_08\"))\n        .alias(\"feature_08\")\n    ).drop('feature_08_null_fill')\n\n    # Fill null as 0\n    test = test.fill_nan(0)\n    \n    # time_based_feat_cols = ['feature_39', 'feature_42', 'feature_53', 'feature_50', 'feature_15', 'feature_44', 'feature_41', 'feature_52', 'feature_55',]\n    # for col in time_based_feat_cols:\n    #     test = test.with_columns(\n    #         pl.when(pl.col(col).is_null()).then(null_fill_dict[col]).otherwise(pl.col(col)).alias(col)\n    #     )\n\n    # print(\"test\")\n    \n    return test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T11:09:17.525397Z","iopub.execute_input":"2024-12-03T11:09:17.525795Z","iopub.status.idle":"2024-12-03T11:09:17.818089Z","shell.execute_reply.started":"2024-12-03T11:09:17.525760Z","shell.execute_reply":"2024-12-03T11:09:17.816737Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags_ : pl.DataFrame | None = None\n\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    feature_08_fill_dict= pickle.load(open(\"../input/jane-street-preprocessing/feature08_fill_dict.pkl\", \"rb\"))\n    \n    global lags_\n    if lags is not None:\n        lags_ = lags\n\n    predictions = test.select(\n        'row_id',\n        pl.lit(0.0).alias('responder_6'),\n    )\n    symbol_ids = test.select('symbol_id').to_numpy()[:, 0]\n\n    if not lags is None:\n        lags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last() # pick up last record of previous date\n        test = test.join(lags, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\n    else:\n        test = test.with_columns(\n            ( pl.lit(0.0).alias(f'responder_{idx}_lag_1') for idx in range(9) )\n        )\n\n    test = preprocess(test, feature_08_fill_dict)\n    dtest = xgb.DMatrix(test.to_pandas()[CONFIG.feature_cols])\n\n    preds = model.predict(dtest)\n    print(f\"predict> preds.shape =\", preds.shape)\n    \n    predictions = \\\n    test.select('row_id').\\\n    with_columns(\n        pl.Series(\n            name   = 'responder_6', \n            values = np.clip(preds, a_min = -5, a_max = 5),\n            dtype  = pl.Float64,\n        )\n    )\n\n    # The predict function must return a DataFrame\n    assert isinstance(predictions, pl.DataFrame | pd.DataFrame)\n    # with columns 'row_id', 'responer_6'\n    assert list(predictions.columns) == ['row_id', 'responder_6']\n    # and as many rows as the test data.\n    assert len(predictions) == len(test)\n\n    return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T10:06:48.675752Z","iopub.execute_input":"2024-12-03T10:06:48.676159Z","iopub.status.idle":"2024-12-03T10:06:48.689513Z","shell.execute_reply.started":"2024-12-03T10:06:48.676124Z","shell.execute_reply":"2024-12-03T10:06:48.688274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T10:06:54.436476Z","iopub.execute_input":"2024-12-03T10:06:54.436876Z","iopub.status.idle":"2024-12-03T10:06:54.928912Z","shell.execute_reply.started":"2024-12-03T10:06:54.436844Z","shell.execute_reply":"2024-12-03T10:06:54.927831Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}