{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport joblib\nimport polars as pl\nimport xgboost as xgb\nimport numpy as np\nimport pandas as pd\nimport kaggle_evaluation.jane_street_inference_server\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-12-18T06:40:54.635601Z","iopub.execute_input":"2024-12-18T06:40:54.636396Z","iopub.status.idle":"2024-12-18T06:40:54.640499Z","shell.execute_reply.started":"2024-12-18T06:40:54.636362Z","shell.execute_reply":"2024-12-18T06:40:54.639652Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Paths and constants\ninput_path = '/kaggle/input/jane-street-real-time-market-data-forecasting'\ndef read_selected_data(input_path):\n    # Define the directory containing your data files\n\n    # List three specific Parquet files you want to read\n    selected_files = [f\"partition_id={i}/part-0.parquet\" for i in range(1)]\n    # Load and filter the data from only the selected Parquet files\n    dfs = []\n    for file_name in selected_files:\n        file_path = f'{input_path}/train.parquet/{file_name}'\n        lazy_df = pl.scan_parquet(file_path)\n        df = lazy_df.collect()\n        dfs.append(df)\n\n    # Concatenate all dataframes into a single dataframe\n    full_df = pl.concat(dfs)\n\n    return full_df","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:40:54.642257Z","iopub.execute_input":"2024-12-18T06:40:54.643072Z","iopub.status.idle":"2024-12-18T06:40:54.652859Z","shell.execute_reply.started":"2024-12-18T06:40:54.643043Z","shell.execute_reply":"2024-12-18T06:40:54.652048Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = read_selected_data(input_path)\ndf = df.fill_null(strategy='forward')\n\n# Prepare feature names\nfeature_names = [f\"feature_{i:02d}\" for i in range(79)]\n\n# Prepare training and validation data\nnum_valid_dates = 100\ndates = df['date_id'].unique().to_numpy()\nvalid_dates = dates[-num_valid_dates:]\ntrain_dates = dates[:-num_valid_dates]\n","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:40:54.653872Z","iopub.execute_input":"2024-12-18T06:40:54.654097Z","iopub.status.idle":"2024-12-18T06:40:55.409769Z","shell.execute_reply.started":"2024-12-18T06:40:54.654074Z","shell.execute_reply":"2024-12-18T06:40:55.409058Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extract features, target, and weights for validation and training sets\nX_valid = df.filter(pl.col('date_id').is_in(valid_dates)).select(feature_names).to_numpy()\ny_valid = df.filter(pl.col('date_id').is_in(valid_dates)).select('responder_6').to_numpy().ravel()\nw_valid = df.filter(pl.col('date_id').is_in(valid_dates)).select('weight').to_numpy().ravel()\n\nX_train = df.filter(pl.col('date_id').is_in(train_dates)).select(feature_names).to_numpy()\ny_train = df.filter(pl.col('date_id').is_in(train_dates)).select('responder_6').to_numpy().ravel()\nw_train = df.filter(pl.col('date_id').is_in(train_dates)).select('weight').to_numpy().ravel()","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:40:55.410707Z","iopub.execute_input":"2024-12-18T06:40:55.410940Z","iopub.status.idle":"2024-12-18T06:40:56.480954Z","shell.execute_reply.started":"2024-12-18T06:40:55.410916Z","shell.execute_reply":"2024-12-18T06:40:56.480227Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Define Customized Evaluation Method\nwhich is R2 that specified by Jane Street","metadata":{}},{"cell_type":"code","source":"def r2_xgb(y_true, y_pred, sample_weight=None):\n    if sample_weight is None:\n        sample_weight = np.ones_like(y_true)\n    r2 = 1 - np.average((y_pred - y_true) ** 2, weights=sample_weight) / (np.average((y_true) ** 2, weights=sample_weight) + 1e-38)\n    return -r2\n","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:40:56.483089Z","iopub.execute_input":"2024-12-18T06:40:56.483371Z","iopub.status.idle":"2024-12-18T06:40:56.487911Z","shell.execute_reply.started":"2024-12-18T06:40:56.483343Z","shell.execute_reply":"2024-12-18T06:40:56.487124Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Train Models","metadata":{}},{"cell_type":"code","source":"# Train the XGBoost model\nmodel = xgb.XGBRegressor(\n    n_estimators=2000,\n    learning_rate=0.1,\n    max_depth=6,\n    tree_method='hist',\n    device=\"cuda\",\n    objective='reg:squarederror',\n    eval_metric=r2_xgb,\n    disable_default_eval_metric=True,\n    early_stopping_rounds=2\n)\n","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:40:56.489268Z","iopub.execute_input":"2024-12-18T06:40:56.490038Z","iopub.status.idle":"2024-12-18T06:40:56.498059Z","shell.execute_reply.started":"2024-12-18T06:40:56.489999Z","shell.execute_reply":"2024-12-18T06:40:56.497419Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.fit(\n    X_train, y_train,\n    sample_weight=w_train,\n    eval_set=[(X_valid, y_valid)],\n    sample_weight_eval_set=[w_valid],\n    verbose=2)","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:40:56.498945Z","iopub.execute_input":"2024-12-18T06:40:56.499174Z","iopub.status.idle":"2024-12-18T06:41:07.622882Z","shell.execute_reply.started":"2024-12-18T06:40:56.499151Z","shell.execute_reply":"2024-12-18T06:41:07.622036Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Save Model to Output","metadata":{}},{"cell_type":"code","source":"if not os.path.exists(\"./model_save\"):\n    # Create the directory if it does not exist\n    os.mkdir(\"./model_save\")\nmodel.save_model('./model_save/xgboost_model_baseline.json')","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:41:07.623915Z","iopub.execute_input":"2024-12-18T06:41:07.624150Z","iopub.status.idle":"2024-12-18T06:41:07.631823Z","shell.execute_reply.started":"2024-12-18T06:41:07.624127Z","shell.execute_reply":"2024-12-18T06:41:07.631109Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Load Models","metadata":{}},{"cell_type":"code","source":"model_loaded = xgb.XGBRegressor()\nmodel_loaded.load_model('/kaggle/working/model_save/xgboost_model_baseline.json')","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:41:07.632826Z","iopub.execute_input":"2024-12-18T06:41:07.633069Z","iopub.status.idle":"2024-12-18T06:41:07.644287Z","shell.execute_reply.started":"2024-12-18T06:41:07.633046Z","shell.execute_reply":"2024-12-18T06:41:07.643608Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Load Test Dataset","metadata":{}},{"cell_type":"code","source":"test = pl.scan_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet\")\ntest = test.collect()\ntest = test.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:41:07.645166Z","iopub.execute_input":"2024-12-18T06:41:07.645402Z","iopub.status.idle":"2024-12-18T06:41:07.655289Z","shell.execute_reply.started":"2024-12-18T06:41:07.645378Z","shell.execute_reply":"2024-12-18T06:41:07.654449Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:41:07.656317Z","iopub.execute_input":"2024-12-18T06:41:07.656658Z","iopub.status.idle":"2024-12-18T06:41:07.679153Z","shell.execute_reply.started":"2024-12-18T06:41:07.656622Z","shell.execute_reply":"2024-12-18T06:41:07.678458Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = test[feature_names].values","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:41:07.679948Z","iopub.execute_input":"2024-12-18T06:41:07.680147Z","iopub.status.idle":"2024-12-18T06:41:07.684642Z","shell.execute_reply.started":"2024-12-18T06:41:07.680126Z","shell.execute_reply":"2024-12-18T06:41:07.683853Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions = model_loaded.predict(test)","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:41:07.685764Z","iopub.execute_input":"2024-12-18T06:41:07.686517Z","iopub.status.idle":"2024-12-18T06:41:07.694582Z","shell.execute_reply.started":"2024-12-18T06:41:07.686480Z","shell.execute_reply":"2024-12-18T06:41:07.693724Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:41:07.697106Z","iopub.execute_input":"2024-12-18T06:41:07.697347Z","iopub.status.idle":"2024-12-18T06:41:07.704764Z","shell.execute_reply.started":"2024-12-18T06:41:07.697323Z","shell.execute_reply":"2024-12-18T06:41:07.703941Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## debug the submission","metadata":{}},{"cell_type":"code","source":"test = pl.scan_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet\")\ntest = test.collect()\ntest = test.to_pandas()\n\ntest_df = test[feature_names].values\npredictions = model_loaded.predict(test_df)\n\noutput_df = pd.DataFrame({\"row_id\": test['row_id'], \"responder_6\": predictions})","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:41:07.705746Z","iopub.execute_input":"2024-12-18T06:41:07.706015Z","iopub.status.idle":"2024-12-18T06:41:07.717963Z","shell.execute_reply.started":"2024-12-18T06:41:07.705987Z","shell.execute_reply":"2024-12-18T06:41:07.717070Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:41:07.718886Z","iopub.execute_input":"2024-12-18T06:41:07.719243Z","iopub.status.idle":"2024-12-18T06:41:07.727085Z","shell.execute_reply.started":"2024-12-18T06:41:07.719174Z","shell.execute_reply":"2024-12-18T06:41:07.726231Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Submission API","metadata":{}},{"cell_type":"code","source":"# Global lags storage\nlags_: pl.DataFrame | None = None\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame:\n    global lags_, model_loaded # Declare models as global\n    \n    # Logic for saving or loading lags\n    if lags is not None:\n        lags_ = lags\n    \n    test = test.to_pandas()\n    test_df = test[feature_names].values\n    predictions = model_loaded.predict(test_df)\n\n    output_df = pd.DataFrame({\"row_id\": test['row_id'], \"responder_6\": predictions})\n\n        \n    return pl.from_pandas(output_df)","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:41:07.728192Z","iopub.execute_input":"2024-12-18T06:41:07.728507Z","iopub.status.idle":"2024-12-18T06:41:07.737104Z","shell.execute_reply.started":"2024-12-18T06:41:07.728472Z","shell.execute_reply":"2024-12-18T06:41:07.736395Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Setup the inference server\ninference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\n# Running the inference server\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway((\n        '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n        '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n    ))","metadata":{"execution":{"iopub.status.busy":"2024-12-18T06:41:07.738068Z","iopub.execute_input":"2024-12-18T06:41:07.738634Z","iopub.status.idle":"2024-12-18T06:41:07.771306Z","shell.execute_reply.started":"2024-12-18T06:41:07.738598Z","shell.execute_reply":"2024-12-18T06:41:07.770696Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}