{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport joblib\nimport polars as pl\nimport xgboost as xgb\nimport numpy as np\nimport pandas as pd\nimport kaggle_evaluation.jane_street_inference_server\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-08T00:03:43.738947Z","iopub.execute_input":"2024-11-08T00:03:43.739783Z","iopub.status.idle":"2024-11-08T00:03:45.511807Z","shell.execute_reply.started":"2024-11-08T00:03:43.739729Z","shell.execute_reply":"2024-11-08T00:03:45.510598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Paths and constants\ninput_path = '/kaggle/input/jane-street-real-time-market-data-forecasting'\ndef read_selected_data(input_path):\n    # Define the directory containing your data files\n\n    # List three specific Parquet files you want to read\n    selected_files = [f\"partition_id={i}/part-0.parquet\" for i in range(1)]\n    # Load and filter the data from only the selected Parquet files\n    dfs = []\n    for file_name in selected_files:\n        file_path = f'{input_path}/train.parquet/{file_name}'\n        lazy_df = pl.scan_parquet(file_path)\n        df = lazy_df.collect()\n        dfs.append(df)\n\n    # Concatenate all dataframes into a single dataframe\n    full_df = pl.concat(dfs)\n\n    return full_df","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:03:45.513868Z","iopub.execute_input":"2024-11-08T00:03:45.514506Z","iopub.status.idle":"2024-11-08T00:03:45.521221Z","shell.execute_reply.started":"2024-11-08T00:03:45.514458Z","shell.execute_reply":"2024-11-08T00:03:45.520101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = read_selected_data(input_path)\ndf = df.fill_null(strategy='forward')\n\n# Prepare feature names\nfeature_names = [f\"feature_{i:02d}\" for i in range(79)]\n\n# Prepare training and validation data\nnum_valid_dates = 100\ndates = df['date_id'].unique().to_numpy()\nvalid_dates = dates[-num_valid_dates:]\ntrain_dates = dates[:-num_valid_dates]\n","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:03:45.522524Z","iopub.execute_input":"2024-11-08T00:03:45.522911Z","iopub.status.idle":"2024-11-08T00:03:47.905390Z","shell.execute_reply.started":"2024-11-08T00:03:45.522877Z","shell.execute_reply":"2024-11-08T00:03:47.904291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extract features, target, and weights for validation and training sets\nX_valid = df.filter(pl.col('date_id').is_in(valid_dates)).select(feature_names).to_numpy()\ny_valid = df.filter(pl.col('date_id').is_in(valid_dates)).select('responder_6').to_numpy().ravel()\nw_valid = df.filter(pl.col('date_id').is_in(valid_dates)).select('weight').to_numpy().ravel()\n\nX_train = df.filter(pl.col('date_id').is_in(train_dates)).select(feature_names).to_numpy()\ny_train = df.filter(pl.col('date_id').is_in(train_dates)).select('responder_6').to_numpy().ravel()\nw_train = df.filter(pl.col('date_id').is_in(train_dates)).select('weight').to_numpy().ravel()","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:03:47.907683Z","iopub.execute_input":"2024-11-08T00:03:47.908067Z","iopub.status.idle":"2024-11-08T00:03:49.407423Z","shell.execute_reply.started":"2024-11-08T00:03:47.908016Z","shell.execute_reply":"2024-11-08T00:03:49.406236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define Customized Evaluation Method\nwhich is R2 that specified by Jane Street","metadata":{}},{"cell_type":"code","source":"def r2_xgb(y_true, y_pred, sample_weight=None):\n    if sample_weight is None:\n        sample_weight = np.ones_like(y_true)\n    r2 = 1 - np.average((y_pred - y_true) ** 2, weights=sample_weight) / (np.average((y_true) ** 2, weights=sample_weight) + 1e-38)\n    return -r2\n","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:03:49.408745Z","iopub.execute_input":"2024-11-08T00:03:49.409120Z","iopub.status.idle":"2024-11-08T00:03:49.415185Z","shell.execute_reply.started":"2024-11-08T00:03:49.409083Z","shell.execute_reply":"2024-11-08T00:03:49.414044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train Models","metadata":{}},{"cell_type":"code","source":"# Train the XGBoost model\nmodel = xgb.XGBRegressor(\n    n_estimators=2000,\n    learning_rate=0.1,\n    max_depth=6,\n    tree_method='hist',\n#     device=\"cuda\",\n    objective='reg:squarederror',\n    eval_metric=r2_xgb,\n    disable_default_eval_metric=True,\n    early_stopping_rounds=2\n)\n","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:03:49.416416Z","iopub.execute_input":"2024-11-08T00:03:49.416754Z","iopub.status.idle":"2024-11-08T00:03:49.430463Z","shell.execute_reply.started":"2024-11-08T00:03:49.416722Z","shell.execute_reply":"2024-11-08T00:03:49.429309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(\n    X_train, y_train,\n    sample_weight=w_train,\n    eval_set=[(X_valid, y_valid)],\n    sample_weight_eval_set=[w_valid],\n    verbose=2)","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:03:49.432120Z","iopub.execute_input":"2024-11-08T00:03:49.432561Z","iopub.status.idle":"2024-11-08T00:04:18.916465Z","shell.execute_reply.started":"2024-11-08T00:03:49.432511Z","shell.execute_reply":"2024-11-08T00:04:18.915354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save Model to Output","metadata":{}},{"cell_type":"code","source":"if not os.path.exists(\"./model_save\"):\n    # Create the directory if it does not exist\n    os.mkdir(\"./model_save\")\nmodel.save_model('./model_save/xgboost_model_baseline.json')","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:04:18.917702Z","iopub.execute_input":"2024-11-08T00:04:18.918041Z","iopub.status.idle":"2024-11-08T00:04:18.931861Z","shell.execute_reply.started":"2024-11-08T00:04:18.918007Z","shell.execute_reply":"2024-11-08T00:04:18.931032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load Models","metadata":{}},{"cell_type":"code","source":"model_loaded = xgb.XGBRegressor()\nmodel_loaded.load_model('/kaggle/working/model_save/xgboost_model_baseline.json')","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:04:18.933104Z","iopub.execute_input":"2024-11-08T00:04:18.933435Z","iopub.status.idle":"2024-11-08T00:04:18.948068Z","shell.execute_reply.started":"2024-11-08T00:04:18.933401Z","shell.execute_reply":"2024-11-08T00:04:18.947268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load Test Dataset","metadata":{}},{"cell_type":"code","source":"test = pl.scan_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet\")\ntest = test.collect()\ntest = test.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:04:18.951895Z","iopub.execute_input":"2024-11-08T00:04:18.952374Z","iopub.status.idle":"2024-11-08T00:04:18.995850Z","shell.execute_reply.started":"2024-11-08T00:04:18.952338Z","shell.execute_reply":"2024-11-08T00:04:18.994867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:04:18.997142Z","iopub.execute_input":"2024-11-08T00:04:18.997477Z","iopub.status.idle":"2024-11-08T00:04:19.036429Z","shell.execute_reply.started":"2024-11-08T00:04:18.997444Z","shell.execute_reply":"2024-11-08T00:04:19.035405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test[feature_names].values","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:04:19.037530Z","iopub.execute_input":"2024-11-08T00:04:19.037854Z","iopub.status.idle":"2024-11-08T00:04:19.044391Z","shell.execute_reply.started":"2024-11-08T00:04:19.037819Z","shell.execute_reply":"2024-11-08T00:04:19.043349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model_loaded.predict(test)","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:04:19.045655Z","iopub.execute_input":"2024-11-08T00:04:19.046072Z","iopub.status.idle":"2024-11-08T00:04:19.058406Z","shell.execute_reply.started":"2024-11-08T00:04:19.046007Z","shell.execute_reply":"2024-11-08T00:04:19.057391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:04:19.060143Z","iopub.execute_input":"2024-11-08T00:04:19.060879Z","iopub.status.idle":"2024-11-08T00:04:19.070587Z","shell.execute_reply.started":"2024-11-08T00:04:19.060838Z","shell.execute_reply":"2024-11-08T00:04:19.069576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## debug the submission","metadata":{}},{"cell_type":"code","source":"test = pl.scan_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet\")\ntest = test.collect()\ntest = test.to_pandas()\n\ntest_df = test[feature_names].values\npredictions = model_loaded.predict(test_df)\n\noutput_df = pd.DataFrame({\"row_id\": test['row_id'], \"responder_6\": predictions})","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:04:19.071807Z","iopub.execute_input":"2024-11-08T00:04:19.072220Z","iopub.status.idle":"2024-11-08T00:04:19.094278Z","shell.execute_reply.started":"2024-11-08T00:04:19.072183Z","shell.execute_reply":"2024-11-08T00:04:19.093313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:04:19.095394Z","iopub.execute_input":"2024-11-08T00:04:19.095690Z","iopub.status.idle":"2024-11-08T00:04:19.105407Z","shell.execute_reply.started":"2024-11-08T00:04:19.095660Z","shell.execute_reply":"2024-11-08T00:04:19.104135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Submission API","metadata":{}},{"cell_type":"code","source":"# Global lags storage\nlags_: pl.DataFrame | None = None\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame:\n    global lags_, model_loaded # Declare models as global\n    \n    # Logic for saving or loading lags\n    if lags is not None:\n        lags_ = lags\n    \n    test = test.to_pandas()\n    test_df = test[feature_names].values\n    predictions = model_loaded.predict(test_df)\n\n    output_df = pd.DataFrame({\"row_id\": test['row_id'], \"responder_6\": predictions})\n\n        \n    return pl.from_pandas(output_df)","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:04:19.106941Z","iopub.execute_input":"2024-11-08T00:04:19.107669Z","iopub.status.idle":"2024-11-08T00:04:19.115283Z","shell.execute_reply.started":"2024-11-08T00:04:19.107615Z","shell.execute_reply":"2024-11-08T00:04:19.113970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Setup the inference server\ninference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\n# Running the inference server\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway((\n        '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n        '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n    ))","metadata":{"execution":{"iopub.status.busy":"2024-11-08T00:04:19.117082Z","iopub.execute_input":"2024-11-08T00:04:19.118069Z","iopub.status.idle":"2024-11-08T00:04:19.254634Z","shell.execute_reply.started":"2024-11-08T00:04:19.118002Z","shell.execute_reply":"2024-11-08T00:04:19.253573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}