{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":213424,"sourceType":"modelInstanceVersion","modelInstanceId":181920,"modelId":204152}],"dockerImageVersionId":30822,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import r2_score\nimport xgboost as xgb\nimport polars as pl\nimport joblib\nimport kaggle_evaluation.jane_street_inference_server\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import MinMaxScaler","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-29T12:24:52.978152Z","iopub.execute_input":"2024-12-29T12:24:52.978526Z","iopub.status.idle":"2024-12-29T12:24:52.983687Z","shell.execute_reply.started":"2024-12-29T12:24:52.978499Z","shell.execute_reply":"2024-12-29T12:24:52.982546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Loading train data...\")\nTrainData = pd.read_parquet(\n    \"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet\", \n    engine=\"pyarrow\"\n)\n\nprint(\"Loading test data...\")\nTestData = pd.read_parquet(\n    \"/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet\",\n    engine=\"pyarrow\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T12:24:52.985148Z","iopub.execute_input":"2024-12-29T12:24:52.985471Z","iopub.status.idle":"2024-12-29T12:24:54.229239Z","shell.execute_reply.started":"2024-12-29T12:24:52.985414Z","shell.execute_reply":"2024-12-29T12:24:54.228265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in [\"feature_00\", \"feature_01\", \"feature_02\"]:\n    for w in [5, 10]:\n        TrainData[f'{col}_rolling_mean_{w}'] = TrainData[col].rolling(window=w).mean()\n        TrainData[f'{col}_rolling_std_{w}'] = TrainData[col].rolling(window=w).std()\n\nTrainData[\"feature_00_lag1\"] = TrainData[\"feature_00\"].shift(1)\n\nfor col in [\"feature_00\", \"feature_01\", \"feature_02\"]:\n    for w in [5, 10]:\n        TestData[f'{col}_rolling_mean_{w}'] = TestData[col].rolling(window=w).mean()\n        TestData[f'{col}_rolling_std_{w}'] = TestData[col].rolling(window=w).std()\n\nTestData[\"feature_00_lag1\"] = TestData[\"feature_00\"].shift(1)\n\nresponder_columns = [f\"responder_{i}\" for i in range(9)]  # 0..8\nexclude_cols = responder_columns + ['date_id', 'time_id', 'symbol_id', 'weight']\n\nfeature_columns = [c for c in TrainData.columns if c not in exclude_cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T12:24:54.231020Z","iopub.execute_input":"2024-12-29T12:24:54.231295Z","iopub.status.idle":"2024-12-29T12:24:54.679887Z","shell.execute_reply.started":"2024-12-29T12:24:54.231273Z","shell.execute_reply":"2024-12-29T12:24:54.679076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MODEL_PATH = \"/kaggle/input/forcaster/other/default/1\"\n\nmodel_xgb = xgb.Booster()\nmodel_xgb.load_model(os.path.join(MODEL_PATH, \"model-2.json\"))\n\nimputer = joblib.load(os.path.join(MODEL_PATH, \"imputer.joblib\"))\nscaler  = joblib.load(os.path.join(MODEL_PATH, \"scaler.joblib\"))\n\nprint(\"Model, imputer, and scaler loaded successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T12:24:54.680958Z","iopub.execute_input":"2024-12-29T12:24:54.681199Z","iopub.status.idle":"2024-12-29T12:24:57.479909Z","shell.execute_reply.started":"2024-12-29T12:24:54.681179Z","shell.execute_reply":"2024-12-29T12:24:57.478727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags_: pl.DataFrame | None = None\n\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction on the incoming test batch.\"\"\"\n    global lags_\n    \n    # Falls lags übergeben werden (z.B. Tageswechsel time_id=0), speichern wir sie global\n    if lags is not None:\n        lags_ = lags\n    \n    # Polars -> Pandas (falls nötig, damit wir problemlos transform() anwenden)\n    test_pd = test.to_pandas()\n    \n    # Sicherstellen, dass alle Feature-Spalten vorhanden sind\n    for col in feature_columns:\n        if col not in test_pd.columns:\n            test_pd[col] = np.nan\n\n    # Features extrahieren\n    X_test = test_pd[feature_columns].copy()\n    \n    # Transformieren mit dem (bereits gefitteten!) Imputer und Scaler\n    X_test_imputed = imputer.transform(X_test)\n    X_test_scaled  = scaler.transform(X_test_imputed)\n\n    # XGBoost-Prediction\n    dtest = xgb.DMatrix(X_test_scaled)\n    y_pred = model_xgb.predict(dtest)\n\n    # Ausgabe als Polars DataFrame (erlaubt ist auch ein Pandas DataFrame)\n    predictions = pl.DataFrame(\n        {\n            \"row_id\": test_pd[\"row_id\"].values,\n            \"responder_6\": y_pred,\n        }\n    )\n    \n    # Validierung, dass die Spalten stimmen\n    assert predictions.columns == [\"row_id\", \"responder_6\"]\n    assert len(predictions) == len(test)\n    \n    return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T12:24:57.481016Z","iopub.execute_input":"2024-12-29T12:24:57.481305Z","iopub.status.idle":"2024-12-29T12:24:57.488721Z","shell.execute_reply.started":"2024-12-29T12:24:57.481273Z","shell.execute_reply":"2024-12-29T12:24:57.487335Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T12:24:57.489883Z","iopub.execute_input":"2024-12-29T12:24:57.490302Z","iopub.status.idle":"2024-12-29T12:24:57.566023Z","shell.execute_reply.started":"2024-12-29T12:24:57.490265Z","shell.execute_reply":"2024-12-29T12:24:57.565123Z"}},"outputs":[],"execution_count":null}]}