{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":10004078,"sourceType":"datasetVersion","datasetId":6158005},{"sourceId":10004352,"sourceType":"datasetVersion","datasetId":6158178},{"sourceId":10004448,"sourceType":"datasetVersion","datasetId":6158247},{"sourceId":177576,"sourceType":"modelInstanceVersion","modelInstanceId":151273,"modelId":173745},{"sourceId":177591,"sourceType":"modelInstanceVersion","modelInstanceId":151288,"modelId":173760},{"sourceId":177620,"sourceType":"modelInstanceVersion","modelInstanceId":151306,"modelId":173777}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport joblib\nimport pandas as pd\nimport numpy as np\nimport polars as pl\nimport xgboost as xgb\nimport sys\nimport kaggle_evaluation.jane_street_inference_server\n\nsys.path.append('/kaggle/input/kaggle-evaluation/')\n\n# Model file\nmodel_path = '/kaggle/input/jane-street-rtdf-submission-xgboost-2-json/scikitlearn/default/1/final_model.json'\n\n# Load the model\nmodel = xgb.Booster()\nmodel.load_model(model_path)\n\n# Load feature columns\nfeature_columns_path = '/kaggle/input/feature-columns-jane-street-rtdf-2/feature_columns.pkl'  # Update if stored in a different directory\nfeature_cols = pd.read_pickle(feature_columns_path)\n\nlags_ = None\n\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    global lags_\n    \n    # Handle lags data if provided\n    if lags is not None:\n        lags_ = lags\n    \n    # Convert Polars DataFrame to Pandas for compatibility with XGBoost\n    test_pd = test.to_pandas()\n    \n    # Select only the feature columns used during training\n    try:\n        X_test = test_pd[feature_cols]\n    except KeyError as e:\n        missing_cols = list(set(feature_cols) - set(test_pd.columns))\n        raise KeyError(f\"The following required feature columns are missing from the test data: {missing_cols}\") from e\n    \n    # Ensure the index is appropriate\n    X_test.reset_index(drop=True, inplace=True)\n    \n    # Handle missing values as done during training\n    X_test_imputed = X_test.copy()\n    for col in feature_cols:\n        if X_test_imputed[col].isna().any():\n            # Forward fill\n            X_test_imputed.loc[:, col] = X_test_imputed[col].ffill()\n            # Rolling mean with window size 800\n            X_test_imputed.loc[:, col] = (\n                X_test_imputed[col]\n                .rolling(window=800, min_periods=1)\n                .mean()\n            )\n            # If still NaNs, fill with overall mean\n            X_test_imputed.loc[:, col] = X_test_imputed[col].fillna(X_test_imputed[col].mean())\n    \n    # Ensure no remaining NaNs\n    X_test_imputed.fillna(0, inplace=True)\n    \n    # Replace infinite values\n    X_test_imputed.replace([np.inf, -np.inf], np.nan, inplace=True)\n    X_test_imputed.fillna(0, inplace=True)\n    \n    # Ensure all columns are of type float64\n    X_test_imputed = X_test_imputed.astype('float64')\n    \n    # Convert to DMatrix with feature names\n    dtest = xgb.DMatrix(X_test_imputed, feature_names=feature_cols)\n    \n    # Make predictions\n    predictions = model.predict(dtest, validate_features=True)\n    \n    # Prepare the output DataFrame\n    if 'row_id' not in test_pd.columns:\n        raise ValueError(\"The test DataFrame must contain a 'row_id' column.\")\n    \n    output_df = pd.DataFrame({\n        'row_id': test_pd['row_id'],\n        'responder_6': predictions\n    })\n    \n    # Convert back to Polars DataFrame\n    output_pl = pl.from_pandas(output_df)\n    \n    # Ensure the output has the correct columns\n    assert output_pl.columns == ['row_id', 'responder_6'], \"Output DataFrame must have 'row_id' and 'responder_6' columns.\"\n    \n    # Confirm the number of rows matches\n    assert len(output_pl) == len(test), \"Number of predictions does not match number of test samples.\"\n    \n    return output_pl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-25T02:59:14.386650Z","iopub.execute_input":"2024-11-25T02:59:14.387007Z","iopub.status.idle":"2024-11-25T02:59:14.398602Z","shell.execute_reply.started":"2024-11-25T02:59:14.386977Z","shell.execute_reply":"2024-11-25T02:59:14.397299Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-25T02:59:14.998623Z","iopub.execute_input":"2024-11-25T02:59:14.999050Z","iopub.status.idle":"2024-11-25T02:59:15.076715Z","shell.execute_reply.started":"2024-11-25T02:59:14.998993Z","shell.execute_reply":"2024-11-25T02:59:15.075784Z"}},"outputs":[],"execution_count":null}]}