{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Jane Street Real-Time Market Data Forecasting","metadata":{"_uuid":"752b07e4-a5e3-4b98-a187-a6f110e530c1","_cell_guid":"45ba5ef8-ddf9-43de-a854-a53094f30544","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"# Import packages","metadata":{"_uuid":"d2a3da04-8d67-4be6-bf25-636952058579","_cell_guid":"b3f2b1ff-c3f7-45aa-bc06-d1a489847d82","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"import pandas as pd, numpy as np, polars as pl\nimport os, joblib, xgboost as xgb, lightgbm as lgb, catboost as cb\nimport kaggle_evaluation.jane_street_inference_server","metadata":{"_uuid":"b10533d8-2ee8-4d1c-950c-7c048ae5da00","_cell_guid":"9f96c4e5-c390-4c20-b32d-406243d7fb5a","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T18:23:11.316342Z","iopub.execute_input":"2024-12-29T18:23:11.316781Z","iopub.status.idle":"2024-12-29T18:23:14.020264Z","shell.execute_reply.started":"2024-12-29T18:23:11.316744Z","shell.execute_reply":"2024-12-29T18:23:14.018850Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Import data","metadata":{"_uuid":"24b9435b-65ff-4205-a3c7-c5db9c9dfced","_cell_guid":"be2b9507-0924-4628-954c-f6aceeedfbab","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"str1 = '../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id='\nstr2 = '/part-0.parquet'\nfile_paths = [f\"{str1}{i}{str2}\" for i in range(10)]","metadata":{"_uuid":"776fc0a5-88ff-4b73-80ef-2d28b5628bef","_cell_guid":"0e25b420-2f65-48f1-9e6a-0a001ad984cd","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T18:23:14.022399Z","iopub.execute_input":"2024-12-29T18:23:14.023034Z","iopub.status.idle":"2024-12-29T18:23:14.028841Z","shell.execute_reply.started":"2024-12-29T18:23:14.022994Z","shell.execute_reply":"2024-12-29T18:23:14.027729Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dts = ['date_id', 'time_id', 'symbol_id']\nfeatures = [f'feature_{i:02}' for i in range(79)]\nresponders = [f'responder_{i:1}' for i in range(9)]","metadata":{"_uuid":"7d02ea9c-d5ce-447f-a3c3-a595b7f70120","_cell_guid":"929e2ba7-e84c-43b6-b862-ed08212a009f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T18:23:14.030326Z","iopub.execute_input":"2024-12-29T18:23:14.030696Z","iopub.status.idle":"2024-12-29T18:23:14.046290Z","shell.execute_reply.started":"2024-12-29T18:23:14.030661Z","shell.execute_reply":"2024-12-29T18:23:14.044770Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ndataframes = [pd.read_parquet(file)[features + dts + ['weight','responder_6']] for file in file_paths[-4:]]\ndf_train = pd.concat(dataframes)\n\ndel dataframes","metadata":{"_uuid":"c6bfe5b7-1edd-4a4b-a62b-0699d7f6129c","_cell_guid":"7794af85-863b-491a-98ea-5bd14ac9e4bd","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T18:23:14.049190Z","iopub.execute_input":"2024-12-29T18:23:14.049539Z","iopub.status.idle":"2024-12-29T18:24:33.332483Z","shell.execute_reply.started":"2024-12-29T18:23:14.049508Z","shell.execute_reply":"2024-12-29T18:24:33.330469Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = df_train[features]\nweights = df_train['weight']\ny = df_train['responder_6']\ndel df_train","metadata":{"_uuid":"ebd32fa7-c70a-418b-99ae-402c9686355b","_cell_guid":"04ff8bcc-22bd-45d0-ac7f-c4a4dc92f709","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T18:24:33.334695Z","iopub.execute_input":"2024-12-29T18:24:33.335160Z","iopub.status.idle":"2024-12-29T18:24:38.009142Z","shell.execute_reply.started":"2024-12-29T18:24:33.335112Z","shell.execute_reply":"2024-12-29T18:24:38.007974Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"memory_usage = X.memory_usage(deep=True).sum()\nprint(f\"Memory usage of X: {memory_usage / (1024 ** 2):.2f} MB\")","metadata":{"_uuid":"62103a54-01ee-4f78-94e4-6e5107ec4580","_cell_guid":"82176a1a-fddd-4ce7-bb8f-622f26c1a82b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T18:24:38.010682Z","iopub.execute_input":"2024-12-29T18:24:38.011043Z","iopub.status.idle":"2024-12-29T18:24:38.027741Z","shell.execute_reply.started":"2024-12-29T18:24:38.011011Z","shell.execute_reply":"2024-12-29T18:24:38.026536Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Shape of X:\", X.shape)\nprint(f\"Shape of y:\", y.shape)\nprint(f\"Shape of sample_weight vector:\", weights.shape)","metadata":{"_uuid":"27d2f97c-8631-4afa-9bee-9cc666eb2056","_cell_guid":"b7b0bff3-c86f-4b16-9eb6-c6ddc1925de5","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T18:24:38.029245Z","iopub.execute_input":"2024-12-29T18:24:38.029616Z","iopub.status.idle":"2024-12-29T18:24:38.046313Z","shell.execute_reply.started":"2024-12-29T18:24:38.029577Z","shell.execute_reply":"2024-12-29T18:24:38.045041Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train-Test Split","metadata":{"_uuid":"5091ecbb-6a84-45c0-af2d-c2212a79676e","_cell_guid":"ffddb159-0b41-4bc3-a086-6e31030e1a13","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"train_size = int(len(X) * 0.8)\n\n# Sequential split\nX_train = X[:train_size]\nX_val = X[train_size:]\ny_train = y[:train_size]\ny_val = y[train_size:]\nweights_train = weights[:train_size]\nweights_val = weights[train_size:]\n\nprint(f\"Train shapes: {X_train.shape}, {y_train.shape}, {weights_train.shape}\")\nprint(f\"Validation shapes: {X_val.shape}, {y_val.shape}, {weights_val.shape}\")","metadata":{"_uuid":"b107639f-fe29-4e71-ba67-758747c437cf","_cell_guid":"03d17917-1543-424d-9495-1aa9376c93f4","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T18:24:38.047792Z","iopub.execute_input":"2024-12-29T18:24:38.048218Z","iopub.status.idle":"2024-12-29T18:24:38.060421Z","shell.execute_reply.started":"2024-12-29T18:24:38.048186Z","shell.execute_reply":"2024-12-29T18:24:38.059275Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Models","metadata":{"_uuid":"00ac76b8-478e-4083-b316-84cfc0392860","_cell_guid":"0b9e57ca-674d-431c-acd6-f59c9c78e77c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Define a learning rate schedule\ndef learning_rate_scheduler_xgb(epoch):\n    initial_rate = 0.3\n    decay_rate = 0.999\n    return initial_rate * (decay_rate ** (np.log(epoch)))","metadata":{"_uuid":"581e316e-3c64-4448-814f-03494db2506f","_cell_guid":"79aa47a8-e35b-4e25-9383-0b4ae4731242","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T18:24:38.062119Z","iopub.execute_input":"2024-12-29T18:24:38.062548Z","iopub.status.idle":"2024-12-29T18:24:38.079093Z","shell.execute_reply.started":"2024-12-29T18:24:38.062502Z","shell.execute_reply":"2024-12-29T18:24:38.077547Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Define models","metadata":{"_uuid":"ff1910de-629c-40ee-99fd-81642b1d331a","_cell_guid":"b92d4c2a-903a-4550-8ee3-d5e49a6504c8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"from xgboost import XGBRegressor\nxgb_params = {\n    'max_depth': 5, \n    'min_child_weight': 5.826292396086343, \n    'colsample_bytree': 0.7771835631254393, \n    'reg_alpha': 4.084816116138748, \n    'reg_lambda': 1.1786086543434473, \n    'subsample': 0.6739210190464564, \n    'n_estimators': 706,\n    'device': 'cuda',\n    'tree_method' : 'hist',\n    'learning_rate':0.01}\nmodel_xgb = xgb.XGBRegressor(**xgb_params)","metadata":{"_uuid":"02e6e5e2-33ce-4bd8-a9d0-30b6bf71df38","_cell_guid":"23d61829-adb1-431f-b6aa-3d04e4585f1a","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T18:24:38.083210Z","iopub.execute_input":"2024-12-29T18:24:38.084597Z","iopub.status.idle":"2024-12-29T18:24:38.094800Z","shell.execute_reply.started":"2024-12-29T18:24:38.084543Z","shell.execute_reply":"2024-12-29T18:24:38.093731Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\nlgb_params = {\n    'max_depth': 11, \n    'num_leaves': 28, \n    'feature_fraction': 0.9845340232901648, \n    'bagging_fraction': 0.9751777653062266, \n    'bagging_freq': 4, \n    'reg_alpha': 1.9794504987659611, \n    'reg_lambda': 1.5074268152177714, \n    'n_estimators': 904,\n    'verbosity':-1,\n    'boosting_type': 'gbdt',\n    'objective': 'regression',\n    'metric': 'rmse',\n    'learning_rate':0.05}\nmodel_lgb = lgb.LGBMRegressor(**lgb_params)","metadata":{"_uuid":"26a579da-1840-4252-b124-5a8a5b1b519f","_cell_guid":"508ca7b6-c863-48cc-aa57-c1473422bb5f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T18:24:38.096341Z","iopub.execute_input":"2024-12-29T18:24:38.096692Z","iopub.status.idle":"2024-12-29T18:24:38.108075Z","shell.execute_reply.started":"2024-12-29T18:24:38.096647Z","shell.execute_reply":"2024-12-29T18:24:38.106870Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from catboost import CatBoostRegressor\ncat_params = {\n    'iterations': 971, \n    'depth': 9, \n    'random_strength': 3.7025175269170347, \n    'bagging_temperature': 0.07181853264492258, \n    'border_count': 209, \n    'l2_leaf_reg': 31.926710123760373,         \n    'eval_metric': 'RMSE',\n    'random_state': 0,\n    'verbose': 0,\n    'learning_rate':0.05}\nmodel_cat = cb.CatBoostRegressor(**cat_params)","metadata":{"_uuid":"8ee14be5-6680-41fb-897a-5e195a2174d1","_cell_guid":"d85cb3ea-c580-47b0-97ff-081e5f935027","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T18:24:38.109419Z","iopub.execute_input":"2024-12-29T18:24:38.109716Z","iopub.status.idle":"2024-12-29T18:24:38.130293Z","shell.execute_reply.started":"2024-12-29T18:24:38.109685Z","shell.execute_reply":"2024-12-29T18:24:38.128945Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Fit models","metadata":{"_uuid":"15203215-97bc-4d1d-b304-5e7cce42a686","_cell_guid":"a9f20e76-898b-4a61-bb62-094b9e8149ba","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"%%time\n\nmodel_xgb.fit(\n    X_train,\n    y_train,\n    eval_set=[(X_val, y_val)],\n    eval_metric='rmse',\n    early_stopping_rounds=10,\n    verbose=False\n)\n\nmodel_lgb.fit(\n    X_train,\n    y_train,\n    eval_set=[(X_val, y_val)]\n)\n\nmodel_cat.fit(\n    X_train,\n    y_train,\n    eval_set=[(X_val, y_val)],\n    early_stopping_rounds=50,\n    cat_features=None,\n    verbose=False\n)","metadata":{"_uuid":"8356b25b-19aa-4612-b030-4bdf99d8ea46","_cell_guid":"7479d1dd-7a3a-4bda-83cb-0104ac77c155","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-29T18:24:38.131978Z","iopub.execute_input":"2024-12-29T18:24:38.132350Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred_xgb = model_xgb.predict(X_val)\ny_pred_lgb = model_lgb.predict(X_val)\ny_pred_cat = model_cat.predict(X_val)","metadata":{"_uuid":"bda3339c-24fe-4f6f-b8bb-5cdf5bb03f24","_cell_guid":"423356a9-34be-4cc7-9e73-bb6ddee1d8a5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(y_pred_xgb)","metadata":{"_uuid":"0e308aa6-d491-4025-b1b1-3fd814713d7d","_cell_guid":"85b0dc81-aea3-4542-afad-74d113bad0ab","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = (y_pred_xgb + y_pred_lgb + y_pred_cat) / 3","metadata":{"_uuid":"84b8fb4d-0147-4798-bfcf-c0392f7451e7","_cell_guid":"56c698e2-6d4a-475a-9f76-ecae36b22f1d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Evaluate models","metadata":{"_uuid":"36e0f1a2-57c7-4c1d-b0b0-9784fc0e904f","_cell_guid":"6fc8a351-eef8-418a-bb4e-b93bc56f80ff","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error, r2_score\nmse = mean_squared_error(y_val, y_pred, squared=False)\nr2 = r2_score(y_val, y_pred)\nprint(f\"RMSE: {mse}\")\nprint(f\"R²: {r2}\")","metadata":{"_uuid":"fb0580da-0c10-4db2-bf55-ea8748de463e","_cell_guid":"806d74aa-0d69-41aa-bc85-a2cf3e0019be","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Save models","metadata":{"_uuid":"6ee89a84-7730-4e9e-b884-706f35c1f83e","_cell_guid":"cdc6ae51-994b-420c-a10c-788b9038f698","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"import joblib\n# Save the model\njoblib.dump(model_xgb, \"xgboost_sklearn.pkl\")\njoblib.dump(model_lgb, \"lightgbm_sklearn.pkl\")\njoblib.dump(model_cat, \"catboost_sklearn.pkl\")","metadata":{"_uuid":"a0b932b1-7b78-442e-b085-95f91506e3a8","_cell_guid":"bda84c51-c26d-4b51-8cf9-0fb929f4637a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del X_train\ndel y_train\ndel X_val\ndel y_val","metadata":{"_uuid":"bee3080f-68e9-49e1-ae66-6aa8f7ecd300","_cell_guid":"82835fba-17ea-4fd9-9516-6683c27a52ba","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission\n\nSee [Jane Street RMF Demo Submission](https://www.kaggle.com/code/ryanholbrook/jane-street-rmf-demo-submission) for details.\n\nDepending on the size of your training set, you will need an [inference notebook](https://www.kaggle.com/code/regisvargas/inference-jane-street-a-beginner-s-notebook).","metadata":{"_uuid":"15333c8d-f9cd-489a-8471-7f1eb29530bb","_cell_guid":"992eba4f-2bae-401e-a298-d6e7a1197bd2","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    global lags_\n    if lags is not None:\n        lags_ = lags\n    # Extract the features for the model input\n    feature_columns = [col for col in test.columns if col.startswith(\"feature_\")]\n    features = test.select(feature_columns).to_numpy()  # Convert to numpy array for model input\n    features = np.nan_to_num(features, nan=0.0, posinf=0.0, neginf=0.0)\n    # Generate predictions using the model\n    responder_6_predictions_xgb = model_xgb.predict(features)\n    responder_6_predictions_lgb = model_lgb.predict(features)\n    responder_6_predictions_cat = model_cat.predict(features)\n    responder_6_predictions = (responder_6_predictions_xgb + responder_6_predictions_lgb + responder_6_predictions_cat) / 3\n    responder_6_predictions = responder_6_predictions.astype(np.float32)\n    print(responder_6_predictions)    \n    # Create a new Polars DataFrame with row_id and responder_6 predictions\n    predictions = test.select(\"row_id\").with_columns(\n        pl.Series(\"responder_6\", responder_6_predictions)\n    )\n    print(predictions)\n    # Ensure the output format and length requirements\n    if isinstance(predictions, pl.DataFrame):\n        assert predictions.columns == ['row_id', 'responder_6']\n    elif isinstance(predictions, pd.DataFrame):\n        assert (predictions.columns == ['row_id', 'responder_6']).all()\n    else:\n        raise TypeError('The predict function must return a DataFrame')\n    \n    assert len(predictions) == len(test)\n    return predictions","metadata":{"_uuid":"8ee7c458-d124-4dc1-b045-26a039f708ba","_cell_guid":"a080965f-715f-4de4-87e4-1dcda31e8d1f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"_uuid":"f4cd3f4e-601b-47f1-99f4-1c7d4454d39c","_cell_guid":"0b97f5dc-a9a7-4394-b70f-da418b160b4b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"6c79710e-5f9e-4318-a39b-0c17964216c5","_cell_guid":"7f49a86e-ef82-41c8-a276-5f7b1b4274d3","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}