{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc , os\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:28:59.528670Z","iopub.execute_input":"2024-12-11T14:28:59.529707Z","iopub.status.idle":"2024-12-11T14:29:00.672032Z","shell.execute_reply.started":"2024-12-11T14:28:59.529659Z","shell.execute_reply":"2024-12-11T14:29:00.671108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# Path to the directory containing Parquet files\npath = \"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n\n# Output file to store the combined data\noutput_file = \"processed_train_subset.parquet\"\n\n# List all files in the directory\ndirs = os.listdir(path)\n\n# Initialize an empty DataFrame to store the combined data\ncombined_data = pd.DataFrame()\n\nfor file in dirs:\n    file_path = os.path.join(path, file)\n    file_path = file_path + '/' + 'part-0.parquet'\n    print(file_path)\n    subset = pd.read_parquet(file_path)\n        \n        # Take 50% random sample of the data\n    sampled_subset = subset.sample(frac=0.45 , random_state=42)  # Adjust `frac` if needed\n        \n        # Concatenate the sampled data into the combined DataFrame\n    combined_data = pd.concat([combined_data, sampled_subset], ignore_index=True)\n\n# Save the combined data to a single Parquet file\ndel subset\ngc.collect()\ncombined_data.to_parquet(output_file, index=False, engine=\"pyarrow\")\n\nprint(f\"Processed data saved to: {output_file}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:29:00.673345Z","iopub.execute_input":"2024-12-11T14:29:00.673741Z","iopub.status.idle":"2024-12-11T14:32:26.053448Z","shell.execute_reply.started":"2024-12-11T14:29:00.673714Z","shell.execute_reply":"2024-12-11T14:32:26.052500Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# combined_data = pd.read_parquet(\"/kaggle/working/processed_train_subset.parquet\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T18:07:17.956851Z","iopub.execute_input":"2024-12-10T18:07:17.957256Z","iopub.status.idle":"2024-12-10T18:07:17.962074Z","shell.execute_reply.started":"2024-12-10T18:07:17.957207Z","shell.execute_reply":"2024-12-10T18:07:17.961097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"combined_data.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:32:26.055130Z","iopub.execute_input":"2024-12-11T14:32:26.055385Z","iopub.status.idle":"2024-12-11T14:32:26.061680Z","shell.execute_reply.started":"2024-12-11T14:32:26.055360Z","shell.execute_reply":"2024-12-11T14:32:26.060813Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"combined_data.dropna(inplace = True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:32:26.062665Z","iopub.execute_input":"2024-12-11T14:32:26.063005Z","iopub.status.idle":"2024-12-11T14:32:37.146755Z","shell.execute_reply.started":"2024-12-11T14:32:26.062978Z","shell.execute_reply":"2024-12-11T14:32:37.145789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:32:37.148568Z","iopub.execute_input":"2024-12-11T14:32:37.148842Z","iopub.status.idle":"2024-12-11T14:32:37.203920Z","shell.execute_reply.started":"2024-12-11T14:32:37.148816Z","shell.execute_reply":"2024-12-11T14:32:37.203006Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## creating dataset for testing ##\npath = \"/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet\"\ndirs = os.listdir(path)\n\n\nfor file in dirs:\n    df_test = pd.read_parquet(os.path.join(path, file))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:32:37.205098Z","iopub.execute_input":"2024-12-11T14:32:37.205442Z","iopub.status.idle":"2024-12-11T14:32:37.236518Z","shell.execute_reply.started":"2024-12-11T14:32:37.205404Z","shell.execute_reply":"2024-12-11T14:32:37.235915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:32:37.237462Z","iopub.execute_input":"2024-12-11T14:32:37.237732Z","iopub.status.idle":"2024-12-11T14:32:37.266364Z","shell.execute_reply.started":"2024-12-11T14:32:37.237707Z","shell.execute_reply":"2024-12-11T14:32:37.265424Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(combined_data.columns.difference(df_test.columns))\nprint()\nprint(df_test.columns.difference(combined_data.columns))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:32:37.267395Z","iopub.execute_input":"2024-12-11T14:32:37.267680Z","iopub.status.idle":"2024-12-11T14:32:37.275301Z","shell.execute_reply.started":"2024-12-11T14:32:37.267654Z","shell.execute_reply":"2024-12-11T14:32:37.274400Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## dropping columns for training and inference\ncombined_data.drop(['responder_0', 'responder_1', 'responder_2', 'responder_3',\n       'responder_4', 'responder_5' , 'responder_7',\n       'responder_8' ] , axis =1 , inplace = True )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:32:37.276387Z","iopub.execute_input":"2024-12-11T14:32:37.276722Z","iopub.status.idle":"2024-12-11T14:32:39.259559Z","shell.execute_reply.started":"2024-12-11T14:32:37.276683Z","shell.execute_reply":"2024-12-11T14:32:39.258601Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## usefuls cols for x and y ##\nx_cols =  combined_data.columns.difference(['responder_6'])\ny = 'responder_6'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:32:39.260924Z","iopub.execute_input":"2024-12-11T14:32:39.261292Z","iopub.status.idle":"2024-12-11T14:32:39.266111Z","shell.execute_reply.started":"2024-12-11T14:32:39.261254Z","shell.execute_reply":"2024-12-11T14:32:39.265254Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:32:39.269025Z","iopub.execute_input":"2024-12-11T14:32:39.269295Z","iopub.status.idle":"2024-12-11T14:32:39.328713Z","shell.execute_reply.started":"2024-12-11T14:32:39.269270Z","shell.execute_reply":"2024-12-11T14:32:39.327844Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Data split for train and test ##\nfrom sklearn.model_selection import train_test_split\n\ntrainx , testx , trainy , testy  = train_test_split(combined_data[x_cols] ,combined_data[y] , test_size = 0.25 ,random_state= 0)\nprint(trainx.shape)\nprint(trainy.shape)\nprint(testx.shape)\nprint(testy.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:32:39.329690Z","iopub.execute_input":"2024-12-11T14:32:39.329938Z","iopub.status.idle":"2024-12-11T14:33:02.443100Z","shell.execute_reply.started":"2024-12-11T14:32:39.329913Z","shell.execute_reply":"2024-12-11T14:33:02.442198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:33:02.444469Z","iopub.execute_input":"2024-12-11T14:33:02.445315Z","iopub.status.idle":"2024-12-11T14:33:02.522416Z","shell.execute_reply.started":"2024-12-11T14:33:02.445274Z","shell.execute_reply":"2024-12-11T14:33:02.521459Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def weighted_zero_mean_r2(y_true, y_pred, weights):\n#     \"\"\"\n#     Calculate the sample weighted zero-mean R-squared score.\n\n#     Parameters:\n#     y_true (numpy.ndarray): Ground-truth values for responder_6.\n#     y_pred (numpy.ndarray): Predicted values for responder_6.\n#     weights (numpy.ndarray): Sample weight vector.\n\n#     Returns:\n#     float: The weighted zero-mean R-squared score.\n#     \"\"\"\n#     numerator = np.sum(weights * (y_true - y_pred)**2)\n#     denominator = np.sum(weights * y_true**2)\n    \n#     r2_score = 1 - numerator / denominator\n#     return r2_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T18:07:56.053036Z","iopub.execute_input":"2024-12-10T18:07:56.053303Z","iopub.status.idle":"2024-12-10T18:07:56.064274Z","shell.execute_reply.started":"2024-12-10T18:07:56.053277Z","shell.execute_reply":"2024-12-10T18:07:56.063371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import optuna\n# from xgboost import XGBRegressor\n# from sklearn.metrics import mean_squared_error\n# import numpy as np\n\n\n# # objective function for Optuna\n# def objective(trial):\n#     # Hyperparameter space to search\n#     n_estimators = trial.suggest_int(\"n_estimators\", 10, 600)\n#     max_depth = trial.suggest_int(\"max_depth\", 3, 20)\n#     learning_rate = trial.suggest_float(\"learning_rate\", 0.01, 0.3)\n#     subsample = trial.suggest_float(\"subsample\", 0.5, 1.0)\n#     colsample_bytree = trial.suggest_float(\"colsample_bytree\", 0.5, 1.0)\n#     min_child_weight = trial.suggest_int(\"min_child_weight\", 1, 10)\n#     gamma = trial.suggest_float(\"gamma\", 0, 10)\n\n#     # Create and train the model with GPU support\n#     model = XGBRegressor(\n#         n_estimators=n_estimators,\n#         max_depth=max_depth,\n#         learning_rate=learning_rate,\n#         subsample=subsample,\n#         colsample_bytree=colsample_bytree,\n#         min_child_weight=min_child_weight,\n#         gamma=gamma,\n#         random_state=42,\n#         tree_method=\"gpu_hist\"  # Enables GPU acceleration\n#     )\n\n#     model.fit(trainx, trainy, verbose=2)\n\n#     # Predict and evaluate\n#     preds = model.predict(testx)\n#     weights2 = testx.weight[0:len(preds)]  # Assuming weights are defined in your dataset\n#     r2_score = weighted_zero_mean_r2(testy, preds, weights2)  # Assuming this function is defined\n#     return -r2_score  # Maximize R2, so minimize negative R2\n\n# # Step 2: Create and run the Optuna study\n# study = optuna.create_study(direction=\"minimize\")\n# study.optimize(objective, n_trials=100, timeout=5600)\n\n# # Step 3: Display the best hyperparameters\n# print(\"Best hyperparameters:\", study.best_params)\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T18:07:56.065427Z","iopub.execute_input":"2024-12-10T18:07:56.065794Z","iopub.status.idle":"2024-12-10T18:07:56.077482Z","shell.execute_reply.started":"2024-12-10T18:07:56.065755Z","shell.execute_reply":"2024-12-10T18:07:56.076565Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## parameters obtained from optuna ##\nbest_params  = {'n_estimators': 546, 'max_depth': 19, 'learning_rate': 0.17956591017191098, \n                'subsample': 0.9584084805509517, 'colsample_bytree': 0.9541469016621238, \n                'min_child_weight': 10, 'gamma': 0.044313355939343324}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:33:02.523616Z","iopub.execute_input":"2024-12-11T14:33:02.523886Z","iopub.status.idle":"2024-12-11T14:33:02.535334Z","shell.execute_reply.started":"2024-12-11T14:33:02.523862Z","shell.execute_reply":"2024-12-11T14:33:02.534444Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"### model training ##\nfrom xgboost import XGBRegressor, callback\nfrom sklearn.metrics import mean_squared_error\n\n# Initialize the model with GPU support\nxgb_model = XGBRegressor(\n    **best_params, \n    n_jobs=-1, \n    tree_method=\"gpu_hist\",  # Ensures GPU usage\n    verbose=-1, \n   \n)\n\n\n\n# Train the model with callbacks\nxgb_model.fit(\n    trainx, trainy\n\n)\n\n# Evaluate on validation set\nfinal_preds = xgb_model.predict(testx)\nfinal_rmse = mean_squared_error(testy, final_preds, squared=False)\nprint(f\"Final RMSE: {final_rmse}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:34:08.973145Z","iopub.execute_input":"2024-12-11T14:34:08.973492Z","iopub.status.idle":"2024-12-11T14:46:20.798364Z","shell.execute_reply.started":"2024-12-11T14:34:08.973462Z","shell.execute_reply":"2024-12-11T14:46:20.797302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_columns = trainx.columns\ndf_columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:46:20.799920Z","iopub.execute_input":"2024-12-11T14:46:20.800217Z","iopub.status.idle":"2024-12-11T14:46:20.806572Z","shell.execute_reply.started":"2024-12-11T14:46:20.800190Z","shell.execute_reply":"2024-12-11T14:46:20.805613Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Inferencing on Test data for submission ##\n\nresult  = xgb_model.predict(df_test[df_columns])\nsubmission  = pd.read_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet\")\nsubmission  = pd.DataFrame({'row_id' :submission['row_id'] , 'responder_6' : result })\nsubmission.to_csv('submission.csv' , index = False)\nsubmission.to_parquet('submission.parquet', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:46:20.807511Z","iopub.execute_input":"2024-12-11T14:46:20.807826Z","iopub.status.idle":"2024-12-11T14:46:21.505732Z","shell.execute_reply.started":"2024-12-11T14:46:20.807800Z","shell.execute_reply":"2024-12-11T14:46:21.505004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## submission for api ##\nimport os\n\nimport pandas as pd\nimport polars as pl\n\nimport kaggle_evaluation.jane_street_inference_server\n\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pd.DataFrame:\n    global lags_ , cols\n    \n    test_pd = test.to_pandas()\n    \n    if lags is not None:\n        lags_ = lags\n\n        # Convert lags to Pandas DataFrame if it's Polars\n        lags_pd = lags.to_pandas()\n        lags_pd = lags_pd.groupby(['date_id', 'symbol_id']).last().reset_index() \n        test_pd = test_pd.merge(lags_pd, on=['date_id', 'symbol_id'], how='left')\n    else:\n        \n        for idx in range(9):\n            test_pd[f'responder_{idx}_lag_1'] = 0.0\n    print(test_pd.head())\n    cols = test_pd.columns\n    test_pd.rename(columns={'time_id_x': 'time_id'}, inplace=True)  \n\n \n    # Prepare the features for prediction\n    # test_pd['time_id'] = np.nan\n    \n    predictions =  xgb_model.predict(test_pd[df_columns])\n    print(predictions)\n\n    # Create a DataFrame for the output\n    output = pd.DataFrame({\n        'row_id': test_pd['row_id'],\n        'responder_6': predictions\n    })\n\n    # Ensure the output DataFrame has the correct format\n    assert output.columns.tolist() == ['row_id', 'responder_6']\n    assert len(output) == len(test)\n\n    return output\n\nimport kaggle_evaluation.jane_street_inference_server\nimport os\n\n# Set up the inference server\ninference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T14:46:21.507286Z","iopub.execute_input":"2024-12-11T14:46:21.507575Z","iopub.status.idle":"2024-12-11T14:46:23.239178Z","shell.execute_reply.started":"2024-12-11T14:46:21.507517Z","shell.execute_reply":"2024-12-11T14:46:23.238208Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}