{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os , gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:40:42.120852Z","iopub.execute_input":"2024-12-06T09:40:42.121950Z","iopub.status.idle":"2024-12-06T09:40:42.479862Z","shell.execute_reply.started":"2024-12-06T09:40:42.121884Z","shell.execute_reply":"2024-12-06T09:40:42.478961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\n\n# Path to the directory containing Parquet files\npath = \"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n\n# Output file to store the combined data\noutput_file = \"processed_train_subset.parquet\"\n\n# List all files in the directory\ndirs = os.listdir(path)\n\n# Initialize an empty DataFrame to store the combined data\ncombined_data = pd.DataFrame()\n\nfor file in dirs:\n    file_path = os.path.join(path, file)\n    file_path = file_path + '/' + 'part-0.parquet'\n    print(file_path)\n    subset = pd.read_parquet(file_path)\n        \n        # Take 50% random sample of the data\n    sampled_subset = subset.sample(frac=0.45 , random_state=42)  # Adjust `frac` if needed\n        \n        # Concatenate the sampled data into the combined DataFrame\n    combined_data = pd.concat([combined_data, sampled_subset], ignore_index=True)\n\n# Save the combined data to a single Parquet file\ndel subset\ngc.collect()\ncombined_data.to_parquet(output_file, index=False, engine=\"pyarrow\")\n\nprint(f\"Processed data saved to: {output_file}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:40:42.481653Z","iopub.execute_input":"2024-12-06T09:40:42.482561Z","iopub.status.idle":"2024-12-06T09:44:27.616876Z","shell.execute_reply.started":"2024-12-06T09:40:42.482516Z","shell.execute_reply":"2024-12-06T09:44:27.615968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"combined_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:44:27.618098Z","iopub.execute_input":"2024-12-06T09:44:27.618393Z","iopub.status.idle":"2024-12-06T09:44:27.644939Z","shell.execute_reply.started":"2024-12-06T09:44:27.618363Z","shell.execute_reply":"2024-12-06T09:44:27.644110Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"combined_data.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:44:27.646976Z","iopub.execute_input":"2024-12-06T09:44:27.647334Z","iopub.status.idle":"2024-12-06T09:44:27.653714Z","shell.execute_reply.started":"2024-12-06T09:44:27.647279Z","shell.execute_reply":"2024-12-06T09:44:27.652683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"combined_data.dropna(inplace = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:44:27.654810Z","iopub.execute_input":"2024-12-06T09:44:27.655058Z","iopub.status.idle":"2024-12-06T09:44:38.557643Z","shell.execute_reply.started":"2024-12-06T09:44:27.655033Z","shell.execute_reply":"2024-12-06T09:44:38.556697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# lags_df  = pd.read_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet/date_id=0/part-0.parquet\")\n# lags_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:44:38.558679Z","iopub.execute_input":"2024-12-06T09:44:38.558939Z","iopub.status.idle":"2024-12-06T09:44:38.562842Z","shell.execute_reply.started":"2024-12-06T09:44:38.558889Z","shell.execute_reply":"2024-12-06T09:44:38.561965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:44:38.563742Z","iopub.execute_input":"2024-12-06T09:44:38.564010Z","iopub.status.idle":"2024-12-06T09:44:38.624402Z","shell.execute_reply.started":"2024-12-06T09:44:38.563984Z","shell.execute_reply":"2024-12-06T09:44:38.623621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n## creating dataset for testing ##\npath = \"/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet\"\ndirs = os.listdir(path)\n\n\nfor file in dirs:\n    df_test = pd.read_parquet(os.path.join(path, file))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:44:38.625291Z","iopub.execute_input":"2024-12-06T09:44:38.625506Z","iopub.status.idle":"2024-12-06T09:44:38.661397Z","shell.execute_reply.started":"2024-12-06T09:44:38.625482Z","shell.execute_reply":"2024-12-06T09:44:38.660798Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:44:38.664412Z","iopub.execute_input":"2024-12-06T09:44:38.664687Z","iopub.status.idle":"2024-12-06T09:44:38.684558Z","shell.execute_reply.started":"2024-12-06T09:44:38.664661Z","shell.execute_reply":"2024-12-06T09:44:38.683731Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(combined_data.columns.difference(df_test.columns))\nprint()\nprint(df_test.columns.difference(combined_data.columns))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:44:38.685727Z","iopub.execute_input":"2024-12-06T09:44:38.686055Z","iopub.status.idle":"2024-12-06T09:44:38.698753Z","shell.execute_reply.started":"2024-12-06T09:44:38.686017Z","shell.execute_reply":"2024-12-06T09:44:38.697976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## dropping columns for training and inference\ncombined_data.drop(['responder_0', 'responder_1', 'responder_2', 'responder_3',\n       'responder_4', 'responder_5' , 'responder_7',\n       'responder_8' ] , axis =1 , inplace = True )\n\n# df_test.drop(['is_scored' , 'row_id' , 'time_id'] , axis =  1 , inplace = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:44:38.699551Z","iopub.execute_input":"2024-12-06T09:44:38.699782Z","iopub.status.idle":"2024-12-06T09:44:43.573562Z","shell.execute_reply.started":"2024-12-06T09:44:38.699751Z","shell.execute_reply":"2024-12-06T09:44:43.572455Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## usefuls cols for x and y ##\nx_cols =  combined_data.columns.difference(['responder_6'])\ny = 'responder_6'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:44:43.574653Z","iopub.execute_input":"2024-12-06T09:44:43.574896Z","iopub.status.idle":"2024-12-06T09:44:43.579453Z","shell.execute_reply.started":"2024-12-06T09:44:43.574871Z","shell.execute_reply":"2024-12-06T09:44:43.578553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:44:43.580458Z","iopub.execute_input":"2024-12-06T09:44:43.580763Z","iopub.status.idle":"2024-12-06T09:44:43.640246Z","shell.execute_reply.started":"2024-12-06T09:44:43.580736Z","shell.execute_reply":"2024-12-06T09:44:43.639338Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Data split for train and test ##\nfrom sklearn.model_selection import train_test_split\n\ntrainx , testx , trainy , testy  = train_test_split(combined_data[x_cols] ,combined_data[y] , test_size = 0.25 ,random_state= 0)\nprint(trainx.shape)\nprint(trainy.shape)\nprint(testx.shape)\nprint(testy.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:44:43.641227Z","iopub.execute_input":"2024-12-06T09:44:43.641522Z","iopub.status.idle":"2024-12-06T09:45:09.363325Z","shell.execute_reply.started":"2024-12-06T09:44:43.641495Z","shell.execute_reply":"2024-12-06T09:45:09.362431Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:45:09.364505Z","iopub.execute_input":"2024-12-06T09:45:09.364964Z","iopub.status.idle":"2024-12-06T09:45:09.435950Z","shell.execute_reply.started":"2024-12-06T09:45:09.364933Z","shell.execute_reply":"2024-12-06T09:45:09.435120Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def weighted_zero_mean_r2(y_true, y_pred, weights):\n#     \"\"\"\n#     Calculate the sample weighted zero-mean R-squared score.\n\n#     Parameters:\n#     y_true (numpy.ndarray): Ground-truth values for responder_6.\n#     y_pred (numpy.ndarray): Predicted values for responder_6.\n#     weights (numpy.ndarray): Sample weight vector.\n\n#     Returns:\n#     float: The weighted zero-mean R-squared score.\n#     \"\"\"\n#     numerator = np.sum(weights * (y_true - y_pred)**2)\n#     denominator = np.sum(weights * y_true**2)\n    \n#     r2_score = 1 - numerator / denominator\n#     return r2_score\n\n\n\n","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-12-06T09:45:09.436955Z","iopub.execute_input":"2024-12-06T09:45:09.437206Z","iopub.status.idle":"2024-12-06T09:45:09.446018Z","shell.execute_reply.started":"2024-12-06T09:45:09.437179Z","shell.execute_reply":"2024-12-06T09:45:09.445212Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Hyper para,eter optimization ##\n\n# gc.collect()\n\n# import optuna\n# import lightgbm as lgb\n# import pandas as pd\n# from sklearn.metrics import mean_squared_error\n\n\n\n# # Optuna objective function\n# def objective(trial):\n#     # Hyperparameter search space\n#     param = {\n#         \"objective\": \"regression\",\n#         \"metric\": \"rmse\",\n#         \"device\": \"gpu\",  # Enable GPU training\n#         \"max_depth\": trial.suggest_int(\"max_depth\", 15, 25 ),\n#         \"num_leaves\": trial.suggest_int(\"num_leaves\", 70 , 255),\n#         \"learning_rate\": trial.suggest_float('learning_rate', 0.005 , 0.1),\n#         \"n_estimators\": trial.suggest_int(\"n_estimators\", 70  , 800 ),\n#         \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 3, 12),\n#         \"subsample\": trial.suggest_float(\"subsample\", 0.6, 1.0),\n#         \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.6, 1.0) , \n#         \"reg_alpha\": trial.suggest_float(\"reg_alpha\", 1e-4, 5.0, log=True),\n#         \"reg_lambda\": trial.suggest_float(\"reg_lambda\", 1e-4, 5.0, log=True),\n#          \"bagging_freq\":  trial.suggest_int('bagging_freq', 4, 6),\n#         \"bagging_fraction\": trial.suggest_float('bagging_fraction', 0.56, 0.57),\n#           'importance_type': 'gain'\n      \n#     }\n\n#     # Train the model\n#     model = lgb.LGBMRegressor(**param  )\n#     model.fit(\n#     trainx,\n#     trainy\n#      )\n\n\n#     #Predict and calculate RMSE\n#     preds = model.predict(testx)\n#     # rmse = mean_squared_error(testy, preds, squared=False)\n#     weights2=testx.weight[0:len(preds)]\n#     r2_score=weighted_zero_mean_r2(testy, preds, weights2)\n#     #rmse = mean_squared_error(test_y, preds,squared=False)\n    \n#     return r2_score\n#     # return rmse\n\n\n# # Run Optuna study\n# study = optuna.create_study(direction=\"minimize\")  # Minimize RMSE\n# study.optimize(objective, n_trials= 100 , timeout=5600)  \n\n# # Best parameters and performance\n# print(\"Best trial:\")\n# print(f\"  Value: {study.best_trial.value}\")\n# print(f\"  Params: {study.best_trial.params}\")","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-12-06T09:45:09.447185Z","iopub.execute_input":"2024-12-06T09:45:09.447530Z","iopub.status.idle":"2024-12-06T09:45:09.455340Z","shell.execute_reply.started":"2024-12-06T09:45:09.447491Z","shell.execute_reply":"2024-12-06T09:45:09.454590Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## params obtained from optuna optimisation technique\nbest_params  = {'max_depth': 25, 'num_leaves': 198, 'learning_rate': 0.005084805644444183, 'n_estimators': 79, 'min_child_samples': 3, \n                'subsample': 0.6100675566547535,'colsample_bytree': 0.7416045138403826, 'reg_alpha': 0.0001958150390817618,\n                'reg_lambda': 4.637964371746215, 'bagging_freq': 5, 'bagging_fraction': 0.5652858281164913}\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:45:09.456213Z","iopub.execute_input":"2024-12-06T09:45:09.456463Z","iopub.status.idle":"2024-12-06T09:45:09.467520Z","shell.execute_reply.started":"2024-12-06T09:45:09.456429Z","shell.execute_reply":"2024-12-06T09:45:09.466793Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train the final model with the best hyperparameters\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error\ncallbacks = [\n    lgb.early_stopping(stopping_rounds= 50,  verbose=True),\n    lgb.callback.log_evaluation(period=50)\n]\n\n\n# best_params = study.best_trial.params\n# best_params[\"device\"] = \"gpu\"  # Ensure GPU is enabled\nlgb_model = lgb.LGBMRegressor(**best_params , n_jobs = -1 , verbose = -1  , device = \"gpu\")\n\nlgb_model.fit(trainx , trainy ,eval_set = [(testx , testy)] , callbacks = callbacks)\n\n\n\n# Evaluate on validation set\nfinal_preds = lgb_model.predict(testx)\nfinal_rmse = mean_squared_error(testy, final_preds, squared=False)\nprint(f\"Final RMSE: {final_rmse}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:45:09.468341Z","iopub.execute_input":"2024-12-06T09:45:09.468669Z","iopub.status.idle":"2024-12-06T09:47:40.877965Z","shell.execute_reply.started":"2024-12-06T09:45:09.468642Z","shell.execute_reply":"2024-12-06T09:47:40.876350Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_columns = trainx.columns\ndf_columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:47:40.879130Z","iopub.execute_input":"2024-12-06T09:47:40.879895Z","iopub.status.idle":"2024-12-06T09:47:40.886568Z","shell.execute_reply.started":"2024-12-06T09:47:40.879839Z","shell.execute_reply":"2024-12-06T09:47:40.885553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Inferencing on Test data for submission ##\n\nresult  = lgb_model.predict(df_test[df_columns])\nsubmission  = pd.read_parquet(\"/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet\")\nsubmission  = pd.DataFrame({'row_id' :submission['row_id'] , 'responder_6' : result })\nsubmission.to_csv('submission.csv' , index = False)\nsubmission.to_parquet('submission.parquet', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:47:40.887560Z","iopub.execute_input":"2024-12-06T09:47:40.887841Z","iopub.status.idle":"2024-12-06T09:47:40.917280Z","shell.execute_reply.started":"2024-12-06T09:47:40.887810Z","shell.execute_reply":"2024-12-06T09:47:40.916513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## submission for api ##\nimport os\n\nimport pandas as pd\nimport polars as pl\n\nimport kaggle_evaluation.jane_street_inference_server\n\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pd.DataFrame:\n    global lags_ , cols\n    \n    test_pd = test.to_pandas()\n    \n    if lags is not None:\n        lags_ = lags\n\n        # Convert lags to Pandas DataFrame if it's Polars\n        lags_pd = lags.to_pandas()\n        lags_pd = lags_pd.groupby(['date_id', 'symbol_id']).last().reset_index() \n        test_pd = test_pd.merge(lags_pd, on=['date_id', 'symbol_id'], how='left')\n    else:\n        \n        for idx in range(9):\n            test_pd[f'responder_{idx}_lag_1'] = 0.0\n    print(test_pd.head())\n    cols = test_pd.columns\n    test_pd.rename(columns={'time_id_x': 'time_id'}, inplace=True)  \n\n    columns =  ['date_id', 'feature_00', 'feature_01', 'feature_02', 'feature_03',\n       'feature_04', 'feature_05', 'feature_06', 'feature_07', 'feature_08',\n       'feature_09', 'feature_10', 'feature_11', 'feature_12', 'feature_13',\n       'feature_14', 'feature_15', 'feature_16', 'feature_17', 'feature_18',\n       'feature_19', 'feature_20', 'feature_21', 'feature_22', 'feature_23',\n       'feature_24', 'feature_25', 'feature_26', 'feature_27', 'feature_28',\n       'feature_29', 'feature_30', 'feature_31', 'feature_32', 'feature_33',\n       'feature_34', 'feature_35', 'feature_36', 'feature_37', 'feature_38',\n       'feature_39', 'feature_40', 'feature_41', 'feature_42', 'feature_43',\n       'feature_44', 'feature_45', 'feature_46', 'feature_47', 'feature_48',\n       'feature_49', 'feature_50', 'feature_51', 'feature_52', 'feature_53',\n       'feature_54', 'feature_55', 'feature_56', 'feature_57', 'feature_58',\n       'feature_59', 'feature_60', 'feature_61', 'feature_62', 'feature_63',\n       'feature_64', 'feature_65', 'feature_66', 'feature_67', 'feature_68',\n       'feature_69', 'feature_70', 'feature_71', 'feature_72', 'feature_73',\n       'feature_74', 'feature_75', 'feature_76', 'feature_77', 'feature_78',\n       'symbol_id', 'time_id', 'weight']\n    # Prepare the features for prediction\n    # test_pd['time_id'] = np.nan\n    \n    predictions =  lgb_model.predict(test_pd[columns])\n\n    # Create a DataFrame for the output\n    output = pd.DataFrame({\n        'row_id': test_pd['row_id'],\n        'responder_6': predictions\n    })\n\n    # Ensure the output DataFrame has the correct format\n    assert output.columns.tolist() == ['row_id', 'responder_6']\n    assert len(output) == len(test)\n\n    return output\n\nimport kaggle_evaluation.jane_street_inference_server\nimport os\n\n# Set up the inference server\ninference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T09:47:40.918546Z","iopub.execute_input":"2024-12-06T09:47:40.918825Z","iopub.status.idle":"2024-12-06T09:47:41.699931Z","shell.execute_reply.started":"2024-12-06T09:47:40.918795Z","shell.execute_reply":"2024-12-06T09:47:41.698859Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}