{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install polars==1.18.0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:11:57.118346Z","iopub.execute_input":"2025-01-03T06:11:57.118791Z","iopub.status.idle":"2025-01-03T06:12:01.67259Z","shell.execute_reply.started":"2025-01-03T06:11:57.118757Z","shell.execute_reply":"2025-01-03T06:12:01.671264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport polars as pl\nimport kaggle_evaluation.jane_street_inference_server\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nfrom sklearn.feature_selection import SelectKBest\nfrom sklearn.linear_model import LogisticRegression, ElasticNet as ENC\nfrom sklearn.svm import SVC, SVR\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\nfrom xgboost import XGBClassifier\nfrom catboost import CatBoostClassifier\nfrom sklearn.neural_network import MLPClassifier, MLPRegressor\nfrom sklearn.linear_model import LinearRegression, ElasticNet as ENR\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn as nn\nimport torch.optim as optim\nimport scipy.stats as stats\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler, OrdinalEncoder, OneHotEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import train_test_split, GridSearchCV\nfrom sklearn.metrics import accuracy_score, log_loss, classification_report, r2_score, mean_squared_error\nimport torchvision.transforms as transforms\nfrom sklearn.model_selection import KFold\nfrom tensorflow.keras import layers, optimizers\nimport tensorflow as tf\nfrom lightgbm import LGBMRegressor, LGBMClassifier\nimport lightgbm as lgbm\nfrom sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor\nfrom sklearn.mixture import GaussianMixture\nimport os\nimport pyarrow.parquet as pa\nfrom sklearn.model_selection import train_test_split\nimport polars as pl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:12:05.971799Z","iopub.execute_input":"2025-01-03T06:12:05.972201Z","iopub.status.idle":"2025-01-03T06:12:05.983079Z","shell.execute_reply.started":"2025-01-03T06:12:05.972172Z","shell.execute_reply":"2025-01-03T06:12:05.981579Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pl.read_parquet('/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:33:51.037936Z","iopub.execute_input":"2025-01-03T06:33:51.038445Z","iopub.status.idle":"2025-01-03T06:33:51.089742Z","shell.execute_reply.started":"2025-01-03T06:33:51.038401Z","shell.execute_reply":"2025-01-03T06:33:51.088564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dftrain","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:34:19.000615Z","iopub.execute_input":"2025-01-03T06:34:19.001141Z","iopub.status.idle":"2025-01-03T06:34:19.022579Z","shell.execute_reply.started":"2025-01-03T06:34:19.001108Z","shell.execute_reply":"2025-01-03T06:34:19.021247Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocessing and EDA","metadata":{}},{"cell_type":"code","source":"# Symbol wise partition create\nos.makedirs('./instrumentwise', exist_ok=True)\ninstruments = [0, 1, 2, 3, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19, 30, 33, 34, 38]\n# for i in instruments:\ni = 0\ndftrain = pl.DataFrame()\nfor j in range(10):\n    temp = pl.read_parquet(f'/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id={j}/part-0.parquet')\n    dftrain = pl.concat([dftrain, temp.filter(temp['symbol_id']==i)], how=\"vertical\")\n# dfi.write_csv(f'instrumentwise/instrument_{i}.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:13:07.449129Z","iopub.execute_input":"2025-01-03T06:13:07.44961Z","iopub.status.idle":"2025-01-03T06:13:23.853251Z","shell.execute_reply.started":"2025-01-03T06:13:07.449575Z","shell.execute_reply":"2025-01-03T06:13:23.851981Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def custom_r2(y_true, y_pred, sample_weight):\n    # if sample_weight is not None:\n    den = np.sum(sample_weight*(y_true**2))\n    # num = mean_squared_error(y_true, y_pred, sample_weight=sample_weight)\n    num = np.sum(sample_weight*(y_true-y_pred)**2)\n    # else:\n    #     den = np.dot(y_true, y_true)\n    #     num = mean_squared_error(y_true, y_pred)\n    # print(num, den)\n    return (1-num/den)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:13:23.854643Z","iopub.execute_input":"2025-01-03T06:13:23.855084Z","iopub.status.idle":"2025-01-03T06:13:23.860576Z","shell.execute_reply.started":"2025-01-03T06:13:23.855023Z","shell.execute_reply":"2025-01-03T06:13:23.859155Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"daterange = {0: (169, 0),\n 1: (339, 170),\n 2: (509, 340),\n 3: (679, 510),\n 4: (849, 680),\n 5: (1019, 850),\n 6: (1189, 1020),\n 7: (1359, 1190),\n 8: (1529, 1360),\n 9: (1698, 1530)}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:13:23.862404Z","iopub.execute_input":"2025-01-03T06:13:23.862745Z","iopub.status.idle":"2025-01-03T06:13:23.881244Z","shell.execute_reply.started":"2025-01-03T06:13:23.862712Z","shell.execute_reply":"2025-01-03T06:13:23.880103Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select specific columns for lagged features\ndflag = dftrain.select([\n    \"date_id\", \"time_id\", \"responder_0\", \"responder_1\", \"responder_2\", \n    \"responder_3\", \"responder_4\", \"responder_5\", \"responder_6\", \n    \"responder_7\", \"responder_8\"\n]).with_columns(\n    (pl.col(\"date_id\") - 1).alias(\"date_id\")\n)\n\n# Rename columns for lagged features\ndflag = dflag.rename({col: f\"{col}_lagged_1\" for col in dflag.columns if col not in [\"date_id\", \"time_id\"]})\n\n# Perform a left join with the original DataFrame\ndftrain = dftrain.join(\n    dflag,\n    on=[\"date_id\", \"time_id\"],\n    how=\"left\"\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:37:27.932217Z","iopub.execute_input":"2025-01-03T06:37:27.932742Z","iopub.status.idle":"2025-01-03T06:37:28.340017Z","shell.execute_reply.started":"2025-01-03T06:37:27.9327Z","shell.execute_reply":"2025-01-03T06:37:28.338809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"nanpercent = dftrain.null_count()/len(dftrain)\nvartodrop = ['date_id', 'time_id', 'symbol_id', 'weight',  'responder_0', 'responder_1', 'responder_2',\n       'responder_3', 'responder_4', 'responder_5', 'responder_6',\n       'responder_7', 'responder_8']\nfor i in dftrain.columns:\n    if nanpercent[i].item()>0.3:\n        vartodrop.append(i)\nvartodrop","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:54:07.453417Z","iopub.execute_input":"2025-01-03T06:54:07.45389Z","iopub.status.idle":"2025-01-03T06:54:07.495787Z","shell.execute_reply.started":"2025-01-03T06:54:07.453855Z","shell.execute_reply":"2025-01-03T06:54:07.49457Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"columnskeep = [col for col in dftrain.columns if col not in vartodrop]\nnrows = len(columnskeep)\nncols = 4\n# gs = GridSpec(rows, cols, figure=fig)\n# Create plots: box, hist, probplot, violinplot\n# fig, ax = plt.subplots(nrows=1, ncols=4, figsize=(4*ncols, 2*nrows))\n# for i, col in enumerate(columnskeep):\n#     print(col)\n#     fig, ax = plt.subplots(nrows=1, ncols=4, figsize=(4*ncols, 2))\n#     # ax[0].set_axis_off()\n#     # ax[0].text(0.5, 0.5, col, fontsize=14, rotation=90, ha='center', va='center')\n#     sns.boxplot(dftrain[col], ax=ax[0])\n#     sns.histplot(dftrain[col], kde=True, ax=ax[1])\n#     stats.probplot(dftrain[col], dist=\"norm\", plot=ax[2])\n#     sns.violinplot(dftrain[col], y=y, ax=ax[3])\n#     fig.tight_layout()\n#     plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:54:07.718034Z","iopub.execute_input":"2025-01-03T06:54:07.718431Z","iopub.status.idle":"2025-01-03T06:54:07.724135Z","shell.execute_reply.started":"2025-01-03T06:54:07.718395Z","shell.execute_reply":"2025-01-03T06:54:07.722693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dftrain = dftrain.with_columns([\n    pl.col(col).fill_null(dftrain[col].median()).cast(pl.Float32) \n    for col in columnskeep\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:54:08.016529Z","iopub.execute_input":"2025-01-03T06:54:08.016948Z","iopub.status.idle":"2025-01-03T06:54:08.860652Z","shell.execute_reply.started":"2025-01-03T06:54:08.016917Z","shell.execute_reply":"2025-01-03T06:54:08.85941Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = dftrain.drop(vartodrop).to_numpy()\nY = dftrain['responder_6'].to_numpy()\nweights = dftrain[\"weight\"].to_numpy()\nx_train, x_test, y_train, y_test, wtrain, wtest = train_test_split(X, Y, weights, test_size=0.25)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:54:09.233406Z","iopub.execute_input":"2025-01-03T06:54:09.233768Z","iopub.status.idle":"2025-01-03T06:54:17.217691Z","shell.execute_reply.started":"2025-01-03T06:54:09.233736Z","shell.execute_reply":"2025-01-03T06:54:17.216417Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training the model","metadata":{}},{"cell_type":"code","source":"# Train the standard models for regression\nmodels = {\n    \"linear\": LinearRegression(),\n\t\"enr\": ENR(),\n\t# \"svr\": SVR(),\n    # \"dt\": DecisionTreeRegressor(),\n    # \"rf\": RandomForestRegressor(),\n    # \"gbdt\": GradientBoostingRegressor(),\n    \"xgb\": XGBRegressor(),\n    # \"cb\": CatBoostRegressor(verbose=0),\n\t\"nn\": MLPRegressor(hidden_layer_sizes=(5,), max_iter=1000, random_state=42),\n\t\"lgbm\": LGBMRegressor(verbose=0),\n\t# \"knn\": KNeighborsRegressor()\n}\n\nfor name in models:\n\tprint(name)\n\tmodels[name].fit(x_train, y_train)\n\tprint(custom_r2(y_train, models[name].predict(x_train), sample_weight=wtrain))\n\tprint(custom_r2(y_test, models[name].predict(x_test), sample_weight=wtest))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:37:44.289641Z","iopub.execute_input":"2025-01-03T06:37:44.289994Z","iopub.status.idle":"2025-01-03T06:41:07.522997Z","shell.execute_reply.started":"2025-01-03T06:37:44.289962Z","shell.execute_reply":"2025-01-03T06:41:07.521742Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# K-Fold Cross validation","metadata":{}},{"cell_type":"code","source":"K_SPLIT = 5\nkf = KFold(n_splits=K_SPLIT, shuffle=True)\n\navgtrain = 0\navgtest = 0\nweights = dftrain[\"weight\"].to_numpy()\nfor train, test in kf.split(X):\n    model = LGBMRegressor(verbose=0)\n    x_train = X[train]\n    x_test = X[test]\n    y_train = Y[train]\n    y_test = Y[test]\n    \n    model.fit(x_train, y_train)\n              # , sample_weight=weights[train])\n    avgtrain += custom_r2(y_train, model.predict(x_train), sample_weight=weights[train])\n    avgtest += custom_r2(y_test, model.predict(x_test), sample_weight=weights[test])\n    print(\"train: \", custom_r2(y_train, model.predict(x_train), sample_weight=weights[train]))\n    print(\"test: \", custom_r2(y_test, model.predict(x_test), sample_weight=weights[test]))\n\navgtrain /= K_SPLIT\navgtest /= K_SPLIT\nprint(\"avg train: \", avgtrain)\nprint(\"avg test: \", avgtest)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:54:34.944212Z","iopub.execute_input":"2025-01-03T06:54:34.944712Z","iopub.status.idle":"2025-01-03T06:57:33.887591Z","shell.execute_reply.started":"2025-01-03T06:54:34.944667Z","shell.execute_reply":"2025-01-03T06:57:33.886452Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Automated for partitioning based on instrument","metadata":{}},{"cell_type":"code","source":"columnstodrop = {}\nparititonmodels = {}\ntrainacc = []\ntestacc = []\nnsamples = []\n\ninstruments = [0, 1, 2, 3, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19, 30, 33, 34, 38]\nfor i in instruments:\n    print(i)\n    \n    dftrain = pl.DataFrame()\n    for j in range(10):\n        temp = pl.read_parquet(f'/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id={j}/part-0.parquet')\n        dftrain = pl.concat([dftrain, temp.filter(temp['symbol_id']==i)], how=\"vertical\")\n    \n    # Select specific columns for lagged features\n    dflag = dftrain.select([\n        \"date_id\", \"time_id\", \"responder_0\", \"responder_1\", \"responder_2\", \n        \"responder_3\", \"responder_4\", \"responder_5\", \"responder_6\", \n        \"responder_7\", \"responder_8\"\n    ]).with_columns(\n        (pl.col(\"date_id\") - 1).alias(\"date_id\")\n    )\n    \n    # Rename columns for lagged features\n    dflag = dflag.rename({col: f\"{col}_lagged_1\" for col in dflag.columns if col not in [\"date_id\", \"time_id\"]})\n    \n    # Perform a left join with the original DataFrame\n    dftrain = dftrain.join(\n        dflag,\n        on=[\"date_id\", \"time_id\"],\n        how=\"left\"\n    )\n\n    \n    vartodrop = ['date_id', 'time_id', 'symbol_id', 'weight', 'responder_0', 'responder_1', 'responder_2',\n       'responder_3', 'responder_4', 'responder_5', 'responder_6',\n       'responder_7', 'responder_8']\n    for i in dftrain.columns:\n        if nanpercent[i].item()>0.3:\n            vartodrop.append(i)\n\n    columnskeep = [col for col in dftrain.columns if col not in vartodrop]\n    # print(vartodrop)\n    \n    columnstodrop[i] = vartodrop\n    \n    dftrain = dftrain.with_columns([\n        pl.col(col).fill_null(dftrain[col].median()).cast(pl.Float32) \n        for col in columnskeep\n    ])\n    \n    X = dftrain.drop(vartodrop).to_numpy()\n    Y = dftrain['responder_6'].to_numpy()\n    weights = dftrain['weight'].to_numpy()\n    x_train, x_test, y_train, y_test, w_train, w_test = train_test_split(X, Y, weights,  test_size=0.25)\n\n    model = LGBMRegressor(verbose=0)\n    model.fit(x_train, y_train, sample_weight=w_train)\n    parititonmodels[i] = model\n    print(\"train: \", custom_r2(y_train, model.predict(x_train), sample_weight=w_train))\n    print(\"test: \", custom_r2(y_test, model.predict(x_test), sample_weight=w_test))\n\n    trainacc.append(custom_r2(y_train, model.predict(x_train), sample_weight=w_train))\n    testacc.append(custom_r2(y_test, model.predict(x_test), sample_weight=w_test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T07:09:20.280249Z","iopub.execute_input":"2025-01-03T07:09:20.28067Z","iopub.status.idle":"2025-01-03T07:29:11.337366Z","shell.execute_reply.started":"2025-01-03T07:09:20.280635Z","shell.execute_reply":"2025-01-03T07:29:11.335984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(len(trainacc)):\n    print(f\"- {instruments[i]}: {trainacc[i]} {testacc[i]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T07:32:43.053921Z","iopub.execute_input":"2025-01-03T07:32:43.054288Z","iopub.status.idle":"2025-01-03T07:32:43.065581Z","shell.execute_reply.started":"2025-01-03T07:32:43.054258Z","shell.execute_reply":"2025-01-03T07:32:43.064071Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Autocorrelation test","metadata":{}},{"cell_type":"code","source":"import statsmodels.api as sm\nfrom statsmodels.stats.stattools import durbin_watson\n\nx = sm.add_constant(X)\n\n# Fit the linear regression model\nmodel = sm.OLS(Y, x).fit()\n\n# Print the statistical summary of the model\nprint(model.summary())\n\n\n#perform Durbin-Watson test\ndurbin_watson(model.resid)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T06:22:25.764009Z","iopub.status.idle":"2025-01-03T06:22:25.764382Z","shell.execute_reply":"2025-01-03T06:22:25.764232Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nlags_ : pl.DataFrame | None = None\n\n\n# Replace this function with your inference code.\n# You can return either a Pandas or Polars dataframe, though Polars is recommended.\n# Each batch of predictions (except the very first) must be returned within 1 minute of the batch features being provided.\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    # All the responders from the previous day are passed in at time_id == 0. We save them in a global variable for access at every time_id.\n    # Use them as extra features, if you like.\n    global lags_\n    if lags is not None:\n        lags_ = lags\n\n    # Replace this section with your own predictions\n    predictions = test.select(\n        'row_id',\n        pl.lit(0.0).alias('responder_6'),\n    )\n\n    if isinstance(predictions, pl.DataFrame):\n        assert predictions.columns == ['row_id', 'responder_6']\n    elif isinstance(predictions, pd.DataFrame):\n        assert (predictions.columns == ['row_id', 'responder_6']).all()\n    else:\n        raise TypeError('The predict function must return a DataFrame')\n    # Confirm has as many rows as the test data.\n    assert len(predictions) == len(test)\n\n    return predictions","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        (\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet',\n            '/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet',\n        )\n    )","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Instrument-0 (without lagged, date, time):\n\n------\nInstrument-0 (including lagged responses features without date, time):\n- linear\n0.012141764163970947\n0.012055575847625732\n- enr\n1.895427703857422e-05\n2.0742416381835938e-05\n- xgb\n0.1925554871559143\n0.14189940690994263\n- nn\n0.024626433849334717\n0.01891404390335083\n- lgbm\n0.07884947199766279\n0.061606859455547225\n\n- k-fold (no shuffle split): [ xgb: [0.20518980026245118,\n-0.20239098072052003], lg\n--------------\nAll instruments shuffle and trained individually:\n- 0: 0.08591097481603416 0.0675794139308693\n- 1: 0.07601666506348115 0.059321343676486005\n- 2: 0.08654843678906898 0.07137907493361473\n- 3: 0.09858768481071734 0.08051084920317964\n- 7: 0.09963596491717064 0.08430064628914546\n- 8: 0.08333802239349508 0.0669012379611772\n- 9: 0.10787378682024518 0.09034529747293174\n- 10: 0.07228309001848432 0.055582023904432565\n- 11: 0.08839853452627067 0.06974840723933606\n- 12: 0.09089290736510691 0.07365965250122664\n- 13: 0.08328137204108588 0.0702376741973263\n- 14: 0.07323916868886171 0.05827583541845871\n- 15: 0.07810992953573526 0.06178979049580724\n- 16: 0.08115143143346049 0.06312943298060048\n- 17: 0.07813833386274815 0.061228933528541085\n- 19: 0.08398492491589793 0.06645034328808519\n- 30: 0.1028521277985397 0.08124022954480636\n- 33: 0.08556749450259071 0.06759250750526025\n- 34: 0.11503668809914969 0.09576235513306508\n- 38: 0.0771829664237873 0.05854977990384658\n","metadata":{}}]}