{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport polars as pl\nimport xgboost as xgb\nimport os\nimport gc\nimport lightgbm as lgb\nfrom sklearn.model_selection import GridSearchCV,TimeSeriesSplit\n#import kaggle_evaluation.jane_street_inference_server\n\n#import seaborn as sb\nimport matplotlib.pyplot as plt\n       ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-17T18:57:50.509787Z","iopub.execute_input":"2024-11-17T18:57:50.510319Z","iopub.status.idle":"2024-11-17T18:57:50.517710Z","shell.execute_reply.started":"2024-11-17T18:57:50.510275Z","shell.execute_reply":"2024-11-17T18:57:50.516398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/jane-street-real-time-market-data-forecasting/'\ntarget = 'responder_6'\nfeatures = [\"date_id\", \"time_id\", \"symbol_id\"] + [f\"feature_{i:02d}\" for i in range(79)]\n\nresponders = [f\"responder_{i}\" for i in range(9)]\n#responders.remove(\"responder_6\")  # Убираем 'responder_6' из списка\n#all_features = features+responders\n","metadata":{"execution":{"iopub.status.busy":"2024-11-17T18:57:52.361565Z","iopub.execute_input":"2024-11-17T18:57:52.362060Z","iopub.status.idle":"2024-11-17T18:57:52.369389Z","shell.execute_reply.started":"2024-11-17T18:57:52.362015Z","shell.execute_reply":"2024-11-17T18:57:52.367973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Модель","metadata":{}},{"cell_type":"code","source":"#parameters\n# path = '/kaggle/input/jane-street-real-time-market-data-forecasting/'\n# n_splits = 8\n# fold_size = 212    # in one parquet around 170 date_id\n#lag_responders = \n","metadata":{"execution":{"iopub.status.busy":"2024-11-17T15:45:30.206688Z","iopub.execute_input":"2024-11-17T15:45:30.207154Z","iopub.status.idle":"2024-11-17T15:45:30.219069Z","shell.execute_reply.started":"2024-11-17T15:45:30.207091Z","shell.execute_reply":"2024-11-17T15:45:30.217937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"$$ R^2=1−∑w_i*(y_i−y_ihat)^2 / ∑w_i*y_i^2 $$","metadata":{}},{"cell_type":"code","source":"# function for weighted R2\ndef weighted_r2(y_true, y_pred, weights):\n   \n    ss_residual = np.sum(weights*(y_true - y_pred)**2)\n    ss_total = np.sum(weights*(y_true)**2)\n    \n    # Вычисление взвешенного R2\n    r2 = 1 - (ss_residual / ss_total)\n    return r2","metadata":{"execution":{"iopub.status.busy":"2024-11-17T18:57:55.402365Z","iopub.execute_input":"2024-11-17T18:57:55.402825Z","iopub.status.idle":"2024-11-17T18:57:55.410288Z","shell.execute_reply.started":"2024-11-17T18:57:55.402784Z","shell.execute_reply":"2024-11-17T18:57:55.408847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model on a batch of data","metadata":{}},{"cell_type":"code","source":"# Data loading\nstart_dt = 1698 - 200\nend_dt = 1698\n\nX_train = (pl.scan_parquet(os.path.join(path, \"train.parquet\"))\n           .filter(pl.col(\"date_id\").is_between(start_dt, end_dt))\n           .drop(responders)\n           .drop('partition_id', 'weight')\n           .collect()\n           .to_numpy()\n           .astype(np.float32))\n\ny_train = (pl.scan_parquet(os.path.join(path, \"train.parquet\"))\n           .filter(pl.col(\"date_id\").is_between(start_dt, end_dt))\n           .select(target)\n           .collect()\n           .to_numpy()\n           .flatten())\n\n# Lags for all responders \ny_lags = (pl.scan_parquet(os.path.join(path, \"train.parquet\"))\n           .filter(pl.col(\"date_id\").is_between(start_dt, end_dt))\n           .select(responders)\n           .collect()\n           .shift(1)) \n\nX_train = np.hstack((X_train, y_lags))\n\nweights = (pl.scan_parquet(os.path.join(path, \"train.parquet\"))\n           .filter(pl.col(\"date_id\").is_between(start_dt, end_dt))\n           .select('weight')\n           .collect()\n           .to_numpy()\n           .flatten())\n\n# X_val = (pl.scan_parquet(os.path.join(path, \"test.parquet\"))\n#          .drop('row_id', 'is_scored', 'weight')\n#          .collect()\n#          .to_numpy()\n#          .astype(np.float32))\n\nmodel = lgb.LGBMRegressor(objective=\"regression\", random_state=42, \n                                                    n_estimators=500,\n                                                    learning_rate=0.05, # 0.01, 0.1],\n                                                    max_depth=6,\n                                                    min_split_gain=1e-6,\n                                                    num_leaves=32)\n\n\n# Fit the best model on the full training set\nmodel.fit(X_train, y_train, sample_weight=weights)\n","metadata":{"execution":{"iopub.status.busy":"2024-11-17T18:57:56.104674Z","iopub.execute_input":"2024-11-17T18:57:56.105605Z","iopub.status.idle":"2024-11-17T19:24:58.781234Z","shell.execute_reply.started":"2024-11-17T18:57:56.105546Z","shell.execute_reply":"2024-11-17T19:24:58.779392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(\"/kaggle/working/model_lags\") ","metadata":{"execution":{"iopub.status.busy":"2024-11-17T19:35:47.586510Z","iopub.execute_input":"2024-11-17T19:35:47.587082Z","iopub.status.idle":"2024-11-17T19:35:47.664259Z","shell.execute_reply.started":"2024-11-17T19:35:47.587037Z","shell.execute_reply":"2024-11-17T19:35:47.662689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\njoblib.dump(model, 'model_lags.pkl')\n#model.save_model('model_lag.json')","metadata":{"execution":{"iopub.status.busy":"2024-11-17T19:36:11.776759Z","iopub.execute_input":"2024-11-17T19:36:11.777229Z","iopub.status.idle":"2024-11-17T19:36:11.865070Z","shell.execute_reply.started":"2024-11-17T19:36:11.777190Z","shell.execute_reply":"2024-11-17T19:36:11.863369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags = (pl.scan_parquet(os.path.join(path, \"/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet/date_id=0/part-0.parquet\"))\n           .collect())","metadata":{"execution":{"iopub.status.busy":"2024-11-17T19:40:30.550756Z","iopub.execute_input":"2024-11-17T19:40:30.551292Z","iopub.status.idle":"2024-11-17T19:40:30.560641Z","shell.execute_reply.started":"2024-11-17T19:40:30.551243Z","shell.execute_reply":"2024-11-17T19:40:30.559369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags[lags.columns[3:]]","metadata":{"execution":{"iopub.status.busy":"2024-11-17T19:41:26.402080Z","iopub.execute_input":"2024-11-17T19:41:26.403095Z","iopub.status.idle":"2024-11-17T19:41:26.414804Z","shell.execute_reply.started":"2024-11-17T19:41:26.403042Z","shell.execute_reply":"2024-11-17T19:41:26.413405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Перенести в отдельный файл для предсказаний","metadata":{}},{"cell_type":"code","source":"# # Global variables\n# model = None  # Global model instance\n\n# # В predict функции\n# def predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n#     global model\n#     if model is None:\n#         #model = joblib.load(\"model.pkl\")  # Загрузка модели из файла\n    \n#     # Ваш код предсказаний\n#     predictions = test.select(\n#         'row_id',\n#         pl.lit(model.predict(test.to_numpy())).alias('responder_6')\n#     )\n#     return predictions","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub = pd.read_csv('/kaggle/input/jane-street-real-time-market-data-forecasting/sample_submission.csv')\n# sub['responder_6'] = model.predict(X_val)\n# sub","metadata":{"execution":{"iopub.status.busy":"2024-11-16T19:31:39.238008Z","iopub.execute_input":"2024-11-16T19:31:39.238511Z","iopub.status.idle":"2024-11-16T19:31:39.255142Z","shell.execute_reply.started":"2024-11-16T19:31:39.238470Z","shell.execute_reply":"2024-11-16T19:31:39.253701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plt.hist(y_train, bins=40);\nplt.plot(range(0,20), y_train[len(y_train)-20:]);\nplt.plot(range(21,60),y_pred, color = 'r');","metadata":{"execution":{"iopub.status.busy":"2024-11-16T19:31:42.820138Z","iopub.execute_input":"2024-11-16T19:31:42.821486Z","iopub.status.idle":"2024-11-16T19:31:43.156185Z","shell.execute_reply.started":"2024-11-16T19:31:42.821426Z","shell.execute_reply":"2024-11-16T19:31:43.154872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model with N-Fold&GridSerch","metadata":{}},{"cell_type":"code","source":"## \n# param_grid = {\n#     'n_estimators': [500],\n#     'learning_rate': [0.01, 0.05, 0.1],\n#     'max_depth': [6],\n#     'min_split_gain': [1e-6],\n#     'num_leaves': [32]\n# }\n\n# Data loading\n# start_dt = 1698 - 200\n# end_dt = 1698\n\n# X_train = (pl.scan_parquet(os.path.join(path, \"train.parquet\"))\n#            .filter(pl.col(\"date_id\").is_between(start_dt, end_dt))\n#            .drop(responders)\n#            .drop('partition_id', 'weight')\n#            .collect()\n#            .to_numpy()\n#            .astype(np.float32))\n\n# y_train = (pl.scan_parquet(os.path.join(path, \"train.parquet\"))\n#            .filter(pl.col(\"date_id\").is_between(start_dt, end_dt))\n#            .select(target)\n#            .collect()\n#            .to_numpy()\n#            .flatten())\n\n# weights = (pl.scan_parquet(os.path.join(path, \"train.parquet\"))\n#            .filter(pl.col(\"date_id\").is_between(start_dt, end_dt))\n#            .select('weight')\n#            .collect()\n#            .to_numpy()\n#            .flatten())\n\n# X_val = (pl.scan_parquet(os.path.join(path, \"test.parquet\"))\n#          .drop('row_id', 'is_scored', 'weight')\n#          .collect()\n#          .to_numpy()\n#          .astype(np.float32))\n\n# GridSearchCV setup\n\n\n# model = lgb.LGBMRegressor(objective=\"regression\", random_state=42)\n#tscv = TimeSeriesSplit(n_splits=3)\n\n# grid_search = GridSearchCV(\n#     model,\n#     param_grid,\n#     cv=tscv,  # Use 3-fold cross-validation, data is not shuffled\n#     scoring='neg_mean_squared_error',\n#     verbose=1\n# )\n\n# Fit GridSearchCV\n# grid_search.fit(X_train, y_train, sample_weight=weights)\n\n# Best parameters and model\n# print(\"Best parameters found:\", grid_search.best_params_)\n# best_model = grid_search.best_estimator_\n\n\n# # Fit the best model on the full training set\n# model.fit(X_train, y_train, sample_weight=weights)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Разбиение по времени\n\n# param_grid = {\n#     'n_estimators': [50, 100, 200],\n#     'learning_rate': [0.01, 0.05, 0.1],\n#     'max_depth': [3, 6, 9],\n#     'num_leaves': [16, 32, 64],\n# }\n\n# for i in range(4;8):\n#     start_dt =  i * 212 #31698 - 250\n#     end_dt = start_dt + 212  #1698\n\n#     # loading train \n#     X_train = (pl.scan_parquet(os.path.join(path, \"train.parquet\"))\n#         .filter(pl.col(\"date_id\").is_between(start_dt, end_dt))\n#         .drop(target)\n#         .collect()\n#         .to_numpy()\n#        .astype(np.float32))\n\n#     y_train = (pl.scan_parquet(os.path.join(path, \"train.parquet\"))\n#         .filter(pl.col(\"date_id\").is_between(start_dt, end_dt))\n#         .select(target)\n#         .collect()\n#         .flatten())\n\n#     weights = (pl.scan_parquet(os.path.join(path, \"train.parquet\"))\n#         .filter(pl.col(\"date_id\").is_between(start_dt, end_dt))\n#         .select('weight')\n#         .collect()\n#         .to_numpy()\n#        .astype(np.float32))\n\n\n#     #X_train_fold = X_train.to_numpy().astype(np.float32)\n#     #y_train_fold = y_train.to_numpy().flatten()\n#     #weights = weights.to_numpy().flatten() #.astype(np.float32)\n\n\n#     # Dividing into train, validation\n#     split_index = int(0.8 * len(X_train))  \n#     X_train, X_val = X_train[:split_index], X_train[split_index:]\n#     y_train, y_val = y_train[:split_index], y_train[split_index:]\n#     weights_val = weights[split_index:]\n#     print('Данные загружены')\n\n#     # Use GridSearchCV to find the best parameters\n#     model = lgb.LGBMRegressor(objective=\"regression\", random_state=42)\n\n#     grid_search = GridSearchCV(\n#         model,\n#         param_grid,\n#         cv=3,  # 3-fold cross-validation on the training set\n#         scoring='neg_mean_squared_error',\n#         verbose=1\n#     )\n\n#     grid_search.fit(X_train_fold, y_train_fold, sample_weight=weights_train)\n\n#     print(\"Best parameters found:\", grid_search.best_params_)\n\n#     # Train the model with the best parameters\n#     best_model = grid_search.best_estimator_\n#     best_model.fit(X_train_fold, y_train_fold, sample_weight=weights_train)\n\n#     print('Model trained on fold:', i + 1)\n\n#     # Free up memory\n#     del X_train, y_train, weights, X_train_fold, y_train_fold\n#     gc.collect()\n\n#     # Make predictions\n#     y_pred = best_model.predict(X_val)\n#     fold_r2 = weighted_r2(y_val, y_pred, weights_val)\n#     fold_errors.append(fold_r2)\n\n#     print(f\"Fold {i+1} Weighted R2: {fold_r2}\")\n\n# # Calculate mean R2 score across all folds\n# mean_r2 = np.mean(fold_errors)\n# print(f\"Mean Weighted R2 across all folds: {mean_r2}\")","metadata":{"execution":{"iopub.status.busy":"2024-11-17T08:35:25.516302Z","iopub.execute_input":"2024-11-17T08:35:25.517016Z","iopub.status.idle":"2024-11-17T08:35:25.527363Z","shell.execute_reply.started":"2024-11-17T08:35:25.516936Z","shell.execute_reply":"2024-11-17T08:35:25.525836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-11-16T15:10:14.716095Z","iopub.execute_input":"2024-11-16T15:10:14.717151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.save(\"/kaggle/working/model\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_test = (pl.scan_parquet(os.path.join(path, \"test.parquet\"))\n#   #  .filter(pl.col(\"date_id\").is_between(start_dt, end_dt))\n#     .drop('row_id', 'is_scored')\n#     .collect())\n    \n# y_pred_submission = model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-11-16T13:15:27.926686Z","iopub.execute_input":"2024-11-16T13:15:27.927217Z","iopub.status.idle":"2024-11-16T13:15:28.178007Z","shell.execute_reply.started":"2024-11-16T13:15:27.927153Z","shell.execute_reply":"2024-11-16T13:15:28.176760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Draft","metadata":{}},{"cell_type":"code","source":"# X_full_train = (pl.scan_parquet(os.path.join(path, \"train.parquet\"))\n#     .drop(target, 'weight')\n#     .collect()\n#     .to_numpy()\n#     .astype(np.float32))\n\n# y_full_train = (pl.scan_parquet(os.path.join(path, \"train.parquet\"))\n#     .select(target)\n#     .collect()\n#     .to_numpy()\n#     .flatten())\n\n# weights_full = (pl.scan_parquet(os.path.join(path, \"train.parquet\"))\n#     .select('weight')\n#     .collect()\n#     .to_numpy()\n#     .flatten())\n\n# final_model = lgb.LGBMRegressor(\n#     objective=\"regression\",\n#     n_estimators=100,\n#     learning_rate=0.1,\n#     max_depth=6,\n#     random_state=42,\n#     num_leaves=64,\n#     min_split_gain=0.0\n# )\n# final_model.fit(X_full_train, y_full_train, sample_weight=weights_full)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tab.info()\n#train.describe()\n#plt.plot(X_train['date_id'], X_train['feature_45']);\n#print('Unique financial instuments: ', X_train['symbol_id'].nunique())","metadata":{"execution":{"iopub.status.busy":"2024-11-09T18:19:50.151664Z","iopub.execute_input":"2024-11-09T18:19:50.152102Z","iopub.status.idle":"2024-11-09T18:19:50.157216Z","shell.execute_reply.started":"2024-11-09T18:19:50.152060Z","shell.execute_reply":"2024-11-09T18:19:50.155944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Уменьшение использование памяти за счет типа данных","metadata":{}},{"cell_type":"code","source":"# def reduce_memory_usage(df, use_float32=True):\n\n\n#     for col in df.columns:\n#         col_type = df[col].dtype\n\n#         # Process numeric columns only\n#         if pd.api.types.is_numeric_dtype(col_type):\n#             col_min = df[col].min()\n#             col_max = df[col].max()\n\n#             # Downcast integer columns\n#             if pd.api.types.is_integer_dtype(col_type):\n#                 if col_min >= np.iinfo(np.int8).min and col_max <= np.iinfo(np.int8).max:\n#                     df[col] = df[col].astype(np.int8)\n#                 elif col_min >= np.iinfo(np.int16).min and col_max <= np.iinfo(np.int16).max:\n#                     df[col] = df[col].astype(np.int16)\n#                 elif col_min >= np.iinfo(np.int32).min and col_max <= np.iinfo(np.int32).max:\n#                     df[col] = df[col].astype(np.int32)\n#                 else:\n#                     df[col] = df[col].astype(np.int64)\n\n#             # Downcast floating-point columns\n#             elif pd.api.types.is_float_dtype(col_type):\n#                 if use_float32:\n#                     if col_min >= np.finfo(np.float32).min and col_max <= np.finfo(np.float32).max:\n#                         df[col] = df[col].astype(np.float32)\n#                 else:\n#                     if col_min >= np.finfo(np.float16).min and col_max <= np.finfo(np.float16).max:\n#                         df[col] = df[col].astype(np.float16)\n\n\n#     return df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Загрузка паркетов","metadata":{}},{"cell_type":"code","source":"# !pip install fastparquet -q\n# import fastparquet\n# i = 9\n# tab  = pd.read_parquet(f'/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id={i}/part-0.parquet'\n#                 , engine = 'fastparquet')\n# tab.tail(2)","metadata":{"execution":{"iopub.status.busy":"2024-11-16T12:51:50.958138Z","iopub.execute_input":"2024-11-16T12:51:50.959462Z","iopub.status.idle":"2024-11-16T12:52:24.598752Z","shell.execute_reply.started":"2024-11-16T12:51:50.959417Z","shell.execute_reply":"2024-11-16T12:52:24.597268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#кол-во дат\n1698/8","metadata":{"execution":{"iopub.status.busy":"2024-11-16T12:54:29.337915Z","iopub.execute_input":"2024-11-16T12:54:29.338361Z","iopub.status.idle":"2024-11-16T12:54:29.345988Z","shell.execute_reply.started":"2024-11-16T12:54:29.338320Z","shell.execute_reply":"2024-11-16T12:54:29.344723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}