{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":10192259,"sourceType":"datasetVersion","datasetId":6297429}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"https://www.kaggle.com/code/tangtunyu/js24-preprocessing-create-lags","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:27.467167Z","iopub.execute_input":"2024-12-20T17:05:27.467517Z","iopub.status.idle":"2024-12-20T17:05:27.508938Z","shell.execute_reply.started":"2024-12-20T17:05:27.467487Z","shell.execute_reply":"2024-12-20T17:05:27.508223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport gc\nfrom matplotlib import pyplot as plt\nimport matplotlib.cm as cm\nfrom sklearn.model_selection import StratifiedGroupKFold\n\nfrom tqdm.auto import tqdm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:27.510178Z","iopub.execute_input":"2024-12-20T17:05:27.510433Z","iopub.status.idle":"2024-12-20T17:05:28.765217Z","shell.execute_reply.started":"2024-12-20T17:05:27.510409Z","shell.execute_reply":"2024-12-20T17:05:28.764556Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CONFIG:\n    target_col = \"responder_6\"\n    lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_rename = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1\" for idx in range(9)}\n    valid_ratio = 0.05\n    #start_dt = 1100\n    start_dt = -1\n\n    seed = 42\n\n    # Model parameters (can be adjusted or expanded for tuning)\n    xgb_params = {\n        'objective': 'reg:squarederror',\n        'learning_rate': 0.05,\n        'max_depth': 6,\n        'n_estimators': 200,\n        'subsample': 0.8,\n        'colsample_bytree': 0.8,\n        'reg_alpha': 1,\n        'reg_lambda': 5,\n        'random_state': seed,\n        'tree_method': 'hist',\n        'device': 'cuda',\n        'eval_metric': 'rmse',\n        'early_stopping_rounds': 10,\n    }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:28.766139Z","iopub.execute_input":"2024-12-20T17:05:28.766458Z","iopub.status.idle":"2024-12-20T17:05:28.771884Z","shell.execute_reply.started":"2024-12-20T17:05:28.766434Z","shell.execute_reply":"2024-12-20T17:05:28.771077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Specify the partition IDs you want to load\nselected_partitions = [0]\n\n# Create a list of file paths for the selected partitions\npaths = [\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id={i}/part-0.parquet\"\n    for i in selected_partitions\n]\n\ntrain = pl.scan_parquet(paths).select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),\n).with_columns(\n    (pl.col(CONFIG.target_col)*2).cast(pl.Int32).alias(\"label\"),\n).filter(\n    pl.col(\"date_id\").gt(CONFIG.start_dt)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:28.772915Z","iopub.execute_input":"2024-12-20T17:05:28.773535Z","iopub.status.idle":"2024-12-20T17:05:28.834002Z","shell.execute_reply.started":"2024-12-20T17:05:28.773496Z","shell.execute_reply":"2024-12-20T17:05:28.833448Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_col = \"responder_6\"\nfeature_cols = [\"symbol_id\", \"time_id\"] \\\n    + [f\"feature_{idx:02d}\" for idx in range(79)] \\\n    + [f\"responder_{idx}_lag_1\" for idx in range(9)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:28.836463Z","iopub.execute_input":"2024-12-20T17:05:28.837071Z","iopub.status.idle":"2024-12-20T17:05:28.840841Z","shell.execute_reply.started":"2024-12-20T17:05:28.837043Z","shell.execute_reply":"2024-12-20T17:05:28.839996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags = train.select(pl.col(CONFIG.lag_cols_original))\nlags = lags.rename(CONFIG.lag_cols_rename)\nlags = lags.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )\nlags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()  # pick up last record of previous date\n#lags.show_graph()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:28.841725Z","iopub.execute_input":"2024-12-20T17:05:28.841945Z","iopub.status.idle":"2024-12-20T17:05:28.851704Z","shell.execute_reply.started":"2024-12-20T17:05:28.841923Z","shell.execute_reply":"2024-12-20T17:05:28.851129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.join(lags, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\n#train.show_graph()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:28.852757Z","iopub.execute_input":"2024-12-20T17:05:28.853004Z","iopub.status.idle":"2024-12-20T17:05:28.868091Z","shell.execute_reply.started":"2024-12-20T17:05:28.852978Z","shell.execute_reply":"2024-12-20T17:05:28.867501Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = train.select(feature_cols).collect().to_numpy()\nnp.save('X.npy', X)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:28.868952Z","iopub.execute_input":"2024-12-20T17:05:28.869211Z","iopub.status.idle":"2024-12-20T17:05:32.371912Z","shell.execute_reply.started":"2024-12-20T17:05:28.869187Z","shell.execute_reply":"2024-12-20T17:05:32.370978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = train.select(target_col).collect()[target_col].to_numpy()\nnp.save('y.npy', y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:32.373035Z","iopub.execute_input":"2024-12-20T17:05:32.373349Z","iopub.status.idle":"2024-12-20T17:05:32.491604Z","shell.execute_reply.started":"2024-12-20T17:05:32.373322Z","shell.execute_reply":"2024-12-20T17:05:32.490529Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"w = train.select('weight').collect()['weight'].to_numpy()\nnp.save('w.npy', w)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:32.492708Z","iopub.execute_input":"2024-12-20T17:05:32.492949Z","iopub.status.idle":"2024-12-20T17:05:32.609934Z","shell.execute_reply.started":"2024-12-20T17:05:32.492925Z","shell.execute_reply":"2024-12-20T17:05:32.609170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len_train   = train.select(pl.col(\"date_id\")).collect().shape[0]\nvalid_records = int(len_train * CONFIG.valid_ratio)\nlen_train_exValid = len_train - valid_records\nlast_tr_dt  = train.select(pl.col(\"date_id\")).collect().row(len_train_exValid)[0]\n\n#X = train.select(feature_cols).collect().to_numpy()\n\n\n#print(f\"\\n len_train = {len_train}\")\n#print(f\"\\n len_train_exValid = {len_train_exValid}\")\n#print(f\"\\n---> Last offline train date = {last_tr_dt}\\n\")\n\n#training_data validation_data both come from train; only cut up by date\ntraining_data = train.filter(pl.col(\"date_id\").le(last_tr_dt))\nvalidation_data   = train.filter(pl.col(\"date_id\").gt(last_tr_dt))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:32.610906Z","iopub.execute_input":"2024-12-20T17:05:32.611173Z","iopub.status.idle":"2024-12-20T17:05:32.824229Z","shell.execute_reply.started":"2024-12-20T17:05:32.611144Z","shell.execute_reply":"2024-12-20T17:05:32.823387Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#training_data: 103 columns in total\n#id, date_id, time_id, symbol_id, weight, \n#feature_00 to feature_78, \n#responder_0 to responder_8, \n#label, \n#responder_0_lag_1 to responder_8_lag_1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:32.825205Z","iopub.execute_input":"2024-12-20T17:05:32.825475Z","iopub.status.idle":"2024-12-20T17:05:32.829240Z","shell.execute_reply.started":"2024-12-20T17:05:32.825450Z","shell.execute_reply":"2024-12-20T17:05:32.828434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = training_data.select(feature_cols).collect()\nX_valid = validation_data.select(feature_cols).collect()\ny_train = training_data.select(target_col).collect()\ny_valid = validation_data.select(target_col).collect()\nw_train = training_data.select('weight').collect()\nw_valid = validation_data.select('weight').collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:32.830304Z","iopub.execute_input":"2024-12-20T17:05:32.830619Z","iopub.status.idle":"2024-12-20T17:05:34.313284Z","shell.execute_reply.started":"2024-12-20T17:05:32.830583Z","shell.execute_reply":"2024-12-20T17:05:34.312513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"training_data.collect().\\\nwrite_parquet(\n    f\"training.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:34.317753Z","iopub.execute_input":"2024-12-20T17:05:34.318034Z","iopub.status.idle":"2024-12-20T17:05:38.417123Z","shell.execute_reply.started":"2024-12-20T17:05:34.318006Z","shell.execute_reply":"2024-12-20T17:05:38.416190Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data.collect().\\\nwrite_parquet(\n    \"validation.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:38.418450Z","iopub.execute_input":"2024-12-20T17:05:38.418873Z","iopub.status.idle":"2024-12-20T17:05:39.241460Z","shell.execute_reply.started":"2024-12-20T17:05:38.418824Z","shell.execute_reply":"2024-12-20T17:05:39.240551Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **XGB Training**","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.metrics import r2_score\nimport pickle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:39.242442Z","iopub.execute_input":"2024-12-20T17:05:39.242697Z","iopub.status.idle":"2024-12-20T17:05:39.583077Z","shell.execute_reply.started":"2024-12-20T17:05:39.242673Z","shell.execute_reply":"2024-12-20T17:05:39.582466Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to initialize XGBoost Model\ndef get_model(config):\n    model = xgb.XGBRegressor(**config.xgb_params)\n    return model\n\n# Initialize Model\nmodel = get_model(CONFIG)\n\n# Train the Model\nprint(\"Training XGBoost Model...\")\nmodel.fit(\n    X_train, y_train,\n    sample_weight=w_train,\n    eval_set=[(X_valid, y_valid)],\n    verbose=True\n)\nprint(\"XGBoost training finished\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:39.584048Z","iopub.execute_input":"2024-12-20T17:05:39.584341Z","iopub.status.idle":"2024-12-20T17:05:53.184190Z","shell.execute_reply.started":"2024-12-20T17:05:39.584314Z","shell.execute_reply":"2024-12-20T17:05:53.182531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Evaluate on Training Set\ny_pred_train = model.predict(X_train)\ntrain_r2 = r2_score(y_train, y_pred_train, sample_weight=w_train)\n#print(f\"Training R² Score: {train_r2:.4f}\")\n\n# Evaluate on Validation Set\ny_pred_valid = model.predict(X_valid)\nvalid_r2 = r2_score(y_valid, y_pred_valid, sample_weight=w_valid)\n#print(f\"Validation R² Score: {valid_r2:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:53.185059Z","iopub.execute_input":"2024-12-20T17:05:53.185336Z","iopub.status.idle":"2024-12-20T17:05:56.557366Z","shell.execute_reply.started":"2024-12-20T17:05:53.185309Z","shell.execute_reply":"2024-12-20T17:05:56.556572Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Saving the Trained Model and Scaler\nresult = {\n    \"model\": model,\n    \"train_r2\": train_r2,\n    \"valid_r2\": valid_r2,\n}\n\nwith open(\"trained_model_0.pkl\", \"wb\") as fp:\n    #pickle.dump(result, fp) \n    pickle.dump(model, fp)   ### save model, not result! ###\n\n#print(\"Model saved to 'trained_model_0.pkl'.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:56.558516Z","iopub.execute_input":"2024-12-20T17:05:56.558858Z","iopub.status.idle":"2024-12-20T17:05:56.569529Z","shell.execute_reply.started":"2024-12-20T17:05:56.558820Z","shell.execute_reply":"2024-12-20T17:05:56.568805Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **XGB Predict** ","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nimport kaggle_evaluation.jane_street_inference_server as js_server\nimport time\nimport random\nimport torch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:05:56.570357Z","iopub.execute_input":"2024-12-20T17:05:56.570614Z","iopub.status.idle":"2024-12-20T17:06:02.593566Z","shell.execute_reply.started":"2024-12-20T17:05:56.570589Z","shell.execute_reply":"2024-12-20T17:06:02.592649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DATA_DIR = Path('/kaggle/input/jane-street-real-time-market-data-forecasting')\nN_PARTITION = 10\n\n#feature_cols = [f'feature_{x:02d}' for x in range(79)]\n#responder_cols = [f'responder_{i}' for i in range(9)]\n\n#meta_cols = ['symbol_id', 'date_id', 'time_id']\n\ntrain_parquets = [\n    DATA_DIR / f\"train.parquet/partition_id={i}/part-0.parquet\" for i in range(N_PARTITION)]\n\nRANDOM_SEED = 2042","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:06:02.594747Z","iopub.execute_input":"2024-12-20T17:06:02.595174Z","iopub.status.idle":"2024-12-20T17:06:02.599697Z","shell.execute_reply.started":"2024-12-20T17:06:02.595146Z","shell.execute_reply":"2024-12-20T17:06:02.598667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n        # Ensure deterministic behavior (may impact performance)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n\ndef lazy_load(par_path):\n    return pl.scan_parquet(par_path).select(\n        pl.int_range(pl.len(), dtype=pl.UInt64).alias(\"index\"),\n        pl.all()\n    )\n\nseed_everything(RANDOM_SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:06:02.600789Z","iopub.execute_input":"2024-12-20T17:06:02.601033Z","iopub.status.idle":"2024-12-20T17:06:02.629125Z","shell.execute_reply.started":"2024-12-20T17:06:02.601002Z","shell.execute_reply":"2024-12-20T17:06:02.628325Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_parquet = DATA_DIR / f'test.parquet'\nlag_parquet =  DATA_DIR / f'lags.parquet'\ntotal_time_steps = 1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:06:02.630014Z","iopub.execute_input":"2024-12-20T17:06:02.630239Z","iopub.status.idle":"2024-12-20T17:06:02.634945Z","shell.execute_reply.started":"2024-12-20T17:06:02.630216Z","shell.execute_reply":"2024-12-20T17:06:02.634028Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_path = \"/kaggle/working/trained_model_0.pkl\"\nwith open(model_path, \"rb\") as fp:\n    model_load = pickle.load(fp)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:06:02.636046Z","iopub.execute_input":"2024-12-20T17:06:02.636327Z","iopub.status.idle":"2024-12-20T17:06:02.651684Z","shell.execute_reply.started":"2024-12-20T17:06:02.636301Z","shell.execute_reply":"2024-12-20T17:06:02.651051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%%time\nX_load = np.load(\"/kaggle/working/X.npy\")\ny_load = np.load(\"/kaggle/working/y.npy\")\nw_load = np.load(\"/kaggle/working/w.npy\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:06:02.652637Z","iopub.execute_input":"2024-12-20T17:06:02.653144Z","iopub.status.idle":"2024-12-20T17:06:02.886554Z","shell.execute_reply.started":"2024-12-20T17:06:02.653115Z","shell.execute_reply":"2024-12-20T17:06:02.885853Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:06:02.887548Z","iopub.execute_input":"2024-12-20T17:06:02.887817Z","iopub.status.idle":"2024-12-20T17:06:03.010921Z","shell.execute_reply.started":"2024-12-20T17:06:02.887791Z","shell.execute_reply":"2024-12-20T17:06:03.010021Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_needed_to_cache = ['date_id', 'weight'] + feature_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:06:03.012075Z","iopub.execute_input":"2024-12-20T17:06:03.012410Z","iopub.status.idle":"2024-12-20T17:06:03.023064Z","shell.execute_reply.started":"2024-12-20T17:06:03.012383Z","shell.execute_reply":"2024-12-20T17:06:03.022217Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"early_stopping_rounds = 5\nearly_stop = xgb.callback.EarlyStopping(\n    rounds=early_stopping_rounds, save_best=True\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:06:03.024088Z","iopub.execute_input":"2024-12-20T17:06:03.024358Z","iopub.status.idle":"2024-12-20T17:06:03.033466Z","shell.execute_reply.started":"2024-12-20T17:06:03.024333Z","shell.execute_reply":"2024-12-20T17:06:03.032826Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class JaneStreetPredictor:\n    \n    def __init__(self, test_parquet, lag_parquet, sequence_len, feature_cols, \n                 X, y, w, X_val, y_val, w_val, model, pbar_length=0):\n        # Initialize model and parameters\n        self.sequence_len = sequence_len\n        self.feature_cols = feature_cols\n        \n        # Initialize parquet data for test and lag\n        self.test_parquet = test_parquet\n        self.lag_parquet = lag_parquet\n        \n        self.X = X        \n        self.y = y\n        self.w = w\n        \n        self.X_val = X_val\n        self.y_val = y_val\n        self.w_val = w_val\n        \n        self.model = model\n        self.refit_model = None\n        self.refit_iter = 0\n        self.scoring = False\n        self.retrained = False\n        self.days_passed = 0\n        \n        # Initialize global variables as class attributes\n        self.history_cache = []\n        self.test_ = None\n        self.lags_ = None\n        self.time_step_count = 0\n        \n        self.lag_stats = None\n\n        # setup pbar:\n        self.pbar = tqdm(total=pbar_length, disable=(pbar_length == 0))\n        self.pbar.clear()\n\n    def run_inference_server(self):\n\n        self.pbar.refresh()\n\n        inference_server = js_server.JSInferenceServer(self.predict)\n        \n        if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n            inference_server.serve()\n        else:\n            inference_server.run_local_gateway((self.test_parquet, self.lag_parquet))\n\n        self.pbar.close()\n\n    def predict(self, test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n        \n        start = time.time()\n        \n        if (lags is not None) and (self.lags_ is not None) :\n            \n            lags_for_merge = self.lags_.clone()\n            print(f'lags_for_merge:{lags_for_merge}')\n            \n            if self.scoring:\n                self.days_passed += 1\n                print(f'self.days_passed: {self.days_passed}')\n\n                lags_for_merge.columns = [i + \"_fitting\" if \"res\" in i else i for i in lags_for_merge.columns]\n                \n                lags_for_merge = lags_for_merge.with_columns([\n                    (pl.col(\"date_id\") - 1).alias(\"date_id\")\n                ])\n                \n                if test['date_id'].to_numpy()[0] > 0:\n                    \n                    test_last_date = pl.concat(self.history_cache)\n\n                    self.history_cache = []\n\n                    test_last_date_merged = test_last_date.join(lags_for_merge.select(['date_id', 'time_id', 'symbol_id', 'responder_6_lag_1_fitting']), how='inner', on=['date_id', 'time_id', 'symbol_id'])\n\n                    X_one = np.array(test_last_date_merged.select(self.feature_cols).to_numpy())\n    \n                    n_to_add = len(X_one)\n                        \n                    self.X = np.vstack([self.X[n_to_add:], X_one])\n                    del X_one\n                \n                    w_one = test_last_date_merged.select(\"weight\")['weight'].to_numpy()\n                    y_one = test_last_date_merged.select('responder_6_lag_1_fitting')['responder_6_lag_1_fitting'].to_numpy() \n\n                    self.w = np.concatenate([self.w[n_to_add:], w_one])\n                    self.y = np.concatenate([self.y[n_to_add:], y_one])\n\n        \n        if self.days_passed > 5:\n            print(\"start retraining\")\n            model = xgb.XGBRegressor(**XGB_Params)\n            self.model = model.fit(self.X, self.y, sample_weight=self.w, xgb_model=self.model, eval_set=[(self.X[-1000000:], self.y[-1000000:])], sample_weight_eval_set=[self.w[-1000000:]], callbacks=[early_stop]) # .get_booster()\n\n            self.days_passed = 0\n            \n            self.retrained = True\n            \n        if lags is not None:\n\n            self.lags_ = lags\n            print(f'self.lags_: {self.lags_}')\n            \n            if self.lags_ is not None:\n\n                lags_for_features = self.lags_.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last() # pick up last record of previous date\n                test = test.join(lags_for_features.drop('time_id'), on=[\"date_id\", \"symbol_id\"],  how=\"left\")\n                #print(f'test, self.lags_ is not None: {test}')\n            \n        else:\n            test = test.with_columns(\n                [pl.lit(0.0).cast(pl.Float32).alias(f'responder_{idx}_lag_1') for idx in range(9)]\n            )\n            print(f'test: {test}')\n\n        \n        self.history_cache.append(test.select(cols_needed_to_cache))\n        #print(f'self.history_cache: {self.history_cache}')\n    \n        X_test = test.select(self.feature_cols).fill_null(-1).to_numpy()\n        #print(f'X_test: {X_test}')\n    \n        preds = self.model.predict(X_test)\n        #print(f'preds: {preds}')\n        \n        predictions = test.select('row_id').with_columns(\n            pl.Series(\n                name   = 'responder_6', \n                values = np.clip(preds, a_min = -5, a_max = 5),\n                dtype  = pl.Float64,\n            )\n        )\n        #print(f'predictions.head(): {predictions.head()}')\n        \n        self.scoring = True\n        \n        # update time_step_count\n        self.time_step_count += 1\n        self.pbar.update(1)\n\n        end = time.time()\n        \n        if self.retrained:\n            print(end - start)\n        \n        self.retrained = False\n        \n        return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:06:03.034446Z","iopub.execute_input":"2024-12-20T17:06:03.034677Z","iopub.status.idle":"2024-12-20T17:06:03.051922Z","shell.execute_reply.started":"2024-12-20T17:06:03.034653Z","shell.execute_reply":"2024-12-20T17:06:03.051232Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"js_predictor = JaneStreetPredictor(\n    test_parquet, \n    lag_parquet, \n    sequence_len = 0, \n    feature_cols = feature_cols,\n    X = X_load,\n    y = y_load,\n    w = w_load,\n    X_val = None,\n    y_val = None,\n    w_val = None,\n    model = model_load,\n    pbar_length = total_time_steps\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:06:03.052777Z","iopub.execute_input":"2024-12-20T17:06:03.052985Z","iopub.status.idle":"2024-12-20T17:06:03.077514Z","shell.execute_reply.started":"2024-12-20T17:06:03.052964Z","shell.execute_reply":"2024-12-20T17:06:03.076584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"js_predictor.run_inference_server()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:06:03.078855Z","iopub.execute_input":"2024-12-20T17:06:03.079074Z","iopub.status.idle":"2024-12-20T17:06:03.317735Z","shell.execute_reply.started":"2024-12-20T17:06:03.079052Z","shell.execute_reply":"2024-12-20T17:06:03.316865Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if os.path.isfile('submission.parquet'):\n    pl_sub = pl.read_parquet('submission.parquet')\n    print(len(pl_sub))\n    display(pl_sub)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T17:06:03.318684Z","iopub.execute_input":"2024-12-20T17:06:03.318927Z","iopub.status.idle":"2024-12-20T17:06:03.326625Z","shell.execute_reply.started":"2024-12-20T17:06:03.318902Z","shell.execute_reply":"2024-12-20T17:06:03.325845Z"}},"outputs":[],"execution_count":null}]}