{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"},{"sourceId":9742909,"sourceType":"datasetVersion","datasetId":4143466},{"sourceId":9970780,"sourceType":"datasetVersion","datasetId":6134141},{"sourceId":201579302,"sourceType":"kernelVersion"},{"sourceId":204142199,"sourceType":"kernelVersion"},{"sourceId":204352628,"sourceType":"kernelVersion"}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# https://www.kaggle.com/code/motono0223/js24-preprocessing-create-lags\n# https://www.kaggle.com/code/shiyili/js24-rmf-submission-api-debug-with-synthetic-test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T09:17:26.485392Z","iopub.execute_input":"2024-11-22T09:17:26.485668Z","iopub.status.idle":"2024-11-22T09:17:26.494001Z","shell.execute_reply.started":"2024-11-22T09:17:26.485642Z","shell.execute_reply":"2024-11-22T09:17:26.493189Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import sys\nprint(sys.version)\n\nimport lightgbm as lgb\nprint(\"lightgbm version: \", lgb.__version__)\n\nimport sklearn\nprint(\"sklearn version: \", sklearn.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T09:57:56.165769Z","iopub.execute_input":"2024-11-22T09:57:56.166575Z","iopub.status.idle":"2024-11-22T09:58:02.032934Z","shell.execute_reply.started":"2024-11-22T09:57:56.166531Z","shell.execute_reply":"2024-11-22T09:58:02.031967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport json \nimport random \nfrom pathlib import Path\nfrom tqdm import tqdm\nimport joblib\nfrom time import time\n\nfrom collections import OrderedDict, defaultdict\nfrom functools import reduce, partial\n\nimport polars as pl\nimport pandas as pd \nimport numpy as np\nfrom matplotlib import pyplot as plt\nimport pickle\nimport torch\nimport xgboost as xgb\nimport kaggle_evaluation.jane_street_inference_server as js_server\nimport time\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-22T09:58:02.034377Z","iopub.execute_input":"2024-11-22T09:58:02.034852Z","iopub.status.idle":"2024-11-22T09:58:06.624860Z","shell.execute_reply.started":"2024-11-22T09:58:02.034823Z","shell.execute_reply":"2024-11-22T09:58:06.624132Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DATA_DIR = Path('/kaggle/input/jane-street-real-time-market-data-forecasting')\nN_PARTITION = 10\n\nfeature_cols = [f'feature_{x:02}' for x in range(79)]\nresponder_cols = [f'responder_{i}' for i in range(9)]\n\nmeta_cols = ['symbol_id', 'date_id', 'time_id']\n\ntrain_parquets = [\n    DATA_DIR / f\"train.parquet/partition_id={i}/part-0.parquet\" for i in range(N_PARTITION)]\n\nRANDOM_SEED = 2042","metadata":{"execution":{"iopub.status.busy":"2024-11-22T09:58:06.626432Z","iopub.execute_input":"2024-11-22T09:58:06.627090Z","iopub.status.idle":"2024-11-22T09:58:06.632842Z","shell.execute_reply.started":"2024-11-22T09:58:06.627046Z","shell.execute_reply":"2024-11-22T09:58:06.631819Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n        # Ensure deterministic behavior (may impact performance)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n\ndef lazy_load(par_path):\n    return pl.scan_parquet(par_path).select(\n        pl.int_range(pl.len(), dtype=pl.UInt64).alias(\"index\"),\n        pl.all()\n    )\n\nseed_everything(RANDOM_SEED)","metadata":{"execution":{"iopub.status.busy":"2024-11-22T09:58:06.634977Z","iopub.execute_input":"2024-11-22T09:58:06.635301Z","iopub.status.idle":"2024-11-22T09:58:06.701079Z","shell.execute_reply.started":"2024-11-22T09:58:06.635259Z","shell.execute_reply":"2024-11-22T09:58:06.700442Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Use synthetic test or real test?","metadata":{}},{"cell_type":"code","source":"USE_STYNTHETIC = False\n\nif USE_STYNTHETIC:\n    syn_dir = '/kaggle/input/js-write-synthetic-tests'\n    test_parquet = f'{syn_dir}/synthetic_test.parquet'\n    lag_parquet = f'{syn_dir}/synthetic_lag.parquet'\n    total_time_steps = pl.scan_parquet(test_parquet).unique(subset=[\"date_id\", \"time_id\"]).select(pl.len()).collect().item()\nelse:\n    test_parquet = DATA_DIR / f'test.parquet'\n    lag_parquet =  DATA_DIR / f'lags.parquet'\n    total_time_steps = 1\n    \nprint(\"Test parquet:\", test_parquet)\nprint(\"Lag parquet:\", lag_parquet)\nprint(\"Total time steps:\", total_time_steps)","metadata":{"execution":{"iopub.status.busy":"2024-11-22T09:58:06.702083Z","iopub.execute_input":"2024-11-22T09:58:06.702386Z","iopub.status.idle":"2024-11-22T09:58:06.708495Z","shell.execute_reply.started":"2024-11-22T09:58:06.702359Z","shell.execute_reply":"2024-11-22T09:58:06.707604Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_path = \"/kaggle/input/js-xgb-v1/xgb.json\"\n\n# with open( model_path, \"rb\") as fp:\n#     model = pickle.load(fp)\n\nmodel = xgb.XGBRegressor({'nthread': 4})  # init model\nmodel.load_model(model_path)  # load data","metadata":{"execution":{"iopub.status.busy":"2024-11-22T09:58:06.709335Z","iopub.execute_input":"2024-11-22T09:58:06.709551Z","iopub.status.idle":"2024-11-22T09:58:06.815575Z","shell.execute_reply.started":"2024-11-22T09:58:06.709528Z","shell.execute_reply":"2024-11-22T09:58:06.814736Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load Model","metadata":{}},{"cell_type":"code","source":"XGB_Params = {\n    'learning_rate': 0.01,\n    'max_depth': 6,\n    'n_estimators': 100,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,\n    'reg_lambda': 5,\n    'random_state': 42,\n    'tree_method': 'hist',\n    'device' : 'cuda',\n}","metadata":{"execution":{"iopub.status.busy":"2024-11-22T09:58:06.816592Z","iopub.execute_input":"2024-11-22T09:58:06.816854Z","iopub.status.idle":"2024-11-22T09:58:06.821059Z","shell.execute_reply.started":"2024-11-22T09:58:06.816829Z","shell.execute_reply":"2024-11-22T09:58:06.820265Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\nX = np.load(\"/kaggle/input/js24-preprocessing-create-lags/X.npy\")\ny = np.load(\"/kaggle/input/js24-preprocessing-create-lags/y.npy\")\nw = np.load(\"/kaggle/input/js24-preprocessing-create-lags/w.npy\")","metadata":{"execution":{"iopub.status.busy":"2024-11-22T09:58:06.822213Z","iopub.execute_input":"2024-11-22T09:58:06.822568Z","iopub.status.idle":"2024-11-22T09:58:42.235487Z","shell.execute_reply.started":"2024-11-22T09:58:06.822532Z","shell.execute_reply":"2024-11-22T09:58:42.234534Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(X) // 10 * 9","metadata":{"execution":{"iopub.status.busy":"2024-11-22T09:59:00.806126Z","iopub.execute_input":"2024-11-22T09:59:00.806489Z","iopub.status.idle":"2024-11-22T09:59:00.811953Z","shell.execute_reply.started":"2024-11-22T09:59:00.806457Z","shell.execute_reply":"2024-11-22T09:59:00.811123Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time \nX = X[19893852:]\ny = y[19893852:]\nw = w[19893852:]","metadata":{"execution":{"iopub.status.busy":"2024-11-22T09:28:29.687501Z","iopub.execute_input":"2024-11-22T09:28:29.688247Z","iopub.status.idle":"2024-11-22T09:28:29.693090Z","shell.execute_reply.started":"2024-11-22T09:28:29.688218Z","shell.execute_reply":"2024-11-22T09:28:29.692127Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-11-22T09:28:47.348899Z","iopub.execute_input":"2024-11-22T09:28:47.349429Z","iopub.status.idle":"2024-11-22T09:28:47.501249Z","shell.execute_reply.started":"2024-11-22T09:28:47.349399Z","shell.execute_reply":"2024-11-22T09:28:47.500265Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_cols = [\"symbol_id\", \"time_id\"] + [f\"feature_{idx:02d}\" for idx in range(79)]+ [f\"responder_{idx}_lag_1\" for idx in range(9)]","metadata":{"execution":{"iopub.status.busy":"2024-11-22T09:28:47.589930Z","iopub.execute_input":"2024-11-22T09:28:47.590204Z","iopub.status.idle":"2024-11-22T09:28:47.594416Z","shell.execute_reply.started":"2024-11-22T09:28:47.590179Z","shell.execute_reply":"2024-11-22T09:28:47.593615Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_needed_to_cache = ['date_id', 'weight'] + feature_cols","metadata":{"execution":{"iopub.status.busy":"2024-11-22T09:28:47.935926Z","iopub.execute_input":"2024-11-22T09:28:47.936299Z","iopub.status.idle":"2024-11-22T09:28:47.940334Z","shell.execute_reply.started":"2024-11-22T09:28:47.936252Z","shell.execute_reply":"2024-11-22T09:28:47.939411Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def r2_xgb(y_true, y_pred, sample_weight=None):\n    r2 = 1 - np.average((y_pred - y_true) ** 2, weights=sample_weight) / (np.average((y_true) ** 2, weights=sample_weight) + 1e-38)\n    return -r2","metadata":{"execution":{"iopub.status.busy":"2024-11-22T09:28:48.332368Z","iopub.execute_input":"2024-11-22T09:28:48.332976Z","iopub.status.idle":"2024-11-22T09:28:48.337269Z","shell.execute_reply.started":"2024-11-22T09:28:48.332945Z","shell.execute_reply":"2024-11-22T09:28:48.336333Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"early_stopping_rounds = 5\nearly_stop = xgb.callback.EarlyStopping(\n    rounds=early_stopping_rounds, save_best=True\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T09:28:48.562251Z","iopub.execute_input":"2024-11-22T09:28:48.562899Z","iopub.status.idle":"2024-11-22T09:28:48.566756Z","shell.execute_reply.started":"2024-11-22T09:28:48.562868Z","shell.execute_reply":"2024-11-22T09:28:48.565768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class JaneStreetPredictor:\n    \n    def __init__(self, test_parquet, lag_parquet, sequence_len, feature_cols, X, y, w, X_val, y_val, w_val, model, pbar_length=0):\n        # Initialize model and parameters\n        self.sequence_len = sequence_len\n        self.feature_cols = feature_cols\n        \n        # Initialize parquet data for test and lag\n        self.test_parquet = test_parquet\n        self.lag_parquet = lag_parquet\n        \n        self.X = X        \n        self.y = y\n        self.w = w\n        \n        self.X_val = X_val\n        self.y_val = y_val\n        self.w_val = w_val\n        \n        self.model = model\n        self.refit_model = None\n        self.refit_iter = 0\n        self.scoring = False\n        self.retrained = False\n        self.days_passed = 0\n        \n        # Initialize global variables as class attributes\n        self.history_cache = []\n        self.test_ = None\n        self.lags_ = None\n        self.time_step_count = 0\n        \n        self.lag_stats = None\n\n        # setup pbar:\n        self.pbar = tqdm(total=pbar_length, disable=(pbar_length == 0))\n        self.pbar.clear()\n\n    def run_inference_server(self):\n\n        self.pbar.refresh()\n\n        inference_server = js_server.JSInferenceServer(self.predict)\n        \n        if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n            inference_server.serve()\n        else:\n            inference_server.run_local_gateway((self.test_parquet, self.lag_parquet))\n\n        self.pbar.close()\n\n    def predict(self, test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n        \n        start = time.time()\n        \n        if (lags is not None) and (self.lags_ is not None) :\n            \n            lags_for_merge = self.lags_.clone()\n            if self.scoring:\n                self.days_passed += 1\n\n                lags_for_merge.columns = [i + \"_fitting\" if \"res\" in i else i for i in lags_for_merge.columns]\n                \n                lags_for_merge = lags_for_merge.with_columns([\n                    (pl.col(\"date_id\") - 1).alias(\"date_id\")\n                ])\n                \n                if test['date_id'].to_numpy()[0] > 0:\n                    \n                    test_last_date = pl.concat(self.history_cache)\n\n                    self.history_cache = []\n\n                    test_last_date_merged = test_last_date.join(lags_for_merge.select(['date_id', 'time_id', 'symbol_id', 'responder_6_lag_1_fitting']), how='inner', on=['date_id', 'time_id', 'symbol_id'])\n\n                    X_one = np.array(test_last_date_merged.select(self.feature_cols).to_numpy())\n    \n                    n_to_add = len(X_one)\n                        \n                    self.X = np.vstack([self.X[n_to_add:], X_one])\n                    del X_one\n                \n                    w_one = test_last_date_merged.select(\"weight\")['weight'].to_numpy()\n                    y_one = test_last_date_merged.select('responder_6_lag_1_fitting')['responder_6_lag_1_fitting'].to_numpy() \n\n                    self.w = np.concatenate([self.w[n_to_add:], w_one])\n                    self.y = np.concatenate([self.y[n_to_add:], y_one])\n        \n        if self.days_passed > 5:\n            print(\"start retraining\")\n            model = xgb.XGBRegressor(**XGB_Params)\n            self.model = model.fit(self.X, self.y, sample_weight=self.w, xgb_model=self.model, eval_set=[(self.X[-1000000:], self.y[-1000000:])], sample_weight_eval_set=[self.w[-1000000:]], callbacks=[early_stop]) # .get_booster()\n\n            self.days_passed = 0\n            \n            self.retrained = True\n            \n        if lags is not None:\n\n            self.lags_ = lags\n            if self.lags_ is not None:\n\n                lags_for_features = self.lags_.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last() # pick up last record of previous date\n                test = test.join(lags_for_features.drop('time_id'), on=[\"date_id\", \"symbol_id\"],  how=\"left\")\n            \n        else:\n            test = test.with_columns(\n                ( pl.lit(0.0).cast(pl.Float32).alias(f'responder_{idx}_lag_1') for idx in range(9) )\n            )\n\n        \n        self.history_cache.append(test.select(cols_needed_to_cache))\n    \n        X_test = test.select(self.feature_cols).fill_null(-1).to_numpy()\n    \n        preds = self.model.predict(X_test)\n        \n        predictions = test.select('row_id').with_columns(\n            pl.Series(\n                name   = 'responder_6', \n                values = np.clip(preds, a_min = -5, a_max = 5),\n                dtype  = pl.Float64,\n            )\n        )\n        \n        self.scoring = True\n        \n        # update time_step_count\n        self.time_step_count += 1\n        self.pbar.update(1)\n\n        end = time.time()\n        \n        if self.retrained:\n            print(end - start)\n        \n        self.retrained = False\n        \n        return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T09:28:48.791298Z","iopub.execute_input":"2024-11-22T09:28:48.791612Z","iopub.status.idle":"2024-11-22T09:28:48.807950Z","shell.execute_reply.started":"2024-11-22T09:28:48.791585Z","shell.execute_reply":"2024-11-22T09:28:48.807113Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Score time in the LB is roughly 30x longer than the inference time with synthetic data.**","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"js_predictor = JaneStreetPredictor(\n    test_parquet, \n    lag_parquet, \n    sequence_len = 0, \n    feature_cols = feature_cols,\n    X = X,\n    y = y,\n    w = w,\n    X_val = None,\n    y_val = None,\n    w_val = None,\n    model = model,\n    pbar_length = total_time_steps\n)\n\njs_predictor.run_inference_server()","metadata":{"execution":{"iopub.status.busy":"2024-11-22T09:28:50.406978Z","iopub.execute_input":"2024-11-22T09:28:50.407713Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if os.path.isfile('submission.parquet'):\n    pl_sub = pl.read_parquet('submission.parquet')\n    print(len(pl_sub))\n    display(pl_sub)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}