{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":84493,"databundleVersionId":9871156},{"sourceType":"datasetVersion","sourceId":10460393,"datasetId":6468162,"databundleVersionId":10782636},{"sourceType":"datasetVersion","sourceId":10460392,"datasetId":6468154,"databundleVersionId":10782634},{"sourceType":"kernelVersion","sourceId":215844687}],"dockerImageVersionId":30823,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":7.594014,"end_time":"2024-10-10T11:58:36.355301","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-10-10T11:58:28.761287","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import os\nos.system('pip install --force-reinstall /kaggle/input/janestreet2025-code/janestreet-0.1-py3-none-any.whl')\n\nimport time\nimport copy\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport torch \n\nfrom kaggle_evaluation import jane_street_inference_server\n\nfrom janestreet.pipeline import FullPipeline, PipelineCV\nfrom janestreet.data_processor import DataProcessor\nfrom janestreet.config import PATH_DATA","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T17:49:13.861507Z","iopub.execute_input":"2025-01-13T17:49:13.861852Z","iopub.status.idle":"2025-01-13T17:49:21.147987Z","shell.execute_reply.started":"2025-01-13T17:49:13.861825Z","shell.execute_reply":"2025-01-13T17:49:21.146946Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Settings","metadata":{}},{"cell_type":"code","source":"RUN_NAME = \"full\"\nMODEL_NAMES = [\n    \"gru_2.0_700\",\n    \"gru_2.1_700\",\n    \"gru_2.2_700\",\n    \"gru_2.3_700\",\n    \"gru_2.4_700\",\n    \"gru_2.5_700\",\n    \"gru_2.6_700\",\n    \"gru_2.7_700\",\n    \"gru_2.8_700\",\n    \"gru_2.9_700\",\n]\nWEIGHTS = np.array([1.0]*len(MODEL_NAMES))/ len(MODEL_NAMES)\nWEIGHTS = WEIGHTS/sum(WEIGHTS)\nN_ROLL = 1000\n\nprint(\"_\".join(MODEL_NAMES))\nprint(WEIGHTS)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T17:49:21.149177Z","iopub.execute_input":"2025-01-13T17:49:21.149460Z","iopub.status.idle":"2025-01-13T17:49:21.156148Z","shell.execute_reply.started":"2025-01-13T17:49:21.149436Z","shell.execute_reply":"2025-01-13T17:49:21.155314Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load models","metadata":{}},{"cell_type":"code","source":"data_processor = DataProcessor(MODEL_NAMES[0]).load()\n\npipelines = {}\nfor model_name in MODEL_NAMES:\n    print(\"-\"*100)\n\n    pipeline = FullPipeline(\n        None,\n        run_name=RUN_NAME,\n        name=model_name,\n        load_model=True,\n        features=None,\n        save_to_disc=False\n    )\n    pipeline.fit(verbose=True)\n    pipeline.model.lr_refit = 0.0006\n    pipelines[model_name] = pipeline\n    print(model_name)\n    print(pipeline.model.get_params())\n    print(f\"Number of features: {len(pipeline.features)}\")\n    print(f\"Number of auxiliary targets: {pipeline.model.model.num_resp}\")\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.018344,"end_time":"2024-10-10T11:58:33.59684","exception":false,"start_time":"2024-10-10T11:58:33.578496","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T17:49:32.480467Z","iopub.execute_input":"2025-01-13T17:49:32.480816Z","iopub.status.idle":"2025-01-13T17:49:34.870779Z","shell.execute_reply.started":"2025-01-13T17:49:32.480790Z","shell.execute_reply":"2025-01-13T17:49:34.869959Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load tail of train data for rolling features calculation","metadata":{}},{"cell_type":"code","source":"MAX_DATE = 1698\nCOLS_ID = ['row_id', 'date_id', 'time_id', 'symbol_id', 'weight', 'is_scored']\n\ndf_raw = pl.scan_parquet(f\"{PATH_DATA}/train.parquet\")\ndf_raw = df_raw.filter(pl.col(\"date_id\")>=MAX_DATE-10)\ndf_raw = df_raw.collect()\ndf_raw = df_raw.with_columns(\n    pl.lit(-1).cast(pl.Int64).alias(\"row_id\"),\n    pl.lit(True).alias(\"is_scored\"),\n    (pl.col(\"date_id\")-MAX_DATE-1).alias(\"date_id\")\n)\ndf_raw = df_raw.select(COLS_ID + data_processor.COLS_FEATURES_INIT)\n\ndf_raw = (\n    df_raw.filter(pl.col(\"date_id\") >= -5)\n    .sort(['date_id', 'time_id', 'symbol_id'])\n)\n\nhidden_states = [None] * len(pipelines)\ndfs = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T17:49:36.022503Z","iopub.execute_input":"2025-01-13T17:49:36.022831Z","iopub.status.idle":"2025-01-13T17:49:36.441496Z","shell.execute_reply.started":"2025-01-13T17:49:36.022807Z","shell.execute_reply":"2025-01-13T17:49:36.440385Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Predict","metadata":{}},{"cell_type":"code","source":"DEBUG = False\n\nCNT_DATES = 9\nCNT_DATES_NOT_SCORED = 4\n\ntime_start = time.time()\ntime_start_not_scored = time.time()\ntime_est = 0\ntime_est_not_scored = 0\ncnt_dates = 0\n\ndef predict(test: pl.DataFrame, lags: pl.DataFrame | None) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    start_time = time.time()\n    \n    global df_raw\n    global hidden_states\n    global pipeline\n    global dfs\n    global time_est, time_start, time_start_not_scored, time_est_not_scored\n    global cnt_dates\n    \n    date_id = test[\"date_id\"][0]\n    time_id = test[\"time_id\"][0]\n    is_scored = test[\"is_scored\"][0]\n\n    # Count time for debug\n    if DEBUG:\n        if not is_scored:\n            time_est_not_scored = time.time()-time_start_not_scored\n        else:\n            time_est = time.time()-time_start\n\n        if time_id == 0:            \n            if date_id == 1:\n                time_start_not_scored = time.time()\n    \n            if date_id == CNT_DATES_NOT_SCORED: \n                time_start = time.time()\n\n    # Reset hidden states and collect data for weights update\n    if time_id == 0:\n        cnt_dates += 1\n        hidden_states = [None for _, p in pipelines.items()]\n        lags = lags.with_columns(\n            pl.col(\"responder_6_lag_1\").alias(\"responder_6\"),\n            pl.lit(date_id-1).cast(pl.Int16).alias(\"date_id\")\n        ).select([\"date_id\", \"time_id\", \"symbol_id\", \"responder_6\"])\n        if cnt_dates > 1:\n            df = pl.concat(dfs)\n            dfs = []\n            df = df.join(lags, on=[\"date_id\", \"time_id\", \"symbol_id\"], how=\"left\")\n            df = df.sort([\"date_id\", \"time_id\", \"symbol_id\"])\n\n    # Add data to the raw dataframe\n    test = test.select(df_raw.columns)\n    df_raw = pl.concat([df_raw, test], how=\"vertical_relaxed\")\n    df_raw = df_raw.select(test.columns)\n    \n    # Cut raw data (keep last N_ROLL time_ids for each symbol)\n    df_raw = (\n        df_raw\n        .group_by([\"symbol_id\"])\n        .tail(N_ROLL)\n    )\n\n    # Calculate features and save\n    df_cur = data_processor.process_test_data(df_raw, fast=True, date_id=date_id, time_id=time_id, symbols=test[\"symbol_id\"])\n    df_cur = df_cur.sort([\"symbol_id\"])\n    dfs.append(df_cur)\n    df_cur = df_cur.with_columns(pl.lit(None).alias(\"responder_6\"))\n    \n    # Update model weights\n    if (time_id == 0) & (cnt_dates > 1):\n        if len(df) > 968:\n            for i, (name, pipeline) in enumerate(pipelines.items()):\n                pipeline.update(df)\n\n    # Make predictions\n    if is_scored:\n        preds = []\n        for i, (name, pipeline) in enumerate(pipelines.items()):\n            pred, hidden_states[i] = pipeline.predict(df_cur, hidden=hidden_states[i], n_times=1)\n            preds.append(pred)\n        pred = np.average(preds, axis=0, weights=WEIGHTS)\n    \n        df_cur = df_cur.with_columns(pl.Series(\"responder_6\", pred))\n        df_cur = test.select([\"date_id\", \"time_id\", \"symbol_id\"]).join(df_cur, on=[\"date_id\", \"time_id\", \"symbol_id\"], how=\"left\")\n        predictions = df_cur.select([\"row_id\", \"responder_6\"])\n    else:\n        predictions = test.select(\n            'row_id',\n            pl.lit(0.0).alias('responder_6'),\n        )\n\n    if DEBUG:\n        if time_id == 0:\n            print(\"-\" * 100)\n        if time_id % 100 == 0:\n            n_nans = sum(sum(predictions.fill_nan(None).null_count().to_numpy()))\n            print(\n                f\"{date_id} {time_id:3.0f} (is_scored {is_scored}): \"\n                f\"{time.time()-start_time:.4f}s/it, # nans {n_nans}\"\n            )\n    else:\n        if (time_id==0)&(date_id==0):\n            print(predictions)\n            print((time_id, time.time()-start_time))\n    \n    return predictions\n\n    \ninference_server = jane_street_inference_server.JSInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    if not DEBUG:\n        inference_server.run_local_gateway(\n            (\n                f'{PATH_DATA}/test.parquet',\n                f'{PATH_DATA}/lags.parquet',\n            )\n        )\n    else:\n        inference_server.run_local_gateway(\n            (\n                '/kaggle/input/js24-rmf-submission-api-debug-with-synthetic-test/synthetic_test.parquet',\n                '/kaggle/input/js24-rmf-submission-api-debug-with-synthetic-test/synthetic_lag.parquet',\n            )\n        )\n\nif DEBUG:\n    time_est_cur = time_est/(CNT_DATES-CNT_DATES_NOT_SCORED)*200/60/60\n    time_est_scored = time_est/(CNT_DATES-CNT_DATES_NOT_SCORED)*120/60/60\n    time_est_not_scored = time_est_not_scored/(CNT_DATES_NOT_SCORED-1)*240/60/60\n    time_est_final = time_est_scored + time_est_not_scored\n    print(\"-\"*100)\n    print(f\"Estimated current time: {time_est_cur:.4f}\")\n    print(f\"Estimated final time (is_score=True): {time_est_scored:.4f}\")\n    print(f\"Estimated final time (is_score=False): {time_est_not_scored:.4f}\")\n    print(f\"Estimated final time: {time_est_final:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-13T17:49:36.442825Z","iopub.execute_input":"2025-01-13T17:49:36.443090Z","iopub.status.idle":"2025-01-13T17:49:36.864450Z","shell.execute_reply.started":"2025-01-13T17:49:36.443068Z","shell.execute_reply":"2025-01-13T17:49:36.863036Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}