{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nimport os\nfrom tqdm import tqdm\n\nvalid_date_from = 1578\n\nall_train_data = pl.scan_parquet(\n    r\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n)\nvalid_data = (\n    all_train_data\n    .filter(pl.col(\"date_id\") >= valid_date_from)\n    # .filter(pl.col(\"date_id\") <= 1580)\n    .collect()\n)\nvalid_data = (\n    valid_data\n    .with_columns([\n        pl.Series(range(len(valid_data))).alias(\"row_id\"),\n        pl.lit(True).alias(\"is_scored\")\n    ])\n)\n\ntest = pl.read_parquet(\n    r\"/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0\"\n)\nvalid_data = (\n    valid_data\n    .select(test.columns)\n)\n\n\nresponder_clos = [f\"responder_{i}\" for i in range(9)]\ndef make_lag(date_id: int):\n    lag = (\n        all_train_data\n        .filter(pl.col(\"date_id\") == date_id)\n        .select([\"date_id\", \"time_id\", \"symbol_id\"] + responder_clos)\n        .rename({i: f\"{i}_lag_1\" for i in responder_clos})\n        # .filter(pl.col(\"time_id\")==0)\n        .with_columns([\n            pl.col(\"date_id\") + 1\n        ])\n        .collect()\n        \n    )\n    return lag\n\ntotal_iterations = len(valid_data[\"date_id\"].unique())\n\n\nfor num_days, df_per_day in tqdm(\n    valid_data.group_by(\"date_id\", maintain_order=True), \n    total=total_iterations,\n    desc=\"Processing\"):\n    \n    day = num_days[0] - valid_date_from\n    os.makedirs(\n        f\"test.parquet/date_id={str(day)}\",exist_ok=True)\n    os.makedirs(\n        f\"lags.parquet/date_id={str(day)}\",exist_ok=True)\n    lag = make_lag(num_days[0] - 1)\n    # print(df_per_day)\n    # print(lag)\n    df_per_day.write_parquet(\n        f\"test.parquet/date_id={str(day)}/part-0.parquet\"\n    )\n    lag.write_parquet(\n        f\"lags.parquet/date_id={str(day)}/part-0.parquet\"\n    )\n    ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:23:06.419117Z","iopub.execute_input":"2024-12-12T07:23:06.420054Z","iopub.status.idle":"2024-12-12T07:23:24.8151Z","shell.execute_reply.started":"2024-12-12T07:23:06.420011Z","shell.execute_reply":"2024-12-12T07:23:24.813763Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}