{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Clear output folder\nimport os\n\ndef remove_folder_contents(folder):\n    if not os.path.exists(folder):\n        print(f\"Directory {folder} does not exist\")\n        return\n        \n    # Add safety check to prevent deleting system directories\n    if not folder.startswith('/kaggle/working'):\n        print(\"Can only delete files in the Kaggle working directory\")\n        return\n        \n    for the_file in os.listdir(folder):\n        file_path = os.path.join(folder, the_file)\n        try:\n            if os.path.isfile(file_path):\n                os.unlink(file_path)\n            elif os.path.isdir(file_path):\n                remove_folder_contents(file_path)\n                os.rmdir(file_path)\n        except Exception as e:\n            print(e)\n\nfolder_path = '/kaggle/working'\nremove_folder_contents(folder_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T19:12:20.540309Z","iopub.execute_input":"2024-12-17T19:12:20.540855Z","iopub.status.idle":"2024-12-17T19:12:20.581188Z","shell.execute_reply.started":"2024-12-17T19:12:20.540798Z","shell.execute_reply":"2024-12-17T19:12:20.580019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport os\nfrom pathlib import Path\n\n\nDATA_PATH = \"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\nclass CONFIG:\n    target = \"responder_6\"\n    lag_col_original = [\"date_id\", \"time_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_col_rename = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1\" for idx in range(9)}\n    valid_ratio = 0.05\n\n\ndata = pl.scan_parquet(DATA_PATH)\\\n.select(CONFIG.lag_col_original)\\\n.with_columns(date_id=pl.col(\"date_id\") + 1)\\\n.rename(CONFIG.lag_col_rename)\n\n# Collect the data and look at basic information\nlag_df = data.collect()\n\noutput_dir = \"processed_data\"\n# Create output directory if it doesn't exist\nPath(output_dir).mkdir(exist_ok=True)\n\n# Convert lag_df to a LazyFrame for efficient joining\nlag_lf = lag_df.lazy()\n\n# Get list of all parquet files in input directory\ninput_files = list(Path(DATA_PATH).rglob(\"*.parquet\"))\n\nfor file_path in input_files:\n    print(f\"Processing {file_path.name}...\")\n    \n    # Read current parquet file\n    current_df = pl.scan_parquet(file_path)\n    \n    # Join with lag features on matching keys\n    augmented_df = current_df.join(\n        lag_lf,\n        on=[\"date_id\", \"time_id\", \"symbol_id\"],\n        how=\"left\"\n    ).collect()\n    \n    # Write to partitioned parquet files\n    output_path = os.path.join(output_dir)\n    augmented_df.write_parquet(\n        output_path,\n        partition_by=[\"date_id\"]\n    )\n    \n    print(f\"Completed processing {file_path}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-17T19:12:20.583598Z","iopub.execute_input":"2024-12-17T19:12:20.583936Z","iopub.status.idle":"2024-12-17T19:16:24.901260Z","shell.execute_reply.started":"2024-12-17T19:12:20.583903Z","shell.execute_reply":"2024-12-17T19:16:24.899513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"FINAL_PATH = Path(\"/kaggle/working/processed_data\")\n\nfiles = [int(p.name.split('=')[-1]) for p in FINAL_PATH.glob(\"*\") if p.is_dir()]\n\nprint(sorted(files))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T19:16:24.903671Z","iopub.execute_input":"2024-12-17T19:16:24.904194Z","iopub.status.idle":"2024-12-17T19:16:24.955189Z","shell.execute_reply.started":"2024-12-17T19:16:24.904134Z","shell.execute_reply":"2024-12-17T19:16:24.953675Z"}},"outputs":[],"execution_count":null}]}