{"metadata":{"kernelspec":{"display_name":"base","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":144.544038,"end_time":"2024-10-28T10:53:12.391324","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-10-28T10:50:47.847286","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"10cb1725","cell_type":"markdown","source":"# Libraries","metadata":{"papermill":{"duration":0.004507,"end_time":"2024-10-28T10:50:50.680870","exception":false,"start_time":"2024-10-28T10:50:50.676363","status":"completed"},"tags":[]}},{"id":"f62add46","cell_type":"code","source":"import polars as pl","metadata":{"execution":{"iopub.execute_input":"2024-10-28T10:50:50.690409Z","iopub.status.busy":"2024-10-28T10:50:50.689985Z","iopub.status.idle":"2024-10-28T10:50:53.445438Z","shell.execute_reply":"2024-10-28T10:50:53.444399Z"},"papermill":{"duration":2.762986,"end_time":"2024-10-28T10:50:53.447975","exception":false,"start_time":"2024-10-28T10:50:50.684989","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"d88bbf85","cell_type":"markdown","source":"# Configurations","metadata":{"papermill":{"duration":0.00385,"end_time":"2024-10-28T10:50:53.456099","exception":false,"start_time":"2024-10-28T10:50:53.452249","status":"completed"},"tags":[]}},{"id":"95b20029","cell_type":"code","source":"class CONFIG:\n    target_col = \"responder_6\"\n    lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_rename = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1\" for idx in range(9)}\n    valid_ratio = 0.05\n    start_dt = 1100","metadata":{"execution":{"iopub.execute_input":"2024-10-28T10:50:53.465769Z","iopub.status.busy":"2024-10-28T10:50:53.465156Z","iopub.status.idle":"2024-10-28T10:50:53.471976Z","shell.execute_reply":"2024-10-28T10:50:53.470759Z"},"papermill":{"duration":0.014164,"end_time":"2024-10-28T10:50:53.474208","exception":false,"start_time":"2024-10-28T10:50:53.460044","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"ac3b364d","cell_type":"markdown","source":"# Load training data","metadata":{"papermill":{"duration":0.003748,"end_time":"2024-10-28T10:50:53.481903","exception":false,"start_time":"2024-10-28T10:50:53.478155","status":"completed"},"tags":[]}},{"id":"107eaeea","cell_type":"code","source":"# Use last 2 parquets\ntrain = pl.scan_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n).select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),\n).with_columns(\n    (pl.col(CONFIG.target_col)*2).cast(pl.Int32).alias(\"label\"),\n).filter(\n    pl.col(\"date_id\").gt(CONFIG.start_dt)\n)","metadata":{"execution":{"iopub.execute_input":"2024-10-28T10:50:53.491659Z","iopub.status.busy":"2024-10-28T10:50:53.491243Z","iopub.status.idle":"2024-10-28T10:50:53.528349Z","shell.execute_reply":"2024-10-28T10:50:53.527277Z"},"papermill":{"duration":0.04486,"end_time":"2024-10-28T10:50:53.530904","exception":false,"start_time":"2024-10-28T10:50:53.486044","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"5d6893d5","cell_type":"markdown","source":"# Create Lags data from training data","metadata":{"papermill":{"duration":0.003601,"end_time":"2024-10-28T10:50:53.538424","exception":false,"start_time":"2024-10-28T10:50:53.534823","status":"completed"},"tags":[]}},{"id":"4888f45b","cell_type":"code","source":"lags = train.select(pl.col(CONFIG.lag_cols_original))\nlags = lags.rename(CONFIG.lag_cols_rename)\nlags = lags.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )\nlags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()  # pick up last record of previous date\n#lags","metadata":{"execution":{"iopub.execute_input":"2024-10-28T10:50:53.547969Z","iopub.status.busy":"2024-10-28T10:50:53.547541Z","iopub.status.idle":"2024-10-28T10:50:53.893220Z","shell.execute_reply":"2024-10-28T10:50:53.892075Z"},"papermill":{"duration":0.353197,"end_time":"2024-10-28T10:50:53.895572","exception":false,"start_time":"2024-10-28T10:50:53.542375","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"2114e63c","cell_type":"markdown","source":"# Merge training data and lags data","metadata":{"papermill":{"duration":0.003961,"end_time":"2024-10-28T10:50:53.904002","exception":false,"start_time":"2024-10-28T10:50:53.900041","status":"completed"},"tags":[]}},{"id":"c9be8650","cell_type":"code","source":"train = train.join(lags, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\n#train","metadata":{"execution":{"iopub.execute_input":"2024-10-28T10:50:53.915910Z","iopub.status.busy":"2024-10-28T10:50:53.915156Z","iopub.status.idle":"2024-10-28T10:50:53.945165Z","shell.execute_reply":"2024-10-28T10:50:53.944004Z"},"papermill":{"duration":0.038525,"end_time":"2024-10-28T10:50:53.947807","exception":false,"start_time":"2024-10-28T10:50:53.909282","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"1f54a3c1","cell_type":"markdown","source":"# Split training data and validation data","metadata":{"papermill":{"duration":0.004368,"end_time":"2024-10-28T10:50:53.957063","exception":false,"start_time":"2024-10-28T10:50:53.952695","status":"completed"},"tags":[]}},{"id":"c37b5d81","cell_type":"code","source":"len_train   = train.select(pl.col(\"date_id\")).collect().shape[0]\nvalid_records = int(len_train * CONFIG.valid_ratio)\nlen_ofl_mdl = len_train - valid_records\nlast_tr_dt  = train.select(pl.col(\"date_id\")).collect().row(len_ofl_mdl)[0]\n\nprint(f\"\\n len_train = {len_train}\")\nprint(f\"\\n len_ofl_mdl = {len_ofl_mdl}\")\nprint(f\"\\n---> Last offline train date = {last_tr_dt}\\n\")\n\ntraining_data = train.filter(pl.col(\"date_id\").le(last_tr_dt))\nvalidation_data   = train.filter(pl.col(\"date_id\").gt(last_tr_dt))","metadata":{"execution":{"iopub.execute_input":"2024-10-28T10:50:53.968024Z","iopub.status.busy":"2024-10-28T10:50:53.967611Z","iopub.status.idle":"2024-10-28T10:50:57.287742Z","shell.execute_reply":"2024-10-28T10:50:57.286619Z"},"papermill":{"duration":3.328535,"end_time":"2024-10-28T10:50:57.290290","exception":false,"start_time":"2024-10-28T10:50:53.961755","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"81fe7c1e","cell_type":"markdown","source":"# Save data as parquets","metadata":{"papermill":{"duration":0.004758,"end_time":"2024-10-28T10:50:57.340849","exception":false,"start_time":"2024-10-28T10:50:57.336091","status":"completed"},"tags":[]}},{"id":"6c7b24f5","cell_type":"code","source":"training_data.collect().\\\nwrite_parquet(\n    f\"training.parquet\", partition_by = \"date_id\",\n)","metadata":{"execution":{"iopub.execute_input":"2024-10-28T10:50:57.352891Z","iopub.status.busy":"2024-10-28T10:50:57.352024Z","iopub.status.idle":"2024-10-28T10:52:19.675611Z","shell.execute_reply":"2024-10-28T10:52:19.674557Z"},"papermill":{"duration":82.332599,"end_time":"2024-10-28T10:52:19.678428","exception":false,"start_time":"2024-10-28T10:50:57.345829","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"8dacc686","cell_type":"code","source":"validation_data.collect().\\\nwrite_parquet(\n    \"validation.parquet\", partition_by = \"date_id\",\n)","metadata":{"execution":{"iopub.execute_input":"2024-10-28T10:52:19.691137Z","iopub.status.busy":"2024-10-28T10:52:19.690690Z","iopub.status.idle":"2024-10-28T10:53:09.731770Z","shell.execute_reply":"2024-10-28T10:53:09.730583Z"},"papermill":{"duration":50.050795,"end_time":"2024-10-28T10:53:09.735054","exception":false,"start_time":"2024-10-28T10:52:19.684259","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}