{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport gc\nfrom matplotlib import pyplot as plt\nimport matplotlib.cm as cm\nfrom sklearn.model_selection import StratifiedGroupKFold","metadata":{"execution":{"iopub.status.busy":"2024-11-21T07:22:56.234712Z","iopub.execute_input":"2024-11-21T07:22:56.235144Z","iopub.status.idle":"2024-11-21T07:22:59.891117Z","shell.execute_reply.started":"2024-11-21T07:22:56.235103Z","shell.execute_reply":"2024-11-21T07:22:59.889846Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Configurations","metadata":{}},{"cell_type":"code","source":"class CONFIG:\n    target_col = \"responder_6\"\n    lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_rename = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1\" for idx in range(9)}\n    valid_ratio = 0.05\n    start_dt = 1100","metadata":{"execution":{"iopub.status.busy":"2024-11-21T07:23:04.95267Z","iopub.execute_input":"2024-11-21T07:23:04.953228Z","iopub.status.idle":"2024-11-21T07:23:04.961703Z","shell.execute_reply.started":"2024-11-21T07:23:04.953193Z","shell.execute_reply":"2024-11-21T07:23:04.960235Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load training data","metadata":{}},{"cell_type":"code","source":"train_native = pl.scan_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n)\ntrain_native.count()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T07:23:06.510761Z","iopub.execute_input":"2024-11-21T07:23:06.511198Z","iopub.status.idle":"2024-11-21T07:23:06.962265Z","shell.execute_reply.started":"2024-11-21T07:23:06.511158Z","shell.execute_reply":"2024-11-21T07:23:06.961004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 実体化\ntrain_native = train_native.collect()\n# 行列表示\nprint(train_native.shape)\n# 内容確認\ntrain_native.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T07:20:50.078689Z","iopub.execute_input":"2024-11-21T07:20:50.079171Z","iopub.status.idle":"2024-11-21T07:21:53.033876Z","shell.execute_reply.started":"2024-11-21T07:20:50.079117Z","shell.execute_reply":"2024-11-21T07:21:53.032535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use last 2 parquets\ntrain = (pl.scan_parquet(f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\")\n# インデックス列 (id) を追加\n.select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),\n# ターゲット列の2倍の値を持つ新しい列 \"label\" をデータフレームに追加\n).with_columns(\n    (pl.col(CONFIG.target_col)*2).cast(pl.Int32).alias(\"label\"),\n# \"date_id\" 列の値が CONFIG.start_dt より大きい行だけ抽出\n).filter(\n    pl.col(\"date_id\").gt(CONFIG.start_dt)\n)\n)","metadata":{"execution":{"iopub.status.busy":"2024-11-21T07:23:09.630849Z","iopub.execute_input":"2024-11-21T07:23:09.631303Z","iopub.status.idle":"2024-11-21T07:23:09.644863Z","shell.execute_reply.started":"2024-11-21T07:23:09.631264Z","shell.execute_reply":"2024-11-21T07:23:09.643465Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create Lags data from training data\n- lag:ラグ\n- 時系列データで、前のデータからの差などを特徴として使うために作る。","metadata":{}},{"cell_type":"code","source":"lags = train.select(pl.col(CONFIG.lag_cols_original)) # lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n# responderそれぞれlagをつくる\nlags = lags.rename(CONFIG.lag_cols_rename) \n\n# lagは一日後とする\nlags = lags.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )\n\n# \nlags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()  # pick up last record of previous date\nlags","metadata":{"execution":{"iopub.status.busy":"2024-11-21T07:23:15.543751Z","iopub.execute_input":"2024-11-21T07:23:15.544185Z","iopub.status.idle":"2024-11-21T07:23:15.593575Z","shell.execute_reply.started":"2024-11-21T07:23:15.544149Z","shell.execute_reply":"2024-11-21T07:23:15.592351Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Merge training data and lags data","metadata":{}},{"cell_type":"code","source":"train = train.join(lags, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\ntrain","metadata":{"execution":{"iopub.status.busy":"2024-11-21T07:23:18.136421Z","iopub.execute_input":"2024-11-21T07:23:18.136877Z","iopub.status.idle":"2024-11-21T07:23:18.173918Z","shell.execute_reply.started":"2024-11-21T07:23:18.136807Z","shell.execute_reply":"2024-11-21T07:23:18.172632Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Split training data and validation data","metadata":{}},{"cell_type":"code","source":"len_train   = train.select(pl.col(\"date_id\")).collect().shape[0]\nvalid_records = int(len_train * CONFIG.valid_ratio) # valid_ratio = 0.05 train:95%, valid:5%\n\n# ofl:off line\nlen_ofl_mdl = len_train - valid_records\nlast_tr_dt  = train.select(pl.col(\"date_id\")).collect().row(len_ofl_mdl)[0]\n\nprint(f\"\\n len_train = {len_train}\")\nprint(f\"\\n len_ofl_mdl = {len_ofl_mdl}\")\nprint(f\"\\n---> Last offline train date = {last_tr_dt}\\n\")\n\ntraining_data = train.filter(pl.col(\"date_id\").le(last_tr_dt))\nvalidation_data   = train.filter(pl.col(\"date_id\").gt(last_tr_dt))","metadata":{"execution":{"iopub.status.busy":"2024-11-21T07:23:20.334165Z","iopub.execute_input":"2024-11-21T07:23:20.334585Z","iopub.status.idle":"2024-11-21T07:23:24.255162Z","shell.execute_reply.started":"2024-11-21T07:23:20.334526Z","shell.execute_reply":"2024-11-21T07:23:24.253882Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data","metadata":{"execution":{"iopub.status.busy":"2024-11-21T07:23:24.257585Z","iopub.execute_input":"2024-11-21T07:23:24.258067Z","iopub.status.idle":"2024-11-21T07:23:24.291445Z","shell.execute_reply.started":"2024-11-21T07:23:24.258017Z","shell.execute_reply":"2024-11-21T07:23:24.290187Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Save data as parquets","metadata":{}},{"cell_type":"code","source":"training_data.collect().\\\nwrite_parquet(\n    f\"training_with_lags.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T07:23:24.292761Z","iopub.execute_input":"2024-11-21T07:23:24.2932Z","iopub.status.idle":"2024-11-21T07:24:50.827761Z","shell.execute_reply.started":"2024-11-21T07:23:24.293165Z","shell.execute_reply":"2024-11-21T07:24:50.826043Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# date_id順にソート\nsorted_data = training_data.sort(\"date_id\")\n\n# データを具体化して件数を取得\nsorted_data_materialized = sorted_data.collect()\nsubset_size = len(sorted_data_materialized) // 6\n\n# 上から1/6のレコードを抽出\nsubset_data = sorted_data_materialized[:subset_size]\n\n# Parquetファイルに保存\nsubset_data.write_parquet(\"training_with_lags_not_partition.parquet\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T08:27:14.031411Z","iopub.execute_input":"2024-11-21T08:27:14.03195Z","iopub.status.idle":"2024-11-21T08:28:35.741584Z","shell.execute_reply.started":"2024-11-21T08:27:14.031868Z","shell.execute_reply":"2024-11-21T08:28:35.740091Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data.collect().\\\nwrite_parquet(\n    \"validation_with_lags.parquet\", partition_by = \"date_id\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T07:24:50.830154Z","iopub.execute_input":"2024-11-21T07:24:50.830542Z","iopub.status.idle":"2024-11-21T07:25:39.854065Z","shell.execute_reply.started":"2024-11-21T07:24:50.830504Z","shell.execute_reply":"2024-11-21T07:25:39.852801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data.collect().write_parquet(\"validation_with_lags_not_partition.parquet\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T08:24:14.835674Z","iopub.execute_input":"2024-11-21T08:24:14.836231Z","iopub.status.idle":"2024-11-21T08:25:15.084718Z","shell.execute_reply.started":"2024-11-21T08:24:14.836193Z","shell.execute_reply":"2024-11-21T08:25:15.081734Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Check","metadata":{}},{"cell_type":"code","source":"train_check_df = pl.scan_parquet(f\"/kaggle/working/training_with_lags.parquet\").collect()\ntrain_check_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T07:42:29.52052Z","iopub.execute_input":"2024-11-21T07:42:29.520869Z","iopub.status.idle":"2024-11-21T07:42:39.291044Z","shell.execute_reply.started":"2024-11-21T07:42:29.520838Z","shell.execute_reply":"2024-11-21T07:42:39.289654Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_check_df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T07:45:37.647986Z","iopub.execute_input":"2024-11-21T07:45:37.648608Z","iopub.status.idle":"2024-11-21T07:45:37.65723Z","shell.execute_reply.started":"2024-11-21T07:45:37.64853Z","shell.execute_reply":"2024-11-21T07:45:37.655994Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 各列ごとの欠損値のカウント\nnull_counts = train_check_df.select([\n    pl.col(col).is_null().sum().alias(f\"{col}_null_count\") for col in train_check_df.columns\n])\n\n# 結果を表示\nprint(null_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T07:45:17.176335Z","iopub.execute_input":"2024-11-21T07:45:17.176832Z","iopub.status.idle":"2024-11-21T07:45:17.191621Z","shell.execute_reply.started":"2024-11-21T07:45:17.176791Z","shell.execute_reply":"2024-11-21T07:45:17.190096Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 各列の欠損率を計算\nmissing_rate = train_check_df.select([\n    (pl.col(col).is_null().sum() / pl.count()).alias(f\"{col}_missing_rate\")\n    for col in train_check_df.columns\n])\n\n# 結果を表示\nprint(missing_rate)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T07:46:42.373476Z","iopub.execute_input":"2024-11-21T07:46:42.374027Z","iopub.status.idle":"2024-11-21T07:46:42.398194Z","shell.execute_reply.started":"2024-11-21T07:46:42.373986Z","shell.execute_reply":"2024-11-21T07:46:42.396851Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pl.Config.set_tbl_cols(len(train_check_df.columns))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T07:47:31.503311Z","iopub.execute_input":"2024-11-21T07:47:31.503797Z","iopub.status.idle":"2024-11-21T07:47:31.512433Z","shell.execute_reply.started":"2024-11-21T07:47:31.503756Z","shell.execute_reply":"2024-11-21T07:47:31.510887Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(missing_rate)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T07:47:37.349426Z","iopub.execute_input":"2024-11-21T07:47:37.350017Z","iopub.status.idle":"2024-11-21T07:47:37.364468Z","shell.execute_reply.started":"2024-11-21T07:47:37.349963Z","shell.execute_reply":"2024-11-21T07:47:37.363016Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_rate = train_check_df.select([\n    (pl.col(col).is_null().sum() / pl.count()).alias(col)\n    for col in train_check_df.columns\n])\n\n# データフレームを転置\nmissing_rate_t = missing_rate.transpose(include_header=True)\n\n# 結果を見やすくソート（高い順に並べる）\n#missing_rate_t = missing_rate_t.sort(\"Missing Rate (%)\", descending=True)\n\n# 結果を表示\nprint(missing_rate_t)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T07:50:08.637195Z","iopub.execute_input":"2024-11-21T07:50:08.637712Z","iopub.status.idle":"2024-11-21T07:50:08.657552Z","shell.execute_reply.started":"2024-11-21T07:50:08.637669Z","shell.execute_reply":"2024-11-21T07:50:08.65612Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pl.Config.set_tbl_rows(100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T07:50:54.904769Z","iopub.execute_input":"2024-11-21T07:50:54.905275Z","iopub.status.idle":"2024-11-21T07:50:54.913173Z","shell.execute_reply.started":"2024-11-21T07:50:54.905237Z","shell.execute_reply":"2024-11-21T07:50:54.911844Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 結果を表示\nprint(missing_rate_t)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T07:50:59.899707Z","iopub.execute_input":"2024-11-21T07:50:59.900151Z","iopub.status.idle":"2024-11-21T07:50:59.907479Z","shell.execute_reply.started":"2024-11-21T07:50:59.900113Z","shell.execute_reply":"2024-11-21T07:50:59.905947Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# END","metadata":{}}]}