{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport gc\nfrom matplotlib import pyplot as plt\nimport matplotlib.cm as cm\nfrom sklearn.model_selection import StratifiedGroupKFold","metadata":{"execution":{"iopub.status.busy":"2025-01-03T01:53:01.735997Z","iopub.execute_input":"2025-01-03T01:53:01.736438Z","iopub.status.idle":"2025-01-03T01:53:05.194993Z","shell.execute_reply.started":"2025-01-03T01:53:01.736390Z","shell.execute_reply":"2025-01-03T01:53:05.193578Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Configurations","metadata":{}},{"cell_type":"code","source":"class CONFIG:\n    target_col = \"responder_6\"\n    lag_cols_original = [\"date_id\", \"symbol_id\"] + [f\"responder_{idx}\" for idx in range(9)]\n    lag_cols_rename = { f\"responder_{idx}\" : f\"responder_{idx}_lag_1\" for idx in range(9)}\n    valid_ratio = 0.05\n    start_dt = 1100\n    chg_date_1 = 484\n    chg_date_2 = 677\n    chg_date_3 = 1360","metadata":{"execution":{"iopub.status.busy":"2025-01-03T01:53:05.198494Z","iopub.execute_input":"2025-01-03T01:53:05.199173Z","iopub.status.idle":"2025-01-03T01:53:05.207886Z","shell.execute_reply.started":"2025-01-03T01:53:05.199117Z","shell.execute_reply":"2025-01-03T01:53:05.206342Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"means = {'feature_00': 0.640198826789856, 'feature_01': 0.03755598142743111, 'feature_02': 0.6368075609207153, 'feature_03': 0.6365063786506653, 'feature_04': 0.013741530478000641, 'feature_05': -0.02173694409430027, 'feature_06': -0.006415014620870352, 'feature_07': -0.010971736162900925, 'feature_08': -0.04653771221637726, 'feature_09': 32.596106194690265, 'feature_10': 4.95929203539823, 'feature_11': 167.6541592920354, 'feature_12': -0.13415881991386414, 'feature_13': -0.07573335617780685, 'feature_14': -0.12015637010335922, 'feature_15': -0.7470195889472961, 'feature_16': -0.6257441639900208, 'feature_17': -0.7294047474861145, 'feature_18': -0.042215555906295776, 'feature_19': -0.08798160403966904, 'feature_20': -0.15741558372974396, 'feature_21': 0.10528526455163956, 'feature_22': 0.018054703250527382, 'feature_23': 0.03165541961789131, 'feature_24': 2.733017921447754, 'feature_25': 0.39958420395851135, 'feature_26': -0.11045943945646286, 'feature_27': -0.5332594513893127, 'feature_28': -0.4522790312767029, 'feature_29': -0.5739678144454956, 'feature_30': -0.7905704975128174, 'feature_31': 0.10600688308477402, 'feature_32': 0.40044134855270386, 'feature_33': -0.021725023165345192, 'feature_34': 0.4226262867450714, 'feature_35': 0.42143046855926514, 'feature_36': -0.00023802756913937628, 'feature_37': 0.027961043640971184, 'feature_38': 0.010258913040161133, 'feature_39': 0.005768273025751114, 'feature_40': 0.017485467717051506, 'feature_41': 0.038347117602825165, 'feature_42': -0.06123563274741173, 'feature_43': -0.11644423753023148, 'feature_44': -0.12342483550310135, 'feature_45': -0.028769943863153458, 'feature_46': -0.015200662426650524, 'feature_47': 0.015717582777142525, 'feature_48': -0.0033910537604242563, 'feature_49': -0.0052393232472240925, 'feature_50': -0.2285808026790619, 'feature_51': -0.3548349440097809, 'feature_52': -0.358092725276947, 'feature_53': 0.2607136368751526, 'feature_54': 0.18796788156032562, 'feature_55': 0.3154229521751404, 'feature_56': -0.1471923440694809, 'feature_57': 0.15730056166648865, 'feature_58': -0.021774644032120705, 'feature_59': -0.0037768862675875425, 'feature_60': -0.010220836848020554, 'feature_61': -0.03178725391626358, 'feature_62': -0.3769100308418274, 'feature_63': -0.3229374587535858, 'feature_64': -0.3718394339084625, 'feature_65': -0.10233989357948303, 'feature_66': -0.13688170909881592, 'feature_67': -0.14402112364768982, 'feature_68': -0.06875362992286682, 'feature_69': -0.11862917989492416, 'feature_70': -0.11789549142122269, 'feature_71': -0.06013699993491173, 'feature_72': -0.10766122490167618, 'feature_73': -0.09921672940254211, 'feature_74': -0.10233042389154434, 'feature_75': -0.05991339311003685, 'feature_76': -0.06349952518939972, 'feature_77': -0.07424316555261612, 'feature_78': -0.07759837061166763}\nstds = {'feature_00': 1.027751088142395, 'feature_01': 1.0967519283294678, 'feature_02': 1.0156300067901611, 'feature_03': 1.0170334577560425, 'feature_04': 1.0726385116577148, 'feature_05': 0.9639211297035217, 'feature_06': 1.0963259935379028, 'feature_07': 1.0789952278137207, 'feature_08': 0.7962697148323059, 'feature_09': 23.72976726545254, 'feature_10': 3.1867162933797224, 'feature_11': 163.44513161352285, 'feature_12': 0.6700984835624695, 'feature_13': 0.5805172920227051, 'feature_14': 0.664044201374054, 'feature_15': 0.37517768144607544, 'feature_16': 0.3393096327781677, 'feature_17': 0.3603287935256958, 'feature_18': 0.9911752939224243, 'feature_19': 1.0550744533538818, 'feature_20': 0.6643751263618469, 'feature_21': 0.38239365816116333, 'feature_22': 0.950261116027832, 'feature_23': 0.8119344711303711, 'feature_24': 1.4362775087356567, 'feature_25': 1.0947270393371582, 'feature_26': 1.077124834060669, 'feature_27': 1.0645726919174194, 'feature_28': 1.0676648616790771, 'feature_29': 0.2640742361545563, 'feature_30': 0.19689509272575378, 'feature_31': 0.3815343976020813, 'feature_32': 1.2996565103530884, 'feature_33': 0.9989405870437622, 'feature_34': 1.3409572839736938, 'feature_35': 1.3365675210952759, 'feature_36': 0.8695492148399353, 'feature_37': 0.7334080934524536, 'feature_38': 0.698810338973999, 'feature_39': 0.7965824604034424, 'feature_40': 0.518515944480896, 'feature_41': 0.6384949088096619, 'feature_42': 0.8168442249298096, 'feature_43': 0.5228385925292969, 'feature_44': 0.6521403193473816, 'feature_45': 0.8666537404060364, 'feature_46': 0.9039222002029419, 'feature_47': 3.2711963653564453, 'feature_48': 0.6570901274681091, 'feature_49': 0.7083076238632202, 'feature_50': 1.0132617950439453, 'feature_51': 0.6081287860870361, 'feature_52': 0.9250587224960327, 'feature_53': 1.0421689748764038, 'feature_54': 0.5859629511833191, 'feature_55': 0.9191848039627075, 'feature_56': 0.9549097418785095, 'feature_57': 1.0204777717590332, 'feature_58': 0.8327276110649109, 'feature_59': 0.8309783339500427, 'feature_60': 0.8389413356781006, 'feature_61': 1.192766547203064, 'feature_62': 1.388945460319519, 'feature_63': 0.09957146644592285, 'feature_64': 0.3396177291870117, 'feature_65': 1.01683509349823, 'feature_66': 1.0824761390686035, 'feature_67': 0.642227828502655, 'feature_68': 0.5312599539756775, 'feature_69': 0.6208390593528748, 'feature_70': 0.6724499464035034, 'feature_71': 0.5356909036636353, 'feature_72': 0.6534596681594849, 'feature_73': 1.0855497121810913, 'feature_74': 1.0880277156829834, 'feature_75': 1.2321789264678955, 'feature_76': 1.2345560789108276, 'feature_77': 1.0921478271484375, 'feature_78': 1.0924347639083862}\n\ndef normalize_dataframe(df: pl.DataFrame, means: dict, stds: dict) -> pl.DataFrame:\n    # 폴라 데이터프레임을 평균과 표준편차로 정규화\n    normalize_exprs = []\n\n    for col in df.columns:\n        if col in means and col in stds: # 정규화 되어야 하는 열만 존재하도록\n            if stds[col] != 0: # 0인 차원 적용 X\n                # 정규화 하되 이름은 유지하도록\n                normalize_exprs.append(\n                    ((pl.col(col) - means[col]) / stds[col]).alias(col)\n                )\n            else:\n                normalize_exprs.append(pl.col(col) - means[col]).alias(col)\n\n    normalized_df = df.select(normalize_exprs) # 정규화된 데이터프레임 생성\n    return normalized_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T01:53:05.209738Z","iopub.execute_input":"2025-01-03T01:53:05.210232Z","iopub.status.idle":"2025-01-03T01:53:05.241941Z","shell.execute_reply.started":"2025-01-03T01:53:05.210178Z","shell.execute_reply":"2025-01-03T01:53:05.240294Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load training data & select","metadata":{}},{"cell_type":"code","source":"# Use last 2 parquets\n\ntrain = pl.scan_parquet(\n    f\"/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet\"\n).select(\n    pl.int_range(pl.len(), dtype=pl.UInt32).alias(\"id\"),\n    pl.all(),\n).with_columns(\n    (pl.col(CONFIG.target_col)*2).cast(pl.Int32).alias(\"label\"),\n).filter(\n    (pl.col(\"date_id\") >= CONFIG.chg_date_1 - 50) & (pl.col(\"date_id\") <= CONFIG.chg_date_1 + 50)\n    | (pl.col(\"date_id\") >= CONFIG.chg_date_2 - 50) & (pl.col(\"date_id\") <= CONFIG.chg_date_2 + 50)\n    | (pl.col(\"date_id\") >= CONFIG.chg_date_3) # 데이터를 초반 두 번의 변화 두 번 전후 50일씩, 그리고 마지막 2 parquet을 가져옴.\n).fill_null(\n    strategy=\"forward\"\n).fill_null(\n    value=0\n)","metadata":{"execution":{"iopub.status.busy":"2025-01-03T01:53:05.243698Z","iopub.execute_input":"2025-01-03T01:53:05.244291Z","iopub.status.idle":"2025-01-03T01:53:05.294532Z","shell.execute_reply.started":"2025-01-03T01:53:05.244221Z","shell.execute_reply":"2025-01-03T01:53:05.292997Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"col_names = train.collect_schema().names()\n\n# col_names를 사용하여 이후 처리\nnormalize_exprs = []\nfor col in col_names:\n    if col in means and col in stds:\n        if stds[col] != 0:\n            normalize_exprs.append(\n                ((pl.col(col) - means[col]) / stds[col]).alias(col)\n            )\n        else:\n            normalize_exprs.append((pl.col(col) - means[col]).alias(col))\n\n# LazyFrame에 정규화 적용\ntrain = train.with_columns(normalize_exprs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T01:53:05.296397Z","iopub.execute_input":"2025-01-03T01:53:05.297671Z","iopub.status.idle":"2025-01-03T01:53:05.570436Z","shell.execute_reply.started":"2025-01-03T01:53:05.297614Z","shell.execute_reply":"2025-01-03T01:53:05.569135Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.collect().to_pandas()\ntrain.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tmp_normed = normalize_dataframe(pl.DataFrame(train), means, stds)\ntmp_normed = tmp_normed.to_pandas()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tmp_normed.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.update(tmp_normed)\ntrain.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pl.DataFrame(train)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create Lags data from training data","metadata":{}},{"cell_type":"code","source":"lags = train.select(pl.col(CONFIG.lag_cols_original))\nlags = lags.rename(CONFIG.lag_cols_rename)\nlags = lags.with_columns(\n    date_id = pl.col('date_id') + 1,  # lagged by 1 day\n    )\nlags = lags.group_by([\"date_id\", \"symbol_id\"], maintain_order=True).last()  # pick up last record of previous date\nlags","metadata":{"execution":{"iopub.status.busy":"2025-01-03T01:53:05.571755Z","iopub.execute_input":"2025-01-03T01:53:05.572143Z","iopub.status.idle":"2025-01-03T01:53:05.849904Z","shell.execute_reply.started":"2025-01-03T01:53:05.572110Z","shell.execute_reply":"2025-01-03T01:53:05.848532Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Merge training data and lags data","metadata":{}},{"cell_type":"code","source":"train = train.join(lags, on=[\"date_id\", \"symbol_id\"],  how=\"left\")\ntrain","metadata":{"execution":{"iopub.status.busy":"2025-01-03T01:53:05.854307Z","iopub.execute_input":"2025-01-03T01:53:05.854875Z","iopub.status.idle":"2025-01-03T01:53:05.915679Z","shell.execute_reply.started":"2025-01-03T01:53:05.854830Z","shell.execute_reply":"2025-01-03T01:53:05.914062Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Split training data and validation data","metadata":{}},{"cell_type":"code","source":"len_train   = train.select(pl.col(\"date_id\")).shape[0]\nvalid_records = int(len_train * CONFIG.valid_ratio)\nlen_ofl_mdl = len_train - valid_records\nlast_tr_dt  = train.select(pl.col(\"date_id\")).row(len_ofl_mdl)[0]\n\nprint(f\"\\n len_train = {len_train}\")\nprint(f\"\\n len_ofl_mdl = {len_ofl_mdl}\")\nprint(f\"\\n---> Last offline train date = {last_tr_dt}\\n\")\n\ntraining_data = train.filter(pl.col(\"date_id\").le(last_tr_dt))\nvalidation_data = train.filter(pl.col(\"date_id\").gt(last_tr_dt))","metadata":{"execution":{"iopub.status.busy":"2025-01-03T01:53:05.917287Z","iopub.execute_input":"2025-01-03T01:53:05.917754Z","iopub.status.idle":"2025-01-03T01:53:09.564696Z","shell.execute_reply.started":"2025-01-03T01:53:05.917704Z","shell.execute_reply":"2025-01-03T01:53:09.562982Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data","metadata":{"execution":{"iopub.status.busy":"2025-01-03T01:53:09.566539Z","iopub.execute_input":"2025-01-03T01:53:09.567235Z","iopub.status.idle":"2025-01-03T01:53:09.630029Z","shell.execute_reply.started":"2025-01-03T01:53:09.567179Z","shell.execute_reply":"2025-01-03T01:53:09.627140Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Save data as parquets","metadata":{}},{"cell_type":"code","source":"training_data.\\\nwrite_parquet(\n    f\"training_data.parquet\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T01:53:09.632586Z","iopub.execute_input":"2025-01-03T01:53:09.633064Z","iopub.status.idle":"2025-01-03T01:55:31.322482Z","shell.execute_reply.started":"2025-01-03T01:53:09.633021Z","shell.execute_reply":"2025-01-03T01:55:31.320157Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"validation_data.\\\nwrite_parquet(\n    f\"validation_data.parquet\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T01:55:31.324996Z","iopub.execute_input":"2025-01-03T01:55:31.325497Z","iopub.status.idle":"2025-01-03T01:56:38.146506Z","shell.execute_reply.started":"2025-01-03T01:55:31.325423Z","shell.execute_reply":"2025-01-03T01:56:38.145032Z"}},"outputs":[],"execution_count":null}]}