{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":11305158,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **기본설정**","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport polars as pl\nfrom matplotlib import pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-17T12:09:31.170064Z","iopub.execute_input":"2025-09-17T12:09:31.170506Z","iopub.status.idle":"2025-09-17T12:09:33.375136Z","shell.execute_reply.started":"2025-09-17T12:09:31.170476Z","shell.execute_reply":"2025-09-17T12:09:33.373567Z"}},"outputs":[{"name":"stdout","text":"/kaggle/input/jane-street-real-time-market-data-forecasting/responders.csv\n/kaggle/input/jane-street-real-time-market-data-forecasting/sample_submission.csv\n/kaggle/input/jane-street-real-time-market-data-forecasting/features.csv\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=4/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=5/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=6/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=3/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=1/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=8/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=2/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=7/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=9/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet/date_id=0/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/jane_street_gateway.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/jane_street_inference_server.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/__init__.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/templates.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/base_gateway.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/relay.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/kaggle_evaluation.proto\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/__init__.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/kaggle_evaluation_pb2.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/kaggle_evaluation_pb2_grpc.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/__init__.py\n","output_type":"stream"}],"execution_count":1},{"cell_type":"markdown","source":"# **데이터 로드**","metadata":{}},{"cell_type":"code","source":"ROOT_DIR = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-17T12:09:41.914901Z","iopub.execute_input":"2025-09-17T12:09:41.915258Z","iopub.status.idle":"2025-09-17T12:09:41.919854Z","shell.execute_reply.started":"2025-09-17T12:09:41.915232Z","shell.execute_reply":"2025-09-17T12:09:41.918904Z"}},"outputs":[],"execution_count":3},{"cell_type":"code","source":"ROOT_DIR = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\n\n# === 경로 ===\ntrain_path = os.path.join(ROOT_DIR, \"train.parquet\")\n\n# === 메타데이터 ===\nlags = os.path.join(ROOT_DIR, \"lags.parquet\")\ntrain_path = os.path.join(ROOT_DIR, \"train.parquet\")\nlags_path = os.path.join(ROOT_DIR, \"lags.parquet\")\nfeatures = pl.read_csv(os.path.join(ROOT_DIR, \"features.csv\"))\nresponders = pl.read_csv(os.path.join(ROOT_DIR, \"responders.csv\"))\nsample_sub = pl.read_csv(os.path.join(ROOT_DIR, \"sample_submission.csv\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-17T12:09:43.755716Z","iopub.execute_input":"2025-09-17T12:09:43.75613Z","iopub.status.idle":"2025-09-17T12:09:43.989761Z","shell.execute_reply.started":"2025-09-17T12:09:43.756101Z","shell.execute_reply":"2025-09-17T12:09:43.988571Z"}},"outputs":[],"execution_count":4},{"cell_type":"markdown","source":"# **데이터 전처리(1)**","metadata":{}},{"cell_type":"code","source":"import os\nimport polars as pl\n\nROOT_DIR = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\n\n# -----------------------------\n# 1) train: 파티션 0~5만 로드\n# -----------------------------\nscan_list = []\nfor pid in range(6): # 0~5만\n f = os.path.join(ROOT_DIR, \"train.parquet\", f\"partition_id={pid}\", \"part-0.parquet\")\n lf = pl.scan_parquet(f).with_columns(pl.lit(pid).alias(\"partition_id\"))\n scan_list.append(lf)\n\ntrain_all_lf = pl.concat(scan_list, how=\"vertical_relaxed\")\n\ntcols = train_all_lf.collect_schema().names()\nbase_cols = [c for c in [\"date_id\",\"time_id\",\"symbol_id\",\"weight\",\"partition_id\"] if c in tcols]\nfeature_cols = sorted([c for c in tcols if c.startswith(\"feature_\")])\ntarget_cols = [\"responder_6\"] if \"responder_6\" in tcols else []\n\n# responder_6만 남김 (y alias 제거)\nraw_data_lf = train_all_lf.select(base_cols + feature_cols + target_cols)\n\n# -----------------------------\n# 2) lags: 단일 파일(기본) + 예외적으로 파티션일 수도 있어 자동대응\n# -----------------------------\nlags_file = os.path.join(ROOT_DIR, \"lags.parquet\") # 단일 파일일 때\nlags_part_glob = os.path.join(ROOT_DIR, \"lags.parquet\", \"date_id=*\", \"part-0.parquet\") # 파티션일 때\n\n# 단일 파일 우선 시도, 실패하면 파티션으로\ntry:\n lags_lf = pl.scan_parquet(lags_file).select(\n [\"date_id\",\"time_id\",\"symbol_id\",\"responder_6_lag_1\"]\n )\nexcept Exception:\n lags_lf = pl.scan_parquet(lags_part_glob).select(\n [\"date_id\",\"time_id\",\"symbol_id\",\"responder_6_lag_1\"]\n )\n\n# -----------------------------\n# 3) join + 정렬 (lag 인덱스 기준: date_id, time_id, symbol_id)\n# -----------------------------\njoined_lf = (\n raw_data_lf\n .join(lags_lf, on=[\"date_id\",\"time_id\",\"symbol_id\"], how=\"left\")\n .sort([\"date_id\",\"time_id\",\"symbol_id\"])\n)\n\nfinal_cols = base_cols + feature_cols + [\"responder_6\",\"responder_6_lag_1\"]\nraw_with_lag_lf = joined_lf.select([c for c in final_cols if c in joined_lf.collect_schema().names()])\n\n# -----------------------------\n# 4) split: 0~3 train / 4~5 test\n# -----------------------------\ntrain_lf = raw_with_lag_lf.filter(pl.col(\"partition_id\") <= 3)\ntest_lf = raw_with_lag_lf.filter((pl.col(\"partition_id\") >= 4) & (pl.col(\"partition_id\") <= 5))\n\n# 확인\ndisplay(train_lf.head(5).collect())\ndisplay(test_lf.head(5).collect())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-17T12:09:47.375319Z","iopub.execute_input":"2025-09-17T12:09:47.375702Z","iopub.status.idle":"2025-09-17T12:10:32.199298Z","shell.execute_reply.started":"2025-09-17T12:09:47.375666Z","shell.execute_reply":"2025-09-17T12:10:32.197958Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"shape: (5, 86)\n┌─────────┬─────────┬───────────┬──────────┬───┬────────────┬────────────┬────────────┬────────────┐\n│ date_id ┆ time_id ┆ symbol_id ┆ weight ┆ … ┆ feature_77 ┆ feature_78 ┆ responder_ ┆ responder_ │\n│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ 6 ┆ 6_lag_1 │\n│ i16 ┆ i16 ┆ i8 ┆ f32 ┆ ┆ f32 ┆ f32 ┆ --- ┆ --- │\n│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ f32 ┆ f32 │\n╞═════════╪═════════╪═══════════╪══════════╪═══╪════════════╪════════════╪════════════╪════════════╡\n│ 0 ┆ 0 ┆ 1 ┆ 3.889038 ┆ … ┆ -0.335556 ┆ -0.281498 ┆ 0.775981 ┆ -1.162801 │\n│ 0 ┆ 0 ┆ 7 ┆ 1.370613 ┆ … ┆ -0.245565 ┆ -0.302441 ┆ 0.703665 ┆ -0.492168 │\n│ 0 ┆ 0 ┆ 9 ┆ 2.285698 ┆ … ┆ -0.106842 ┆ -0.096792 ┆ 2.109352 ┆ -0.716492 │\n│ 0 ┆ 0 ┆ 10 ┆ 0.690606 ┆ … ┆ -0.215522 ┆ -0.296244 ┆ 1.114137 ┆ -0.263427 │\n│ 0 ┆ 0 ┆ 14 ┆ 0.44057 ┆ … ┆ 2.61825 ┆ 3.418133 ┆ -3.57282 ┆ -0.507432 │\n└─────────┴─────────┴───────────┴──────────┴───┴────────────┴────────────┴────────────┴────────────┘","text/html":"
\nshape: (5, 86)
date_idtime_idsymbol_idweightpartition_idfeature_00feature_01feature_02feature_03feature_04feature_05feature_06feature_07feature_08feature_09feature_10feature_11feature_12feature_13feature_14feature_15feature_16feature_17feature_18feature_19feature_20feature_21feature_22feature_23feature_24feature_25feature_26feature_27feature_28feature_29feature_30feature_31…feature_44feature_45feature_46feature_47feature_48feature_49feature_50feature_51feature_52feature_53feature_54feature_55feature_56feature_57feature_58feature_59feature_60feature_61feature_62feature_63feature_64feature_65feature_66feature_67feature_68feature_69feature_70feature_71feature_72feature_73feature_74feature_75feature_76feature_77feature_78responder_6responder_6_lag_1
i16i16i8f32i32f32f32f32f32f32f32f32f32f32i8i8i16f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32…f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32
0013.8890380nullnullnullnullnull0.8510330.2429710.2634-0.89168711776-0.8830280.003067-0.744703null-0.169586null-1.335938-1.7078030.91013null1.6364311.522133-1.551398-0.229627nullnull1.378301-0.2837120.123196null…nullnullnull0.5640212.0885060.832022null0.204797nullnull-0.808103null-2.0376830.727661null-0.989118-0.345213-1.36224nullnullnullnullnull-1.251104-0.110252-0.491157-1.022690.152241-0.659864nullnull-0.261412-0.211486-0.335556-0.2814980.775981-1.162801
0071.3706130nullnullnullnullnull0.6769610.1519840.192465-0.52172911776-0.865307-0.225629-0.582163null0.317467null-1.250016-1.6829291.412757null0.5203780.744132-0.7886580.641776nullnull0.22720.5809071.128879null…nullnullnull-10.835207-0.002704-0.621836null1.172836nullnull-1.625862null-1.4100171.063013null0.8883550.467994-1.36224nullnullnullnullnull-1.0657590.013322-0.592855-1.052685-0.393726-0.741603nullnull-0.281207-0.182894-0.245565-0.3024410.703665-0.492168
0092.2856980nullnullnullnullnull1.0562850.1872270.249901-0.7730511776-0.675719-0.199404-0.586798null-0.814909null-1.296782-2.0402340.639589null1.5973590.657514-1.3501480.364215nullnull-0.017751-0.317361-0.122379null…nullnullnull-1.420632-3.515137-4.67776null0.535897nullnull-0.72542null-2.294171.764551null-0.120789-0.063458-1.36224nullnullnullnullnull-0.882604-0.072482-0.617934-0.86323-0.241892-0.709919nullnull0.3771310.300724-0.106842-0.0967922.109352-0.716492
00100.6906060nullnullnullnullnull1.1393660.2733280.306549-1.262223425150-0.6940083.0040910.114809null-0.251882null-1.902009-0.9794470.241165null-0.392359-0.224699-2.129397-0.855287nullnull0.404142-0.5781560.105702null…nullnullnull0.3820742.6691350.611711null2.413415nullnull1.313203null-0.8101252.939022null3.9888011.834661-1.36224nullnullnullnullnull-0.6975951.074309-0.206929-0.5306024.7652150.571554nullnull-0.226891-0.251412-0.215522-0.2962441.114137-0.263427
00140.440570nullnullnullnullnull0.95520.2624040.344457-0.61381344316-0.947351-0.030018-0.502379null0.646086null-1.844685-1.58656-0.182024null-0.969949-0.673813-1.282132-1.399894nullnull0.043815-0.320225-0.031713null…nullnullnull-2.0146-2.321076-3.711265null1.253902nullnull0.476195null-0.7717322.843421null1.3798150.411827-1.36224nullnullnullnullnull-0.948601-0.136814-0.447704-1.1417610.099631-0.661928nullnull3.6780762.7935812.618253.418133-3.57282-0.507432
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"shape: (5, 86)\n┌─────────┬─────────┬───────────┬──────────┬───┬────────────┬────────────┬────────────┬────────────┐\n│ date_id ┆ time_id ┆ symbol_id ┆ weight ┆ … ┆ feature_77 ┆ feature_78 ┆ responder_ ┆ responder_ │\n│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ 6 ┆ 6_lag_1 │\n│ i16 ┆ i16 ┆ i8 ┆ f32 ┆ ┆ f32 ┆ f32 ┆ --- ┆ --- │\n│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ f32 ┆ f32 │\n╞═════════╪═════════╪═══════════╪══════════╪═══╪════════════╪════════════╪════════════╪════════════╡\n│ 680 ┆ 0 ┆ 0 ┆ 2.29816 ┆ … ┆ -0.178144 ┆ -0.1951 ┆ 0.929775 ┆ null │\n│ 680 ┆ 0 ┆ 1 ┆ 3.928745 ┆ … ┆ -0.266495 ┆ -0.470271 ┆ 0.826995 ┆ null │\n│ 680 ┆ 0 ┆ 2 ┆ 1.340433 ┆ … ┆ 0.198109 ┆ 0.152837 ┆ -0.296969 ┆ null │\n│ 680 ┆ 0 ┆ 3 ┆ 1.695526 ┆ … ┆ 9.818627 ┆ 11.179185 ┆ 2.047506 ┆ null │\n│ 680 ┆ 0 ┆ 5 ┆ 2.700766 ┆ … ┆ -0.700707 ┆ -0.770234 ┆ 0.103898 ┆ null │\n└─────────┴─────────┴───────────┴──────────┴───┴────────────┴────────────┴────────────┴────────────┘","text/html":"
\nshape: (5, 86)
date_idtime_idsymbol_idweightpartition_idfeature_00feature_01feature_02feature_03feature_04feature_05feature_06feature_07feature_08feature_09feature_10feature_11feature_12feature_13feature_14feature_15feature_16feature_17feature_18feature_19feature_20feature_21feature_22feature_23feature_24feature_25feature_26feature_27feature_28feature_29feature_30feature_31…feature_44feature_45feature_46feature_47feature_48feature_49feature_50feature_51feature_52feature_53feature_54feature_55feature_56feature_57feature_58feature_59feature_60feature_61feature_62feature_63feature_64feature_65feature_66feature_67feature_68feature_69feature_70feature_71feature_72feature_73feature_74feature_75feature_76feature_77feature_78responder_6responder_6_lag_1
i16i16i8f32i32f32f32f32f32f32f32f32f32f32i8i8i16f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32…f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32f32
680002.2981640.8518141.1975910.2194220.4116982.057359-0.542597-3.4331-1.0901650.15188811776-0.971420.670215-0.502896null-0.070096null-1.308236-2.1201281.747068-0.2196610.7916020.114922-0.3116720.1565481.4763461.3013411.235173-0.259882-0.30125-0.264237…null-1.5434090.809453-2.679432-1.730216-1.349158null0.322135nullnull-2.273168null-1.6275071.155271null-3.847984-2.230864-0.171579-0.273791-0.301876-0.432933-1.215778-1.670469-0.6379630.803874-0.21269-0.7647020.435278-0.619145nullnull-0.021034-0.045094-0.178144-0.19510.929775null
680013.92874540.5344411.079740.0387480.2753432.135057-0.541966-2.774344-1.0480890.16376811776-0.8738471.794426-0.226819null-0.328627null-0.870575-1.2042920.935869-0.0643651.2731090.752816-0.0622810.6870360.5466031.2293731.535235-0.483758-0.431013-0.058963…null-1.6873221.475159-0.946082-0.245348-0.320477null0.505224nullnull-1.074216null-1.9096051.566016null-1.044804-0.807186-0.171579-0.466983-0.250124-0.342297-1.896279-2.157645-0.6987551.743311-0.069648-0.9334451.653637-0.348816nullnull-0.154413-0.301091-0.266495-0.4702710.826995null
680021.3404334-0.2276430.764146-0.2433490.2470272.347248-0.478477-2.660244-1.2616130.23442581259-0.952889-0.04806-0.791763null-0.140953null-1.691419-2.242023-0.459649-0.241993-0.47108-1.056603-0.387841-0.4089620.3595140.7071610.669771-0.641928-0.645097-0.362472…null-0.3398841.743191-1.4159997.0399612.292301null2.115924nullnull0.095438null-0.6648232.016616null2.0853580.690842-0.1715790.022096-0.018111-0.015769-2.270972-1.826189-0.908704-0.051331-0.658539-1.011692-0.008993-0.363811nullnull1.6776421.7052280.1981090.152837-0.296969null
680031.69552640.2676861.193612-0.3887980.0306732.175273-0.408371-1.859344-0.7719720.1048854311-1.0051840.546772-0.587481null-0.628245null-1.243775-2.907238-0.0983980.004987-0.320296-1.1932561.2604761.621998-0.255902-1.644356-1.292619-0.541355-0.7031210.00681…null-1.5331411.7504440.208898-0.3225150.188045null-0.442253nullnull-1.021894null-3.0886171.440495null0.192640.270458-0.171579-0.387987-0.30065-0.169365-1.324387-2.20877-0.9758380.605086-0.303872-1.0714950.445026-0.465118nullnull3.8200254.3352689.81862711.1791852.047506null
680052.70076640.9523720.861269-0.3754050.2590992.497325-0.618828-2.754378-0.4799920.108627210171-0.8926010.207765-0.420839null-0.522783null-1.711776-2.0786920.1632080.0235840.598875-0.4972260.2319891.630302-1.233744-0.4344980.37159-0.586595-0.5126720.017691…null-1.0781322.067645-0.571223-0.124274-0.282079null1.003178nullnull-0.92965null-1.6211221.520507null-0.532552-0.482004-0.171579-0.164008-0.194068-0.257875-1.623473-1.959616-0.7485920.022282-0.561959-0.9959840.385183-0.312552nullnull-0.552556-0.460608-0.700707-0.7702340.103898null
"},"metadata":{}}],"execution_count":5},{"cell_type":"markdown","source":"***x,y로 나누는 것 아직 안함***","metadata":{}},{"cell_type":"code","source":"# === 1) 결측률 확인 (train 기준) ===\n# 대상 컬럼: feature_cols (+ 원하면 lag 포함)\nfeature_cols_in_train = [c for c in feature_cols if c in train_lf.collect_schema().names()]\ninclude_lag = \"responder_6_lag_1\" in train_lf.collect_schema().names()\ncheck_cols = feature_cols_in_train + ([\"responder_6_lag_1\"] if include_lag else [])\n\n# 전체 행 수\nn_rows = train_lf.select(pl.len()).collect().item()\n\n# 각 컬럼별 null 개수 집계 → melt → 결측률 계산\nmissing_rate_df = (\n train_lf\n .select([pl.col(c).is_null().sum().alias(c) for c in check_cols])\n .collect()\n .melt(variable_name=\"column\", value_name=\"nulls\")\n .with_columns(\n pl.lit(n_rows).alias(\"rows\"),\n (pl.col(\"nulls\") / pl.lit(n_rows)).alias(\"missing_rate\")\n )\n .sort(\"missing_rate\", descending=True)\n)\n\n# 상위 몇 개 확인\nprint(\"총 행수:\", n_rows)\nprint(\"컬럼별 결측률 top 20\")\ndisplay(missing_rate_df.head(20))\n\npl.Config.set_tbl_rows(200) # 최대 200행까지 출력\ndisplay(missing_rate_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-17T12:11:38.762801Z","iopub.execute_input":"2025-09-17T12:11:38.764131Z","iopub.status.idle":"2025-09-17T12:11:49.798255Z","shell.execute_reply.started":"2025-09-17T12:11:38.764085Z","shell.execute_reply":"2025-09-17T12:11:49.797344Z"}},"outputs":[{"name":"stdout","text":"총 행수: 11802114\n컬럼별 결측률 top 20\n","output_type":"stream"},{"name":"stderr","text":"/tmp/ipykernel_36/2593423922.py:15: DeprecationWarning: `DataFrame.melt` is deprecated. Use `unpivot` instead, with `index` instead of `id_vars` and `on` instead of `value_vars`\n .melt(variable_name=\"column\", value_name=\"nulls\")\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"shape: (20, 4)\n┌───────────────────┬──────────┬──────────┬──────────────┐\n│ column ┆ nulls ┆ rows ┆ missing_rate │\n│ --- ┆ --- ┆ --- ┆ --- │\n│ str ┆ u32 ┆ i32 ┆ f64 │\n╞═══════════════════╪══════════╪══════════╪══════════════╡\n│ responder_6_lag_1 ┆ 11802106 ┆ 11802114 ┆ 0.999999 │\n│ feature_21 ┆ 8212377 ┆ 11802114 ┆ 0.695839 │\n│ feature_26 ┆ 8212377 ┆ 11802114 ┆ 0.695839 │\n│ feature_27 ┆ 8212377 ┆ 11802114 ┆ 0.695839 │\n│ feature_31 ┆ 8212377 ┆ 11802114 ┆ 0.695839 │\n│ … ┆ … ┆ … ┆ … │\n│ feature_44 ┆ 435922 ┆ 11802114 ┆ 0.036936 │\n│ feature_52 ┆ 388024 ┆ 11802114 ┆ 0.032877 │\n│ feature_55 ┆ 388024 ┆ 11802114 ┆ 0.032877 │\n│ feature_15 ┆ 333462 ┆ 11802114 ┆ 0.028254 │\n│ feature_45 ┆ 306677 ┆ 11802114 ┆ 0.025985 │\n└───────────────────┴──────────┴──────────┴──────────────┘","text/html":"
\nshape: (20, 4)
columnnullsrowsmissing_rate
stru32i32f64
"responder_6_lag_1"11802106118021140.999999
"feature_21"8212377118021140.695839
"feature_26"8212377118021140.695839
"feature_27"8212377118021140.695839
"feature_31"8212377118021140.695839
…………
"feature_44"435922118021140.036936
"feature_52"388024118021140.032877
"feature_55"388024118021140.032877
"feature_15"333462118021140.028254
"feature_45"306677118021140.025985
"},"metadata":{}}],"execution_count":6},{"cell_type":"code","source":"# null 비율 확인하기\n\npl.Config.set_tbl_rows(200) # 최대 200행까지 출력\ndisplay(missing_rate_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-17T12:13:24.982917Z","iopub.execute_input":"2025-09-17T12:13:24.983329Z","iopub.status.idle":"2025-09-17T12:13:24.996762Z","shell.execute_reply.started":"2025-09-17T12:13:24.983298Z","shell.execute_reply":"2025-09-17T12:13:24.995515Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"shape: (80, 4)\n┌───────────────────┬──────────┬──────────┬──────────────┐\n│ column ┆ nulls ┆ rows ┆ missing_rate │\n│ --- ┆ --- ┆ --- ┆ --- │\n│ str ┆ u32 ┆ i32 ┆ f64 │\n╞═══════════════════╪══════════╪══════════╪══════════════╡\n│ responder_6_lag_1 ┆ 11802106 ┆ 11802114 ┆ 0.999999 │\n│ feature_21 ┆ 8212377 ┆ 11802114 ┆ 0.695839 │\n│ feature_26 ┆ 8212377 ┆ 11802114 ┆ 0.695839 │\n│ feature_27 ┆ 8212377 ┆ 11802114 ┆ 0.695839 │\n│ feature_31 ┆ 8212377 ┆ 11802114 ┆ 0.695839 │\n│ feature_00 ┆ 3182052 ┆ 11802114 ┆ 0.269617 │\n│ feature_01 ┆ 3182052 ┆ 11802114 ┆ 0.269617 │\n│ feature_02 ┆ 3182052 ┆ 11802114 ┆ 0.269617 │\n│ feature_03 ┆ 3182052 ┆ 11802114 ┆ 0.269617 │\n│ feature_04 ┆ 3182052 ┆ 11802114 ┆ 0.269617 │\n│ feature_39 ┆ 1818907 ┆ 11802114 ┆ 0.154117 │\n│ feature_42 ┆ 1818907 ┆ 11802114 ┆ 0.154117 │\n│ feature_50 ┆ 1772574 ┆ 11802114 ┆ 0.150191 │\n│ feature_53 ┆ 1772574 ┆ 11802114 ┆ 0.150191 │\n│ feature_41 ┆ 435922 ┆ 11802114 ┆ 0.036936 │\n│ feature_44 ┆ 435922 ┆ 11802114 ┆ 0.036936 │\n│ feature_52 ┆ 388024 ┆ 11802114 ┆ 0.032877 │\n│ feature_55 ┆ 388024 ┆ 11802114 ┆ 0.032877 │\n│ feature_15 ┆ 333462 ┆ 11802114 ┆ 0.028254 │\n│ feature_45 ┆ 306677 ┆ 11802114 ┆ 0.025985 │\n│ feature_46 ┆ 306677 ┆ 11802114 ┆ 0.025985 │\n│ feature_65 ┆ 306677 ┆ 11802114 ┆ 0.025985 │\n│ feature_66 ┆ 306677 ┆ 11802114 ┆ 0.025985 │\n│ feature_62 ┆ 288291 ┆ 11802114 ┆ 0.024427 │\n│ feature_64 ┆ 235171 ┆ 11802114 ┆ 0.019926 │\n│ feature_63 ┆ 225417 ┆ 11802114 ┆ 0.0191 │\n│ feature_73 ┆ 131538 ┆ 11802114 ┆ 0.011145 │\n│ feature_74 ┆ 131538 ┆ 11802114 ┆ 0.011145 │\n│ feature_32 ┆ 131527 ┆ 11802114 ┆ 0.011144 │\n│ feature_33 ┆ 131527 ┆ 11802114 ┆ 0.011144 │\n│ feature_58 ┆ 131522 ┆ 11802114 ┆ 0.011144 │\n│ feature_40 ┆ 67559 ┆ 11802114 ┆ 0.005724 │\n│ feature_43 ┆ 67559 ┆ 11802114 ┆ 0.005724 │\n│ feature_17 ┆ 55853 ┆ 11802114 ┆ 0.004732 │\n│ feature_08 ┆ 19527 ┆ 11802114 ┆ 0.001655 │\n│ feature_51 ┆ 13805 ┆ 11802114 ┆ 0.00117 │\n│ feature_54 ┆ 13805 ┆ 11802114 ┆ 0.00117 │\n│ feature_37 ┆ 849 ┆ 11802114 ┆ 0.000072 │\n│ feature_75 ┆ 303 ┆ 11802114 ┆ 0.000026 │\n│ feature_76 ┆ 303 ┆ 11802114 ┆ 0.000026 │\n│ feature_16 ┆ 250 ┆ 11802114 ┆ 0.000021 │\n│ feature_18 ┆ 219 ┆ 11802114 ┆ 0.000019 │\n│ feature_19 ┆ 219 ┆ 11802114 ┆ 0.000019 │\n│ feature_56 ┆ 219 ┆ 11802114 ┆ 0.000019 │\n│ feature_57 ┆ 219 ┆ 11802114 ┆ 0.000019 │\n│ feature_47 ┆ 87 ┆ 11802114 ┆ 0.000007 │\n│ feature_05 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_06 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_07 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_09 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_10 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_11 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_12 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_13 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_14 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_20 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_22 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_23 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_24 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_25 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_28 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_29 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_30 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_34 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_35 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_36 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_38 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_48 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_49 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_59 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_60 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_61 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_67 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_68 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_69 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_70 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_71 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_72 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_77 ┆ 0 ┆ 11802114 ┆ 0.0 │\n│ feature_78 ┆ 0 ┆ 11802114 ┆ 0.0 │\n└───────────────────┴──────────┴──────────┴──────────────┘","text/html":"
\nshape: (80, 4)
columnnullsrowsmissing_rate
stru32i32f64
"responder_6_lag_1"11802106118021140.999999
"feature_21"8212377118021140.695839
"feature_26"8212377118021140.695839
"feature_27"8212377118021140.695839
"feature_31"8212377118021140.695839
"feature_00"3182052118021140.269617
"feature_01"3182052118021140.269617
"feature_02"3182052118021140.269617
"feature_03"3182052118021140.269617
"feature_04"3182052118021140.269617
"feature_39"1818907118021140.154117
"feature_42"1818907118021140.154117
"feature_50"1772574118021140.150191
"feature_53"1772574118021140.150191
"feature_41"435922118021140.036936
"feature_44"435922118021140.036936
"feature_52"388024118021140.032877
"feature_55"388024118021140.032877
"feature_15"333462118021140.028254
"feature_45"306677118021140.025985
"feature_46"306677118021140.025985
"feature_65"306677118021140.025985
"feature_66"306677118021140.025985
"feature_62"288291118021140.024427
"feature_64"235171118021140.019926
"feature_63"225417118021140.0191
"feature_73"131538118021140.011145
"feature_74"131538118021140.011145
"feature_32"131527118021140.011144
"feature_33"131527118021140.011144
"feature_58"131522118021140.011144
"feature_40"67559118021140.005724
"feature_43"67559118021140.005724
"feature_17"55853118021140.004732
"feature_08"19527118021140.001655
"feature_51"13805118021140.00117
"feature_54"13805118021140.00117
"feature_37"849118021140.000072
"feature_75"303118021140.000026
"feature_76"303118021140.000026
"feature_16"250118021140.000021
"feature_18"219118021140.000019
"feature_19"219118021140.000019
"feature_56"219118021140.000019
"feature_57"219118021140.000019
"feature_47"87118021140.000007
"feature_05"0118021140.0
"feature_06"0118021140.0
"feature_07"0118021140.0
"feature_09"0118021140.0
"feature_10"0118021140.0
"feature_11"0118021140.0
"feature_12"0118021140.0
"feature_13"0118021140.0
"feature_14"0118021140.0
"feature_20"0118021140.0
"feature_22"0118021140.0
"feature_23"0118021140.0
"feature_24"0118021140.0
"feature_25"0118021140.0
"feature_28"0118021140.0
"feature_29"0118021140.0
"feature_30"0118021140.0
"feature_34"0118021140.0
"feature_35"0118021140.0
"feature_36"0118021140.0
"feature_38"0118021140.0
"feature_48"0118021140.0
"feature_49"0118021140.0
"feature_59"0118021140.0
"feature_60"0118021140.0
"feature_61"0118021140.0
"feature_67"0118021140.0
"feature_68"0118021140.0
"feature_69"0118021140.0
"feature_70"0118021140.0
"feature_71"0118021140.0
"feature_72"0118021140.0
"feature_77"0118021140.0
"feature_78"0118021140.0
"},"metadata":{}}],"execution_count":8},{"cell_type":"markdown","source":"- 선형관계 봐봄 -> 처참...","metadata":{}},{"cell_type":"code","source":"# ==== RFECV로 피처 선택 (LightGBM) ====\nimport numpy as np\nimport polars as pl\nimport pandas as pd\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_selection import RFECV\nfrom sklearn.metrics import make_scorer, mean_squared_error\nimport lightgbm as lgb\n\n# 0) 사용 피처 정의: feature_* + (lag 있으면 포함)\nschema = train_lf.collect_schema().names()\nbase_keep = [\"date_id\",\"time_id\",\"symbol_id\",\"weight\",\"partition_id\"]\ntarget = \"responder_6\"\nlag_feats = [c for c in schema if c.startswith(\"responder_6_lag_\")]\nFEATURES = sorted([c for c in schema if c.startswith(\"feature_\")]) + lag_feats\n\n# 1) Pandas로 물질화 (메모리 아끼려 float32)\ncols_needed = base_keep + [target] + FEATURES\ntrain_pd = train_lf.select(cols_needed).collect().to_pandas()\ntest_pd = test_lf.select([c for c in cols_needed if c in test_lf.collect_schema().names()]).collect().to_pandas()\n\nfor c in FEATURES + [target, \"weight\"]:\n if c in train_pd.columns:\n if pd.api.types.is_integer_dtype(train_pd[c]):\n train_pd[c] = train_pd[c].astype(\"float32\")\n if c in test_pd.columns:\n if pd.api.types.is_integer_dtype(test_pd[c]):\n test_pd[c] = test_pd[c].astype(\"float32\")\n\n# 2) X / y / groups / sample_weight\nX_train = train_pd[FEATURES]\ny_train = train_pd[target].astype(\"float32\")\ngroups = train_pd[\"date_id\"].to_numpy() # 날짜 단위로 폴드 분리\nsw = train_pd[\"weight\"].fillna(1.0).to_numpy()\n\n# 3) 평가지표 (RMSE ↓) → RFECV는 \"높을수록 좋은 점수\"가 필요해서 음수로 변환\nneg_rmse = make_scorer(lambda yt, yp: -np.sqrt(mean_squared_error(yt, yp)), greater_is_better=True)\n\n# 4) LightGBM + RFECV 파이프라인\nest = lgb.LGBMRegressor(\n n_estimators=400,\n learning_rate=0.05,\n max_depth=-1,\n num_leaves=64,\n subsample=0.8,\n colsample_bytree=0.8,\n random_state=42,\n n_jobs=-1,\n)\n\npipe = Pipeline([\n (\"imputer\", SimpleImputer(strategy=\"median\")), # 결측치 대체\n (\"selector\", RFECV(\n estimator=est,\n step=0.2, # 20%씩 제거 → 속도↑ (더 촘촘히 하려면 0.1)\n min_features_to_select=20, # 최소 남길 개수 (원하면 조정)\n cv=GroupKFold(n_splits=5),\n scoring=neg_rmse,\n n_jobs=-1,\n verbose=0,\n ))\n])\n\n# 5) 학습 (가중치/그룹 반영)\npipe.fit(X_train, y_train, selector__groups=groups, selector__estimator__sample_weight=sw)\n\n# 6) 선택된 피처 목록\nselector = pipe.named_steps[\"selector\"]\nsupport_mask = selector.support_\nselected_features = [f for f, keep in zip(FEATURES, support_mask) if keep]\n\nprint(f\"[RFECV] 선택된 피처 수: {len(selected_features)} / {len(FEATURES)}\")\nprint(\"예시 20개:\", selected_features[:20])\n\n# 7) 변환된 학습/테스트 세트 만들기 (같은 전처리/선택 적용)\n# 주의: 파이프라인에 imputer가 들어있기 때문에 transform을 통과시켜야 함.\nX_train_sel = pipe[:-1].transform(X_train) # imputer까지 적용\nX_train_sel = X_train_sel[:, support_mask] # 선택된 피처만\n# 테스트에도 동일 적용 (컬럼 정렬 보장)\nX_test = test_pd.reindex(columns=FEATURES, fill_value=np.nan)\nX_test_sel = pipe[:-1].transform(X_test)\nX_test_sel = X_test_sel[:, support_mask]\n\ny_train_vec = y_train.to_numpy().astype(\"float32\")\ny_test_vec = test_pd[target].to_numpy().astype(\"float32\") if target in test_pd.columns else None\ntest_weight = test_pd[\"weight\"].fillna(1.0).to_numpy() if \"weight\" in test_pd.columns else None\n\n# 8) 최종 산출물 정리\nprint(\"X_train_sel shape:\", X_train_sel.shape)\nprint(\"X_test_sel shape:\", X_test_sel.shape if X_test_sel is not None else None)\n\n# (선택) 선택 피처 이름을 파일로 저장\nimport json, os\nwith open(\"selected_features_by_rfecv.json\", \"w\", encoding=\"utf-8\") as f:\n json.dump(selected_features, f, ensure_ascii=False, indent=2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-17T13:21:08.107779Z","iopub.execute_input":"2025-09-17T13:21:08.10821Z","execution_failed":"2025-09-17T13:22:36.069Z"}},"outputs":[],"execution_count":null}]}