{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":11305158,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport polars as pl\nfrom matplotlib import pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T01:31:19.417985Z","iopub.execute_input":"2025-09-20T01:31:19.418331Z","iopub.status.idle":"2025-09-20T01:31:19.438367Z","shell.execute_reply.started":"2025-09-20T01:31:19.418306Z","shell.execute_reply":"2025-09-20T01:31:19.437525Z"}},"outputs":[{"name":"stdout","text":"/kaggle/input/jane-street-real-time-market-data-forecasting/responders.csv\n/kaggle/input/jane-street-real-time-market-data-forecasting/sample_submission.csv\n/kaggle/input/jane-street-real-time-market-data-forecasting/features.csv\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=4/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=5/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=6/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=3/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=1/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=8/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=2/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=7/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=9/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/lags.parquet/date_id=0/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/test.parquet/date_id=0/part-0.parquet\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/jane_street_gateway.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/jane_street_inference_server.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/__init__.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/templates.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/base_gateway.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/relay.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/kaggle_evaluation.proto\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/__init__.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/kaggle_evaluation_pb2.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/kaggle_evaluation_pb2_grpc.py\n/kaggle/input/jane-street-real-time-market-data-forecasting/kaggle_evaluation/core/generated/__init__.py\n","output_type":"stream"}],"execution_count":7},{"cell_type":"code","source":"ROOT_DIR = \"/kaggle/input/jane-street-real-time-market-data-forecasting\"\n\n# === 경로 ===\ntrain_path = os.path.join(ROOT_DIR, \"train.parquet\")\n\n# === 메타데이터 ===\nlags = os.path.join(ROOT_DIR, \"lags.parquet\")\ntrain_path = os.path.join(ROOT_DIR, \"train.parquet\")\nlags_path  = os.path.join(ROOT_DIR, \"lags.parquet\")\nfeatures   = pl.read_csv(os.path.join(ROOT_DIR, \"features.csv\"))\nresponders = pl.read_csv(os.path.join(ROOT_DIR, \"responders.csv\"))\nsample_sub = pl.read_csv(os.path.join(ROOT_DIR, \"sample_submission.csv\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T01:31:26.204429Z","iopub.execute_input":"2025-09-20T01:31:26.204786Z","iopub.status.idle":"2025-09-20T01:31:26.217636Z","shell.execute_reply.started":"2025-09-20T01:31:26.204761Z","shell.execute_reply":"2025-09-20T01:31:26.216708Z"}},"outputs":[],"execution_count":8},{"cell_type":"code","source":"file_path = os.path.join(ROOT_DIR, \"train.parquet\", \"partition_id=0\", \"part-0.parquet\")\ndata_0 = pl.read_parquet(file_path)\n\n# responder 계열 컬럼 확인\nresp_cols = [c for c in data_0.columns if c.startswith(\"responder_\")]\n\n# responder_6만 남기고 나머지 drop\nresp_drop = [c for c in resp_cols if c != \"responder_6\"]\ndata_0 = data_0.drop(resp_drop)\n\n\n# 삭제할 feature 컬럼들 : 결측치 많은 애들\ndrop_features = [\"feature_00\", \"feature_01\", \"feature_02\", \"feature_03\",\"feature_04\", \"feature_26\", \"feature_27\", \"feature_31\"]\n\ndata_0 = data_0.drop(drop_features)\n\n# 버전1 데이터셋 구성 미리보기 : x 데이터에 feature만 남기기\ndata_0_y = data_0[\"responder_6\"]\ndata_0_X = data_0.drop(\"responder_6\",\"date_id\",\"time_id\",\"symbol_id\", \"weight\" )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T01:31:30.143564Z","iopub.execute_input":"2025-09-20T01:31:30.143918Z","iopub.status.idle":"2025-09-20T01:31:30.998187Z","shell.execute_reply.started":"2025-09-20T01:31:30.14389Z","shell.execute_reply":"2025-09-20T01:31:30.997154Z"}},"outputs":[],"execution_count":9},{"cell_type":"code","source":"data_0_X","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T01:31:35.433478Z","iopub.execute_input":"2025-09-20T01:31:35.433852Z","iopub.status.idle":"2025-09-20T01:31:35.467382Z","shell.execute_reply.started":"2025-09-20T01:31:35.433825Z","shell.execute_reply":"2025-09-20T01:31:35.46636Z"}},"outputs":[{"execution_count":10,"output_type":"execute_result","data":{"text/plain":"shape: (1_944_210, 71)\n┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n│ feature_0 ┆ feature_0 ┆ feature_0 ┆ feature_0 ┆ … ┆ feature_7 ┆ feature_7 ┆ feature_7 ┆ feature_ │\n│ 5         ┆ 6         ┆ 7         ┆ 8         ┆   ┆ 5         ┆ 6         ┆ 7         ┆ 78       │\n│ ---       ┆ ---       ┆ ---       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ ---       ┆ ---      │\n│ f32       ┆ f32       ┆ f32       ┆ f32       ┆   ┆ f32       ┆ f32       ┆ f32       ┆ f32      │\n╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n│ 0.851033  ┆ 0.242971  ┆ 0.2634    ┆ -0.891687 ┆ … ┆ -0.261412 ┆ -0.211486 ┆ -0.335556 ┆ -0.28149 │\n│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 8        │\n│ 0.676961  ┆ 0.151984  ┆ 0.192465  ┆ -0.521729 ┆ … ┆ -0.281207 ┆ -0.182894 ┆ -0.245565 ┆ -0.30244 │\n│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 1        │\n│ 1.056285  ┆ 0.187227  ┆ 0.249901  ┆ -0.77305  ┆ … ┆ 0.377131  ┆ 0.300724  ┆ -0.106842 ┆ -0.09679 │\n│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 2        │\n│ 1.139366  ┆ 0.273328  ┆ 0.306549  ┆ -1.262223 ┆ … ┆ -0.226891 ┆ -0.251412 ┆ -0.215522 ┆ -0.29624 │\n│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 4        │\n│ 0.9552    ┆ 0.262404  ┆ 0.344457  ┆ -0.613813 ┆ … ┆ 3.678076  ┆ 2.793581  ┆ 2.61825   ┆ 3.418133 │\n│ …         ┆ …         ┆ …         ┆ …         ┆ … ┆ …         ┆ …         ┆ …         ┆ …        │\n│ -0.028087 ┆ 0.287438  ┆ 0.118074  ┆ -0.644495 ┆ … ┆ -0.267972 ┆ -0.253485 ┆ -0.147347 ┆ -0.16696 │\n│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 4        │\n│ -0.022584 ┆ 0.442352  ┆ 0.140746  ┆ -0.571057 ┆ … ┆ -0.476703 ┆ -0.373956 ┆ -0.356012 ┆ -0.35281 │\n│ -0.024804 ┆ 0.420692  ┆ 0.136259  ┆ -0.809642 ┆ … ┆ -0.339679 ┆ -0.301338 ┆ -0.323033 ┆ -0.23971 │\n│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 6        │\n│ -0.016138 ┆ 0.303561  ┆ 0.14997   ┆ -0.727993 ┆ … ┆ -0.497245 ┆ -0.320908 ┆ -0.486542 ┆ -0.44285 │\n│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 9        │\n│ -0.017634 ┆ 0.271368  ┆ 0.128993  ┆ -0.611178 ┆ … ┆ -0.194399 ┆ -0.230857 ┆ -0.219675 ┆ -0.17446 │\n│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 1        │\n└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘","text/html":"<div><style>\n.dataframe > thead > tr,\n.dataframe > tbody > tr {\n  text-align: right;\n  white-space: pre-wrap;\n}\n</style>\n<small>shape: (1_944_210, 71)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>feature_05</th><th>feature_06</th><th>feature_07</th><th>feature_08</th><th>feature_09</th><th>feature_10</th><th>feature_11</th><th>feature_12</th><th>feature_13</th><th>feature_14</th><th>feature_15</th><th>feature_16</th><th>feature_17</th><th>feature_18</th><th>feature_19</th><th>feature_20</th><th>feature_21</th><th>feature_22</th><th>feature_23</th><th>feature_24</th><th>feature_25</th><th>feature_28</th><th>feature_29</th><th>feature_30</th><th>feature_32</th><th>feature_33</th><th>feature_34</th><th>feature_35</th><th>feature_36</th><th>feature_37</th><th>feature_38</th><th>feature_39</th><th>feature_40</th><th>feature_41</th><th>feature_42</th><th>feature_43</th><th>feature_44</th><th>feature_45</th><th>feature_46</th><th>feature_47</th><th>feature_48</th><th>feature_49</th><th>feature_50</th><th>feature_51</th><th>feature_52</th><th>feature_53</th><th>feature_54</th><th>feature_55</th><th>feature_56</th><th>feature_57</th><th>feature_58</th><th>feature_59</th><th>feature_60</th><th>feature_61</th><th>feature_62</th><th>feature_63</th><th>feature_64</th><th>feature_65</th><th>feature_66</th><th>feature_67</th><th>feature_68</th><th>feature_69</th><th>feature_70</th><th>feature_71</th><th>feature_72</th><th>feature_73</th><th>feature_74</th><th>feature_75</th><th>feature_76</th><th>feature_77</th><th>feature_78</th></tr><tr><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>i8</td><td>i8</td><td>i16</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td></tr></thead><tbody><tr><td>0.851033</td><td>0.242971</td><td>0.2634</td><td>-0.891687</td><td>11</td><td>7</td><td>76</td><td>-0.883028</td><td>0.003067</td><td>-0.744703</td><td>null</td><td>-0.169586</td><td>null</td><td>-1.335938</td><td>-1.707803</td><td>0.91013</td><td>null</td><td>1.636431</td><td>1.522133</td><td>-1.551398</td><td>-0.229627</td><td>1.378301</td><td>-0.283712</td><td>0.123196</td><td>null</td><td>null</td><td>0.28118</td><td>0.269163</td><td>0.349028</td><td>-0.012596</td><td>-0.225932</td><td>null</td><td>-1.073602</td><td>null</td><td>null</td><td>-0.181716</td><td>null</td><td>null</td><td>null</td><td>0.564021</td><td>2.088506</td><td>0.832022</td><td>null</td><td>0.204797</td><td>null</td><td>null</td><td>-0.808103</td><td>null</td><td>-2.037683</td><td>0.727661</td><td>null</td><td>-0.989118</td><td>-0.345213</td><td>-1.36224</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>-1.251104</td><td>-0.110252</td><td>-0.491157</td><td>-1.02269</td><td>0.152241</td><td>-0.659864</td><td>null</td><td>null</td><td>-0.261412</td><td>-0.211486</td><td>-0.335556</td><td>-0.281498</td></tr><tr><td>0.676961</td><td>0.151984</td><td>0.192465</td><td>-0.521729</td><td>11</td><td>7</td><td>76</td><td>-0.865307</td><td>-0.225629</td><td>-0.582163</td><td>null</td><td>0.317467</td><td>null</td><td>-1.250016</td><td>-1.682929</td><td>1.412757</td><td>null</td><td>0.520378</td><td>0.744132</td><td>-0.788658</td><td>0.641776</td><td>0.2272</td><td>0.580907</td><td>1.128879</td><td>null</td><td>null</td><td>-1.512286</td><td>-1.414357</td><td>-1.823322</td><td>-0.082763</td><td>-0.184119</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>-10.835207</td><td>-0.002704</td><td>-0.621836</td><td>null</td><td>1.172836</td><td>null</td><td>null</td><td>-1.625862</td><td>null</td><td>-1.410017</td><td>1.063013</td><td>null</td><td>0.888355</td><td>0.467994</td><td>-1.36224</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>-1.065759</td><td>0.013322</td><td>-0.592855</td><td>-1.052685</td><td>-0.393726</td><td>-0.741603</td><td>null</td><td>null</td><td>-0.281207</td><td>-0.182894</td><td>-0.245565</td><td>-0.302441</td></tr><tr><td>1.056285</td><td>0.187227</td><td>0.249901</td><td>-0.77305</td><td>11</td><td>7</td><td>76</td><td>-0.675719</td><td>-0.199404</td><td>-0.586798</td><td>null</td><td>-0.814909</td><td>null</td><td>-1.296782</td><td>-2.040234</td><td>0.639589</td><td>null</td><td>1.597359</td><td>0.657514</td><td>-1.350148</td><td>0.364215</td><td>-0.017751</td><td>-0.317361</td><td>-0.122379</td><td>null</td><td>null</td><td>-0.320921</td><td>-0.95809</td><td>-2.436589</td><td>0.070999</td><td>-0.245239</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>-1.420632</td><td>-3.515137</td><td>-4.67776</td><td>null</td><td>0.535897</td><td>null</td><td>null</td><td>-0.72542</td><td>null</td><td>-2.29417</td><td>1.764551</td><td>null</td><td>-0.120789</td><td>-0.063458</td><td>-1.36224</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>-0.882604</td><td>-0.072482</td><td>-0.617934</td><td>-0.86323</td><td>-0.241892</td><td>-0.709919</td><td>null</td><td>null</td><td>0.377131</td><td>0.300724</td><td>-0.106842</td><td>-0.096792</td></tr><tr><td>1.139366</td><td>0.273328</td><td>0.306549</td><td>-1.262223</td><td>42</td><td>5</td><td>150</td><td>-0.694008</td><td>3.004091</td><td>0.114809</td><td>null</td><td>-0.251882</td><td>null</td><td>-1.902009</td><td>-0.979447</td><td>0.241165</td><td>null</td><td>-0.392359</td><td>-0.224699</td><td>-2.129397</td><td>-0.855287</td><td>0.404142</td><td>-0.578156</td><td>0.105702</td><td>null</td><td>null</td><td>0.544138</td><td>-0.087091</td><td>-1.500147</td><td>-0.201288</td><td>-0.038042</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0.382074</td><td>2.669135</td><td>0.611711</td><td>null</td><td>2.413415</td><td>null</td><td>null</td><td>1.313203</td><td>null</td><td>-0.810125</td><td>2.939022</td><td>null</td><td>3.988801</td><td>1.834661</td><td>-1.36224</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>-0.697595</td><td>1.074309</td><td>-0.206929</td><td>-0.530602</td><td>4.765215</td><td>0.571554</td><td>null</td><td>null</td><td>-0.226891</td><td>-0.251412</td><td>-0.215522</td><td>-0.296244</td></tr><tr><td>0.9552</td><td>0.262404</td><td>0.344457</td><td>-0.613813</td><td>44</td><td>3</td><td>16</td><td>-0.947351</td><td>-0.030018</td><td>-0.502379</td><td>null</td><td>0.646086</td><td>null</td><td>-1.844685</td><td>-1.58656</td><td>-0.182024</td><td>null</td><td>-0.969949</td><td>-0.673813</td><td>-1.282132</td><td>-1.399894</td><td>0.043815</td><td>-0.320225</td><td>-0.031713</td><td>null</td><td>null</td><td>-0.08842</td><td>-0.995003</td><td>-2.635336</td><td>-0.196461</td><td>-0.618719</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>-2.0146</td><td>-2.321076</td><td>-3.711265</td><td>null</td><td>1.253902</td><td>null</td><td>null</td><td>0.476195</td><td>null</td><td>-0.771732</td><td>2.843421</td><td>null</td><td>1.379815</td><td>0.411827</td><td>-1.36224</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>-0.948601</td><td>-0.136814</td><td>-0.447704</td><td>-1.141761</td><td>0.099631</td><td>-0.661928</td><td>null</td><td>null</td><td>3.678076</td><td>2.793581</td><td>2.61825</td><td>3.418133</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>-0.028087</td><td>0.287438</td><td>0.118074</td><td>-0.644495</td><td>4</td><td>3</td><td>11</td><td>3.419741</td><td>0.997164</td><td>1.706084</td><td>-0.318914</td><td>-0.461119</td><td>-0.294792</td><td>1.929656</td><td>1.213011</td><td>1.16521</td><td>null</td><td>3.431933</td><td>0.792091</td><td>-0.961323</td><td>0.197048</td><td>0.258624</td><td>-0.538528</td><td>-0.306105</td><td>0.12894</td><td>-0.022014</td><td>0.749626</td><td>0.547256</td><td>1.428185</td><td>2.043587</td><td>1.260826</td><td>1.837012</td><td>1.289607</td><td>1.129898</td><td>-0.712776</td><td>1.209243</td><td>-0.337086</td><td>-0.205467</td><td>-2.51014</td><td>-1.281748</td><td>-0.383061</td><td>0.99984</td><td>1.178574</td><td>-1.942191</td><td>0.445592</td><td>-1.414607</td><td>-1.362255</td><td>-0.760373</td><td>-0.199131</td><td>-2.412528</td><td>-1.033167</td><td>-2.285751</td><td>0.406769</td><td>0.300441</td><td>-0.072687</td><td>0.114416</td><td>0.200983</td><td>2.923207</td><td>1.137057</td><td>5.132564</td><td>0.726686</td><td>1.304514</td><td>3.247437</td><td>0.631181</td><td>2.437627</td><td>-0.341539</td><td>-0.360914</td><td>-0.267972</td><td>-0.253485</td><td>-0.147347</td><td>-0.166964</td></tr><tr><td>-0.022584</td><td>0.442352</td><td>0.140746</td><td>-0.571057</td><td>81</td><td>2</td><td>534</td><td>0.385333</td><td>0.732816</td><td>0.155896</td><td>0.202543</td><td>0.930962</td><td>0.721754</td><td>-0.865098</td><td>0.108584</td><td>-0.640984</td><td>null</td><td>-0.876378</td><td>-1.044793</td><td>-1.282868</td><td>-0.355638</td><td>0.776003</td><td>-0.126703</td><td>0.290924</td><td>-0.123493</td><td>0.995562</td><td>-0.466438</td><td>-0.349856</td><td>-0.892137</td><td>-0.370237</td><td>-0.69819</td><td>2.02505</td><td>3.178006</td><td>2.461243</td><td>1.439713</td><td>2.302129</td><td>2.046488</td><td>0.117269</td><td>-0.105732</td><td>0.343813</td><td>-0.132489</td><td>0.081151</td><td>0.247478</td><td>1.210106</td><td>0.012775</td><td>-0.743068</td><td>0.18959</td><td>0.866432</td><td>0.588312</td><td>-0.599171</td><td>0.269633</td><td>0.248799</td><td>0.33105</td><td>0.300441</td><td>1.796084</td><td>2.408884</td><td>1.916744</td><td>0.658252</td><td>0.340445</td><td>0.652627</td><td>-0.216905</td><td>-0.185448</td><td>0.521877</td><td>1.35906</td><td>0.494098</td><td>-0.20115</td><td>-0.289628</td><td>-0.476703</td><td>-0.373956</td><td>-0.356012</td><td>-0.35281</td></tr><tr><td>-0.024804</td><td>0.420692</td><td>0.136259</td><td>-0.809642</td><td>11</td><td>7</td><td>76</td><td>-0.200244</td><td>-0.409379</td><td>-0.185599</td><td>0.234811</td><td>0.160243</td><td>0.110508</td><td>-0.959411</td><td>1.392733</td><td>2.161604</td><td>null</td><td>-0.219788</td><td>-0.040412</td><td>-1.940056</td><td>-0.637217</td><td>0.481473</td><td>-0.211353</td><td>0.417266</td><td>-0.111355</td><td>0.543678</td><td>0.090486</td><td>-0.336618</td><td>1.615937</td><td>-0.890028</td><td>-0.535917</td><td>-1.156149</td><td>-0.947188</td><td>-1.175608</td><td>0.210434</td><td>-0.663065</td><td>-0.184241</td><td>0.453498</td><td>-0.814056</td><td>0.076486</td><td>0.384369</td><td>0.880885</td><td>0.198965</td><td>1.093185</td><td>1.355752</td><td>0.86659</td><td>-0.543416</td><td>-0.557341</td><td>1.018626</td><td>-0.53542</td><td>-0.299109</td><td>-0.135551</td><td>0.302495</td><td>0.300441</td><td>1.178458</td><td>1.963925</td><td>2.115989</td><td>2.300967</td><td>1.213972</td><td>-0.375353</td><td>-0.315288</td><td>-0.377317</td><td>-0.018499</td><td>-0.303676</td><td>0.042694</td><td>-0.304563</td><td>-0.274469</td><td>-0.339679</td><td>-0.301338</td><td>-0.323033</td><td>-0.239716</td></tr><tr><td>-0.016138</td><td>0.303561</td><td>0.14997</td><td>-0.727993</td><td>42</td><td>5</td><td>150</td><td>0.089536</td><td>-0.318639</td><td>0.229737</td><td>-0.359855</td><td>-0.135196</td><td>-0.361641</td><td>-1.061814</td><td>1.642337</td><td>0.124642</td><td>null</td><td>-0.523115</td><td>-0.88477</td><td>-1.407083</td><td>-0.564234</td><td>1.599255</td><td>-0.414362</td><td>-0.21513</td><td>-0.042788</td><td>-0.010577</td><td>0.227581</td><td>-0.202145</td><td>-0.719912</td><td>-1.727418</td><td>-1.848666</td><td>-1.12455</td><td>-0.059247</td><td>-0.87092</td><td>1.309015</td><td>0.680563</td><td>0.854657</td><td>1.16664</td><td>1.677758</td><td>0.523151</td><td>-0.28115</td><td>0.002627</td><td>1.15112</td><td>0.002777</td><td>0.034642</td><td>0.414592</td><td>0.217866</td><td>-0.189608</td><td>1.401886</td><td>0.302604</td><td>0.711625</td><td>-0.181922</td><td>0.214608</td><td>0.300441</td><td>0.255725</td><td>0.576029</td><td>0.445959</td><td>-0.53101</td><td>1.331284</td><td>0.162807</td><td>-0.20295</td><td>0.06306</td><td>0.219878</td><td>-0.271476</td><td>0.315763</td><td>-0.519676</td><td>-0.716094</td><td>-0.497245</td><td>-0.320908</td><td>-0.486542</td><td>-0.442859</td></tr><tr><td>-0.017634</td><td>0.271368</td><td>0.128993</td><td>-0.611178</td><td>50</td><td>1</td><td>522</td><td>0.858832</td><td>2.573905</td><td>1.143107</td><td>-0.071363</td><td>0.14384</td><td>-0.015339</td><td>-0.791542</td><td>2.087125</td><td>1.837453</td><td>null</td><td>-0.051464</td><td>0.011945</td><td>-0.639944</td><td>0.34263</td><td>0.914254</td><td>-0.400201</td><td>-0.11904</td><td>0.013682</td><td>0.922743</td><td>-0.273362</td><td>-0.935538</td><td>-0.367452</td><td>-0.622919</td><td>-0.77998</td><td>0.09443</td><td>0.803884</td><td>0.62649</td><td>0.361963</td><td>0.602378</td><td>0.764111</td><td>0.452386</td><td>1.129728</td><td>0.129606</td><td>-0.000329</td><td>-0.112252</td><td>-0.525418</td><td>0.289495</td><td>-0.10213</td><td>0.188333</td><td>0.511473</td><td>-0.710926</td><td>0.661158</td><td>0.226204</td><td>0.044134</td><td>0.386667</td><td>0.087507</td><td>0.300441</td><td>-0.32837</td><td>-0.096137</td><td>-0.271381</td><td>-0.75118</td><td>2.268737</td><td>0.12411</td><td>0.063276</td><td>0.248291</td><td>2.183975</td><td>4.698049</td><td>3.342008</td><td>-0.18195</td><td>-0.143228</td><td>-0.194399</td><td>-0.230857</td><td>-0.219675</td><td>-0.174461</td></tr></tbody></table></div>"},"metadata":{}}],"execution_count":10},{"cell_type":"markdown","source":"# **(part0만 해봄)**","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\nimport pandas as pd\n\n# Polars → Pandas\nX = data_0_X.to_pandas()\ny = data_0_y.to_pandas()\n\n# LightGBM 모델 (회귀 예시)\nmodel = lgb.LGBMRegressor(\n    n_estimators=500,\n    learning_rate=0.05,\n    subsample=0.8,\n    colsample_bytree=0.8,\n    random_state=42,\n    n_jobs=-1\n)\n\n# 학습 (train만)\nmodel.fit(X, y)\n\n# 피처 중요도 추출\nbooster = model.booster_\nimp_df = pd.DataFrame({\n    \"feature\": model.feature_name_,\n    \"gain\": booster.feature_importance(importance_type=\"gain\"),\n    \"split\": booster.feature_importance(importance_type=\"split\"),\n})\nimp_df[\"gain_norm\"] = imp_df[\"gain\"] / (imp_df[\"gain\"].sum() + 1e-12)\nimp_df = imp_df.sort_values(\"gain\", ascending=False).reset_index(drop=True)\n\nprint(imp_df.head(30))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **for문으로 part0~6까지 돌리는 코드**","metadata":{}},{"cell_type":"code","source":"# 첫 번째 코드에서 정의한 drop_features 재사용\ndrop_features = [\"feature_00\", \"feature_01\", \"feature_02\", \"feature_03\",\n                 \"feature_04\", \"feature_26\", \"feature_27\", \"feature_31\"]\n\n# X에서 제외할 메타 컬럼 (첫 번째 코드박스 기준)\nmeta_cols = [\"date_id\", \"time_id\", \"symbol_id\", \"weight\"]\n\n# 결과 저장 dict\nhigh_features_dict = {} \n\nfor pid in range(7):  # partition_id=0~6\n    print(f\"=== Partition {pid} ===\")\n\n    # --------------------\n    # 1) 데이터 로드\n    # --------------------\n    f = os.path.join(ROOT_DIR, \"train.parquet\", f\"partition_id={pid}\", \"part-0.parquet\")\n    df = pl.read_parquet(f)\n\n    # responder 처리 (responder_6만 남기기)\n    df = df.drop([c for c in df.columns if c.startswith(\"responder_\") and c != \"responder_6\"])\n\n    # 결측치 많은 feature drop\n    df = df.drop(drop_features)\n\n    # --------------------\n    # 2) X / y 준비 (첫 번째 코드 로직 재사용)\n    # --------------------\n    df_y = df[\"responder_6\"]\n    df_X = df.drop([\"responder_6\"] + [c for c in meta_cols if c in df.columns])\n\n    X = df_X.to_pandas()\n    y = df_y.to_pandas()\n\n    # --------------------\n    # 3) LightGBM 학습 (partition별)\n    # --------------------\n    model = lgb.LGBMRegressor(\n        n_estimators=500,\n        learning_rate=0.05,\n        subsample=0.8,\n        colsample_bytree=0.8,\n        random_state=42,\n        n_jobs=-1\n    )\n    model.fit(X, y)\n\n    booster = model.booster_\n    imp_df = pd.DataFrame({\n        \"feature\": model.feature_name_,\n        \"gain\": booster.feature_importance(importance_type=\"gain\"),\n    })\n    imp_df[\"gain_norm\"] = imp_df[\"gain\"] / (imp_df[\"gain\"].sum() + 1e-12)\n    imp_df = imp_df.sort_values(\"gain\", ascending=False).reset_index(drop=True)\n\n    # --------------------\n    # 4) 상위 30개 feature 추출 (리스트만 저장)\n    TOP_N = 30\n    high_features = imp_df[\"feature\"].iloc[:TOP_N].tolist()\n\n    # 파티션별로 리스트 저장\n    high_features_dict[f\"high_features_{pid}\"] = high_features\n\n    print(f\"→ high_features_{pid}: {len(high_features)} features\")\n\n    # 이제 high_features_dict[\"high_features_0\"], high_features_dict[\"high_features_1\"], ... 접근 가능\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T01:32:09.178021Z","iopub.execute_input":"2025-09-20T01:32:09.178383Z","iopub.status.idle":"2025-09-20T01:50:31.740278Z","shell.execute_reply.started":"2025-09-20T01:32:09.178358Z","shell.execute_reply":"2025-09-20T01:50:31.73529Z"}},"outputs":[{"name":"stdout","text":"=== Partition 0 ===\n[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.840094 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 17033\n[LightGBM] [Info] Number of data points in the train set: 1944210, number of used features: 70\n[LightGBM] [Info] Start training from score 0.001488\n→ high_features_0: 30 features\n=== Partition 1 ===\n[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 1.337461 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 17033\n[LightGBM] [Info] Number of data points in the train set: 2804247, number of used features: 70\n[LightGBM] [Info] Start training from score -0.006968\n→ high_features_1: 30 features\n=== Partition 2 ===\n[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 1.383380 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 17048\n[LightGBM] [Info] Number of data points in the train set: 3036873, number of used features: 70\n[LightGBM] [Info] Start training from score -0.008083\n→ high_features_2: 30 features\n=== Partition 3 ===\n[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 1.817569 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 17306\n[LightGBM] [Info] Number of data points in the train set: 4016784, number of used features: 71\n[LightGBM] [Info] Start training from score 0.002036\n→ high_features_3: 30 features\n=== Partition 4 ===\n[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 2.308800 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 17306\n[LightGBM] [Info] Number of data points in the train set: 5022952, number of used features: 71\n[LightGBM] [Info] Start training from score 0.000059\n→ high_features_4: 30 features\n=== Partition 5 ===\n[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 2.523469 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 17310\n[LightGBM] [Info] Number of data points in the train set: 5348200, number of used features: 71\n[LightGBM] [Info] Start training from score -0.006323\n→ high_features_5: 30 features\n=== Partition 6 ===\n[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 2.862829 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 17315\n[LightGBM] [Info] Number of data points in the train set: 6203912, number of used features: 71\n[LightGBM] [Info] Start training from score -0.004675\n→ high_features_6: 30 features\n","output_type":"stream"}],"execution_count":12},{"cell_type":"markdown","source":"# **뽑은 피처들 교집합 구하기**","metadata":{}},{"cell_type":"code","source":"print(high_features_dict.keys())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T01:50:38.110704Z","iopub.execute_input":"2025-09-20T01:50:38.11236Z","iopub.status.idle":"2025-09-20T01:50:38.135045Z","shell.execute_reply.started":"2025-09-20T01:50:38.112227Z","shell.execute_reply":"2025-09-20T01:50:38.130369Z"}},"outputs":[{"name":"stdout","text":"dict_keys(['high_features_0', 'high_features_1', 'high_features_2', 'high_features_3', 'high_features_4', 'high_features_5', 'high_features_6'])\n","output_type":"stream"}],"execution_count":13},{"cell_type":"code","source":"# 모든 high_features 리스트의 교집합 구하기\ncommon_features = set(high_features_dict[\"high_features_0\"])\n\nfor pid in range(1, 7):  # 1~6까지\n    common_features &= set(high_features_dict[f\"high_features_{pid}\"])\n\ncommon_features = list(common_features)\n\nprint(f\"모든 파티션에 공통으로 등장한 feature 개수: {len(common_features)}\")\nprint(\"공통 feature 목록:\", common_features)\n\nfrom collections import Counter\n\n# 모든 피처를 하나의 리스트로 합치기\nall_features = []\nfor pid in range(7):\n    all_features.extend(high_features_dict[f\"high_features_{pid}\"])\n\n# 등장 횟수 세기\nfeature_counts = Counter(all_features)\n\n# 3개 이상 리스트에서 등장한 피처만 추출\ncommon_3plus_features = [f for f, cnt in feature_counts.items() if cnt >= 3]\n\nprint(f\"3개 이상 리스트에 나타난 feature 개수: {len(common_3plus_features)}\")\nprint(\"feature 목록:\", common_3plus_features)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}