{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\nfrom datetime import datetime\nimport os\nfrom tqdm import tqdm\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom matplotlib import cycler\nfrom matplotlib.colors import LinearSegmentedColormap\ncolors = [\"#068D9D\", \"#53599A\", \"#607BB0\", \"#6D9DC5\", \"#77BECF\", \"#80DED9\", \"#AEECEF\"]\nplt.rc('axes', facecolor='#E6E6E6', edgecolor='none', axisbelow=True, grid=True, prop_cycle=cycler('color', colors))\nSEED=42","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T17:28:41.432302Z","iopub.execute_input":"2025-06-25T17:28:41.432646Z","iopub.status.idle":"2025-06-25T17:28:41.440046Z","shell.execute_reply.started":"2025-06-25T17:28:41.432626Z","shell.execute_reply":"2025-06-25T17:28:41.438702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def reduce_mem_usage(dataframe,dataset):\n    print(\"Reducing memory usage fo:\",dataset)\n    initial_mem_usage=dataframe.memory_usage().sum()/1024**2\n    for col in dataframe.columns:\n        col_type=dataframe[col].dtype\n        c_min=dataframe[col].min()\n        c_max=dataframe[col].max()\n        if str(col_type)[:3]=='int':\n            if c_min>np.iinfo(np.int8).min and c_max<np.iinfo(np.int8).max:\n                dataframe[col]=dataframe[col].astype(np.int8)\n            elif c_min>np.iinfo(np.int16).min and c_max<np.iinfo(np.int16).max:\n                dataframe[col]=dataframe[col].astype(np.int16)\n            elif c_min>np.iinfo(np.int32).min and c_max<np.iinfo(np.int32).max:\n                dataframe[col]=fataframe[col].astype(np.int32)\n            elif c_min>np.iinfo(np.int64).min and c_max<np.iinfo(np.int64).max:\n                dataframe[col]=dataframe[col].astype(np.int64)\n        else:\n            if c_min>np.finfo(np.float16).min and c_min<np.finfo(np.float16).max:\n                dataframe[col]=dataframe[col].astype(np.float16)\n            elif c_min>np.finfo(np.float32).min and c_min<np.finfo(np.float32).max():\n                dataframe[col]=dataframe[col].astype(np.float32)\n            else:\n                dataframe[col]=dataframe[col].astype(np.float64)\n    final_mem_usage=dataframe.memory_usage().sum()/1024**2\n    print(\"--memory usage before: {:.2f}MB\".format(initial_mem_usage))\n    print(\"--memory usage after: {:.2f}MB\".format(final_mem_usage))\n    print(\"--decreased memory usage by {:.2f}MB%\\n\".format(100*(initial_mem_usage-final_mem_usage)/initial_mem_usage))\n    return dataframe\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T17:28:41.442138Z","iopub.execute_input":"2025-06-25T17:28:41.442427Z","iopub.status.idle":"2025-06-25T17:28:41.469140Z","shell.execute_reply.started":"2025-06-25T17:28:41.442405Z","shell.execute_reply":"2025-06-25T17:28:41.468172Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\ntrain_df=reduce_mem_usage(train_df,\"train\")\ntrain_df=train_df.reset_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T17:28:41.470835Z","iopub.execute_input":"2025-06-25T17:28:41.471519Z","iopub.status.idle":"2025-06-25T17:29:10.793223Z","shell.execute_reply.started":"2025-06-25T17:28:41.471457Z","shell.execute_reply":"2025-06-25T17:29:10.792126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T17:29:10.794167Z","iopub.execute_input":"2025-06-25T17:29:10.794442Z","iopub.status.idle":"2025-06-25T17:29:10.802306Z","shell.execute_reply.started":"2025-06-25T17:29:10.794418Z","shell.execute_reply":"2025-06-25T17:29:10.801451Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['exp_855P289M125'] = np.exp(train_df['X855'] + train_df['X289'] - train_df['X125'])\ntrain_df['868xexp_289M125'] = train_df['X868'] * np.exp(train_df['X289'] - train_df['X125'])\ntrain_df['289xexp_289M125'] = train_df['X289'] * np.exp(train_df['X289'] - train_df['X125'])\ntrain_df['exp_862P289M125'] = np.exp(train_df['X862'] + train_df['X289'] - train_df['X125'])\ntrain_df['302xexp_289M125'] = train_df['X302'] * np.exp(train_df['X289'] - train_df['X125'])\ntrain_df['exp_856P289M125'] = np.exp(train_df['X856'] + train_df['X289'] - train_df['X125'])\ntrain_df['exp_868P855P289'] = np.exp(train_df['X868'] + train_df['X855'] + train_df['X289'])\ntrain_df['exp_302P289M125'] = np.exp(train_df['X302'] + train_df['X289'] - train_df['X125'])\ntrain_df['exp_289P855P21'] = np.exp(train_df['X289'] + train_df['X855'] + train_df['X21'])\ntrain_df['385xexp_289M125'] = train_df['X385'] * np.exp(train_df['X289'] - train_df['X125'])\ntrain_df['465x862x465'] = train_df['X465'] * train_df['X862'] * train_df['X465']\ntrain_df['301xexp_289M125'] = train_df['X301'] * np.exp(train_df['X289'] - train_df['X125'])\ntrain_df['exp_786P289M125'] = np.exp(train_df['X786'] + train_df['X289'] - train_df['X125'])\ntrain_df['125x862x465'] = train_df['X125'] * train_df['X862'] * train_df['X465']\ntrain_df['exp_603P863P153'] = np.exp(train_df['X603'] + train_df['X863'] + train_df['X153'])\ntrain_df['exp_465P863P153'] = np.exp(train_df['X465'] + train_df['X863'] + train_df['X153'])\ntrain_df['125x855x289'] = train_df['X125'] * train_df['X855'] * train_df['X289']\ntrain_df['exp_598P868P855P289'] = np.exp(train_df['X598'] + train_df['X868'] + train_df['X855'] + train_df['X289'])\ntrain_df['exp_862P868P855P289'] = np.exp(train_df['X862'] + train_df['X868'] + train_df['X855'] + train_df['X289'])\ntrain_df['exp_860P868P855P289'] = np.exp(train_df['X860'] + train_df['X868'] + train_df['X855'] + train_df['X289'])\ntrain_df['exp_612P868P855P289'] = np.exp(train_df['X612'] + train_df['X868'] + train_df['X855'] + train_df['X289'])\ntrain_df['exp_868P855P289'] = np.exp(train_df['X868'] + train_df['X855'] + train_df['X289'])\ntrain_df['852x868x855x289'] = train_df['X852'] * train_df['X868'] * train_df['X855'] * train_df['X289']\ntrain_df['exp_174P868P855P289'] = np.exp(train_df['X174'] + train_df['X868'] + train_df['X855'] + train_df['X289'])\ntrain_df['exp_465P868P855P289'] = np.exp(train_df['X465'] + train_df['X868'] + train_df['X855'] + train_df['X289'])\ntrain_df['301x868x855x289'] = train_df['X301'] * train_df['X868'] * train_df['X855'] * train_df['X289']\ntrain_df['302x868x855x289'] = train_df['X302'] * train_df['X868'] * train_df['X855'] * train_df['X289']\ntrain_df['exp_168P868P855P289'] = np.exp(train_df['X168'] + train_df['X868'] + train_df['X855'] + train_df['X289'])\ntrain_df['289x868x855x289'] = train_df['X289'] * train_df['X868'] * train_df['X855'] * train_df['X289']\ntrain_df['855x868x855x289'] = train_df['X855'] * train_df['X868'] * train_df['X855'] * train_df['X289']\ntrain_df['exp_603P868P855P289'] = np.exp(train_df['X603'] + train_df['X868'] + train_df['X855'] + train_df['X289'])\ntrain_df['exp_856P868P855P289'] = np.exp(train_df['X856'] + train_df['X868'] + train_df['X855'] + train_df['X289'])\ntrain_df['612x868x855x289'] = train_df['X612'] * train_df['X868'] * train_df['X855'] * train_df['X289']\ntrain_df['168x868x855x289'] = train_df['X168'] * train_df['X868'] * train_df['X855'] * train_df['X289']\ntrain_df['exp_125P862P289M125'] = np.exp(train_df['X125'] + train_df['X862'] + train_df['X289'] - train_df['X125'])\ntrain_df['21x868x855x289'] = train_df['X21'] * train_df['X868'] * train_df['X855'] * train_df['X289']\ntrain_df['868x868x855x289'] = train_df['X868'] * train_df['X868'] * train_df['X855'] * train_df['X289']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T17:29:10.804143Z","iopub.execute_input":"2025-06-25T17:29:10.804393Z","iopub.status.idle":"2025-06-25T17:29:11.598120Z","shell.execute_reply.started":"2025-06-25T17:29:10.804377Z","shell.execute_reply":"2025-06-25T17:29:11.597104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from collections import Counter\ntarget = 'label'\n\nsubset_features = [\n    \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n    \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\",\n    \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\",\n    'X465', 'X153', 'X289', 'X125', 'X21',\n    'exp_855P289M125',\n    '868xexp_289M125',\n    '289xexp_289M125',\n    'exp_862P289M125',\n    '302xexp_289M125',\n    'exp_856P289M125',\n    'exp_868P855P289',\n    'exp_302P289M125',\n    'exp_289P855P21',\n    '385xexp_289M125',\n    '465x862x465',\n    '301xexp_289M125',\n    'exp_786P289M125',\n    '125x862x465',\n    'exp_603P863P153',\n    'exp_465P863P153',\n    '125x855x289',\n    'exp_598P868P855P289',\n    'exp_862P868P855P289',\n    'exp_860P868P855P289',\n    'exp_612P868P855P289',\n    'exp_868P855P289',\n    '852x868x855x289',\n    'exp_174P868P855P289',\n    'exp_465P868P855P289',\n    '301x868x855x289',\n    '302x868x855x289',\n    'exp_168P868P855P289',\n    '289x868x855x289',\n    '855x868x855x289',\n    'exp_603P868P855P289',\n    'exp_856P868P855P289',\n    '612x868x855x289',\n    '168x868x855x289',\n    'exp_125P862P289M125',\n    '21x868x855x289',\n    '868x868x855x289'\n]\n\nn_splits = 10\nsplit_size = len(train_df) // n_splits\nsplit_corr_dict = {}\n\nfor i in range(n_splits):\n    start_idx = i * split_size\n    if i == n_splits - 1:\n        df_split = train_df.iloc[start_idx:]\n    else:\n        df_split = train_df.iloc[start_idx:start_idx + split_size]\n\n    # 해당 split에서 상관관계 계산\n    corr_matrix = df_split[subset_features + [target]].corr()\n    target_corr = corr_matrix[target].drop(target).abs().sort_values(ascending=False)\n    top50 = target_corr.head(50)\n    split_corr_dict[f'Split_{i+1}'] = top50\n\n# 모든 split에서 나온 상위 피처들 모으기\nall_top_features = set()\nfor s in split_corr_dict.values():\n    all_top_features.update(s.index.tolist())\nall_top_features = list(all_top_features)\n\n# 빈 DataFrame 생성 (index는 Split_1~Split_10, columns는 all_top_features)\ncorr_trend_df = pd.DataFrame(index=[f'Split_{i+1}' for i in range(n_splits)], columns=all_top_features)\n\n# 상관계수 값 채우기\nfor split_name, corr_series in split_corr_dict.items():\n    for feature in all_top_features:\n        value = corr_series.get(feature, np.nan)\n        # value가 Series나 list 형태일 경우 NaN으로 변경\n        if isinstance(value, (pd.Series, list, np.ndarray)):\n            value = np.nan\n        corr_trend_df.loc[split_name, feature] = value\n\n# 숫자형으로 변환 (NaN 유지)\ncorr_trend_df = corr_trend_df.astype(float)\n\n# 각 feature가 몇 번 등장했는지 카운트\nfeature_counter = Counter()\nfor s in split_corr_dict.values():\n    feature_counter.update(s.index.tolist())\n\n# 10번 이상 등장한 feature 필터링\ntop_features_by_frequency = [f for f, count in feature_counter.items() if count >= 10]\n\nprint(f\"10번 이상 등장한 feature 개수: {len(top_features_by_frequency)}\")\n\nx = range(1, n_splits+1)\nplt.figure(figsize=(15, 8))\nfor feature in top_features_by_frequency:\n    plt.plot(x, corr_trend_df.loc[:, feature], marker='o', label=feature)\n\nplt.xticks(x, corr_trend_df.index, rotation=45)\nplt.xlabel('Data Split')\nplt.ylabel('Absolute Correlation with Target')\nplt.title('Feature Correlation with Target over 10 Data Splits')\nplt.legend(loc='best', fontsize='small')\nplt.grid(True)\nplt.tight_layout()\nplt.show()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-25T17:29:11.598894Z","iopub.execute_input":"2025-06-25T17:29:11.599112Z","iopub.status.idle":"2025-06-25T17:29:19.239849Z","shell.execute_reply.started":"2025-06-25T17:29:11.599097Z","shell.execute_reply":"2025-06-25T17:29:19.239019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"split_corr_dict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T17:34:52.844045Z","iopub.execute_input":"2025-06-25T17:34:52.844329Z","iopub.status.idle":"2025-06-25T17:34:52.861774Z","shell.execute_reply.started":"2025-06-25T17:34:52.844311Z","shell.execute_reply":"2025-06-25T17:34:52.860825Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"qualified_features = [f for f, count in feature_counter.items() if count == 10]\nprint(f\"\\n🔢 10개 분할에서만 등장한 피처 수: {len(qualified_features)}\")\nprint(qualified_features)\nqualified_features = [f for f, count in feature_counter.items() if count == 9]\nprint(f\"\\n🔢 9개 분할에서만 등장한 피처 수: {len(qualified_features)}\")\nprint(qualified_features)\nqualified_features = [f for f, count in feature_counter.items() if count == 8]\nprint(f\"\\n🔢 8개 분할에서만 등장한 피처 수: {len(qualified_features)}\")\nprint(qualified_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-25T17:35:34.881363Z","iopub.execute_input":"2025-06-25T17:35:34.881821Z","iopub.status.idle":"2025-06-25T17:35:34.888343Z","shell.execute_reply.started":"2025-06-25T17:35:34.881792Z","shell.execute_reply":"2025-06-25T17:35:34.887505Z"}},"outputs":[],"execution_count":null}]}