{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ANALYSIS | CAT + LGBM + XGB | OPTUNE\n\nThere is Japanese here and there","metadata":{}},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport polars as pl\nimport pandas as pd\nfrom sklearn.base import clone\nfrom copy import deepcopy\nimport optuna\nfrom scipy.optimize import minimize\nimport matplotlib.pyplot as plt\nimport missingno as msno\nimport re\nfrom colorama import Fore, Style\n\nfrom tqdm import tqdm\nfrom IPython.display import clear_output\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor, CatBoostClassifier\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nimport xgboost as xgb\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\n\nSEED = 42\nn_splits = 5","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-27T03:52:32.337549Z","iopub.execute_input":"2024-09-27T03:52:32.338022Z","iopub.status.idle":"2024-09-27T03:52:38.098709Z","shell.execute_reply.started":"2024-09-27T03:52:32.337976Z","shell.execute_reply":"2024-09-27T03:52:38.097502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain = train.drop('id',axis=1)\ntest = test.drop('id',axis=1)\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n       'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n       'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n       'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n       'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n       'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n       'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n       'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n       'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n       'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n       'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n       'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n       'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n       'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n       'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n       'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n       'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n       'PreInt_EduHx-computerinternet_hoursday','sii']\n\ntrain = train[featuresCols]\n#train = train.dropna(subset='sii')\n\ncat_c = ['Basic_Demos-Enroll_Season','CGAS-Season','Physical-Season','Fitness_Endurance-Season','FGC-Season',\n 'BIA-Season','PAQ_A-Season','PAQ_C-Season','SDS-Season','PreInt_EduHx-Season']\n\ndef update(df):\n    global cat_c\n    for c in cat_c : \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n        \n    return df\n        \n#train = update(train)\n#test = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\n    \nfor col in cat_c:\n    all_values = pd.concat([train[col], test[col]]).unique()\n    mapping = {value: idx for idx, value in enumerate(all_values)}\n\n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mapping).astype(int)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:52:38.101266Z","iopub.execute_input":"2024-09-27T03:52:38.102134Z","iopub.status.idle":"2024-09-27T03:52:38.312921Z","shell.execute_reply.started":"2024-09-27T03:52:38.102071Z","shell.execute_reply":"2024-09-27T03:52:38.311287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ANALYSIS","metadata":{}},{"cell_type":"markdown","source":"## histgrams","metadata":{}},{"cell_type":"code","source":"%%time\n\ntrain","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:52:38.314621Z","iopub.execute_input":"2024-09-27T03:52:38.315133Z","iopub.status.idle":"2024-09-27T03:52:38.417446Z","shell.execute_reply.started":"2024-09-27T03:52:38.31509Z","shell.execute_reply":"2024-09-27T03:52:38.416315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom lightgbm import LGBMRegressor, LGBMClassifier\n\ndef fill_missing_with_lgbm(train, test, target_column, n_estimators=1000, random_state=42):\n    \"\"\"\n    使用LightGBM模型填补train和test的某个特征中的缺失值：\n    1. 先合并train和test，\n    2. 训练模型补全缺失值，\n    3. 再拆分出补全后的train和test。\n    \n    参数:\n    train (pd.DataFrame): 训练集数据框\n    test (pd.DataFrame): 测试集数据框\n    target_column (str): 需要填补缺失值的目标特征（列名称）\n    model_type (str): 'regression' 或 'classification' 来指定模型类型\n    n_estimators (int): LightGBM基学习器数量\n    random_state (int): 随机种子\n    \n    返回:\n    train_filled, test_filled: 两个DataFrame（train和test的缺失值已被填充）\n    \"\"\"\n    global cat_c\n    if target_column in cat_c:\n        model_type = 'classification'\n    else:\n        model_type = 'regression'\n    \n    # 1. 添加标记列来标识 train 和 test\n    train['is_train'] = 1\n    test['is_train'] = 0\n\n    # 合并train和test数据集\n    df = pd.concat([train, test], ignore_index=True)\n\n    # 2. 找出不包含目标列的特征列\n    features_columns = df.columns[df.columns != target_column].tolist()\n\n    # 提取缺失值的行和不缺失的行\n    df_missing = df[df[target_column].isnull()]  # 缺失值部分\n    df_not_missing = df[~df[target_column].isnull()]  # 不缺失部分\n\n    if df_missing.empty or df_not_missing.empty:\n        print(f\"No missing data in '{target_column}' column or all data are missing.\")\n        return train, test\n\n    # 3. 准备训练集和特征\n    X_train = df_not_missing[features_columns]  # 非缺失值行的特征\n    y_train = df_not_missing[target_column]     # 非缺失值行的目标列\n\n    # 4. 初始化 LGBM 模型\n    if model_type == 'regression':\n        model = LGBMRegressor(n_estimators=n_estimators, random_state=random_state)\n    elif model_type == 'classification':\n        model = LGBMClassifier(n_estimators=n_estimators, random_state=random_state)\n    else:\n        raise ValueError(\"model_type should be either 'regression' or 'classification'.\")\n\n    # 5. 训练模型\n    model.fit(X_train, y_train)\n\n    # 6. 使用模型预测缺失值\n    X_pred = df_missing[features_columns]\n    y_pred = model.predict(X_pred)\n\n    # 7. 用预测值填充缺失值部分\n    df.loc[df[target_column].isnull(), target_column] = y_pred\n\n    # 8. 将数据集重新拆分为原来的 train 和 test\n    train_filled = df[df['is_train'] == 1].drop(columns=['is_train'])\n    test_filled = df[df['is_train'] == 0].drop(columns=['is_train'])\n\n    return train_filled, test_filled\n\n\ntianbu_cols = ['CGAS-CGAS_Score','Physical-BMI','Physical-Height','Physical-Weight', \\\n               'Physical-Diastolic_BP','Physical-HeartRate','Physical-Systolic_BP', \\\n               'SDS-SDS_Total_Raw','SDS-SDS_Total_T','PreInt_EduHx-computerinternet_hoursday']\n\nfor col in tianbu_cols:\n    print(\"开始填补特征：\"+ col)\n    train, test = fill_missing_with_lgbm(train, test, col)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:52:38.420265Z","iopub.execute_input":"2024-09-27T03:52:38.421056Z","iopub.status.idle":"2024-09-27T03:53:06.767195Z","shell.execute_reply.started":"2024-09-27T03:52:38.42101Z","shell.execute_reply":"2024-09-27T03:53:06.765505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nFGC_cols = [\n  'FGC-FGC_CU',\n  'FGC-FGC_CU_Zone',\n  'FGC-FGC_GSND',\n  'FGC-FGC_GSND_Zone',\n  'FGC-FGC_GSD',\n  'FGC-FGC_GSD_Zone',\n  'FGC-FGC_PU',\n  'FGC-FGC_PU_Zone',\n  'FGC-FGC_SRL',\n  'FGC-FGC_SRL_Zone',\n  'FGC-FGC_SRR',\n  'FGC-FGC_SRR_Zone',\n  'FGC-FGC_TL',\n  'FGC-FGC_TL_Zone'\n]\nfor col in FGC_cols:\n    print(\"开始填补特征：\"+ col)\n    train, test = fill_missing_with_lgbm(train, test, col)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:06.769276Z","iopub.execute_input":"2024-09-27T03:53:06.770285Z","iopub.status.idle":"2024-09-27T03:53:43.263987Z","shell.execute_reply.started":"2024-09-27T03:53:06.770195Z","shell.execute_reply":"2024-09-27T03:53:43.262641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BIA = [\"BIA-BIA_TBW\",\"BIA-BIA_TBW\",\"BIA-BIA_DEE\",\"BIA-BIA_BMC\",\"BIA-BIA_Fat\", \"BIA-BIA_BMI\",]\nfor col in BIA:\n    print(\"开始填补特征：\"+ col)\n    train, test = fill_missing_with_lgbm(train, test, col)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:43.265898Z","iopub.execute_input":"2024-09-27T03:53:43.266331Z","iopub.status.idle":"2024-09-27T03:53:43.271949Z","shell.execute_reply.started":"2024-09-27T03:53:43.266287Z","shell.execute_reply":"2024-09-27T03:53:43.270682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = update(train)\ntest = update(test)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:43.273225Z","iopub.execute_input":"2024-09-27T03:53:43.273646Z","iopub.status.idle":"2024-09-27T03:53:43.301106Z","shell.execute_reply.started":"2024-09-27T03:53:43.273589Z","shell.execute_reply":"2024-09-27T03:53:43.299877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain.hist(figsize=(15, 10), bins=20, xlabelsize=8, ylabelsize=8)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:43.302664Z","iopub.execute_input":"2024-09-27T03:53:43.303031Z","iopub.status.idle":"2024-09-27T03:53:53.824925Z","shell.execute_reply.started":"2024-09-27T03:53:43.302994Z","shell.execute_reply":"2024-09-27T03:53:53.823478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.dropna(subset='sii')\ntrain[\"sii\"].hist()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:53.826887Z","iopub.execute_input":"2024-09-27T03:53:53.827392Z","iopub.status.idle":"2024-09-27T03:53:54.161321Z","shell.execute_reply.started":"2024-09-27T03:53:53.827336Z","shell.execute_reply":"2024-09-27T03:53:54.159975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## missing values","metadata":{}},{"cell_type":"code","source":"missing_percent_train = train.isnull().mean() * 100\nmissing_percent_train\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:54.166295Z","iopub.execute_input":"2024-09-27T03:53:54.166878Z","iopub.status.idle":"2024-09-27T03:53:54.186048Z","shell.execute_reply.started":"2024-09-27T03:53:54.166834Z","shell.execute_reply":"2024-09-27T03:53:54.184829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_percent_test = test.isnull().mean() * 100\nmissing_percent_test","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:54.187726Z","iopub.execute_input":"2024-09-27T03:53:54.188171Z","iopub.status.idle":"2024-09-27T03:53:54.205645Z","shell.execute_reply.started":"2024-09-27T03:53:54.188128Z","shell.execute_reply":"2024-09-27T03:53:54.204345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"msno.matrix(train)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:54.207052Z","iopub.execute_input":"2024-09-27T03:53:54.207473Z","iopub.status.idle":"2024-09-27T03:53:54.907681Z","shell.execute_reply.started":"2024-09-27T03:53:54.207431Z","shell.execute_reply":"2024-09-27T03:53:54.906211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"msno.matrix(test)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:54.909371Z","iopub.execute_input":"2024-09-27T03:53:54.909827Z","iopub.status.idle":"2024-09-27T03:53:55.521964Z","shell.execute_reply.started":"2024-09-27T03:53:54.909783Z","shell.execute_reply":"2024-09-27T03:53:55.520653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## interaction feature","metadata":{}},{"cell_type":"code","source":"def create_interaction_features(df, feature_pairs):\n    global cat_c\n    for feature1, feature2 in feature_pairs:\n        if(feature1 not in cat_c or feature2 not in cat_c):\n            print(\"feature1:\" + feature1 + \",feature2:\" + feature2)\n            new_feature_name = f\"{feature1}_x_{feature2}\"\n            df[new_feature_name] = df[feature1] * df[feature2]\n    return df\n\ndef create_chufa_features(df, feature_pairs):\n    global cat_c\n    for feature1, feature2 in feature_pairs:\n        if feature1 not in cat_c or feature2 not in cat_c:\n            print(f\"feature1: {feature1}, feature2: {feature2}\")\n            \n            # 构造 A/B 特征，先进行除法运算，但在除法之前控制出现0或NaN的情况\n            new_feature_name1 = f\"{feature1}_div_{feature2}\"\n            df[new_feature_name1] = df[feature1] / df[feature2]\n            \n            # 当A或B中有0或NaN时保留NaN\n            df[new_feature_name1] = df[new_feature_name1].mask((df[feature1] == 0) | (df[feature2] == 0) | \n                                                               df[feature1].isna() | df[feature2].isna())\n            \n            # 构造 B/A 特征，类似方式处理\n            new_feature_name2 = f\"{feature2}_div_{feature1}\"\n            df[new_feature_name2] = df[feature2] / df[feature1]\n            df[new_feature_name2] = df[new_feature_name2].mask((df[feature1] == 0) | (df[feature2] == 0) | \n                                                               df[feature1].isna() | df[feature2].isna())\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:55.523653Z","iopub.execute_input":"2024-09-27T03:53:55.524102Z","iopub.status.idle":"2024-09-27T03:53:55.538079Z","shell.execute_reply.started":"2024-09-27T03:53:55.524056Z","shell.execute_reply":"2024-09-27T03:53:55.53683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_pairs = [\n    ('PreInt_EduHx-computerinternet_hoursday', 'Basic_Demos-Age'),\n    ('Basic_Demos-Age', 'SDS-SDS_Total_T'),\n    ('FGC-FGC_SRR_Zone', 'SDS-SDS_Total_T'),\n    ('BIA-BIA_BMC', 'Physical-HeartRate'),\n    #('Fitness_Endurance-Season', 'Physical-Waist_Circumference'),\n    ('BIA-BIA_Fat', 'Physical-BMI'),\n    ('PreInt_EduHx-Season', 'Fitness_Endurance-Season'),\n    ('SDS-SDS_Total_T', 'Physical-Systolic_BP'),\n    ('Basic_Demos-Sex', 'FGC-FGC_PU_Zone')\n]\n\n\ntrain = create_interaction_features(train, feature_pairs)\ntest = create_interaction_features(test, feature_pairs)\ntrain = create_chufa_features(train, feature_pairs)\ntest = create_chufa_features(test, feature_pairs)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:55.539719Z","iopub.execute_input":"2024-09-27T03:53:55.540149Z","iopub.status.idle":"2024-09-27T03:53:55.563451Z","shell.execute_reply.started":"2024-09-27T03:53:55.540108Z","shell.execute_reply":"2024-09-27T03:53:55.561839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:55.565033Z","iopub.execute_input":"2024-09-27T03:53:55.565446Z","iopub.status.idle":"2024-09-27T03:53:55.96807Z","shell.execute_reply.started":"2024-09-27T03:53:55.565379Z","shell.execute_reply":"2024-09-27T03:53:55.96688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Importance","metadata":{}},{"cell_type":"code","source":"X = train.drop(['sii'], axis=1)\ny = train['sii']\n# モデルを学習した後のコード\nXGBoost = xgb.XGBRegressor(random_state=SEED,enable_categorical=True)\nXGBoost.fit(X, y)\n\n# 特徴量重要度を取得\nimportance = XGBoost.feature_importances_\n\n# 特徴量の名前を取得\nfeatures = X.columns\n\n# データフレームとして整理\nimportance_df = pd.DataFrame({'Feature': features, 'Importance': importance})\n\n# 特徴量重要度を降順に並び替え\nimportance_df = importance_df.sort_values(by='Importance', ascending=False)\n\n# プロット\nplt.figure(figsize=(10, 20))\nplt.barh(importance_df['Feature'], importance_df['Importance'])\nplt.xlabel('Feature Importance')\nplt.ylabel('Features')\nplt.title('Feature Importance in XGBoost')\nplt.gca().invert_yaxis()  # 重要度が高いものを上に\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:55.969515Z","iopub.execute_input":"2024-09-27T03:53:55.969963Z","iopub.status.idle":"2024-09-27T03:53:58.722081Z","shell.execute_reply.started":"2024-09-27T03:53:55.96992Z","shell.execute_reply":"2024-09-27T03:53:58.7198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This plot shows the feature importance for an XGBoost model, ranked from the most important feature to the least important. Key insights from this visualization include:\n\nTop Features:\n\nThe most important feature is PreInt_EduHx-computerinternet_hoursday, indicating that the time spent on computer/internet usage per day plays the largest role in the model's predictions.\nOther highly ranked features include Basic_Demos-Age, FGC-FGC_SRR_Zone, and SDS-SDS_Total_T, which suggest that age and specific health or fitness-related scores are also crucial for the model's decisions.\n\nFeature Distribution:\n\nThere is a wide range of feature importances, but most of them contribute relatively equally beyond the top few features.\nThe long tail of features implies that while the top few features dominate, a large number of other features still contribute in small amounts to the predictions.\n","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 特徴量の名前を取得\nfeatures = X.columns\n\n# データフレームとして整理（特徴量重要度と欠損値の割合を結合）\nimportance_df = pd.DataFrame({'Feature': features, 'Importance': importance})\nmissing_df = pd.DataFrame({'Feature': missing_percent_train.index, 'MissingPercent': missing_percent_train.values})\ncombined_df = pd.merge(importance_df, missing_df, on='Feature')\n\n# 散布図を作成\nplt.figure(figsize=(10, 6))\nplt.scatter(combined_df['MissingPercent'], combined_df['Importance'], alpha=0.7)\nplt.xlabel('Missing Percentage (%)')\nplt.ylabel('Feature Importance')\nplt.title('Feature Importance vs Missing Percentage')\nplt.grid(True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:58.724795Z","iopub.execute_input":"2024-09-27T03:53:58.725395Z","iopub.status.idle":"2024-09-27T03:53:59.110049Z","shell.execute_reply.started":"2024-09-27T03:53:58.725333Z","shell.execute_reply":"2024-09-27T03:53:59.10874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 获取各自列的集合\ntrain_columns = set(train.columns)\ntest_columns = set(test.columns)\n\n# 查找只在 train 中但不在 test 中的列\ntrain_only = train_columns - test_columns\n\n# 查找只在 test 中但不在 train 中的列\ntest_only = test_columns - train_columns\n\n# 输出差异列\nprint(f\"Columns only in train: {train_only}\")\nprint(f\"Columns only in test: {test_only}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:59.111648Z","iopub.execute_input":"2024-09-27T03:53:59.112036Z","iopub.status.idle":"2024-09-27T03:53:59.1251Z","shell.execute_reply.started":"2024-09-27T03:53:59.111997Z","shell.execute_reply":"2024-09-27T03:53:59.123761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODEL","metadata":{}},{"cell_type":"code","source":"%%time\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    \n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n    test_data = test_data.drop(['sii'], axis=1)\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead') # Nelder-Mead | # Powell\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission, tKappa","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:59.127151Z","iopub.execute_input":"2024-09-27T03:53:59.127714Z","iopub.status.idle":"2024-09-27T03:53:59.152085Z","shell.execute_reply.started":"2024-09-27T03:53:59.127653Z","shell.execute_reply":"2024-09-27T03:53:59.150824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_params_lgbm = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,\n    'lambda_l2': 0.01\n}\nbest_params_xgb = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,\n    'reg_lambda': 5\n}\nbest_params_catboost = {\n    'iterations': 804,\n    'learning_rate': 0.007849710402582562, \n    'l2_leaf_reg': 7.31183636902306, \n    'subsample': 0.5630297785016092, \n    'random_strength': 1.7097065892440113, \n    'bagging_temperature': 0.026593521316435192,\n    'border_count': 12\n}","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:53:59.153904Z","iopub.execute_input":"2024-09-27T03:53:59.154488Z","iopub.status.idle":"2024-09-27T03:53:59.164325Z","shell.execute_reply.started":"2024-09-27T03:53:59.154399Z","shell.execute_reply":"2024-09-27T03:53:59.163234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# LightGBM\nLight = lgb.LGBMRegressor(**best_params_lgbm, random_state=SEED, verbose=-1)\nSubmission_LGBM, k_lgbm = TrainML(Light, test)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:56:30.575329Z","iopub.execute_input":"2024-09-27T03:56:30.575774Z","iopub.status.idle":"2024-09-27T03:57:22.649097Z","shell.execute_reply.started":"2024-09-27T03:56:30.575734Z","shell.execute_reply":"2024-09-27T03:57:22.647891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# XGBoost\nXGBoost = xgb.XGBRegressor(**best_params_xgb, random_state=SEED,enable_categorical=True)\nSubmission_XGB, k_xgb = TrainML(XGBoost, test)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:54:51.777436Z","iopub.execute_input":"2024-09-27T03:54:51.777879Z","iopub.status.idle":"2024-09-27T03:55:54.956985Z","shell.execute_reply.started":"2024-09-27T03:54:51.777832Z","shell.execute_reply":"2024-09-27T03:55:54.955791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CatBoost\nCatBoost = CatBoostRegressor(**best_params_catboost, random_state=SEED, verbose=0,cat_features=cat_c)\nSubmission_CatBoost , k_cat= TrainML(CatBoost, test)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:55:54.958638Z","iopub.execute_input":"2024-09-27T03:55:54.95915Z","iopub.status.idle":"2024-09-27T03:56:24.776818Z","shell.execute_reply.started":"2024-09-27T03:55:54.959091Z","shell.execute_reply":"2024-09-27T03:56:24.775547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n\n# Params = {'learning_rate': 0.07975474666326936, 'max_depth': 10, 'num_leaves': 207, 'min_data_in_leaf': 41,\n#                 'feature_fraction': 0.6385678848225935, 'bagging_fraction': 0.9042038292349021, 'bagging_freq': 6, \n#                             'lambda_l1': 9.920617415343463, 'lambda_l2': 4.351491475117983} # LB : 0.452\n\n# Light = lgb.LGBMRegressor(**Params,random_state=SEED, verbose=-1,n_estimators=200)\n# Submission_LGBM = TrainML(Light,test)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:56:24.778564Z","iopub.execute_input":"2024-09-27T03:56:24.778966Z","iopub.status.idle":"2024-09-27T03:56:24.784634Z","shell.execute_reply.started":"2024-09-27T03:56:24.778925Z","shell.execute_reply":"2024-09-27T03:56:24.783318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SUBMIT","metadata":{}},{"cell_type":"code","source":"print(Submission_LGBM['sii'].value_counts())\nprint(Submission_XGB['sii'].value_counts())\nprint(Submission_CatBoost['sii'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:56:24.786344Z","iopub.execute_input":"2024-09-27T03:56:24.786823Z","iopub.status.idle":"2024-09-27T03:56:24.803213Z","shell.execute_reply.started":"2024-09-27T03:56:24.786779Z","shell.execute_reply":"2024-09-27T03:56:24.802046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# k_xgb = 0.6\n# k_cat = 0.2\n# k_lgbm = 0.2\n# k値の合計を計算して、各モデルの重みを計算\ntotal_k = k_cat + k_xgb + k_lgbm\n\nweight_cat = k_cat / total_k    # CatBoost の重み\nweight_xgb = k_xgb / total_k    # XGBoost の重み\nweight_lgbm = k_lgbm / total_k  # LightGBM の重み\n\n# 各モデルの予測結果（submission）を用意\n# 'sii' がカテゴリラベルであることを前提とします\nensemble_df = pd.DataFrame({\n    'id': Submission_LGBM['id'],\n    'cat': Submission_CatBoost[\"sii\"],\n    'xgb': Submission_XGB[\"sii\"],\n    'lgbm': Submission_LGBM[\"sii\"]\n})\n\n# 予測結果を長い形式に変換\nmelted = ensemble_df.melt(id_vars='id', value_vars=['cat', 'xgb', 'lgbm'], \n                          var_name='model', value_name='sii')\n\n# 各モデルに対応する重みを割り当て\nmelted['weight'] = melted['model'].map({\n    'cat': weight_cat,\n    'xgb': weight_xgb,\n    'lgbm': weight_lgbm\n})\n\n# 各idごと、siiごとに重みを集計\ngrouped = melted.groupby(['id', 'sii'])['weight'].sum().reset_index()\n\n# 各idごとに最大の重みを持つsiiを選択\nbest_submission = grouped.loc[grouped.groupby('id')['weight'].idxmax()][['id', 'sii']]\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:56:24.804758Z","iopub.execute_input":"2024-09-27T03:56:24.805143Z","iopub.status.idle":"2024-09-27T03:56:24.834666Z","shell.execute_reply.started":"2024-09-27T03:56:24.805103Z","shell.execute_reply":"2024-09-27T03:56:24.833507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"comparison_df = best_submission.merge(ensemble_df, on='id', how='left')","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:56:24.840631Z","iopub.execute_input":"2024-09-27T03:56:24.841349Z","iopub.status.idle":"2024-09-27T03:56:24.854212Z","shell.execute_reply.started":"2024-09-27T03:56:24.841292Z","shell.execute_reply":"2024-09-27T03:56:24.852799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"comparison_df","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:56:24.855364Z","iopub.execute_input":"2024-09-27T03:56:24.855824Z","iopub.status.idle":"2024-09-27T03:56:24.876219Z","shell.execute_reply.started":"2024-09-27T03:56:24.85578Z","shell.execute_reply":"2024-09-27T03:56:24.874919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nbest_submission.to_csv('submission.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T03:56:24.877822Z","iopub.execute_input":"2024-09-27T03:56:24.878246Z","iopub.status.idle":"2024-09-27T03:56:24.891631Z","shell.execute_reply.started":"2024-09-27T03:56:24.878201Z","shell.execute_reply":"2024-09-27T03:56:24.8903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}