{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder, LabelEncoder\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import KFold, cross_val_score\nfrom sklearn.metrics import classification_report, accuracy_score, confusion_matrix\nfrom sklearn.feature_selection import SelectKBest, f_classif, VarianceThreshold\nfrom sklearn.decomposition import PCA\nfrom tqdm import tqdm\nfrom concurrent.futures import ThreadPoolExecutor\nfrom matplotlib import pyplot as plt\nfrom imblearn.over_sampling import SMOTE\nimport seaborn as sns\nimport joblib","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T07:49:05.135471Z","iopub.execute_input":"2024-12-27T07:49:05.135747Z","iopub.status.idle":"2024-12-27T07:49:05.140943Z","shell.execute_reply.started":"2024-12-27T07:49:05.135729Z","shell.execute_reply":"2024-12-27T07:49:05.139798Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import ExtraTreesRegressor\nfrom lightgbm import LGBMRegressor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:52:38.463430Z","iopub.execute_input":"2024-12-27T06:52:38.463711Z","iopub.status.idle":"2024-12-27T06:52:40.466867Z","shell.execute_reply.started":"2024-12-27T06:52:38.463693Z","shell.execute_reply":"2024-12-27T06:52:40.465986Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:52:40.467795Z","iopub.execute_input":"2024-12-27T06:52:40.468397Z","iopub.status.idle":"2024-12-27T06:52:40.471784Z","shell.execute_reply.started":"2024-12-27T06:52:40.468371Z","shell.execute_reply":"2024-12-27T06:52:40.470823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:52:40.472784Z","iopub.execute_input":"2024-12-27T06:52:40.473073Z","iopub.status.idle":"2024-12-27T06:52:40.549443Z","shell.execute_reply.started":"2024-12-27T06:52:40.473050Z","shell.execute_reply":"2024-12-27T06:52:40.548508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def perform_pca(train, test, n_components=None, random_state=42):\n    \n    pca = PCA(n_components=n_components, random_state=random_state)\n    train_pca = pca.fit_transform(train)\n    test_pca = pca.transform(test)\n    \n    explained_variance_ratio = pca.explained_variance_ratio_\n    print(f\"Explained variance ratio of the components:\\n {explained_variance_ratio}\")\n    print(np.sum(explained_variance_ratio))\n    \n    train_pca_df = pd.DataFrame(train_pca, columns=[f'PC_{i+1}' for i in range(train_pca.shape[1])])\n    test_pca_df = pd.DataFrame(test_pca, columns=[f'PC_{i+1}' for i in range(test_pca.shape[1])])\n    \n    return train_pca_df, test_pca_df, pca","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:52:40.550151Z","iopub.execute_input":"2024-12-27T06:52:40.550364Z","iopub.status.idle":"2024-12-27T06:52:40.555411Z","shell.execute_reply.started":"2024-12-27T06:52:40.550344Z","shell.execute_reply":"2024-12-27T06:52:40.554214Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"Stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    \n    return df\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\nscaler = StandardScaler()\ndf_train = pd.DataFrame(scaler.fit_transform(df_train), columns=df_train.columns)\ndf_test = pd.DataFrame(scaler.transform(df_test), columns=df_test.columns)\n\nfor c in df_train.columns:\n    m = np.mean(df_train[c])\n    df_train[c].fillna(m, inplace=True)\n    df_test[c].fillna(m, inplace=True)\n\nprint(df_train.shape)\n\ndf_train_pca, df_test_pca, pca = perform_pca(df_train, df_test, n_components=15, random_state=SEED)\n\ndf_train_pca['id'] = train_ts['id']\ndf_test_pca['id'] = test_ts['id']\n\ntrain = pd.merge(train, df_train_pca, how=\"left\", on='id')\ntest = pd.merge(test, df_test_pca, how=\"left\", on='id')\ntrain.shape, test.shape\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:52:40.557546Z","iopub.execute_input":"2024-12-27T06:52:40.557753Z","iopub.status.idle":"2024-12-27T06:53:39.531750Z","shell.execute_reply.started":"2024-12-27T06:52:40.557733Z","shell.execute_reply":"2024-12-27T06:53:39.530389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\ntrain = train.dropna(subset='sii')\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', \n          'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ntest_ids = test['id'].copy()\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\ndef update(df):\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('object')\n    return df\n        \ntrain = update(train)\ntest = update(test)\ntrain.shape, test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:39.532779Z","iopub.execute_input":"2024-12-27T06:53:39.532964Z","iopub.status.idle":"2024-12-27T06:53:39.590531Z","shell.execute_reply.started":"2024-12-27T06:53:39.532944Z","shell.execute_reply":"2024-12-27T06:53:39.589076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 檢視 train 和 test 資料的基本資訊\nprint(\"Train Dataset Info:\")\nprint(train.info())\nprint(\"\\nTest Dataset Info:\")\nprint(test.info())\n\n# 顯示缺失值分布\nprint(\"\\nMissing Values in Train Dataset:\")\nprint(train.isnull().sum().sort_values(ascending=False).head(10))\n\nprint(\"\\nMissing Values in Test Dataset:\")\nprint(test.isnull().sum().sort_values(ascending=False).head(10))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 缺失值處理\n","metadata":{}},{"cell_type":"markdown","source":"- 針對連續型的資料特徵 ➡️ 使用中位數填補\n","metadata":{}},{"cell_type":"code","source":"num_imputer = SimpleImputer(strategy='median')\nnumerical_features = train.select_dtypes(include=['float64', 'int64']).columns\nnumerical_in_test = [col for col in test.columns if col in numerical_features]\ntrain[numerical_features] = num_imputer.fit_transform(train[numerical_features])\ntest[numerical_in_test] = num_imputer.fit_transform(test[numerical_in_test])\nprint(\"✅ Continuous numerical missing values filled with median.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:39.628368Z","iopub.execute_input":"2024-12-27T06:53:39.628611Z","iopub.status.idle":"2024-12-27T06:53:39.671743Z","shell.execute_reply.started":"2024-12-27T06:53:39.628583Z","shell.execute_reply":"2024-12-27T06:53:39.670981Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- 針對類別型的資料特徵 ➡️ 使用多數填補\n","metadata":{}},{"cell_type":"code","source":"cat_imputer = SimpleImputer(strategy='most_frequent')\ncategorical_features = train.select_dtypes(include=['object']).columns\ncategorical_in_test = [col for col in test.columns if col in categorical_features]\n\ntrain[categorical_features] = cat_imputer.fit_transform(train[categorical_features])\ntest[categorical_in_test] = cat_imputer.fit_transform(test[categorical_in_test])\n\nprint(\"✅ Categorical missing values filled with mode.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:39.672654Z","iopub.execute_input":"2024-12-27T06:53:39.672918Z","iopub.status.idle":"2024-12-27T06:53:39.692253Z","shell.execute_reply.started":"2024-12-27T06:53:39.672899Z","shell.execute_reply":"2024-12-27T06:53:39.691486Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n🔍 Missing Values in Train Dataset After Filling:\")\nprint(train.isnull().sum().sort_values(ascending=False).head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:39.693182Z","iopub.execute_input":"2024-12-27T06:53:39.693557Z","iopub.status.idle":"2024-12-27T06:53:39.705566Z","shell.execute_reply.started":"2024-12-27T06:53:39.693538Z","shell.execute_reply":"2024-12-27T06:53:39.704751Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 將類別型特徵進行編碼\n","metadata":{}},{"cell_type":"code","source":"le = LabelEncoder()\nfor col in categorical_in_test:\n    if col == \"id\":\n        continue\n    train[col] = le.fit_transform(train[col])\n    test[col] = le.transform(test[col])\n\nprint(\"✅ 類別型特徵已進行 Label Encoding\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:39.706154Z","iopub.execute_input":"2024-12-27T06:53:39.706350Z","iopub.status.idle":"2024-12-27T06:53:39.727155Z","shell.execute_reply.started":"2024-12-27T06:53:39.706333Z","shell.execute_reply":"2024-12-27T06:53:39.726008Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 將資料進行標準化\n","metadata":{}},{"cell_type":"code","source":"numerical_features = [col for col in numerical_in_test if col not in ['sii', 'id']]\nscaler = StandardScaler()\ntrain[numerical_features] = scaler.fit_transform(train[numerical_features])\ntest[numerical_features] = scaler.transform(test[numerical_features])\nprint(\"✅ 數值特徵已進行標準化\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:39.727997Z","iopub.execute_input":"2024-12-27T06:53:39.728339Z","iopub.status.idle":"2024-12-27T06:53:39.756494Z","shell.execute_reply.started":"2024-12-27T06:53:39.728313Z","shell.execute_reply":"2024-12-27T06:53:39.755632Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.shape, test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:39.757138Z","iopub.execute_input":"2024-12-27T06:53:39.757318Z","iopub.status.idle":"2024-12-27T06:53:39.761940Z","shell.execute_reply.started":"2024-12-27T06:53:39.757301Z","shell.execute_reply":"2024-12-27T06:53:39.761190Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 異常值檢測\n","metadata":{}},{"cell_type":"code","source":"# for col in numerical_features:\n#     Q1 = train[col].quantile(0.25)\n#     Q3 = train[col].quantile(0.75)\n#     IQR = Q3 - Q1\n#     lower_bound = Q1 - 1.5 * IQR\n#     upper_bound = Q3 + 1.5 * IQR\n#     train = train[(train[col] >= lower_bound) & (train[col] <= upper_bound)]\n\n# print(\"✅ 異常值已使用 IQR 方法處理\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:39.762596Z","iopub.execute_input":"2024-12-27T06:53:39.762773Z","iopub.status.idle":"2024-12-27T06:53:39.776250Z","shell.execute_reply.started":"2024-12-27T06:53:39.762756Z","shell.execute_reply":"2024-12-27T06:53:39.775500Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 儲存處理後的資料\n","metadata":{}},{"cell_type":"code","source":"test['id'] = test_ids\n# train.to_csv('../data/cleaned_train.csv', index=False)\n# test.to_csv('../data/cleaned_test.csv', index=False)\n# print(\"✅ 處理後的資料已保存為 'cleaned_train.csv', 'cleaned_test.csv'\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:39.776885Z","iopub.execute_input":"2024-12-27T06:53:39.777104Z","iopub.status.idle":"2024-12-27T06:53:39.794398Z","shell.execute_reply.started":"2024-12-27T06:53:39.777062Z","shell.execute_reply":"2024-12-27T06:53:39.793618Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train = pd.read_csv('../data/cleaned_train.csv')\n# test = pd.read_csv('../data/cleaned_test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:39.794908Z","iopub.execute_input":"2024-12-27T06:53:39.795141Z","iopub.status.idle":"2024-12-27T06:53:39.809326Z","shell.execute_reply.started":"2024-12-27T06:53:39.795119Z","shell.execute_reply":"2024-12-27T06:53:39.808501Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"smote = SMOTE(random_state=SEED)\ntrain","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:39.810141Z","iopub.execute_input":"2024-12-27T06:53:39.810361Z","iopub.status.idle":"2024-12-27T06:53:39.845225Z","shell.execute_reply.started":"2024-12-27T06:53:39.810340Z","shell.execute_reply":"2024-12-27T06:53:39.844494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"exclude = ['PCIAT-Season', 'PCIAT-PCIAT_01', 'PCIAT-PCIAT_02', 'PCIAT-PCIAT_03',\n           'PCIAT-PCIAT_04', 'PCIAT-PCIAT_05', 'PCIAT-PCIAT_06', 'PCIAT-PCIAT_07',\n           'PCIAT-PCIAT_08', 'PCIAT-PCIAT_09', 'PCIAT-PCIAT_10', 'PCIAT-PCIAT_11',\n           'PCIAT-PCIAT_12', 'PCIAT-PCIAT_13', 'PCIAT-PCIAT_14', 'PCIAT-PCIAT_15',\n           'PCIAT-PCIAT_16', 'PCIAT-PCIAT_17', 'PCIAT-PCIAT_18', 'PCIAT-PCIAT_19',\n           'PCIAT-PCIAT_20', 'PCIAT-PCIAT_Total', 'sii', 'id']\ny_model = \"PCIAT-PCIAT_Total\" # Score, target for the model\ny_comp = \"sii\" # Index, target of the competition\nfeatures = [col for col in train.columns if col not in exclude]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:39.845902Z","iopub.execute_input":"2024-12-27T06:53:39.846157Z","iopub.status.idle":"2024-12-27T06:53:39.850458Z","shell.execute_reply.started":"2024-12-27T06:53:39.846135Z","shell.execute_reply":"2024-12-27T06:53:39.849727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.set_theme(style=\"whitegrid\")\nplt.figure(figsize=(6, 3))\nplt.hist(train['PCIAT-PCIAT_Total'], bins=50, color=\"darkorange\")\nplt.title('Score Distribution')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:39.853389Z","iopub.execute_input":"2024-12-27T06:53:39.853581Z","iopub.status.idle":"2024-12-27T06:53:40.216647Z","shell.execute_reply.started":"2024-12-27T06:53:39.853564Z","shell.execute_reply":"2024-12-27T06:53:40.215862Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define thresholds for the `sii` target\nsii_thresholds = [30, 49, 79, 100]\n\ndef pciat_to_sii(pciat_score):\n    if pciat_score <= sii_thresholds[0]:\n        return 0\n    elif pciat_score <= sii_thresholds[1]:\n        return 1\n    elif pciat_score <= sii_thresholds[2]:\n        return 2\n    else:\n        return 3\n\n# Map `PCIAT-PCIAT_Total` to `sii`\ndef map_sii_targets(df):\n    return df[\"PCIAT-PCIAT_Total\"].apply(pciat_to_sii)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:40.217693Z","iopub.execute_input":"2024-12-27T06:53:40.217855Z","iopub.status.idle":"2024-12-27T06:53:40.222221Z","shell.execute_reply.started":"2024-12-27T06:53:40.217838Z","shell.execute_reply":"2024-12-27T06:53:40.221313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Parameters for LGBM, XGB and CatBoost\nlgb_params = {\n    'objective': 'poisson', \n    'n_estimators': 295, \n    'max_depth': 4, \n    'learning_rate': 0.04505693066482616, \n    'subsample': 0.6042489155604022, \n    'colsample_bytree': 0.5021876720502726, \n    'min_data_in_leaf': 100\n}\n\nxgb_params = {\n    'objective': 'reg:tweedie', \n    'num_parallel_tree': 18, \n    'n_estimators': 175, \n    'max_depth': 3, \n    'learning_rate': 0.032620453423049305, \n    'subsample': 0.6155579670568023, \n    'colsample_bytree': 0.5988773292417443, \n    'reg_alpha': 0.0028895066837627205, \n    'reg_lambda': 0.002232531512636924, \n    'tweedie_variance_power': 1.1708678482038286\n}\n\ncat_params = {\n    'objective': 'RMSE', \n    'iterations': 238, \n    'depth': 4, \n    'learning_rate': 0.044523361750173816, \n    'l2_leaf_reg': 0.09301285673435761, \n    'subsample': 0.6902492783438681, \n    'bagging_temperature': 0.3007304771330199, \n    'random_strength': 3.562201626987314, \n    'min_data_in_leaf': 60\n}\n\nxtrees_params = {\n    'n_estimators': 500, \n    'max_depth': 15, \n    'min_samples_leaf': 20, \n    'bootstrap': False\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:40.222944Z","iopub.execute_input":"2024-12-27T06:53:40.223225Z","iopub.status.idle":"2024-12-27T06:53:40.238308Z","shell.execute_reply.started":"2024-12-27T06:53:40.223199Z","shell.execute_reply":"2024-12-27T06:53:40.237485Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Base models\nbase_models = {\n    \"xgboost\": XGBRegressor(**xgb_params,random_state=SEED),\n    \"catboost\": CatBoostRegressor(**cat_params,verbose=0, random_state=SEED),\n    \"extratrees\": ExtraTreesRegressor(**xtrees_params,random_state=42)\n}\n\n# Meta model\nmeta_model = LGBMRegressor(random_state=42)\n\n# Training function\ndef train_stacking_ensemble(train, target_column, features, n_splits=10):\n    kf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n    meta_features = np.zeros((train.shape[0], len(base_models)))\n    target_sii = map_sii_targets(train)\n\n    final_results = {}\n\n    for name, model in base_models.items():\n        print(f\"Evaluating {name} model\")\n        model_results = []\n\n        for fold, (train_idx, val_idx) in enumerate(kf.split(train)):\n            train_data, val_data = train.iloc[train_idx], train.iloc[val_idx]\n            X_train, y_train = train_data[features], train_data[target_column]\n            X_val, y_val = val_data[features], val_data[target_column]\n\n            model.fit(X_train, y_train)\n            val_pred = model.predict(X_val)\n            meta_features[val_idx, list(base_models.keys()).index(name)] = val_pred\n\n            val_pred_sii = np.array([pciat_to_sii(p) for p in val_pred])\n            val_true_sii = target_sii.iloc[val_idx].values\n\n            score = accuracy_score(val_true_sii, val_pred_sii)\n            model_results.append(score)\n\n        cross_val_mean = np.mean(model_results)\n        cross_val_std = np.std(model_results)\n\n        print(f\"{name} model cross_val_score: {cross_val_mean:.4f} ± {cross_val_std:.4f}\")\n        final_results[name] = {\n            \"cross_val_score\": (cross_val_mean, cross_val_std),\n            \"classification_report\": classification_report(target_sii, np.array([pciat_to_sii(p) for p in model.predict(train[features])]), zero_division=0)\n        }\n\n    print(\"Training meta-model\")\n    meta_targets = train[target_column].values\n    meta_model.fit(meta_features, meta_targets)\n\n    # Save meta-model checkpoint\n    joblib.dump(meta_model, 'meta_model_checkpoint.pkl')\n    \n    # Evaluate meta-model\n    meta_predictions = meta_model.predict(meta_features)\n    meta_predictions_sii = np.array([pciat_to_sii(p) for p in meta_predictions])\n    meta_accuracy = accuracy_score(target_sii, meta_predictions_sii)\n    meta_report = classification_report(target_sii, meta_predictions_sii, zero_division=0)\n\n    print(f\"Meta-model accuracy: {meta_accuracy:.4f}\")\n    print(f\"Meta-model classification report:\\n{meta_report}\")\n\n    final_results[\"meta_model\"] = {\n        \"cross_val_score\": meta_accuracy,\n        \"classification_report\": meta_report\n    }\n\n\n    return final_results, meta_features\n\n# Test function\ndef test_stacking_ensemble(test, features):\n    # Load meta-model checkpoint\n    meta_model = joblib.load('meta_model_checkpoint.pkl')\n\n    # Generate predictions for test data\n    test_meta_features = np.zeros((test.shape[0], len(base_models)))\n\n    for name, model in base_models.items():\n        test_meta_features[:, list(base_models.keys()).index(name)] = model.predict(test[features])\n\n    test_meta_predictions = meta_model.predict(test_meta_features)\n    y_pred = np.array([pciat_to_sii(p) for p in test_meta_predictions])\n\n    # Save predictions\n    submission = pd.DataFrame({'id': test['id'], 'sii': y_pred})\n    submission.to_csv('submission.csv', index=False)\n\n    print(\"\\u2705 Predictions saved as 'final_submission.csv'\")\n    return y_pred","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:40.239060Z","iopub.execute_input":"2024-12-27T06:53:40.239319Z","iopub.status.idle":"2024-12-27T06:53:40.258385Z","shell.execute_reply.started":"2024-12-27T06:53:40.239300Z","shell.execute_reply":"2024-12-27T06:53:40.257401Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# results, meta_features = train_stacking_ensemble(train, \"PCIAT-PCIAT_Total\", features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:53:40.259201Z","iopub.execute_input":"2024-12-27T06:53:40.259565Z","iopub.status.idle":"2024-12-27T06:54:44.292208Z","shell.execute_reply.started":"2024-12-27T06:53:40.259535Z","shell.execute_reply":"2024-12-27T06:54:44.291466Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for result in results:\n#     print(result)\n#     print(results[result]['cross_val_score'])\n#     print(results[result]['classification_report'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:54:44.292905Z","iopub.execute_input":"2024-12-27T06:54:44.293144Z","iopub.status.idle":"2024-12-27T06:54:44.298676Z","shell.execute_reply.started":"2024-12-27T06:54:44.293122Z","shell.execute_reply":"2024-12-27T06:54:44.297727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# y_pred = test_stacking_ensemble(meta_features, test, features)\n# y_pred\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T06:54:44.299293Z","iopub.execute_input":"2024-12-27T06:54:44.299539Z","iopub.status.idle":"2024-12-27T06:54:44.351663Z","shell.execute_reply.started":"2024-12-27T06:54:44.299514Z","shell.execute_reply":"2024-12-27T06:54:44.350487Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb_params = {\n    'objective': 'multiclass',         # Suitable for multi-class classification\n    'num_class': 4,                    # Specify the number of classes\n    'n_estimators': 100,\n    'max_depth': 6,                    # Allow deeper trees for more splits\n    'learning_rate': 0.04505693066482616,\n    'subsample': 0.6042489155604022,\n    'colsample_bytree': 0.5021876720502726,\n    'min_gain_to_split': 0.01,         # Set a small positive value for valid splits\n    'min_data_in_leaf': 5,             # Allow smaller leaf nodes\n}\n\n\nxgb_params = {\n    'objective': 'multi:softprob',  # Suitable for multi-class classification\n    'num_class': 4,                # Specify the number of classes\n    'num_parallel_tree': 18,\n    'n_estimators': 175,\n    'max_depth': 3,\n    'learning_rate': 0.032620453423049305,\n    'subsample': 0.6155579670568023,\n    'colsample_bytree': 0.5988773292417443,\n    'reg_alpha': 0.0028895066837627205,\n    'reg_lambda': 0.002232531512636924\n}\n\ncat_params = {\n    'objective': 'MultiClass',    # Suitable for multi-class classification\n    'iterations': 238,\n    'depth': 4,\n    'learning_rate': 0.044523361750173816,\n    'l2_leaf_reg': 0.09301285673435761,\n    # 'subsample': 0.6902492783438681,\n    'bagging_temperature': 0.3007304771330199,\n    'random_strength': 3.562201626987314,\n    'min_data_in_leaf': 60\n}\n\nxtrees_params = {\n    'n_estimators': 500,\n    'max_depth': 15,\n    'min_samples_leaf': 20,\n    'bootstrap': False\n}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T08:17:46.623572Z","iopub.execute_input":"2024-12-27T08:17:46.623840Z","iopub.status.idle":"2024-12-27T08:17:46.629494Z","shell.execute_reply.started":"2024-12-27T08:17:46.623821Z","shell.execute_reply":"2024-12-27T08:17:46.628308Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import accuracy_score, classification_report\nfrom imblearn.over_sampling import SMOTE\nfrom xgboost import XGBClassifier\nfrom catboost import CatBoostClassifier\nfrom lightgbm import LGBMClassifier\nimport numpy as np\nimport joblib\n\n# Base models\nbase_models = {\n    \"xgboost\": XGBClassifier(**xgb_params, random_state=SEED),\n    \"catboost\": CatBoostClassifier(**cat_params, verbose=0, random_state=SEED),\n    \"extratrees\": ExtraTreesClassifier(**xtrees_params, random_state=42),\n}\n\n# Meta model\nmeta_model = LGBMClassifier(random_state=42)\n\n# Training function\nfrom sklearn.model_selection import StratifiedKFold, cross_val_score\nfrom sklearn.metrics import accuracy_score, f1_score, classification_report\nfrom sklearn.ensemble import StackingClassifier\nfrom imblearn.over_sampling import SMOTE\nimport numpy as np\nimport pandas as pd\n\n# Training function with detailed reports\ndef train_stacking_ensemble_class(train, target_column, features, n_splits=10):\n    X = train[features]\n    y = train[target_column]\n\n    # Apply SMOTE to address class imbalance\n    smote = SMOTE(random_state=42)\n    X_resampled, y_resampled = smote.fit_resample(X, y)\n\n    # Initialize stratified K-fold cross-validation\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n\n    # Store out-of-fold predictions for meta-model training\n    meta_features = np.zeros((X_resampled.shape[0], len(base_models)))\n\n    # Dictionary to store detailed reports for each model\n    model_reports = {}\n\n    for idx, (model_name, model) in enumerate(base_models.items()):\n        print(f\"Training base model: {model_name}\")\n        oof_predictions = np.zeros(X_resampled.shape[0])\n\n        for fold, (train_idx, val_idx) in enumerate(skf.split(X_resampled, y_resampled)):\n            X_train, X_val = X_resampled.iloc[train_idx], X_resampled.iloc[val_idx]\n            y_train, y_val = y_resampled.iloc[train_idx], y_resampled.iloc[val_idx]\n\n            # Train base model\n            model.fit(X_train, y_train)\n\n            # Make predictions\n            val_predictions = model.predict(X_val)\n\n            # Ensure predictions have correct shape for meta-features\n            if val_predictions.ndim == 1:\n                oof_predictions[val_idx] = val_predictions\n            else:\n                oof_predictions[val_idx] = val_predictions[:, 0]\n\n        # Store predictions as meta-features\n        meta_features[:, idx] = oof_predictions\n\n        # Generate a classification report on the full training set\n        train_report = classification_report(\n            y_resampled,\n            model.predict(X_resampled),\n            zero_division=0\n        )\n\n        # Store report for the model\n        model_reports[model_name] = {\n            \"cross_val_score\": (np.mean(cross_val_score(model, X_resampled, y_resampled, cv=skf)),\n                                 np.std(cross_val_score(model, X_resampled, y_resampled, cv=skf))),\n            \"classification_report\": train_report,\n        }\n\n    # Train meta-model using meta-features\n    print(\"Training meta-model\")\n    meta_model.fit(meta_features, y_resampled)\n\n    # Evaluate meta-model\n    meta_predictions = meta_model.predict(meta_features)\n    meta_report = classification_report(y_resampled, meta_predictions, zero_division=0)\n    print(\"Meta-Model Classification Report:\")\n    print(meta_report)\n\n    # Combine base models and meta-model into stacking classifier\n    stacking_clf = StackingClassifier(\n        estimators=[(name, model) for name, model in base_models.items()],\n        final_estimator=meta_model,\n        passthrough=True\n    )\n\n    return stacking_clf, model_reports\n\n\n# Testing function for the test set\ndef test_stacking_ensemble(train, target_column, test, features, stacking_clf):\n    X = train[features]\n    y = train[target_column]\n\n    # Apply SMOTE to address class imbalance\n    smote = SMOTE(random_state=42)\n    X_resampled, y_resampled = smote.fit_resample(X, y)\n    stacking_clf.fit(X_resampled, y_resampled)\n    X_test = test[features]\n    predictions = stacking_clf.predict(X_test)\n    return predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T09:07:19.751230Z","iopub.execute_input":"2024-12-27T09:07:19.751548Z","iopub.status.idle":"2024-12-27T09:07:19.767005Z","shell.execute_reply.started":"2024-12-27T09:07:19.751519Z","shell.execute_reply":"2024-12-27T09:07:19.766185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stacking_model, reports= train_stacking_ensemble_class(train, \"sii\", features)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for model_name, report in reports.items():\n     print(f\"Report for {model_name}:\")\n     print(f\"Cross-validation Mean: {report['cross_val_score'][0]:.4f}\")\n     print(f\"Cross-validation Std: {report['cross_val_score'][1]:.4f}\")\n     print(report['classification_report'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T09:04:13.249562Z","iopub.execute_input":"2024-12-27T09:04:13.249983Z","iopub.status.idle":"2024-12-27T09:04:13.256853Z","shell.execute_reply.started":"2024-12-27T09:04:13.249959Z","shell.execute_reply":"2024-12-27T09:04:13.255962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions = test_stacking_ensemble(train,\"sii\",test, features, stacking_model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T09:07:28.929489Z","iopub.execute_input":"2024-12-27T09:07:28.929752Z","iopub.status.idle":"2024-12-27T09:09:26.243920Z","shell.execute_reply.started":"2024-12-27T09:07:28.929732Z","shell.execute_reply":"2024-12-27T09:09:26.243081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({'id': test['id'], 'sii': predictions})\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T09:12:16.167679Z","iopub.execute_input":"2024-12-27T09:12:16.167945Z","iopub.status.idle":"2024-12-27T09:12:16.174751Z","shell.execute_reply.started":"2024-12-27T09:12:16.167925Z","shell.execute_reply":"2024-12-27T09:12:16.173378Z"}},"outputs":[],"execution_count":null}]}