{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1.Import lib","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nimport seaborn as sns\nfrom sklearn.base import clone, BaseEstimator, RegressorMixin\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score, mean_squared_error\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.decomposition import PCA\nfrom sklearn.datasets import make_classification\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nimport missingno as msno\n\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import RandomForestRegressor\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nfrom bayes_opt import BayesianOptimization\nfrom sklearn.metrics import mean_squared_error","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:40:17.345037Z","iopub.execute_input":"2024-12-18T05:40:17.345398Z","iopub.status.idle":"2024-12-18T05:40:17.353301Z","shell.execute_reply.started":"2024-12-18T05:40:17.345370Z","shell.execute_reply":"2024-12-18T05:40:17.352352Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Helper function to read parquet data","metadata":{}},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:40:17.354622Z","iopub.execute_input":"2024-12-18T05:40:17.354893Z","iopub.status.idle":"2024-12-18T05:40:17.374709Z","shell.execute_reply.started":"2024-12-18T05:40:17.354867Z","shell.execute_reply":"2024-12-18T05:40:17.373852Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Read data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\ndict = pd.read_csv('../input/child-mind-institute-problematic-internet-use/data_dictionary.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:40:17.376134Z","iopub.execute_input":"2024-12-18T05:40:17.376450Z","iopub.status.idle":"2024-12-18T05:41:28.486001Z","shell.execute_reply.started":"2024-12-18T05:40:17.376423Z","shell.execute_reply":"2024-12-18T05:41:28.485114Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Kiểm tra xem các data trong timeseries có label không\ncombine = pd.merge(train_ts, train[['id', 'sii']], on='id', how='left')\n# Kiểm tra số lượng data không nhãn\ncombine['sii'].isna().sum() \n# => Tất cả data trong timne series đều có nhãn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:28.487256Z","iopub.execute_input":"2024-12-18T05:41:28.487603Z","iopub.status.idle":"2024-12-18T05:41:28.499574Z","shell.execute_reply.started":"2024-12-18T05:41:28.487563Z","shell.execute_reply":"2024-12-18T05:41:28.498786Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Check data","metadata":{}},{"cell_type":"code","source":"dict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:28.501219Z","iopub.execute_input":"2024-12-18T05:41:28.501495Z","iopub.status.idle":"2024-12-18T05:41:28.521864Z","shell.execute_reply.started":"2024-12-18T05:41:28.501467Z","shell.execute_reply":"2024-12-18T05:41:28.521138Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:28.522937Z","iopub.execute_input":"2024-12-18T05:41:28.523199Z","iopub.status.idle":"2024-12-18T05:41:28.543418Z","shell.execute_reply.started":"2024-12-18T05:41:28.523174Z","shell.execute_reply":"2024-12-18T05:41:28.542239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:28.544464Z","iopub.execute_input":"2024-12-18T05:41:28.544725Z","iopub.status.idle":"2024-12-18T05:41:28.555988Z","shell.execute_reply.started":"2024-12-18T05:41:28.544698Z","shell.execute_reply":"2024-12-18T05:41:28.555120Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3.Preprocess data","metadata":{}},{"cell_type":"markdown","source":"## 3.1 Preprocess csv","metadata":{}},{"cell_type":"code","source":"# Kiểm tra tình trạng dữ liệu trong train\nmsno.bar(train.iloc[:, :train.shape[1]], sort='ascending') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:28.557322Z","iopub.execute_input":"2024-12-18T05:41:28.557603Z","iopub.status.idle":"2024-12-18T05:41:30.892293Z","shell.execute_reply.started":"2024-12-18T05:41:28.557574Z","shell.execute_reply":"2024-12-18T05:41:30.891409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Kiểm tra tình trạng dữ liệu trong test\nmsno.bar(test.iloc[:, :test.shape[1]], sort='ascending') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:30.893576Z","iopub.execute_input":"2024-12-18T05:41:30.894261Z","iopub.status.idle":"2024-12-18T05:41:32.795337Z","shell.execute_reply.started":"2024-12-18T05:41:30.894218Z","shell.execute_reply":"2024-12-18T05:41:32.794429Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Kiểm tra các cột dư thừa ở trong train không có trong test\ndifferent_columns = set(train) - set(test)\ndifferent_columns # Cột sii là label, cần dữ lại","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:32.796469Z","iopub.execute_input":"2024-12-18T05:41:32.796740Z","iopub.status.idle":"2024-12-18T05:41:32.802398Z","shell.execute_reply.started":"2024-12-18T05:41:32.796713Z","shell.execute_reply":"2024-12-18T05:41:32.801572Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"threshold = 0.7\nnan_columns = test.columns[test.isna().mean() > threshold]\n# Loại bỏ các cột có quá nhiều Nan\n# common_columns = train.columns.intersection(test.columns).difference(nan_columns)\n# Hoặc không\ncommon_columns = train.columns.intersection(test.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:32.805013Z","iopub.execute_input":"2024-12-18T05:41:32.805312Z","iopub.status.idle":"2024-12-18T05:41:32.819035Z","shell.execute_reply.started":"2024-12-18T05:41:32.805286Z","shell.execute_reply":"2024-12-18T05:41:32.818126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Lấy các cột chung giữa train và test\n\n# Tạo lại data chỉ có các cột của test\ntrain_df = train[common_columns]\n\n# Gắn lại label\ntrain_df['sii'] = train['sii']\ntrain = train_df\n\ntest = test[common_columns]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:32.820390Z","iopub.execute_input":"2024-12-18T05:41:32.821211Z","iopub.status.idle":"2024-12-18T05:41:32.839837Z","shell.execute_reply.started":"2024-12-18T05:41:32.821182Z","shell.execute_reply":"2024-12-18T05:41:32.838957Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Encode Season","metadata":{}},{"cell_type":"code","source":"def season_encode(df, kill_season=False):\n    if kill_season:\n        season_cols = [col for col in df.columns if 'Season' in col]\n        df_ = df.drop(season_cols, axis=1)\n        return df_\n        \n    df_no_id = df.drop(columns='id')\n    string_columns = df_no_id.select_dtypes(include=['object']).columns.tolist()\n    \n    season_encode_map = {\n        'Spring': 1,\n        'Summer': 2,\n        'Fall': 3,\n        'Winter': 4,\n        np.nan: 0\n    }\n\n    # Áp dụng mapping cho tất cả các cột kiểu string\n    df_no_id[string_columns] = df_no_id[string_columns].apply(lambda col: col.map(season_encode_map))\n    df_no_id['id'] = df['id']\n    return df_no_id","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:32.841029Z","iopub.execute_input":"2024-12-18T05:41:32.841317Z","iopub.status.idle":"2024-12-18T05:41:32.847805Z","shell.execute_reply.started":"2024-12-18T05:41:32.841292Z","shell.execute_reply":"2024-12-18T05:41:32.846990Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Feature Engineering","metadata":{}},{"cell_type":"code","source":"def feature_engineering(df_):\n    df = df_.copy()\n    # Các chỉ số liên quan giữa thể chất và thời gian dùng internet\n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    \n    # Tỉ lệ lượng mỡ\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    # Quan hệ giữa cơ và mỡ\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    # Quan hệ giữa mỡ cơ thể và mỡ tổng thế\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    # Tỷ lệ mô mềm và lượng nước\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    # Mỡ * Tốc độ chuyển hóa\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    # Mỡ * Mức năng lượng\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    # Tỷ lệ BMR với cân nặng\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    # Tỷ lệ tiêu thụ năng lượng và cân nặng\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    # Tỷ lệ phát triển cơ bắp và chiều cao\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    # Tỷ lệ co bắp xương / cân nặng\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    # Mức độ Hydrat hóa cơ thể\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    \n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    \n    df['BMI_PHR'] = df['Physical-BMI'] * df['Physical-HeartRate']\n\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:32.848774Z","iopub.execute_input":"2024-12-18T05:41:32.849017Z","iopub.status.idle":"2024-12-18T05:41:32.859615Z","shell.execute_reply.started":"2024-12-18T05:41:32.848993Z","shell.execute_reply":"2024-12-18T05:41:32.858978Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.2 Preprocess parquet(time series)","metadata":{}},{"cell_type":"code","source":"# Kiểm tra kích thước time series\nprint(f\"2 tập cùng số features: {train_ts.shape[1] == test_ts.shape[1]}\") \n# True => Các fetures tồn tại trong cả train và test\nprint(f\"Số features: {train_ts.shape[1]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:32.860590Z","iopub.execute_input":"2024-12-18T05:41:32.860828Z","iopub.status.idle":"2024-12-18T05:41:32.873845Z","shell.execute_reply.started":"2024-12-18T05:41:32.860805Z","shell.execute_reply":"2024-12-18T05:41:32.873031Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Helper func auto encoder","metadata":{}},{"cell_type":"code","source":"class AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n        \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:32.874755Z","iopub.execute_input":"2024-12-18T05:41:32.875019Z","iopub.status.idle":"2024-12-18T05:41:32.887513Z","shell.execute_reply.started":"2024-12-18T05:41:32.874993Z","shell.execute_reply":"2024-12-18T05:41:32.886817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    # Scale lại data\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    \n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n\n    # Huấn luyện mô hình Autoencoder\n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n\n    # Encoder dữ liệu - Chỉ lấy phần encoder\n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:32.889009Z","iopub.execute_input":"2024-12-18T05:41:32.889416Z","iopub.status.idle":"2024-12-18T05:41:32.907834Z","shell.execute_reply.started":"2024-12-18T05:41:32.889374Z","shell.execute_reply":"2024-12-18T05:41:32.906947Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Encode data","metadata":{}},{"cell_type":"code","source":"df_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\nprint(\"Train encode\")\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=100, batch_size=32)\nprint(\"Test encode\")\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=100, batch_size=32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:32.908966Z","iopub.execute_input":"2024-12-18T05:41:32.909249Z","iopub.status.idle":"2024-12-18T05:41:43.646151Z","shell.execute_reply.started":"2024-12-18T05:41:32.909204Z","shell.execute_reply":"2024-12-18T05:41:43.645137Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Loss ko giảm, ko cần nhiều epoch","metadata":{}},{"cell_type":"code","source":"# Lấy lại id\ntrain_ts_encoded[\"id\"]=train_ts[\"id\"]\ntest_ts_encoded['id']=test_ts[\"id\"]\n\n# Lấy các cột\ntime_series_cols = train_ts_encoded.columns.tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.647369Z","iopub.execute_input":"2024-12-18T05:41:43.647659Z","iopub.status.idle":"2024-12-18T05:41:43.654112Z","shell.execute_reply.started":"2024-12-18T05:41:43.647629Z","shell.execute_reply":"2024-12-18T05:41:43.653135Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.3 Combine data","metadata":{}},{"cell_type":"code","source":"# Hợp nhất dữ liệu dựa trên id\ntrain_has_season = season_encode(train, kill_season=False)\ntest_has_season = season_encode(test, kill_season=False)\n\ntrain_combine = pd.merge(train_has_season, train_ts_encoded, how='left', on='id')\ntest_combine = pd.merge(test_has_season, test_ts_encoded, how='left', on='id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.655686Z","iopub.execute_input":"2024-12-18T05:41:43.655973Z","iopub.status.idle":"2024-12-18T05:41:43.695387Z","shell.execute_reply.started":"2024-12-18T05:41:43.655946Z","shell.execute_reply":"2024-12-18T05:41:43.694587Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Drop nan\n","metadata":{}},{"cell_type":"code","source":"train_combine = train_combine.dropna(subset='sii')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.696466Z","iopub.execute_input":"2024-12-18T05:41:43.696759Z","iopub.status.idle":"2024-12-18T05:41:43.704050Z","shell.execute_reply.started":"2024-12-18T05:41:43.696727Z","shell.execute_reply":"2024-12-18T05:41:43.703032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Xử lý dữ liệu sau khi fill\n# train_combine = train_imputed\n# test_combine = test_imputed\ntrain_combine = feature_engineering(train_combine)\ntest_combine = feature_engineering(test_combine)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.705094Z","iopub.execute_input":"2024-12-18T05:41:43.705379Z","iopub.status.idle":"2024-12-18T05:41:43.730080Z","shell.execute_reply.started":"2024-12-18T05:41:43.705346Z","shell.execute_reply":"2024-12-18T05:41:43.729294Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.4 Select feature","metadata":{}},{"cell_type":"code","source":"train_featuresCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\ntrain_featuresCols += time_series_cols\n\ntest_featuresCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday']\n\ntest_featuresCols += time_series_cols\n\ntrain_combine = train_combine[train_featuresCols]\ntest_combine = test_combine[test_featuresCols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.731831Z","iopub.execute_input":"2024-12-18T05:41:43.732248Z","iopub.status.idle":"2024-12-18T05:41:43.745503Z","shell.execute_reply.started":"2024-12-18T05:41:43.732206Z","shell.execute_reply":"2024-12-18T05:41:43.744719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Loại bỏ cột id ở cả 2 tập\ntrain_combine.drop(columns=['id'], inplace=True)\ntest_combine.drop(columns=['id'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.746507Z","iopub.execute_input":"2024-12-18T05:41:43.746831Z","iopub.status.idle":"2024-12-18T05:41:43.762134Z","shell.execute_reply.started":"2024-12-18T05:41:43.746802Z","shell.execute_reply":"2024-12-18T05:41:43.761302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Phân phối data sau khi resolve NaN\ntrain_combine['sii'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.763621Z","iopub.execute_input":"2024-12-18T05:41:43.763996Z","iopub.status.idle":"2024-12-18T05:41:43.776888Z","shell.execute_reply.started":"2024-12-18T05:41:43.763954Z","shell.execute_reply":"2024-12-18T05:41:43.775955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Phân phối data trước khi resolve NaN\ntrain['sii'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.777950Z","iopub.execute_input":"2024-12-18T05:41:43.778259Z","iopub.status.idle":"2024-12-18T05:41:43.790463Z","shell.execute_reply.started":"2024-12-18T05:41:43.778225Z","shell.execute_reply":"2024-12-18T05:41:43.789597Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_combine.replace([np.inf, -np.inf], np.nan, inplace=True)\ntest_combine.replace([np.inf, -np.inf], np.nan, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.791479Z","iopub.execute_input":"2024-12-18T05:41:43.791821Z","iopub.status.idle":"2024-12-18T05:41:43.800971Z","shell.execute_reply.started":"2024-12-18T05:41:43.791782Z","shell.execute_reply":"2024-12-18T05:41:43.800062Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Train model","metadata":{}},{"cell_type":"code","source":"# ## Hyperparameters\n# N_SPLITS = 5\n# SEED = 42\n\n# # Parameter for 3 model\n# LightGBM_Params = {\n#     'random_state': SEED, \n#     'verbose':-1,\n#     # 'n_estimators': 200,\n#     # 'learning_rate': 0.046,\n#     # 'max_depth': 12,\n#     # 'num_leaves': 478,\n#     'min_data_in_leaf': 13,\n#     'feature_fraction': 0.893,\n#     'bagging_fraction': 0.784,\n#     'bagging_freq': 4,\n#     'lambda_l1': 10,\n#     'lambda_l2': 0.01,\n#     'device': 'cpu',\n#     'learning_rate': 0.0194,          # Làm tròn đến 4 chữ số thập phân\n#     'max_depth': 4,                   # Làm tròn xuống từ 4.169925057285209\n#     'min_child_samples': 19,          # Làm tròn lên từ 18.512843017102462\n#     'n_estimators': 277,              # Làm tròn từ 277.1412524913512\n#     'num_leaves': 43,                 # Làm tròn từ 43.0111475413638\n#     'subsample': 0.6135               # Làm tròn đến 4 chữ số thập phân\n# }\n\n\n# XGB_Params = {\n#     'reg_alpha': 1,\n#     'reg_lambda': 5,\n#     'random_state': SEED,\n#     'tree_method': 'gpu_hist',\n#     'colsample_bytree': 0.8617,\n#     'learning_rate': 0.0197,\n#     'max_depth': int(6.4756),  \n#     'min_child_weight': 9.8952,\n#     'n_estimators': int(153.25),  \n#     'subsample': 0.5427\n\n#      # 'learning_rate': 0.05,\n#     # 'max_depth': 6,\n#     # 'n_estimators': 200,\n#     # 'subsample': 0.8,\n#     # 'colsample_bytree': 0.8,\n#     # 'reg_alpha': 1,\n#     # 'reg_lambda': 5,\n#     # 'random_state': SEED,\n#     # 'tree_method': 'gpu_hist',\n# }\n\n\n# CatBoost_Params = {\n#     'random_seed': 42,\n#     'verbose': 0,\n#     'task_type': 'GPU',\n#     'depth': 7,                         \n#     'iterations': 147,                \n#     'l2_leaf_reg': 2.0,                 \n#     'learning_rate': 0.0325             \n\n#     # 'learning_rate': 0.05,\n#     # 'depth': 6,\n#     # 'iterations': 200,\n#     # 'random_seed': 42,\n#     # 'verbose': 0,\n#     # 'l2_leaf_reg': 10,\n#     # 'task_type': 'GPU'\n# }\n\n# RF_Params = {\n#     # 'n_estimators': 200,             # Số lượng cây\n#     # 'max_depth': 10,                 # Độ sâu tối đa của cây (trung bình của 6-12)\n#     # 'min_samples_split': 5,          # Số mẫu tối thiểu để chia nhánh\n#     # 'min_samples_leaf': 3,           # Số mẫu tối thiểu trên một lá\n#     'max_features': 0.8,             # Tỷ lệ chọn thuộc tính tại mỗi node\n#     'max_samples': 0.8,              # Tỷ lệ mẫu sử dụng để xây dựng mỗi cây\n#     'random_state': SEED,            # Khóa ngẫu nhiên\n#     'bootstrap': True,               # Bagging (tương đương với subsample)\n#     'max_depth': 8,                  # Làm tròn xuống từ 8.203917319696927\n#     'min_samples_leaf': 5,           # Làm tròn lên từ 4.843674344989926\n#     'min_samples_split': 3,          # Làm tròn xuống từ 3.287214210584705\n#     'n_estimators': 227              # Làm tròn lên từ 226.74955692359586\n# }\n\n# GB_Params = {\n#     \"learning_rate\": 0.03,\n#     \"max_depth\": 4,\n#     \"min_samples_leaf\": 4,\n#     \"min_samples_split\": 7,\n#     \"n_estimators\": 250,\n#     'random_state': SEED, \n# }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:05:37.869843Z","iopub.execute_input":"2024-12-18T06:05:37.870221Z","iopub.status.idle":"2024-12-18T06:05:37.878057Z","shell.execute_reply.started":"2024-12-18T06:05:37.870189Z","shell.execute_reply":"2024-12-18T06:05:37.877257Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Hyperparameters\nN_SPLITS = 5\nSEED = 42\n\n# Parameter for 3 model\nLightGBM_Params = {\n    'random_state': SEED, \n    'verbose':-1,\n    'n_estimators': 200,\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,\n    'lambda_l2': 0.01,\n    'device': 'cpu',\n}\n\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,\n    'reg_lambda': 5,\n    'random_state': SEED,\n    'tree_method': 'gpu_hist',\n}\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': 42,\n    'verbose': 0,\n    'l2_leaf_reg': 10,\n    'task_type': 'GPU'\n}\n\nRF_Params = {\n    'n_estimators': 200,             # Số lượng cây\n    'max_depth': 10,                 # Độ sâu tối đa của cây (trung bình của 6-12)\n    'min_samples_split': 5,          # Số mẫu tối thiểu để chia nhánh\n    'min_samples_leaf': 3,           # Số mẫu tối thiểu trên một lá\n    'max_features': 0.8,             # Tỷ lệ chọn thuộc tính tại mỗi node\n    'max_samples': 0.8,              # Tỷ lệ mẫu sử dụng để xây dựng mỗi cây\n    'random_state': SEED,            # Khóa ngẫu nhiên\n    'bootstrap': True,               # Bagging (tương đương với subsample)\n}\n\nGB_Params = {\n    \"learning_rate\": 0.03,\n    \"max_depth\": 4,\n    \"min_samples_leaf\": 4,\n    \"min_samples_split\": 7,\n    \"n_estimators\": 250,\n    'random_state': SEED, \n}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Useful function\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n    \ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:05:41.028617Z","iopub.execute_input":"2024-12-18T06:05:41.028978Z","iopub.status.idle":"2024-12-18T06:05:41.035086Z","shell.execute_reply.started":"2024-12-18T06:05:41.028944Z","shell.execute_reply":"2024-12-18T06:05:41.034103Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Train and get predict function\ndef train_predict(model, train_data, test_data):\n    # Khớp giữ liệu input của train và test\n    X = train_data.drop(columns=['sii'])\n    y = train_data['sii']\n\n    # Định nghĩa K-Fold\n    SKF = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\n\n    train_his = []\n    val_his = []\n\n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), N_SPLITS))\n\n    for fold, (train_index, val_index) in enumerate(tqdm(SKF.split(X, y), desc=\"Train progress\", total = N_SPLITS)):\n        # Xác định dữ liệu của fold\n        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n        # Train\n        model_ = clone(model) # tạo bản sao làm mới tại mỗi fold -> độc lập\n        model_.fit(X_train, y_train)\n\n        # Tính toán sai số\n        y_train_pred = model_.predict(X_train)\n        y_val_pred = model_.predict(X_val)\n\n        oof_non_rounded[val_index] = y_val_pred\n        y_train_pred_rounded = y_train_pred.round(0).astype(int)\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[val_index] = y_val_pred_rounded\n\n        # Kiểm tra sai số giữa mô hình đánh giá và thực tế\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred_rounded)\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_his.append(train_kappa)\n        val_his.append(val_kappa)\n\n        test_preds[:, fold] = model_.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n\n    print(f\"Mean Train QWK --> {np.mean(train_his):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(val_his):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission, model_, np.mean(val_his)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:05:45.731831Z","iopub.execute_input":"2024-12-18T06:05:45.732474Z","iopub.status.idle":"2024-12-18T06:05:45.742709Z","shell.execute_reply.started":"2024-12-18T06:05:45.732436Z","shell.execute_reply":"2024-12-18T06:05:45.741724Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from sklearn.model_selection import cross_val_score, KFold\n# from sklearn.ensemble import GradientBoostingRegressor\n# from bayes_opt import BayesianOptimization\n\n# # Định nghĩa hàm mục tiêu (objective function)\n# def gb_evaluate(n_estimators, learning_rate, max_depth, min_samples_split, min_samples_leaf):\n\n#     # Dữ liệu\n#     X = train_combine.drop(columns=['sii'])\n#     y = train_combine['sii']\n\n#     # Điền giá trị NaN nếu có (hoặc bạn có thể sử dụng .dropna() nếu không muốn giữ giá trị NaN)\n#     imputer = SimpleImputer(strategy='mean')\n#     X = imputer.fit_transform(X)\n    \n#     # Tạo mô hình GradientBoostingRegressor với các tham số đã tối ưu\n#     model = GradientBoostingRegressor(\n#         n_estimators=int(n_estimators),         # Chuyển về kiểu int\n#         learning_rate=learning_rate,\n#         max_depth=int(max_depth),                # Chuyển về kiểu int\n#         min_samples_split=int(min_samples_split),# Chuyển về kiểu int\n#         min_samples_leaf=int(min_samples_leaf),  # Chuyển về kiểu int\n#         random_state=42\n#     )\n    \n#     # Đánh giá mô hình bằng cross-validation (sử dụng độ chính xác hoặc QWK)\n#     kf = KFold(n_splits=5, shuffle=True, random_state=42)\n#     score = cross_val_score(model, X, y, cv=kf, scoring='neg_mean_squared_error')\n    \n#     # Trả về kết quả (giảm sai số để tối ưu hóa)\n#     return score.mean()\n\n# # Định nghĩa các tham số cần tối ưu hóa\n# param_bounds = {\n#     'n_estimators': (100, 300),            # Số lượng cây\n#     'learning_rate': (0.01, 0.1),          # Tốc độ học\n#     'max_depth': (3, 10),                   # Độ sâu tối đa của cây\n#     'min_samples_split': (2, 10),           # Số mẫu tối thiểu để chia nhánh\n#     'min_samples_leaf': (1, 5),             # Số mẫu tối thiểu trong lá\n# }\n\n# # Sử dụng Bayesian Optimization\n# optimizer = BayesianOptimization(\n#     f=gb_evaluate,\n#     pbounds=param_bounds,\n#     random_state=42,\n#     verbose=2\n# )\n\n# # Tối ưu hóa với 10 lần khởi tạo và 50 vòng lặp\n# optimizer.maximize(init_points=10, n_iter=50)\n\n# # Kết quả tối ưu hóa\n# print(\"Best parameters found: \", optimizer.max)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.824061Z","iopub.status.idle":"2024-12-18T05:41:43.824510Z","shell.execute_reply.started":"2024-12-18T05:41:43.824287Z","shell.execute_reply":"2024-12-18T05:41:43.824305Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from xgboost import XGBRegressor\n\n# def xgb_evaluate(n_estimators, learning_rate, max_depth, colsample_bytree, subsample, min_child_weight):\n#     X = train_combine.drop(columns=['sii'])\n#     y = train_combine['sii']\n#     imputer = SimpleImputer(strategy='mean')\n#     X = imputer.fit_transform(X)\n\n#     model = XGBRegressor(\n#         n_estimators=int(n_estimators),\n#         learning_rate=learning_rate,\n#         max_depth=int(max_depth),\n#         colsample_bytree=colsample_bytree,\n#         subsample=subsample,\n#         min_child_weight=min_child_weight,\n#         random_state=42\n#     )\n#     kf = KFold(n_splits=5, shuffle=True, random_state=42)\n#     score = cross_val_score(model, X, y, cv=kf, scoring='neg_mean_squared_error')\n#     return score.mean()\n\n# xgb_bounds = {\n#     'n_estimators': (100, 300),\n#     'learning_rate': (0.01, 0.3),\n#     'max_depth': (3, 10),\n#     'colsample_bytree': (0.5, 1),\n#     'subsample': (0.5, 1),\n#     'min_child_weight': (1, 10),\n# }\n\n# xgb_optimizer = BayesianOptimization(\n#     f=xgb_evaluate,\n#     pbounds=xgb_bounds,\n#     random_state=42,\n#     verbose=2\n# )\n\n# xgb_optimizer.maximize(init_points=10, n_iter=50)\n# print(\"Best XGBoost parameters: \", xgb_optimizer.max)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.825739Z","iopub.status.idle":"2024-12-18T05:41:43.826720Z","shell.execute_reply.started":"2024-12-18T05:41:43.826539Z","shell.execute_reply":"2024-12-18T05:41:43.826559Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# #bayes for catboost\n# from catboost import CatBoostRegressor\n\n# def catboost_evaluate(iterations, learning_rate, depth, l2_leaf_reg):\n#     X = train_combine.drop(columns=['sii'])\n#     y = train_combine['sii']\n#     imputer = SimpleImputer(strategy='mean')\n#     X = imputer.fit_transform(X)\n\n#     model = CatBoostRegressor(\n#         iterations=int(iterations),\n#         learning_rate=learning_rate,\n#         depth=int(depth),\n#         l2_leaf_reg=l2_leaf_reg,\n#         random_state=42,\n#         verbose=0\n#     )\n#     kf = KFold(n_splits=5, shuffle=True, random_state=42)\n#     score = cross_val_score(model, X, y, cv=kf, scoring='neg_mean_squared_error')\n#     return score.mean()\n\n# catboost_bounds = {\n#     'iterations': (100, 300),\n#     'learning_rate': (0.01, 0.3),\n#     'depth': (3, 10),\n#     'l2_leaf_reg': (1, 10),\n# }\n\n# catboost_optimizer = BayesianOptimization(\n#     f=catboost_evaluate,\n#     pbounds=catboost_bounds,\n#     random_state=42,\n#     verbose=2\n# )\n\n# catboost_optimizer.maximize(init_points=10, n_iter=50)\n# print(\"Best CatBoost parameters: \", catboost_optimizer.max)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.827861Z","iopub.status.idle":"2024-12-18T05:41:43.828258Z","shell.execute_reply.started":"2024-12-18T05:41:43.828050Z","shell.execute_reply":"2024-12-18T05:41:43.828090Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# #bayes Opt cho LGBM\n# from lightgbm import LGBMRegressor\n\n# def lightgbm_evaluate(n_estimators, learning_rate, max_depth, num_leaves, min_child_samples, subsample):\n#     X = train_combine.drop(columns=['sii'])\n#     y = train_combine['sii']\n#     imputer = SimpleImputer(strategy='mean')\n#     X = imputer.fit_transform(X)\n\n#     model = LGBMRegressor(\n#         n_estimators=int(n_estimators),\n#         learning_rate=learning_rate,\n#         max_depth=int(max_depth),\n#         num_leaves=int(num_leaves),\n#         min_child_samples=int(min_child_samples),\n#         subsample=subsample,\n#         random_state=42\n#     )\n#     kf = KFold(n_splits=5, shuffle=True, random_state=42)\n#     score = cross_val_score(model, X, y, cv=kf, scoring='neg_mean_squared_error')\n#     return score.mean()\n\n# lightgbm_bounds = {\n#     'n_estimators': (100, 300),\n#     'learning_rate': (0.01, 0.3),\n#     'max_depth': (3, 10),\n#     'num_leaves': (20, 50),\n#     'min_child_samples': (5, 20),\n#     'subsample': (0.5, 1),\n# }\n\n# lightgbm_optimizer = BayesianOptimization(\n#     f=lightgbm_evaluate,\n#     pbounds=lightgbm_bounds,\n#     random_state=42,\n#     verbose=2\n# )\n\n# lightgbm_optimizer.maximize(init_points=10, n_iter=50)\n# print(\"Best LightGBM parameters: \", lightgbm_optimizer.max)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.829847Z","iopub.status.idle":"2024-12-18T05:41:43.830276Z","shell.execute_reply.started":"2024-12-18T05:41:43.830037Z","shell.execute_reply":"2024-12-18T05:41:43.830054Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# #bayes cho RF\n# from sklearn.ensemble import RandomForestRegressor\n\n# def rf_evaluate(n_estimators, max_depth, min_samples_split, min_samples_leaf):\n#     X = train_combine.drop(columns=['sii'])\n#     y = train_combine['sii']\n#     imputer = SimpleImputer(strategy='mean')\n#     X = imputer.fit_transform(X)\n\n#     model = RandomForestRegressor(\n#         n_estimators=int(n_estimators),\n#         max_depth=int(max_depth),\n#         min_samples_split=int(min_samples_split),\n#         min_samples_leaf=int(min_samples_leaf),\n#         random_state=42\n#     )\n#     kf = KFold(n_splits=5, shuffle=True, random_state=42)\n#     score = cross_val_score(model, X, y, cv=kf, scoring='neg_mean_squared_error')\n#     return score.mean()\n\n# rf_bounds = {\n#     'n_estimators': (100, 300),\n#     'max_depth': (5, 20),\n#     'min_samples_split': (2, 10),\n#     'min_samples_leaf': (1, 5),\n# }\n\n# rf_optimizer = BayesianOptimization(\n#     f=rf_evaluate,\n#     pbounds=rf_bounds,\n#     random_state=42,\n#     verbose=2\n# )\n\n# rf_optimizer.maximize(init_points=10, n_iter=50)\n# print(\"Best Random Forest parameters: \", rf_optimizer.max)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.831374Z","iopub.status.idle":"2024-12-18T05:41:43.831720Z","shell.execute_reply.started":"2024-12-18T05:41:43.831570Z","shell.execute_reply":"2024-12-18T05:41:43.831587Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4.1 XGboost  + LightGBM + CatBoost + RF","metadata":{}},{"cell_type":"markdown","source":"### Model","metadata":{}},{"cell_type":"code","source":"# Create model instances\nLightGBM_Model = LGBMRegressor(**LightGBM_Params)\nXGBoost_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n# RF_model = RandomForestRegressor(**RF_Params)\n# GB_Model = GradientBoostingRegressor(**GB_Params)\n\nGB_pipeline = Pipeline([\n    ('imputer', SimpleImputer(strategy='mean')),  # Hoặc 'median'\n    ('gradient_boost', GradientBoostingRegressor(**GB_Params))\n])\n\nRF_pipeline = Pipeline([\n    ('imputer', SimpleImputer(strategy='mean')),  # Hoặc 'median'\n    ('random_forest', RandomForestRegressor(**RF_Params))\n])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T09:35:04.724593Z","iopub.execute_input":"2024-12-18T09:35:04.724882Z","iopub.status.idle":"2024-12-18T09:35:04.970320Z","shell.execute_reply.started":"2024-12-18T09:35:04.724845Z","shell.execute_reply":"2024-12-18T09:35:04.968901Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Calc weight for each model to vote","metadata":{}},{"cell_type":"code","source":"def evaluate_model(model, X_test, y_test):\n    # Dự đoán giá trị\n    y_pred = model.predict(X_test)\n    \n    # Tính toán các chỉ số\n    rmse = np.sqrt(mean_squared_error(y_test, y_pred))\n    mae = mean_absolute_error(y_test, y_pred)\n    r2 = r2_score(y_test, y_pred)\n    \n    return rmse, mae, r2\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:05:57.001882Z","iopub.execute_input":"2024-12-18T06:05:57.002302Z","iopub.status.idle":"2024-12-18T06:05:57.007178Z","shell.execute_reply.started":"2024-12-18T06:05:57.002248Z","shell.execute_reply":"2024-12-18T06:05:57.006202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n# Định nghĩa các mô hình\nmodels = {\n    \"LightGBM_Model\": LightGBM_Model,\n    \"XGBoost_Model\": XGBoost_Model,\n    \"CatBoost_Model\": CatBoost_Model,\n    \"RF_pipeline\": RF_pipeline,\n    \"GB_pipeline\": GB_pipeline,\n}\n\nX = train_combine.drop(columns=['sii'])\ny = train_combine['sii']\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n# Lưu kết quả\nresults = []\n\n# Huấn luyện và đánh giá từng mô hình\nfor name, model in models.items():\n    # Huấn luyện mô hình\n    model.fit(X_train, y_train)\n    \n    # Đánh giá hiệu suất\n    rmse, mae, r2 = evaluate_model(model, X_test, y_test)\n    \n    # Lưu kết quả\n    results.append((name, rmse, mae, r2))\n    \n    # In kết quả cho mô hình hiện tại\n    print(f\"Model: {name}\")\n    print(f\"  RMSE: {rmse:.4f}\")\n    print(f\"  MAE: {mae:.4f}\")\n    print(f\"  R2: {r2:.4f}\")\n    print(\"-\" * 30)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:06:00.042023Z","iopub.execute_input":"2024-12-18T06:06:00.042393Z","iopub.status.idle":"2024-12-18T06:06:16.164551Z","shell.execute_reply.started":"2024-12-18T06:06:00.042362Z","shell.execute_reply":"2024-12-18T06:06:16.163614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results_df = pd.DataFrame(results, columns=[\"Model\", \"RMSE\", \"MAE\", \"R2\"]).sort_values(by=\"RMSE\")\n\n# Hiển thị kết quả\nprint(results_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:06:20.712578Z","iopub.execute_input":"2024-12-18T06:06:20.712943Z","iopub.status.idle":"2024-12-18T06:06:20.722697Z","shell.execute_reply.started":"2024-12-18T06:06:20.712909Z","shell.execute_reply":"2024-12-18T06:06:20.721734Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Tính toán lại thủ công trọng số theo nghịch đảo RMSE\n\n# # RMSE values\n# rmse_values = [0.6712, 0.6747, 0.6764, 0.6767, 0.6822]\n\n# # Tính nghịch đảo của RMSE\n# inverse_rmse = [1 / rmse for rmse in rmse_values]\n\n# # Tổng nghịch đảo RMSE\n# sum_inverse_rmse = sum(inverse_rmse)\n\n# # Chuẩn hóa trọng số\n# weights = [val / sum_inverse_rmse for val in inverse_rmse]\n\n# # Kết quả gắn với tên mô hình\n# models = [\"CatBoost_Model\", \"XGBoost_Model\", \"LightGBM_Model\", \"GB_pipeline\", \"RF_pipeline\"]\n# result = [{\"Model\": model, \"RMSE\": rmse, \"Weight\": weight} for model, rmse, weight in zip(models, rmse_values, weights)]\n\n# result\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.841536Z","iopub.status.idle":"2024-12-18T05:41:43.841858Z","shell.execute_reply.started":"2024-12-18T05:41:43.841709Z","shell.execute_reply":"2024-12-18T05:41:43.841726Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### CatBoost là có hiệu suất tốt nhất , GB là kém nhất\n--> do fill mean các model xử lý được NaN có hiệu suất cao hơn","metadata":{}},{"cell_type":"code","source":"# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', LightGBM_Model),\n    ('xgboost', XGBoost_Model),\n    ('catboost', CatBoost_Model),\n    ('random_forest', RF_pipeline),\n     # ('gradient_boosting', GB_pipeline) \n# ], weights=[4.0, 3.5, 4.0, 3.5, 3.0]) \n], weights=[4.0,4.0,5.0,4.0])\n# ], weights=[4.0, 4.0, 5.5, 4.0, 4.0]) \n# ], weights=[4.0, 3.5, 5.0, 2.0, 1.5])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:09:05.208466Z","iopub.execute_input":"2024-12-18T06:09:05.209293Z","iopub.status.idle":"2024-12-18T06:09:05.214579Z","shell.execute_reply.started":"2024-12-18T06:09:05.209252Z","shell.execute_reply":"2024-12-18T06:09:05.213511Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submission\n","metadata":{}},{"cell_type":"code","source":"Submission1, model, val = train_predict(voting_model, train_combine, test_combine)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:09:10.524304Z","iopub.execute_input":"2024-12-18T06:09:10.524636Z","iopub.status.idle":"2024-12-18T06:10:32.386158Z","shell.execute_reply.started":"2024-12-18T06:09:10.524607Z","shell.execute_reply":"2024-12-18T06:10:32.385339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Huấn luyện VotingRegressor\n# voting_model.fit(X_train, y_train)\n# #Kiểm tra hiệu suất voting model\n# rmse, mae, r2 = evaluate_model(voting_model, X_test, y_test)\n# print(f\"Voting Model Performance:\")\n# print(f\"  RMSE: {rmse:.4f}\")\n# print(f\"  MAE: {mae:.4f}\")\n# print(f\"  R2: {r2:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T05:41:43.846953Z","iopub.status.idle":"2024-12-18T05:41:43.847295Z","shell.execute_reply.started":"2024-12-18T05:41:43.847142Z","shell.execute_reply":"2024-12-18T05:41:43.847160Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission1['sii'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:11:43.214429Z","iopub.execute_input":"2024-12-18T06:11:43.215283Z","iopub.status.idle":"2024-12-18T06:11:43.222315Z","shell.execute_reply.started":"2024-12-18T06:11:43.215248Z","shell.execute_reply":"2024-12-18T06:11:43.221481Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission1.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:11:44.750376Z","iopub.execute_input":"2024-12-18T06:11:44.751382Z","iopub.status.idle":"2024-12-18T06:11:44.756956Z","shell.execute_reply.started":"2024-12-18T06:11:44.751346Z","shell.execute_reply":"2024-12-18T06:11:44.756093Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T06:11:46.613863Z","iopub.execute_input":"2024-12-18T06:11:46.614772Z","iopub.status.idle":"2024-12-18T06:11:46.623950Z","shell.execute_reply.started":"2024-12-18T06:11:46.614714Z","shell.execute_reply":"2024-12-18T06:11:46.622868Z"}},"outputs":[],"execution_count":null}]}