{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":7453542,"sourceType":"datasetVersion","datasetId":921302}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip -q install /kaggle/input/pytorchtabnet/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:53:41.373346Z","iopub.execute_input":"2024-11-11T05:53:41.374061Z","iopub.status.idle":"2024-11-11T05:54:12.899466Z","shell.execute_reply.started":"2024-11-11T05:53:41.374016Z","shell.execute_reply":"2024-11-11T05:54:12.898372Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone, BaseEstimator, RegressorMixin\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score, mean_squared_error\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.decomposition import PCA\nfrom sklearn.datasets import make_classification\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom pytorch_tabnet.tab_model import TabNetRegressor\nfrom pytorch_tabnet.callbacks import Callback\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nSEED = 42\nn_splits = 5","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-11T05:54:12.901778Z","iopub.execute_input":"2024-11-11T05:54:12.902144Z","iopub.status.idle":"2024-11-11T05:54:12.913468Z","shell.execute_reply.started":"2024-11-11T05:54:12.902092Z","shell.execute_reply":"2024-11-11T05:54:12.912574Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\nseed_everything(2024)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-11T05:54:12.914955Z","iopub.execute_input":"2024-11-11T05:54:12.915367Z","iopub.status.idle":"2024-11-11T05:54:12.926004Z","shell.execute_reply.started":"2024-11-11T05:54:12.915323Z","shell.execute_reply":"2024-11-11T05:54:12.925167Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n\nclass AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n        \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n    \ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    \n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n                 \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded\n\ndef feature_engineering(df):\n    season_cols = [col for col in df.columns if 'Season' in col]\n    df = df.drop(season_cols, axis=1) \n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    df['BMI_PHR'] = df['Physical-BMI'] * df['Physical-HeartRate']\n    \n    return df\n\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=100, batch_size=32)\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=100, batch_size=32)\n\ntime_series_cols = train_ts_encoded.columns.tolist()\ntrain_ts_encoded[\"id\"]=train_ts[\"id\"]\ntest_ts_encoded['id']=test_ts[\"id\"]\n\ntrain = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\ntest = pd.merge(test, test_ts_encoded, how=\"left\", on='id')","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:54:12.927575Z","iopub.execute_input":"2024-11-11T05:54:12.927936Z","iopub.status.idle":"2024-11-11T05:55:44.347225Z","shell.execute_reply.started":"2024-11-11T05:54:12.927893Z","shell.execute_reply":"2024-11-11T05:55:44.345929Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train\n\n","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:55:44.350422Z","iopub.execute_input":"2024-11-11T05:55:44.350824Z","iopub.status.idle":"2024-11-11T05:55:44.537846Z","shell.execute_reply.started":"2024-11-11T05:55:44.35078Z","shell.execute_reply":"2024-11-11T05:55:44.536853Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train = feature_engineering(train)\n# train = train.dropna(thresh=10, axis=0)\n# test = feature_engineering(test)\n\n# # Chuyển đổi các cột category sang kiểu dữ liệu số nếu có thể, hoặc loại bỏ chúng\n# numeric_cols = train.select_dtypes(include=[np.number]).columns\n# train_numeric = train[numeric_cols]\n\n# # Kiểm tra và thay thế các giá trị vô hạn\n# if np.any(np.isinf(train_numeric)):\n#     train_numeric = train_numeric.replace([np.inf, -np.inf], np.nan)\n\n# # Gộp lại với các cột không phải là số\n# train = train_numeric.join(train.drop(columns=numeric_cols))\n\n# # Chuyển đổi các cột category sang kiểu dữ liệu số nếu có thể, hoặc loại bỏ chúng trong tập kiểm tra\n# numeric_cols_test = test.select_dtypes(include=[np.number]).columns\n# test_numeric = test[numeric_cols_test]\n\n# # Kiểm tra và thay thế các giá trị vô hạn trong tập kiểm tra\n# if np.any(np.isinf(test_numeric)):\n#     test_numeric = test_numeric.replace([np.inf, -np.inf], np.nan)\n\n# # Gộp lại với các cột không phải là số trong tập kiểm tra\n# test = test_numeric.join(test.drop(columns=numeric_cols_test))\n\n# imputer = KNNImputer(n_neighbors=5)\n# numeric_cols = train.select_dtypes(include=['int32', 'int64', 'float64', 'int64']).columns\n# imputed_data = imputer.fit_transform(train[numeric_cols])\n# train_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\n# train_imputed['sii'] = train_imputed['sii'].round().astype(int)\n# for col in train.columns:\n#     if col not in numeric_cols:\n#         train_imputed[col] = train[col]\n        \n# train = train_imputed\n\n# Giả sử train và test là DataFrame của bạn\n# Các cột đóng góp\n# contributing_cols = [\n#     'Physical-BMI', 'Basic_Demos-Age', 'PreInt_EduHx-computerinternet_hoursday',\n#     'BIA-BIA_Fat', 'BIA-BIA_BMI', 'BIA-BIA_FFMI', 'BIA-BIA_FMI',\n#     'BIA-BIA_LST', 'BIA-BIA_TBW', 'BIA-BIA_BMR', 'BIA-BIA_DEE',\n#     'Physical-Weight', 'BIA-BIA_SMM', 'Physical-Height', 'Physical-HeartRate',\n#     'BIA-BIA_ICW'\n# ]\n\n# # Sử dụng KNNImputer để điền thiếu các cột đóng góp trong tập huấn luyện\n# imputer = KNNImputer(n_neighbors=5)\n# imputed_data_train = imputer.fit_transform(train[contributing_cols])\n# train_imputed = pd.DataFrame(imputed_data_train, columns=contributing_cols)\n\n# # Gộp lại với các cột không phải là số trong tập huấn luyện\n# for col in train.columns:\n#     if col not in contributing_cols:\n#         train_imputed[col] = train[col]\n\n# # Sử dụng KNNImputer để điền thiếu các cột đóng góp trong tập kiểm tra\n# imputed_data_test = imputer.transform(test[contributing_cols])\n\ntrain = feature_engineering(train)\ntrain = train.dropna(thresh=10, axis=0)\ntest = feature_engineering(test)","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:55:44.539227Z","iopub.execute_input":"2024-11-11T05:55:44.539611Z","iopub.status.idle":"2024-11-11T05:55:52.137232Z","shell.execute_reply.started":"2024-11-11T05:55:44.539565Z","shell.execute_reply":"2024-11-11T05:55:52.136414Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.drop('id', axis=1)\ntrain","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:55:52.138386Z","iopub.execute_input":"2024-11-11T05:55:52.138699Z","iopub.status.idle":"2024-11-11T05:55:52.302969Z","shell.execute_reply.started":"2024-11-11T05:55:52.138664Z","shell.execute_reply":"2024-11-11T05:55:52.302037Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.drop('id', axis=1)\ntest","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:55:52.304095Z","iopub.execute_input":"2024-11-11T05:55:52.304447Z","iopub.status.idle":"2024-11-11T05:55:52.502739Z","shell.execute_reply.started":"2024-11-11T05:55:52.304411Z","shell.execute_reply":"2024-11-11T05:55:52.501861Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featuresCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW', 'BMI_PHR'\n          #       ,\n          #      'Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          # 'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          # 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season'\n               ]\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW', 'BMI_PHR'\n          #       ,\n          #      'Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          # 'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          # 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season'\n               ]\n\nfeaturesCols += time_series_cols\ntest = test[featuresCols]\n\n# cat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n#           'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n#           'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\n# def update(df):\n#     global cat_c\n#     for c in cat_c: \n#         df[c] = df[c].fillna('Missing')\n#         df[c] = df[c].astype('category')\n#     return df\n        \n# train = update(train)\n# test = update(test)","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:55:52.503872Z","iopub.execute_input":"2024-11-11T05:55:52.504201Z","iopub.status.idle":"2024-11-11T05:55:52.519102Z","shell.execute_reply.started":"2024-11-11T05:55:52.504167Z","shell.execute_reply":"2024-11-11T05:55:52.51832Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:55:52.520347Z","iopub.execute_input":"2024-11-11T05:55:52.521348Z","iopub.status.idle":"2024-11-11T05:55:52.651666Z","shell.execute_reply.started":"2024-11-11T05:55:52.521304Z","shell.execute_reply":"2024-11-11T05:55:52.650862Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:55:52.652795Z","iopub.execute_input":"2024-11-11T05:55:52.65307Z","iopub.status.idle":"2024-11-11T05:55:52.849758Z","shell.execute_reply.started":"2024-11-11T05:55:52.653039Z","shell.execute_reply":"2024-11-11T05:55:52.848805Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train['Basic_Demos-Sex_log'] = np.log1p(train['Basic_Demos-Sex'])\n# test['Basic_Demos-Sex_log'] = np.log1p(test['Basic_Demos-Sex'])\n\n# train['Physical-Height_log'] = np.log1p(train['Physical-Height'])\n# test['Physical-Height_log'] = np.log1p(test['Physical-Height'])\n\n# train['Physical-Weight_log'] = np.log1p(train['Physical-Weight'])\n# test['Physical-Weight_log'] = np.log1p(test['Physical-Weight'])\n\ntrain['Log_BMI'] = np.log(train['Physical-BMI'] + 1)\n# train['Log_Weight'] = np.log(train['Physical-Weight'] + 1)\ntrain['Log_Waist'] = np.log(train['Physical-Waist_Circumference'] + 1)\n\ntest['Log_BMI'] = np.log(test['Physical-BMI'] + 1)\n# test['Log_Weight'] = np.log(test['Physical-Weight'] + 1)\ntest['Log_Waist'] = np.log(test['Physical-Waist_Circumference'] + 1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if np.any(np.isinf(train)):\n    train = train.replace([np.inf, -np.inf], np.nan)\n\n# # Chuyển đổi các cột category sang kiểu dữ liệu số nếu có thể, hoặc loại bỏ chúng\n# numeric_cols = train.select_dtypes(include=[np.number]).columns\n# train_numeric = train[numeric_cols]\n\n# # Kiểm tra và thay thế các giá trị vô hạn\n# if np.any(np.isinf(train_numeric)):\n#     train_numeric = train_numeric.replace([np.inf, -np.inf], np.nan)\n\n# # Gộp lại với các cột không phải là số\n# train = train_numeric.join(train.drop(columns=numeric_cols))\n\n# from sklearn.preprocessing import OneHotEncoder\n\n# # Giả sử train và test là DataFrame của bạn\n# categorical_cols = train.select_dtypes(include=['category', 'object']).columns\n\n# # One-Hot Encoding cho tập huấn luyện\n# encoder = OneHotEncoder(sparse=False, drop='first')\n# encoded_train = pd.DataFrame(encoder.fit_transform(train[categorical_cols]), columns=encoder.get_feature_names_out(categorical_cols))\n\n# # One-Hot Encoding cho tập kiểm tra\n# encoded_test = pd.DataFrame(encoder.transform(test[categorical_cols]), columns=encoder.get_feature_names_out(categorical_cols))\n\n# # Kết hợp lại với các cột số\n# train = train.drop(columns=categorical_cols).join(encoded_train)\n# test = test.drop(columns=categorical_cols).join(encoded_test)","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:55:52.850922Z","iopub.execute_input":"2024-11-11T05:55:52.851239Z","iopub.status.idle":"2024-11-11T05:55:52.859527Z","shell.execute_reply.started":"2024-11-11T05:55:52.851206Z","shell.execute_reply":"2024-11-11T05:55:52.85874Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:55:52.860902Z","iopub.execute_input":"2024-11-11T05:55:52.861242Z","iopub.status.idle":"2024-11-11T05:55:52.876369Z","shell.execute_reply.started":"2024-11-11T05:55:52.861197Z","shell.execute_reply":"2024-11-11T05:55:52.87541Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TabNetWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, **kwargs):\n        self.model = TabNetRegressor(**kwargs)\n        self.kwargs = kwargs\n        self.imputer = SimpleImputer(strategy='median')\n        self.best_model_path = 'best_tabnet_model.pt'\n        \n    def fit(self, X, y):\n        X_imputed = self.imputer.fit_transform(X)\n            \n        if hasattr(y, 'values'):\n            y = y.values\n\n        X_train, X_valid, y_train, y_valid = train_test_split(X_imputed, y, test_size=0.2, random_state=SEED)\n\n        history = self.model.fit(\n            X_train = X_train,\n            y_train = y_train.reshape(-1, 1),\n            eval_set = [(X_valid, y_valid.reshape(-1, 1))],\n            eval_name = ['valid'],\n            eval_metric = ['mse'],\n            max_epochs = 200,\n            patience = 20,\n            batch_size = 1024,\n            virtual_batch_size = 128,\n            num_workers = 0,\n            drop_last = False,\n            callbacks = [\n                TabNetPretrainedModelCheckpoint(\n                    filepath = self.best_model_path,\n                    monitor = 'valid_mse',\n                    mode = 'min',\n                    save_best_only = True,\n                    verbose = True\n                )\n            ]\n        )\n\n        if os.path.exists(self.best_model_path):\n            self.model.load_model(self.best_model_path)\n            os.remove(self.best_model_path)\n\n        return self\n\n    def predict(self, X):\n        X_imputed = self.imputer.transform(X)\n        return self.model.predict(X_imputed).flatten()\n\n    def __deepcopy__(self, memo):\n        cls = self.__class__\n        result = self.__new__(cls)\n        memo[id(self)] = result\n        for k, v in self.__dict__.items():\n            setattr(result, k, deepcopy(v, memo))\n\n        return result\n\nTabNet_Params = {\n    'n_d': 64,              # Width of the decision prediction layer\n    'n_a': 64,              # Width of the attention embedding for each step\n    'n_steps': 5,           # Number of steps in the architecture\n    'gamma': 1.5,           # Coefficient for feature selection regularization\n    'n_independent': 2,     # Number of independent GLU layer in each GLU block\n    'n_shared': 2,          # Number of shared GLU layer in each GLU block\n    'lambda_sparse': 1e-4,  # Sparsity regularization\n    'optimizer_fn': torch.optim.Adam,\n    'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n    'mask_type': 'entmax',\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'verbose': 1,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n    def __init__(self, filepath, monitor='val_loss', mode='min', save_best_only=True, verbose=1):\n        super().__init__()\n        self.filepath = filepath\n        self.monitor = monitor\n        self.mode = mode\n        self.save_best_only = save_best_only\n        self.verbose = verbose\n        self.best = float('inf') if mode == 'min' else -float('inf')\n        \n    def on_train_begin(self, logs=None):\n        self.model = self.trainer\n        \n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is None:\n            return\n        \n        if (self.mode == 'min' and current < self.best) or (self.mode == 'max' and current > self.best):\n            if self.verbose:\n                print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n            self.best = current\n            if self.save_best_only:\n                self.model.save_model(self.filepath)","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:55:52.879941Z","iopub.execute_input":"2024-11-11T05:55:52.88025Z","iopub.status.idle":"2024-11-11T05:55:52.899231Z","shell.execute_reply.started":"2024-11-11T05:55:52.880218Z","shell.execute_reply":"2024-11-11T05:55:52.898337Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Params = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01,  # Increased from 2.68e-06\n    'device': 'cpu',\n}\n\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED,\n    'tree_method': 'gpu_hist'\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU',\n}\n\n# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\nTabNet_Model = TabNetWrapper(**TabNet_Params)\n#ODT_Model = ObliqueDecisionTreeRegressor(**ODT_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model),\n    ('tabnet', TabNet_Model),\n    #('odt', ODT_Model),\n], weights=[4.0,4.0,5.0,4.0])","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:55:52.935331Z","iopub.execute_input":"2024-11-11T05:55:52.935638Z","iopub.status.idle":"2024-11-11T05:55:52.946221Z","shell.execute_reply.started":"2024-11-11T05:55:52.935606Z","shell.execute_reply":"2024-11-11T05:55:52.945312Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission1 = TrainML(voting_model, test)\n\n# Save submission\n# Submission1.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:55:52.947359Z","iopub.execute_input":"2024-11-11T05:55:52.947667Z","iopub.status.idle":"2024-11-11T05:57:31.234041Z","shell.execute_reply.started":"2024-11-11T05:55:52.947621Z","shell.execute_reply":"2024-11-11T05:57:31.232987Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission1","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:57:31.235545Z","iopub.execute_input":"2024-11-11T05:57:31.235888Z","iopub.status.idle":"2024-11-11T05:57:31.246904Z","shell.execute_reply.started":"2024-11-11T05:57:31.23585Z","shell.execute_reply":"2024-11-11T05:57:31.245884Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ensemble = VotingRegressor(estimators=[\n#     ('lightgbm', Light),\n#     ('xgboost', XGB_Model),\n#     ('catboost', CatBoost_Model),\n#     # ('lgb',    Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(random_state=SEED))])),\n#     # ('xgb',    Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(random_state=SEED))])),\n#     # ('cat',    Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(random_state=SEED, silent=True))])),\n#     ('rf',     Pipeline(steps=[('imputer', imputer), ('regressor', RandomForestRegressor(random_state=SEED))])),\n#     ('gb',     Pipeline(steps=[('imputer', imputer), ('regressor', GradientBoostingRegressor(random_state=SEED))])),\n#     #('tabnet', Pipeline(steps=[('imputer', imputer), ('regressor', TabNetWrapper(**TabNet_Params))])),\n#     #('odt',    Pipeline(steps=[('imputer', imputer), ('regressor', ObliqueDecisionTreeRegressor(**ODT_Params))])),\n# ])\n\n\n# Submission3 = TrainML(ensemble, test)\n\n# # Submission3 = pd.DataFrame({\n# #     'id': sample['id'],\n# #     'sii': Submission3\n# # })\n\n# Submission3","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n        \ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\n# # mới thêm\n# train = feature_engineering(train)\n# # train = train.dropna(thresh=10, axis=0)\n# test = feature_engineering(test)\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)   \n# \nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        \ntrain = update(train)\ntest = update(test)\n\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n\n# train['Log_BMI'] = np.log(train['Physical-BMI'] + 1)\n# # train['Log_Weight'] = np.log(train['Physical-Weight'] + 1)\n# train['Log_Waist'] = np.log(train['Physical-Waist_Circumference'] + 1)\n\n# test['Log_BMI'] = np.log(test['Physical-BMI'] + 1)\n# # test['Log_Weight'] = np.log(test['Physical-Weight'] + 1)\n# test['Log_Waist'] = np.log(test['Physical-Waist_Circumference'] + 1)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission\n\n# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01  # Increased from 2.68e-06\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'cat_features': cat_c,\n    'verbose': 0,\n    'l2_leaf_reg': 10  # Increase this value\n}\n\n# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n# TabNet_Model = TabNetWrapper(**TabNet_Params)\n#ODT_Model = ObliqueDecisionTreeRegressor(**ODT_Params)\n\n# # Điền giá trị thiếu với -99999\n# train = train.fillna(-99999)\n# test = test.fillna(-99999)\n\n# # Với XGBoost\n# XGB_Model = XGBRegressor(missing=-99999, **XGB_Params)\n\n# # Với LightGBM\n# Light = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\n\n# # Với CatBoost (không cần xử lý NaN thủ công, CatBoost tự động nhận diện)\n# CatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\n\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model),\n    #('tabnet', TabNet_Model),\n    #('odt', ODT_Model),\n])\n\n# Train the ensemble model\nSubmission2 = TrainML(voting_model, test)","metadata":{"execution":{"iopub.status.busy":"2024-11-11T05:57:31.248588Z","iopub.execute_input":"2024-11-11T05:57:31.2489Z","iopub.status.idle":"2024-11-11T05:59:04.766276Z","shell.execute_reply.started":"2024-11-11T05:57:31.248865Z","shell.execute_reply":"2024-11-11T05:59:04.764783Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-11T05:59:04.767213Z","iopub.status.idle":"2024-11-11T05:59:04.767619Z","shell.execute_reply.started":"2024-11-11T05:59:04.767428Z","shell.execute_reply":"2024-11-11T05:59:04.767453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n\ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n\n# train['Log_BMI'] = np.log(train['Physical-BMI'] + 1)\n# # train['Log_Weight'] = np.log(train['Physical-Weight'] + 1)\n# train['Log_Waist'] = np.log(train['Physical-Waist_Circumference'] + 1)\n\n# test['Log_BMI'] = np.log(test['Physical-BMI'] + 1)\n# # test['Log_Weight'] = np.log(test['Physical-Weight'] + 1)\n# test['Log_Waist'] = np.log(test['Physical-Waist_Circumference'] + 1)\n\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tp_rounded = threshold_Rounder(tpm, KappaOPtimizer.x)\n\n    return tp_rounded\n\nimputer = SimpleImputer(strategy='median')\n\n# imputer = KNNImputer(n_neighbors=5)\n# numeric_cols = train.select_dtypes(include=['int32', 'int64', 'float64', 'int64']).columns\n# imputed_data = imputer.fit_transform(train[numeric_cols])\n# train_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\n# train_imputed['sii'] = train_imputed['sii'].round().astype(int)\n# for col in train.columns:\n#     if col not in numeric_cols:\n#         train_imputed[col] = train[col]\n        \n# train = train_imputed\n\nensemble = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model),\n    ('lgb',    Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(random_state=SEED))])),\n    ('xgb',    Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(random_state=SEED))])),\n    ('cat',    Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(random_state=SEED, silent=True))])),\n    ('rf',     Pipeline(steps=[('imputer', imputer), ('regressor', RandomForestRegressor(random_state=SEED))])),\n    ('gb',     Pipeline(steps=[('imputer', imputer), ('regressor', GradientBoostingRegressor(random_state=SEED))])),\n    #('tabnet', Pipeline(steps=[('imputer', imputer), ('regressor', TabNetWrapper(**TabNet_Params))])),\n    #('odt',    Pipeline(steps=[('imputer', imputer), ('regressor', ObliqueDecisionTreeRegressor(**ODT_Params))])),\n])\n\n\nSubmission3 = TrainML(ensemble, test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-11T05:59:04.769156Z","iopub.status.idle":"2024-11-11T05:59:04.769503Z","shell.execute_reply.started":"2024-11-11T05:59:04.769332Z","shell.execute_reply":"2024-11-11T05:59:04.76935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission3 = pd.DataFrame({\n    'id': sample['id'],\n    'sii': Submission3\n})\n\nSubmission3","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-11T05:59:04.770577Z","iopub.status.idle":"2024-11-11T05:59:04.770908Z","shell.execute_reply.started":"2024-11-11T05:59:04.770736Z","shell.execute_reply":"2024-11-11T05:59:04.770753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import pandas as pd\n\n# # Giả định các DataFrame sub1, sub2, sub3 đã được nạp\n# # Dữ liệu đã được cung cấp, chỉ cần thay vào.\n\n# # Sắp xếp và đặt lại chỉ mục\n# sub1 = sub1.sort_values(by='id').reset_index(drop=True)\n# sub2 = sub2.sort_values(by='id').reset_index(drop=True)\n# sub3 = sub3.sort_values(by='id').reset_index(drop=True)\n\n# # Kết hợp các dữ liệu thành một DataFrame\n# combined = pd.DataFrame({\n#     'id': sub1['id'],\n#     'sii_1': sub1['sii'],\n#     'sii_2': sub2['sii'],\n#     'sii_3': sub3['sii']\n# })\n\n# # Hàm lấy giá trị theo số đông (majority vote)\n# def majority_vote(row):\n#     # Lấy mode của các giá trị trong hàng\n#     mode = row.mode()\n#     # Nếu có nhiều giá trị mode (trường hợp hòa), chọn giá trị đầu tiên\n#     return mode.iloc[0] if not mode.empty else None\n\n# # Áp dụng majority vote cho các cột `sii_1`, `sii_2`, `sii_3`\n# combined['final_sii'] = combined[['sii_1', 'sii_2', 'sii_3']].apply(majority_vote, axis=1)\n\n# # Tạo DataFrame cuối cùng\n# final_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\n# # Lưu ra file CSV\n# final_submission.to_csv('submission.csv', index=False)\n\n# print(\"Majority voting completed and saved to 'submission.csv'\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub1 = Submission1\nsub2 = Submission2\nsub3 = Submission3\n\nsub1 = sub1.sort_values(by='id').reset_index(drop=True)\nsub2 = sub2.sort_values(by='id').reset_index(drop=True)\nsub3 = sub3.sort_values(by='id').reset_index(drop=True)\n\ncombined = pd.DataFrame({\n    'id': sub1['id'],\n    'sii_1': sub1['sii'],\n    'sii_2': sub2['sii'],\n    'sii_3': sub3['sii']\n})\n\n# def majority_vote(row):\n#     return row.mode()[0]\n\ndef majority_vote(row):\n    # Nếu không, lấy majority vote từ submission 1, 2, 3\n    mode = row[['sii_1', 'sii_2', 'sii_3']].mode()\n    if len(mode) == 1:  # Nếu có một mode duy nhất\n        return mode[0]\n    else:\n        # Nếu không có giá trị mode duy nhất, lấy giá trị trung bình làm tròn\n        return round(row[['sii_1', 'sii_2', 'sii_3']].mean())\n        \n\ncombined['final_sii'] = combined[['sii_1', 'sii_2', 'sii_3']].apply(majority_vote, axis=1)\n\nfinal_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\nfinal_submission.to_csv('submission.csv', index=False)\n\nprint(\"Majority voting completed and saved to 'Final_Submission.csv'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-11T05:59:04.772824Z","iopub.status.idle":"2024-11-11T05:59:04.773347Z","shell.execute_reply.started":"2024-11-11T05:59:04.773056Z","shell.execute_reply":"2024-11-11T05:59:04.773082Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-11T05:59:04.7747Z","iopub.status.idle":"2024-11-11T05:59:04.775191Z","shell.execute_reply.started":"2024-11-11T05:59:04.774921Z","shell.execute_reply":"2024-11-11T05:59:04.774946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}