{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":10238317,"sourceType":"datasetVersion","datasetId":6331304},{"sourceId":10263899,"sourceType":"datasetVersion","datasetId":6349641}],"dockerImageVersionId":30805,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip -q install /kaggle/input/pytorchtabnet/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"_uuid":"fca6d6b7-ae6b-4d3b-a179-aa9bf0281365","_cell_guid":"093a3bda-6368-4791-ab0e-fb86939aff46","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-21T08:23:19.536544Z","iopub.execute_input":"2024-12-21T08:23:19.536862Z","iopub.status.idle":"2024-12-21T08:23:22.825312Z","shell.execute_reply.started":"2024-12-21T08:23:19.536829Z","shell.execute_reply":"2024-12-21T08:23:22.824297Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Các thư viện cần thiết\nimport os\nimport re\nimport copy\nimport pickle\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.base import clone, BaseEstimator, RegressorMixin\nfrom sklearn.svm import SVR\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import ElasticNet, Ridge\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor\nfrom sklearn.ensemble import BaggingRegressor, GradientBoostingRegressor\nfrom sklearn.ensemble import AdaBoostRegressor, ExtraTreesRegressor, HistGradientBoostingRegressor\n\n\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam, AdamW\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom pytorch_tabnet.callbacks import Callback\n\nimport warnings\nimport seaborn as sns\nimport polars as pl\nimport polars.selectors as cs\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\n\nfrom tqdm import tqdm\nimport plotly.express as px\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\n\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom pytorch_tabnet.tab_model import TabNetRegressor\n\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None","metadata":{"_uuid":"3e19eeb0-3dac-4ea8-9da4-e4289240c369","_cell_guid":"cc65a69c-e36d-49dd-8852-d201ce2083ce","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-21T08:23:19.52758Z","iopub.execute_input":"2024-12-21T08:23:19.527911Z","iopub.status.idle":"2024-12-21T08:23:19.535257Z","shell.execute_reply.started":"2024-12-21T08:23:19.527882Z","shell.execute_reply":"2024-12-21T08:23:19.534197Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pytorch_tabnet.tab_model import TabNetRegressor\nimport torch","metadata":{"_uuid":"f0c29ecb-7909-46ed-a622-c973e8c2e513","_cell_guid":"354ddc99-51f7-4637-a935-3e6703eb9904","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-21T08:23:22.827452Z","iopub.execute_input":"2024-12-21T08:23:22.827782Z","iopub.status.idle":"2024-12-21T08:23:22.83168Z","shell.execute_reply.started":"2024-12-21T08:23:22.827752Z","shell.execute_reply":"2024-12-21T08:23:22.830889Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\nseed_everything(2024)","metadata":{"_uuid":"086b9b16-3284-4e8f-8e3b-d90a861ef8b8","_cell_guid":"709f2b98-a581-4e66-a43d-912628d205cd","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-21T08:23:22.832838Z","iopub.execute_input":"2024-12-21T08:23:22.833136Z","iopub.status.idle":"2024-12-21T08:23:22.852573Z","shell.execute_reply.started":"2024-12-21T08:23:22.833115Z","shell.execute_reply":"2024-12-21T08:23:22.851641Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42\nn_splits = 5","metadata":{"_uuid":"6cac5514-8687-445d-87c8-4a4582a6109e","_cell_guid":"dc2d0441-4843-467a-aa8e-39de008be6cc","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-21T08:23:22.853587Z","iopub.execute_input":"2024-12-21T08:23:22.853886Z","iopub.status.idle":"2024-12-21T08:23:22.867423Z","shell.execute_reply.started":"2024-12-21T08:23:22.853855Z","shell.execute_reply":"2024-12-21T08:23:22.866683Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"9b2d60be-494a-47df-8d41-d4ddd0b26adc","_cell_guid":"9ad9609d-83fc-4fa9-8305-453b77425dd4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n\n\nclass AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Dropout(p=0.1), \n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n        \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    \n    criterion = nn.MSELoss()\n    optimizer = optim.AdamW(autoencoder.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n                 \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded","metadata":{"_uuid":"fb7ddbab-56d5-4448-8c4c-21fbade08601","_cell_guid":"571a3bba-9b64-4c38-bc7c-664dcd79e1b9","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-21T08:23:22.868191Z","iopub.execute_input":"2024-12-21T08:23:22.868407Z","iopub.status.idle":"2024-12-21T08:23:22.880753Z","shell.execute_reply.started":"2024-12-21T08:23:22.868388Z","shell.execute_reply":"2024-12-21T08:23:22.879733Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"_uuid":"935bc055-1121-4d80-bcb7-9bf495ddac5a","_cell_guid":"13ef668e-7d41-4d6f-9e2e-f99bec73a006","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-21T08:23:22.883049Z","iopub.execute_input":"2024-12-21T08:23:22.883256Z","iopub.status.idle":"2024-12-21T08:23:22.939596Z","shell.execute_reply.started":"2024-12-21T08:23:22.883237Z","shell.execute_reply":"2024-12-21T08:23:22.93895Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_data(train):\n    #Thay thế giá trị 999 trong cột CGAS-CGAS_Score thành NaN.\n    train.loc[train['CGAS-CGAS_Score'] > 100, 'CGAS-CGAS_Score'] = np.nan\n    \n    wh_cols = [\n        'Physical-BMI', 'Physical-Height',\n        'Physical-Weight', 'Physical-Waist_Circumference'\n    ]\n    # Thay thế các giá trị 0 trong các cột chiều cao, cân nặng và vòng eo thành NaN.\n    train[wh_cols] = train[wh_cols].replace(0, np.nan)\n    \n    bp_hr_cols = [\n        'Physical-Diastolic_BP', 'Physical-Systolic_BP',\n        'Physical-HeartRate'\n    ]\n    # Thay thế các giá trị <= 0 trong các cột huyết áp, nhịp tim thành NaN\n    train.loc[(train[bp_hr_cols] <= 0).any(axis=1), bp_hr_cols] = np.nan\n    # Đặt NaN khi 'Systolic_BP' <= 'Diastolic_BP'\n    train.loc[train['Physical-Systolic_BP'] <= train['Physical-Diastolic_BP'], bp_hr_cols] = np.nan\n\n    BIA_cols = ['BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', \n                 'BIA-BIA_ECW', 'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', \n                 'BIA-BIA_Fat', 'BIA-BIA_Frame_num', 'BIA-BIA_ICW', 'BIA-BIA_LDM', \n                 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW']\n    # Thay thế các giá trị <= 0 trong các cột BIA thành NaN\n    train.loc[(train[BIA_cols] <= 0).any(axis=1), BIA_cols] = np.nan\n\n    return train","metadata":{"_uuid":"67c53ec9-d842-49fa-a7ef-0fd1ecb90d2a","_cell_guid":"ed630b15-4c34-40a2-903b-e71be651cbd6","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-21T08:23:22.941371Z","iopub.execute_input":"2024-12-21T08:23:22.941616Z","iopub.status.idle":"2024-12-21T08:23:22.946962Z","shell.execute_reply.started":"2024-12-21T08:23:22.941594Z","shell.execute_reply":"2024-12-21T08:23:22.94625Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = clean_data(train)\ntest = clean_data(test)","metadata":{"_uuid":"3f92c4a8-8f35-4f5c-b6da-02ba719b3fdb","_cell_guid":"4b04f467-d50d-4dbc-bd25-231fff72a6a0","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-21T08:23:22.947668Z","iopub.execute_input":"2024-12-21T08:23:22.948022Z","iopub.status.idle":"2024-12-21T08:23:22.984959Z","shell.execute_reply.started":"2024-12-21T08:23:22.947976Z","shell.execute_reply":"2024-12-21T08:23:22.984383Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")","metadata":{"_uuid":"e26df0db-9eb4-4ea8-8a91-eb9195897d8a","_cell_guid":"deea7aa4-af3a-4c0a-a6f7-f3f9effba5a1","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-21T08:23:22.985632Z","iopub.execute_input":"2024-12-21T08:23:22.985845Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"time_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\ndisplay(test.head())","metadata":{"_uuid":"18e32122-c61a-4c9d-ba70-7e9c1482e38b","_cell_guid":"0412c6fe-5b5a-427a-8234-f1be1298fc76","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Fitness_Endurance-Total_Time_Sec'] = train['Fitness_Endurance-Time_Mins'] * 60 + train['Fitness_Endurance-Time_Sec']\n\ntest['Fitness_Endurance-Total_Time_Sec'] = test['Fitness_Endurance-Time_Mins'] * 60 + test['Fitness_Endurance-Time_Sec']\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Total_Time_Sec', 'FGC-Season',\n                'FGC-FGC_CU', 'FGC-FGC_GSND',\n                'FGC-FGC_GSD', 'FGC-FGC_PU',\n                'FGC-FGC_SRL', 'FGC-FGC_SRR',\n                'FGC-FGC_TL', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Total_Time_Sec', 'FGC-Season',\n                'FGC-FGC_CU', 'FGC-FGC_GSND',\n                'FGC-FGC_GSD', 'FGC-FGC_PU',\n                'FGC-FGC_SRL', 'FGC-FGC_SRR',\n                'FGC-FGC_TL', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday']\n\nfeaturesCols += time_series_cols\ntest = test[featuresCols]\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        \ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\n# def evaluate_predictions(thresholds, y_true, oof_non_rounded):\n#     rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n#     return -quadratic_weighted_kappa(y_true, rounded_p)\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded, reg_lambda=0):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    kappa_score = quadratic_weighted_kappa(y_true, rounded_p)\n    \n    # Thêm regularization term (làm giảm sự thay đổi lớn của ngưỡng)\n    reg_term = reg_lambda * np.sum(np.diff(thresholds)**2)\n    \n    return -(kappa_score - reg_term)\n\ndef TrainML(model_class, test_data, sample):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission","metadata":{"_uuid":"525530de-35ff-4c53-9c69-9c77b204d878","_cell_guid":"f66366c6-ee63-4fd6-a7db-9e83e6d2da50","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.09224259399998393,\n    'max_depth': 4,\n    'num_leaves': 193,\n    'min_data_in_leaf': 28,\n    'feature_fraction': 0.7514556531262206,\n    'bagging_fraction': 0.9255846697658219,\n    'bagging_freq': 4,\n    'lambda_l1': 15.86587835516145,\n    'lambda_l2': 3.492177649094307\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.07977342261117466,\n    'max_depth': 3,\n    'n_estimators': 461,\n    'subsample': 0.9370654528316807,\n    'colsample_bytree': 0.5790844655067284,\n    'reg_alpha': 1,\n    'reg_lambda': 5.640597131481835,\n    'random_state': SEED\n}\n\nCatBoost_Params = {\n    'learning_rate': 0.0763882332924562,\n    'depth': 5,\n    'iterations': 200,\n    'random_seed': SEED,\n    'cat_features': cat_c,\n    'verbose': 0,\n    'l2_leaf_reg': 10  # Increase this value\n}\n\n# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\n# Combine models using Voting Regressor","metadata":{"_uuid":"edd8da23-bf9c-4d58-9402-a35d822b0ce9","_cell_guid":"c97699d1-46ea-415e-85b1-5be64439c5e1","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"voting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)\n])\n\n# Train the ensemble model\nSubmission2 = TrainML(voting_model, test, sample)\n\nSubmission2","metadata":{"_uuid":"ea5235a4-cf7a-47ef-b566-14e1d5773e1b","_cell_guid":"77a75253-e04c-4199-8bf8-2b15d4b9c6a9","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"f6fe0ff4-8f89-4bce-be4c-a982a8963b8b","_cell_guid":"6ae923ba-917e-4548-bca1-2ad2cac9d8e3","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntrain = clean_data(train)\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\n\n# Lưu lại id trước khi xử lý\ntrain_ids = train['id'].copy()\ntrain = train.drop('id', axis=1)\n\n# Tính toán tổng thời gian\ntrain['Fitness_Endurance-Total_Time_Sec'] = train['Fitness_Endurance-Time_Mins'] * 60 + train['Fitness_Endurance-Time_Sec']\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference', \n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Total_Time_Sec', 'FGC-Season',\n                'FGC-FGC_CU', 'FGC-FGC_GSND',\n                'FGC-FGC_GSD', 'FGC-FGC_PU', \n                'FGC-FGC_SRL', 'FGC-FGC_SRR',\n                'FGC-FGC_TL', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\n\n# Tạo DataFrame id riêng cho tập thiếu sii\nmissing_sii_ids = pd.DataFrame({'id': train_ids[train['sii'].isna()]})\n\ntrain_data = train.dropna(subset='sii')\nsii_imputed_data = train[train['sii'].isna()].drop(columns=['sii'])\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n         'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n         'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n    global cat_c\n    for c in cat_c:\n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n\ntrain_data = update(train_data)\nsii_imputed_data = update(sii_imputed_data)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train_data)\n    mappingTe = create_mapping(col, sii_imputed_data)\n    \n    train_data[col] = train_data[col].replace(mapping).astype(int)\n    sii_imputed_data[col] = sii_imputed_data[col].replace(mappingTe).astype(int)\n\ndef TrainML(model_class, test_data):\n    X = train_data.drop(['sii'], axis=1)\n    y = train_data['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n        \n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    submission = pd.DataFrame({\n        'id': missing_sii_ids['id'],\n        'sii': tpTuned\n    })\n    \n    return submission\n\n# Sau đó có thể chạy model như bình thường\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)\n])\n\nsii_imputed = TrainML(voting_model, sii_imputed_data)","metadata":{"_uuid":"29c13db8-5af1-4d1f-8dd4-7526d1065a93","_cell_guid":"97cdf8cd-deee-4b2e-8a54-4aa2f91595a6","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"2f91438c-9bbc-46c8-b638-191a9dc32336","_cell_guid":"9d79fc17-6d3d-4b8b-9f4b-629e6bdbc97c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"_uuid":"e6a464ca-5392-4a5e-92c2-f9847ebee4a6","_cell_guid":"d275beb7-0a03-478a-ae29-12da506efb26","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = clean_data(train)\ntest = clean_data(test)\n\nsii_imputed.set_index('id', inplace=True)\ntrain.set_index('id', inplace=True)\n\ntrain.loc[sii_imputed.index, 'sii'] = sii_imputed['sii']","metadata":{"_uuid":"246bbd08-0994-4a38-a32a-adf0e3acd480","_cell_guid":"eb0a2115-7ed2-4210-8e07-855b09f016f7","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = train_ts.drop(columns='id')\ndf_test = test_ts.drop(columns='id')\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=50, epochs=100, batch_size=32)\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=50, epochs=100, batch_size=32)","metadata":{"_uuid":"eed8173c-d1c9-4c35-9794-7a7df77d4212","_cell_guid":"579c37be-2618-44a6-b397-16dce8e046c1","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"6069835a-9095-47df-b534-42c3c4e4e6ad","_cell_guid":"13dffb55-4f12-4185-b08e-1bd69262413c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"time_series_cols = train_ts_encoded.columns.tolist()\ntrain_ts_encoded[\"id\"]=train_ts[\"id\"]\ntest_ts_encoded['id']=test_ts[\"id\"]\ndisplay(train_ts.head())","metadata":{"_uuid":"b9fe5f18-0041-41a0-9f0d-e68e5e325df8","_cell_guid":"813d9552-ab5c-4d01-b5c3-7dffbb6d841b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_engineering(df):\n    # Tạo bản sao để tránh thay đổi dữ liệu gốc\n    df = df.copy()\n    \n    # Thêm epsilon để tránh chia cho 0\n    eps = 1e-7\n    \n    try:\n        # Basic features\n        df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n        df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n        df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n        \n        # BIA related features\n        if 'BIA-BIA_Fat' in df.columns and 'BIA-BIA_BMI' in df.columns:\n            df['BFP_BMI'] = df['BIA-BIA_Fat'] / (df['BIA-BIA_BMI'] + eps)\n        \n        if 'BIA-BIA_FFMI' in df.columns and 'BIA-BIA_Fat' in df.columns:\n            df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / (df['BIA-BIA_Fat'] + eps)\n        \n        if 'BIA-BIA_FMI' in df.columns and 'BIA-BIA_Fat' in df.columns:\n            df['FMI_BFP'] = df['BIA-BIA_FMI'] / (df['BIA-BIA_Fat'] + eps)\n        \n        if 'BIA-BIA_LST' in df.columns and 'BIA-BIA_TBW' in df.columns:\n            df['LST_TBW'] = df['BIA-BIA_LST'] / (df['BIA-BIA_TBW'] + eps)\n        \n        if 'BIA-BIA_Fat' in df.columns and 'BIA-BIA_BMR' in df.columns:\n            df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n        \n        if 'BIA-BIA_Fat' in df.columns and 'BIA-BIA_DEE' in df.columns:\n            df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n        \n        if 'BIA-BIA_BMR' in df.columns and 'Physical-Weight' in df.columns:\n            df['BMR_Weight'] = df['BIA-BIA_BMR'] / (df['Physical-Weight'] + eps)\n        \n        if 'BIA-BIA_DEE' in df.columns and 'Physical-Weight' in df.columns:\n            df['DEE_Weight'] = df['BIA-BIA_DEE'] / (df['Physical-Weight'] + eps)\n        \n        if 'BIA-BIA_SMM' in df.columns and 'Physical-Height' in df.columns:\n            df['SMM_Height'] = df['BIA-BIA_SMM'] / (df['Physical-Height'] + eps)\n        \n        if 'BIA-BIA_SMM' in df.columns and 'BIA-BIA_FMI' in df.columns:\n            df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / (df['BIA-BIA_FMI'] + eps)\n        \n        if 'BIA-BIA_TBW' in df.columns and 'Physical-Weight' in df.columns:\n            df['Hydration_Status'] = df['BIA-BIA_TBW'] / (df['Physical-Weight'] + eps)\n        \n        if 'BIA-BIA_ICW' in df.columns and 'BIA-BIA_TBW' in df.columns:\n            df['ICW_TBW'] = df['BIA-BIA_ICW'] / (df['BIA-BIA_TBW'] + eps)\n        \n        # BMI and Heart Rate\n        if 'Physical-BMI' in df.columns and 'Physical-HeartRate' in df.columns:\n            df['BMI_PHR'] = df['Physical-BMI'] * df['Physical-HeartRate']\n        \n        # Internet usage related features\n        hours_day = df['PreInt_EduHx-computerinternet_hoursday'].fillna(0) + 1\n        \n        if 'SDS-SDS_Total_T' in df.columns:\n            df['Internet_Hours_SDS'] = hours_day * df['SDS-SDS_Total_T']\n        \n        if 'PAQ_C-PAQ_C_Total' in df.columns:\n            df['Internet_Hours_PAQ'] = hours_day * df['PAQ_C-PAQ_C_Total']\n            df['Internet_to_Activity_Ratio'] = hours_day / (df['PAQ_C-PAQ_C_Total'] + eps)\n        \n        # Heart Rate related features\n        if 'Physical-HeartRate' in df.columns:\n            df['HeartRate_Age_Ratio'] = df['Physical-HeartRate'] / (df['Basic_Demos-Age'] + eps)\n            \n            if 'SDS-SDS_Total_T' in df.columns:\n                df['HeartRate_SDS'] = df['Physical-HeartRate'] * df['SDS-SDS_Total_T']\n                df['SDS_HeartRate_Ratio'] = df['SDS-SDS_Total_T'] / (df['Physical-HeartRate'] + eps)\n        \n        # SDS related features\n        if 'SDS-SDS_Total_T' in df.columns:\n            if 'CGAS-CGAS_Score' in df.columns:\n                df['SDS_CGAS_Ratio'] = df['SDS-SDS_Total_T'] / (df['CGAS-CGAS_Score'] + eps)\n            \n            if 'PAQ_C-PAQ_C_Total' in df.columns:\n                df['SDS_PAQ_Interaction'] = df['SDS-SDS_Total_T'] * df['PAQ_C-PAQ_C_Total']\n            \n            if 'Physical-Weight' in df.columns:\n                df['SDS_Weight_Interaction'] = df['SDS-SDS_Total_T'] * df['Physical-Weight']\n            \n            if 'Physical-Waist_Circumference' in df.columns:\n                df['SDS_Waist_Interaction'] = df['SDS-SDS_Total_T'] * df['Physical-Waist_Circumference']\n        \n        # Internet and Heart Rate ratio\n        if 'PreInt_EduHx-computerinternet_hoursday' in df.columns and 'Physical-HeartRate' in df.columns:\n            df['Internet_Hours_HeartRate_Ratio'] = hours_day / (df['Physical-HeartRate'] + eps)\n        \n        # CGAS and PAQ interaction\n        if 'CGAS-CGAS_Score' in df.columns and 'PAQ_C-PAQ_C_Total' in df.columns:\n            df['CGAS_PAQ_Interaction'] = df['CGAS-CGAS_Score'] * df['PAQ_C-PAQ_C_Total']\n\n        df['Fitness_Endurance-Total_Time_Sec'] = df['Fitness_Endurance-Time_Mins'] * 60 + df['Fitness_Endurance-Time_Sec']\n        \n        # Replace infinite values with NaN\n        df = df.replace([np.inf, -np.inf], np.nan)\n        \n    except Exception as e:\n        print(f\"Error in feature engineering: {str(e)}\")\n        # Return original dataframe if error occurs\n        return df\n    \n    return df","metadata":{"_uuid":"919cc6e3-6467-43e0-9124-9d65948fa88e","_cell_guid":"0019775d-6967-497a-ab60-67b913af17b8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#time_series_cols = train_ts.columns.tolist()\ntrain = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\ntest = pd.merge(test, test_ts_encoded, how=\"left\", on='id')\n\n\ntrain = train.drop('id', axis=1)\ntest  = test .drop('id', axis=1)","metadata":{"_uuid":"8e06090b-a5ce-400f-9ce4-2366cb7bd426","_cell_guid":"e5aa1465-753a-4a76-8224-04d60e5cff96","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" \n\n\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_GSND',\n                'FGC-FGC_GSD', 'FGC-FGC_PU',\n                'FGC-FGC_SRL', 'FGC-FGC_SRR',\n                'FGC-FGC_TL',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii'\n               ]\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_GSND',\n                'FGC-FGC_GSD', 'FGC-FGC_PU',\n                'FGC-FGC_SRL', 'FGC-FGC_SRR',\n                'FGC-FGC_TL',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday'\n               ]\n\nfeaturesCols += time_series_cols\n\ntest = test[featuresCols]","metadata":{"_uuid":"fe677068-83f2-4a31-a428-c2300b4035dd","_cell_guid":"9b8cc1a0-72a2-46e9-aa8b-3ac30e97ea67","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if np.any(np.isinf(train)):\n    train = train.replace([np.inf, -np.inf], np.nan)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\n# def evaluate_predictions(thresholds, y_true, oof_non_rounded):\n#     rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n#     return -quadratic_weighted_kappa(y_true, rounded_p)\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded, reg_lambda=0):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    kappa_score = quadratic_weighted_kappa(y_true, rounded_p)\n    \n    # Thêm regularization term (làm giảm sự thay đổi lớn của ngưỡng)\n    reg_term = reg_lambda * np.sum(np.diff(thresholds)**2)\n    \n    return -(kappa_score - reg_term)","metadata":{"_uuid":"4e5a0f3a-f9f3-4cf9-9f6e-e439e1c77eab","_cell_guid":"d276371f-0412-4883-8d41-0b17974d7b66","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    feature_names = X.columns\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n        # Thêm vào trước khi fit model\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n        \n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0, 1, 2], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission","metadata":{"_uuid":"663409e3-eac8-4b66-beb8-1576b31eb9e2","_cell_guid":"86ce5808-9dfd-4428-80cc-b4aeb0b61b24","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# New: TabNet\n\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom pytorch_tabnet.callbacks import Callback\nimport os\nimport torch\nfrom pytorch_tabnet.callbacks import Callback\n\nclass TabNetWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, **kwargs):\n        self.model = TabNetRegressor(**kwargs)\n        self.kwargs = kwargs\n        self.imputer = SimpleImputer(strategy='median')\n        self.best_model_path = 'best_tabnet_model.pt'\n        \n    def fit(self, X, y):\n        # Handle missing values\n        X = X.values if isinstance(X, (pd.DataFrame, pd.Series)) else X\n        y = y.values if isinstance(y, (pd.DataFrame, pd.Series)) else y\n        X_imputed = self.imputer.fit_transform(X)\n        \n        if hasattr(y, 'values'):\n            y = y.values\n            \n        # Create internal validation set\n        X_train, X_valid, y_train, y_valid = train_test_split(\n            X_imputed, \n            y, \n            test_size=0.2,\n            random_state=42\n        )\n        \n        # Train TabNet model\n        history = self.model.fit(\n            X_train=X_train,\n            y_train=y_train.reshape(-1, 1),\n            eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n            eval_name=['valid'],\n            eval_metric=['mse'],\n            max_epochs=200,\n            patience=20,\n            batch_size=1024,\n            virtual_batch_size=128,\n            num_workers=0,\n            drop_last=False,\n            callbacks=[\n                TabNetPretrainedModelCheckpoint(\n                    filepath=self.best_model_path,\n                    monitor='valid_mse',\n                    mode='min',\n                    save_best_only=True,\n                    verbose=True\n                )\n            ]\n        )\n        \n        # Load the best model\n        if os.path.exists(self.best_model_path):\n            self.model.load_model(self.best_model_path)\n            os.remove(self.best_model_path)  # Remove temporary file\n\n        self.feature_importances_ = self.model.feature_importances_\n        \n        return self\n    \n    def predict(self, X):\n        X_imputed = self.imputer.transform(X)\n        return self.model.predict(X_imputed).flatten()\n    \n    def __deepcopy__(self, memo):\n        # Add deepcopy support for scikit-learn\n        cls = self.__class__\n        result = cls.__new__(cls)\n        memo[id(self)] = result\n        for k, v in self.__dict__.items():\n            setattr(result, k, deepcopy(v, memo))\n        return result\n\n# TabNet hyperparameters\nTabNet_Params = {\n    'n_d': 114,              # Width of the decision prediction layer\n    'n_a': 88,               # Width of the attention embedding for each step\n    'n_steps': 5,            # Number of steps in the architecture\n    'gamma': 1.8169342156636743,  # Coefficient for feature selection regularization\n    'n_independent': 2,      # Number of independent GLU layers in each GLU block\n    'n_shared': 2,           # Number of shared GLU layers in each GLU block\n    'lambda_sparse': 0.0005341374006791508,  # Sparsity regularization\n    'optimizer_fn': torch.optim.AdamW,\n    'optimizer_params': dict(lr=0.056686291040246195, weight_decay=1e-5),\n    'mask_type': 'entmax',   # Mask type as determined by the study\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'verbose': 1,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n    def __init__(self, filepath, monitor='val_loss', mode='min', \n                 save_best_only=True, verbose=1):\n        super().__init__()  # Initialize parent class\n        self.filepath = filepath\n        self.monitor = monitor\n        self.mode = mode\n        self.save_best_only = save_best_only\n        self.verbose = verbose\n        self.best = float('inf') if mode == 'min' else -float('inf')\n        \n    def on_train_begin(self, logs=None):\n        self.model = self.trainer  # Use trainer itself as model\n        \n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is None:\n            return\n        \n        # Check if current metric is better than best\n        if (self.mode == 'min' and current < self.best) or \\\n           (self.mode == 'max' and current > self.best):\n            if self.verbose:\n                print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n            self.best = current\n            if self.save_best_only:\n                self.model.save_model(self.filepath)  # Save the entire model","metadata":{"_uuid":"875d7e68-e72e-423d-967b-5d65eb3952c9","_cell_guid":"0c077257-eafe-41af-8cfc-ed39133b46eb","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 18,\n    'lambda_l2': 0.015,\n    'device': 'cpu'\n\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 6,\n    'reg_lambda': 5,\n    'random_state': SEED,\n    'tree_method': 'gpu_hist',\n\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU'\n\n}\n# Params = {\n#     'learning_rate': 0.09224259399998393,\n#     'max_depth': 4,\n#     'num_leaves': 193,\n#     'min_data_in_leaf': 28,\n#     'feature_fraction': 0.7514556531262206,\n#     'bagging_fraction': 0.9255846697658219,\n#     'bagging_freq': 4,\n#     'lambda_l1': 15.86587835516145,\n#     'lambda_l2': 3.492177649094307\n# }\n\n\n# # XGBoost parameters\n# XGB_Params = {\n#     'learning_rate': 0.07977342261117466,\n#     'max_depth': 3,\n#     'n_estimators': 461,\n#     'subsample': 0.9370654528316807,\n#     'colsample_bytree': 0.5790844655067284,\n#     'reg_alpha': 1,\n#     'reg_lambda': 5.640597131481835,\n#     'random_state': SEED\n# }\n\n# CatBoost_Params = {\n#     'learning_rate': 0.0763882332924562,\n#     'depth': 5,\n#     'iterations': 200,\n#     'random_seed': SEED,\n#     'cat_features': cat_c,\n#     'verbose': 0,\n#     'l2_leaf_reg': 10  # Increase this value\n# }","metadata":{"_uuid":"6b287c96-be9f-490a-9564-af4790d39055","_cell_guid":"774419f8-c87d-455b-ba75-c809bf99355a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model1 = CatBoostRegressor(**CatBoost_Params)\nTabNet_Model = TabNetWrapper(**TabNet_Params)","metadata":{"_uuid":"50dd80e3-f272-401b-a485-c4a407c57e13","_cell_guid":"e893df4f-50f7-4635-8e95-a96820872c70","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.compose import ColumnTransformer\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.preprocessing import FunctionTransformer\n\n\nfeaturesCols1 = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Total_Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_GSND',\n                'FGC-FGC_GSD', 'FGC-FGC_PU',\n                'FGC-FGC_SRL', 'FGC-FGC_SRR',\n                'FGC-FGC_TL',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'PreInt_EduHx-computerinternet_hoursday', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW','BMI_PHR'\n                ,'Internet_Hours_SDS','Internet_Hours_PAQ','Internet_to_Activity_Ratio'\n                ,'HeartRate_Age_Ratio','HeartRate_SDS','SDS_CGAS_Ratio','SDS_PAQ_Interaction','Internet_Hours_HeartRate_Ratio'\n                ,'CGAS_PAQ_Interaction','SDS_HeartRate_Ratio', 'SDS_Weight_Interaction', 'SDS_Waist_Interaction'\n               ]\nfeaturesCols1 += time_series_cols\n\nclass FeatureEngineering(BaseEstimator, TransformerMixin):\n    def fit(self, X, y=None):\n        return self\n    \n    def transform(self, X):\n        return feature_engineering(X)\n        \n# Modified pipeline setup\nnumeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\nnumeric_cols = numeric_cols.drop('sii')\n\n\nimputer = KNNImputer(n_neighbors=5).set_output(transform=\"pandas\")\n\n\nselect_features = ColumnTransformer(\n    transformers=[\n        ('features', 'passthrough', featuresCols1)\n    ],\n    remainder='drop'\n).set_output(transform=\"pandas\")\n\npreprocessor = Pipeline(steps=[\n    ('imputer', ColumnTransformer(\n        transformers=[\n            ('knn_imputer', imputer, numeric_cols)\n        ],\n        remainder='passthrough',\n        verbose_feature_names_out=False\n    ).set_output(transform=\"pandas\")),\n    ('feature_engineering', FeatureEngineering()),\n    ('select_features', select_features)\n], verbose=True)\n\n# Modified voting model setup\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Pipeline(steps=[\n        ('preprocessor', clone(preprocessor)), \n        ('regressor', Light)\n    ])),\n    ('xgboost', Pipeline(steps=[\n        ('preprocessor', clone(preprocessor)), \n        ('regressor', XGB_Model)\n    ])),\n    ('catboost', Pipeline(steps=[\n        ('preprocessor', clone(preprocessor)), \n        ('regressor', CatBoost_Model1)\n    ])),\n    ('tabnet', Pipeline(steps=[\n        ('preprocessor', clone(preprocessor)), \n        ('regressor', TabNet_Model)\n    ]))\n], weights=[4.5, 5.0, 7.7, 0.4])\n\nSubmission1 = TrainML(voting_model, test)","metadata":{"_uuid":"38c048a9-7843-42f6-9ddd-35ea3ffcff41","_cell_guid":"2356c719-9572-494f-b672-9d4fc7c95911","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"e8ebebe1-b154-4d0f-9cf7-eb7e7a0cad4b","_cell_guid":"9c770a0e-190b-4905-99bd-176b015d00ca","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Mean Train QWK --> 0.6946\nMean Validation QWK ---> 0.4898\n\n----> || Optimized QWK SCORE ::  0.548","metadata":{"_uuid":"3c3ddd4a-1fde-4b6f-b1da-f58986b7a1e9","_cell_guid":"45d33e44-5449-4ea6-b861-5c74e2c56648","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"Mean Train QWK --> 0.6930\r\nMean Validation QWK ---> 0.4900\r\n----> || Optimized QWK SCORE ::  0.540","metadata":{"_uuid":"d8e6c27e-c644-42c7-b9ba-ecac7e4de618","_cell_guid":"18db6d63-6a92-43f8-bcb3-12d212124963","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"_uuid":"ab747ced-d043-4dfc-b32e-4b700e760c14","_cell_guid":"07e08bb9-1f03-494c-a753-2300868a4eea","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = clean_data(train)\ntest = clean_data(test)","metadata":{"_uuid":"3f4b0f95-0dd4-4790-b10f-2855b4398f6d","_cell_guid":"58fcc2f5-9201-4818-b8d5-a54664326b4d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"time_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)","metadata":{"_uuid":"629341b5-f624-448c-8030-3f72da1b5fa4","_cell_guid":"15bebcf3-0bc2-4ccf-bad4-a947ee137d74","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n\ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\n# def evaluate_predictions(thresholds, y_true, oof_non_rounded):\n#     rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n#     return -quadratic_weighted_kappa(y_true, rounded_p)\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded, reg_lambda=0.01):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    kappa_score = quadratic_weighted_kappa(y_true, rounded_p)\n    \n    # Thêm regularization term (làm giảm sự thay đổi lớn của ngưỡng)\n    reg_term = reg_lambda * np.sum(np.diff(thresholds)**2)\n    \n    return -(kappa_score - reg_term)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tp_rounded = threshold_Rounder(tpm, KappaOPtimizer.x)\n\n    return tp_rounded\n\nimputer = SimpleImputer(strategy='median')\n\nensemble = VotingRegressor(estimators=[\n    ('lgb', Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(random_state=SEED))])),\n    ('xgb', Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(random_state=SEED))])),\n    ('cat', Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(random_state=SEED, silent=True))])),\n    ('rf', Pipeline(steps=[('imputer', imputer), ('regressor', RandomForestRegressor(random_state=SEED))])),\n    ('gb', Pipeline(steps=[('imputer', imputer), ('regressor', GradientBoostingRegressor(random_state=SEED))])),\n    ('elastic_net', Pipeline(steps=[('imputer', imputer), ('regressor', ElasticNet(alpha=0.1, l1_ratio=0.09, random_state=SEED))])),\n    ('svr', Pipeline(steps=[('imputer', imputer), ('scaler', StandardScaler()), ('regressor', SVR(kernel='rbf', C=1.0, epsilon=0.1))])),\n    ('extra_trees', Pipeline(steps=[('imputer', imputer), ('regressor', ExtraTreesRegressor(n_estimators=100, random_state=SEED))])),\n    ('ridge', Pipeline(steps=[('imputer', imputer), ('regressor', Ridge(alpha=1.0, random_state=SEED))])), #0.3618\n    ('adaboost', Pipeline(steps=[('imputer', imputer), ('regressor', AdaBoostRegressor(n_estimators=50, random_state=SEED))])),\n    ('bagging', Pipeline(steps=[('imputer', imputer), ('regressor', BaggingRegressor(base_estimator=RandomForestRegressor(random_state=SEED), n_estimators=10, random_state=SEED))]))\n])\n\nSubmission3 = TrainML(ensemble, test)\nSubmission3 = pd.DataFrame({\n    'id': sample['id'],\n    'sii': Submission3\n})\nSubmission3","metadata":{"_uuid":"af27b43e-4004-494c-bb7c-4bb00cfd2196","_cell_guid":"9ad06561-2215-48c4-91ca-845eafaed1c2","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub1 = Submission1\nsub2 = Submission2\nsub3 = Submission3\n\nsub1 = sub1.sort_values(by='id').reset_index(drop=True)\nsub2 = sub2.sort_values(by='id').reset_index(drop=True)\nsub3 = sub3.sort_values(by='id').reset_index(drop=True)\n\ncombined = pd.DataFrame({\n    'id': sub1['id'],\n    'sii_1': sub1['sii'],\n    'sii_2': sub1['sii'],\n    'sii_3': sub1['sii']\n})\ncombined","metadata":{"_uuid":"3c937c1e-b9e4-4d3c-bb49-2b1d6ae67981","_cell_guid":"9e41b874-8861-45d3-8f3d-189ee46749a5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def majority_vote(row):\n    return row.mode()[0]\n\ncombined['final_sii'] = combined[['sii_1', 'sii_2', 'sii_3']].apply(majority_vote, axis=1)\n\nfinal_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\nfinal_submission.to_csv('submission.csv', index=False)\n\nprint(\"Majority voting completed and saved to 'Final_Submission.csv'\")","metadata":{"_uuid":"2b147616-737b-479f-9806-af1ee0ceff38","_cell_guid":"d5442744-93d8-43fa-a4b2-fb53ff76832d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"72caa805-a905-420a-8a2e-8f57b1c7bb6c","_cell_guid":"3d60fb11-48c4-4ee5-9c41-60b3ae0a42e8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}