{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":10228995,"sourceType":"datasetVersion","datasetId":6324370}],"dockerImageVersionId":30805,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport polars as pl\nimport pandas as pd\nfrom sklearn.base import clone\nfrom copy import deepcopy\nimport optuna\nfrom scipy.optimize import minimize\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport re\nfrom colorama import Fore, Style\n\nfrom tqdm import tqdm\nfrom IPython.display import clear_output\nfrom concurrent.futures import ThreadPoolExecutor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor, CatBoostClassifier\nfrom xgboost import XGBRegressor\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom sklearn.preprocessing import StandardScaler\n\nn_splits = 5\nSEED = 42","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:02:27.433415Z","iopub.execute_input":"2024-12-17T17:02:27.433740Z","iopub.status.idle":"2024-12-17T17:02:44.260028Z","shell.execute_reply.started":"2024-12-17T17:02:27.433710Z","shell.execute_reply":"2024-12-17T17:02:44.259347Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:02:51.020082Z","iopub.execute_input":"2024-12-17T17:02:51.020930Z","iopub.status.idle":"2024-12-17T17:02:51.067678Z","shell.execute_reply.started":"2024-12-17T17:02:51.020896Z","shell.execute_reply":"2024-12-17T17:02:51.066704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:03:03.914159Z","iopub.execute_input":"2024-12-17T17:03:03.914536Z","iopub.status.idle":"2024-12-17T17:03:04.114693Z","shell.execute_reply.started":"2024-12-17T17:03:03.914506Z","shell.execute_reply":"2024-12-17T17:03:04.113765Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['id'].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:03:15.930565Z","iopub.execute_input":"2024-12-17T17:03:15.931167Z","iopub.status.idle":"2024-12-17T17:03:15.938064Z","shell.execute_reply.started":"2024-12-17T17:03:15.931132Z","shell.execute_reply":"2024-12-17T17:03:15.937238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"Stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:03:29.041083Z","iopub.execute_input":"2024-12-17T17:03:29.041729Z","iopub.status.idle":"2024-12-17T17:03:29.047763Z","shell.execute_reply.started":"2024-12-17T17:03:29.041697Z","shell.execute_reply":"2024-12-17T17:03:29.046868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:03:38.983972Z","iopub.execute_input":"2024-12-17T17:03:38.984861Z","iopub.status.idle":"2024-12-17T17:04:52.743648Z","shell.execute_reply.started":"2024-12-17T17:03:38.984829Z","shell.execute_reply":"2024-12-17T17:04:52.743007Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:05:16.727549Z","iopub.execute_input":"2024-12-17T17:05:16.728234Z","iopub.status.idle":"2024-12-17T17:05:16.795567Z","shell.execute_reply.started":"2024-12-17T17:05:16.728199Z","shell.execute_reply":"2024-12-17T17:05:16.794602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.LeakyReLU(0.2),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.LeakyReLU(0.2),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.LeakyReLU(0.2)\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.LeakyReLU(0.2),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.LeakyReLU(0.2),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:05:23.238144Z","iopub.execute_input":"2024-12-17T17:05:23.238954Z","iopub.status.idle":"2024-12-17T17:05:23.245041Z","shell.execute_reply.started":"2024-12-17T17:05:23.238919Z","shell.execute_reply":"2024-12-17T17:05:23.244159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n\n    data_tensor = torch.FloatTensor(df_scaled)\n\n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n\n    criterion = F.smooth_l1_loss\n    optimizer = optim.Adam(autoencoder.parameters())\n\n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n\n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n\n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n\n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n\n    return df_encoded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:05:29.709868Z","iopub.execute_input":"2024-12-17T17:05:29.710215Z","iopub.status.idle":"2024-12-17T17:05:29.717225Z","shell.execute_reply.started":"2024-12-17T17:05:29.710173Z","shell.execute_reply":"2024-12-17T17:05:29.716263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=100, batch_size=32)\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=100, batch_size=32)\n\ntime_series_cols = train_ts_encoded.columns.tolist()\n\ntrain_ts_encoded[\"id\"] = train_ts[\"id\"]\ntest_ts_encoded[\"id\"] = test_ts[\"id\"]\ntrain_ts = train_ts_encoded\ntest_ts = test_ts_encoded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:05:32.459336Z","iopub.execute_input":"2024-12-17T17:05:32.459677Z","iopub.status.idle":"2024-12-17T17:05:42.270001Z","shell.execute_reply.started":"2024-12-17T17:05:32.459649Z","shell.execute_reply":"2024-12-17T17:05:42.269049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:05:48.116563Z","iopub.execute_input":"2024-12-17T17:05:48.117242Z","iopub.status.idle":"2024-12-17T17:05:48.150334Z","shell.execute_reply.started":"2024-12-17T17:05:48.117170Z","shell.execute_reply":"2024-12-17T17:05:48.149549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:05:51.632083Z","iopub.execute_input":"2024-12-17T17:05:51.632944Z","iopub.status.idle":"2024-12-17T17:05:51.642293Z","shell.execute_reply.started":"2024-12-17T17:05:51.632907Z","shell.execute_reply":"2024-12-17T17:05:51.641325Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:05:54.802306Z","iopub.execute_input":"2024-12-17T17:05:54.802680Z","iopub.status.idle":"2024-12-17T17:05:54.890141Z","shell.execute_reply.started":"2024-12-17T17:05:54.802649Z","shell.execute_reply":"2024-12-17T17:05:54.889255Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featuresCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:05:57.965148Z","iopub.execute_input":"2024-12-17T17:05:57.965538Z","iopub.status.idle":"2024-12-17T17:05:57.973550Z","shell.execute_reply.started":"2024-12-17T17:05:57.965508Z","shell.execute_reply":"2024-12-17T17:05:57.972470Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.dropna(subset='sii')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:06:10.466738Z","iopub.execute_input":"2024-12-17T17:06:10.467093Z","iopub.status.idle":"2024-12-17T17:06:10.476467Z","shell.execute_reply.started":"2024-12-17T17:06:10.467064Z","shell.execute_reply":"2024-12-17T17:06:10.475413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', \n          'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        \ntrain = update(train)\ntest = update(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:06:23.090973Z","iopub.execute_input":"2024-12-17T17:06:23.091667Z","iopub.status.idle":"2024-12-17T17:06:23.121401Z","shell.execute_reply.started":"2024-12-17T17:06:23.091632Z","shell.execute_reply":"2024-12-17T17:06:23.120440Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping_train = create_mapping(col, train)\n    mapping_test = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping_train).astype(int)\n    test[col] = test[col].replace(mapping_test).astype(int)\n\nprint(f'Train Shape : {train.shape} || Test Shape : {test.shape}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:06:34.918093Z","iopub.execute_input":"2024-12-17T17:06:34.918921Z","iopub.status.idle":"2024-12-17T17:06:34.962596Z","shell.execute_reply.started":"2024-12-17T17:06:34.918889Z","shell.execute_reply":"2024-12-17T17:06:34.961743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_engineering(df):\n    season_cols = [col for col in df.columns if 'Season' in col]\n    df = df.drop(season_cols, axis=1) \n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:06:45.214730Z","iopub.execute_input":"2024-12-17T17:06:45.215663Z","iopub.status.idle":"2024-12-17T17:06:45.222147Z","shell.execute_reply.started":"2024-12-17T17:06:45.215625Z","shell.execute_reply":"2024-12-17T17:06:45.221246Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = feature_engineering(train)\ntest = feature_engineering(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:06:57.234428Z","iopub.execute_input":"2024-12-17T17:06:57.234771Z","iopub.status.idle":"2024-12-17T17:06:57.252921Z","shell.execute_reply.started":"2024-12-17T17:06:57.234741Z","shell.execute_reply":"2024-12-17T17:06:57.252243Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def remove_outliers(df):\n    \n    df = df.drop(df[df['Physical-BMI'] <= 0].index)\n    df = df.drop(df[df['Physical-Diastolic_BP'] <= 0].index)\n    df = df.drop(df[df['Physical-Systolic_BP'] <= 0].index)\n    df = df.drop(df[df['Physical-Diastolic_BP'] > 160].index)\n\n    children = df[df['Basic_Demos-Age'] <= 12]\n    df = df.drop(children[children['FGC-FGC_CU'] > 80].index)\n    df = df.drop(children[children['FGC-FGC_GSND'] > 80].index)\n\n    df = df.drop(df[df['BIA-BIA_BMI'] <= 0].index)\n    df = df.drop(df[df['BIA-BIA_BMC'] > 1000].index)\n    df = df.drop(df[df['BIA-BIA_BMR'] > 40000].index)\n    df = df.drop(df[df['BIA-BIA_DEE'] > 60000].index)\n    df = df.drop(df[df['BIA-BIA_ECW'] > 2000].index)\n    df = df.drop(df[df['BIA-BIA_FFM'] > 2000].index)\n    df = df.drop(df[df['BIA-BIA_ICW'] > 2000].index)\n    df = df.drop(df[df['BIA-BIA_LDM'] > 2000].index)\n    df = df.drop(df[df['BIA-BIA_LST'] > 2000].index)\n    df = df.drop(df[df['BIA-BIA_SMM'] > 2000].index)\n    df = df.drop(df[df['BIA-BIA_TBW'] > 2000].index)\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:07:06.673841Z","iopub.execute_input":"2024-12-17T17:07:06.674222Z","iopub.status.idle":"2024-12-17T17:07:06.682044Z","shell.execute_reply.started":"2024-12-17T17:07:06.674165Z","shell.execute_reply":"2024-12-17T17:07:06.681134Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temp_train = remove_outliers(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:07:14.886067Z","iopub.execute_input":"2024-12-17T17:07:14.886440Z","iopub.status.idle":"2024-12-17T17:07:14.919606Z","shell.execute_reply.started":"2024-12-17T17:07:14.886410Z","shell.execute_reply":"2024-12-17T17:07:14.918841Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temp_train[temp_train['Physical-BMI'] == 0]\ntrain = temp_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:07:21.895297Z","iopub.execute_input":"2024-12-17T17:07:21.895999Z","iopub.status.idle":"2024-12-17T17:07:21.901170Z","shell.execute_reply.started":"2024-12-17T17:07:21.895967Z","shell.execute_reply":"2024-12-17T17:07:21.900352Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featuresCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score',\n                'Physical-Season', 'Physical-Height', 'Physical-Weight', 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND', 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR', 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-Season', 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'PAQ_A-Season','PAQ_A-PAQ_A_Total', 'PAQ_C-Season', 'PAQ_C-PAQ_C_Total',\n                'SDS-Season', 'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T',\n                'PreInt_EduHx-Season', 'PreInt_EduHx-computerinternet_hoursday',\n                'sii']\n\nremovedCols = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\nfeaturesCols = [feature for feature in featuresCols if feature not in removedCols]\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\nfeaturesColstest = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score',\n                'Physical-Season', 'Physical-Height', 'Physical-Weight', 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND', 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR', 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-Season', 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'PAQ_A-Season','PAQ_A-PAQ_A_Total', 'PAQ_C-Season', 'PAQ_C-PAQ_C_Total',\n                'SDS-Season', 'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T',\n                'PreInt_EduHx-Season', 'PreInt_EduHx-computerinternet_hoursday']\n\nfeaturesColstest = [feature for feature in featuresColstest if feature not in removedCols]\n\nfeaturesColstest += time_series_cols\ntest = test[featuresColstest]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:07:44.267107Z","iopub.execute_input":"2024-12-17T17:07:44.267469Z","iopub.status.idle":"2024-12-17T17:07:44.277945Z","shell.execute_reply.started":"2024-12-17T17:07:44.267441Z","shell.execute_reply":"2024-12-17T17:07:44.277079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:07:53.908159Z","iopub.execute_input":"2024-12-17T17:07:53.908781Z","iopub.status.idle":"2024-12-17T17:07:53.931051Z","shell.execute_reply.started":"2024-12-17T17:07:53.908749Z","shell.execute_reply":"2024-12-17T17:07:53.929864Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:08:08.122654Z","iopub.execute_input":"2024-12-17T17:08:08.122995Z","iopub.status.idle":"2024-12-17T17:08:08.181206Z","shell.execute_reply.started":"2024-12-17T17:08:08.122967Z","shell.execute_reply":"2024-12-17T17:08:08.180268Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['sii'].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:08:34.242351Z","iopub.execute_input":"2024-12-17T17:08:34.242696Z","iopub.status.idle":"2024-12-17T17:08:34.249547Z","shell.execute_reply.started":"2024-12-17T17:08:34.242667Z","shell.execute_reply":"2024-12-17T17:08:34.248728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:09:03.789078Z","iopub.execute_input":"2024-12-17T17:09:03.789996Z","iopub.status.idle":"2024-12-17T17:09:03.794088Z","shell.execute_reply.started":"2024-12-17T17:09:03.789946Z","shell.execute_reply":"2024-12-17T17:09:03.793431Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:09:12.831423Z","iopub.execute_input":"2024-12-17T17:09:12.831769Z","iopub.status.idle":"2024-12-17T17:09:12.836349Z","shell.execute_reply.started":"2024-12-17T17:09:12.831739Z","shell.execute_reply":"2024-12-17T17:09:12.835471Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:09:21.261618Z","iopub.execute_input":"2024-12-17T17:09:21.262512Z","iopub.status.idle":"2024-12-17T17:09:21.266371Z","shell.execute_reply.started":"2024-12-17T17:09:21.262477Z","shell.execute_reply":"2024-12-17T17:09:21.265450Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def TrainML(model_class, test_data):\n    \n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    # Apply K-Fold\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        # Train model\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        # Round to integer values\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n\n        #Predict with test dataset\n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    # Using optimizer to find the best threshold\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead') # Nelder-Mead | # Powell\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n\n    # Use the threshold retrive from the optimizer to predict again to evaluate\n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    # Use the threshold retrive from the optimizer to predict test\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n\n    # Create submition\n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission,model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:10:15.717820Z","iopub.execute_input":"2024-12-17T17:10:15.718653Z","iopub.status.idle":"2024-12-17T17:10:15.728644Z","shell.execute_reply.started":"2024-12-17T17:10:15.718619Z","shell.execute_reply":"2024-12-17T17:10:15.727874Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip -q install /kaggle/input/pytorch/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T18:01:01.590354Z","iopub.execute_input":"2024-12-17T18:01:01.591345Z","iopub.status.idle":"2024-12-17T18:01:09.946756Z","shell.execute_reply.started":"2024-12-17T18:01:01.591307Z","shell.execute_reply":"2024-12-17T18:01:09.945685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pytorch_tabnet.tab_model import TabNetRegressor\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.model_selection import train_test_split\nfrom pytorch_tabnet.callbacks import Callback\nimport os\nimport torch\nfrom pytorch_tabnet.callbacks import Callback\n\nclass TabNetWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, **kwargs):\n        self.model = TabNetRegressor(**kwargs)\n        self.kwargs = kwargs\n        self.imputer = KNNImputer(n_neighbors=5)\n        #self.imputer = SimpleImputer(strategy='median')\n        self.best_model_path = 'best_tabnet_model.pt'\n\n    def fit(self, X, y):\n        X_imputed = self.imputer.fit_transform(X)\n\n        if hasattr(y, 'values'):\n            y = y.values\n\n        X_train, X_valid, y_train, y_valid = train_test_split(\n            X_imputed,\n            y,\n            test_size=0.2,\n            random_state=42\n        )\n\n        # Train TabNet model\n        history = self.model.fit(\n            X_train=X_train,\n            y_train=y_train.reshape(-1, 1),\n            eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n            eval_name=['valid'],\n            eval_metric=['mse', 'mae', 'rmse'],\n            max_epochs=500,\n            patience=50,\n            batch_size=1024,\n            virtual_batch_size=128,\n            num_workers=0,\n            drop_last=False,\n            callbacks=[\n                TabNetPretrainedModelCheckpoint(\n                    filepath=self.best_model_path,\n                    monitor='valid_mse',\n                    mode='min',\n                    save_best_only=True,\n                    verbose=True\n                )\n            ]\n        )\n\n        # Load the best model\n        if os.path.exists(self.best_model_path):\n            self.model.load_model(self.best_model_path)\n            os.remove(self.best_model_path)  # Remove temporary file\n\n        return self\n\n\n    def predict(self, X):\n        X_imputed = self.imputer.transform(X)\n        return self.model.predict(X_imputed).flatten()\n\n    def __deepcopy__(self, memo):\n        cls = self.__class__\n        result = cls.__new__(cls)\n        memo[id(self)] = result\n        for k, v in self.__dict__.items():\n            setattr(result, k, deepcopy(v, memo))\n        return result\n\nTabNet_Params = {\n    'n_d': 64,\n    'n_a': 64,\n    'n_steps': 5,\n    'gamma': 1.5,\n    'n_independent': 2,\n    'n_shared': 2,\n    'lambda_sparse': 1e-4,\n    'optimizer_fn': torch.optim.Adam,\n    'optimizer_params': dict(lr=1e-4, weight_decay=1e-5),\n    'mask_type': 'entmax',\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'verbose': 1,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n    def __init__(self, filepath, monitor='val_loss', mode='min',\n                 save_best_only=True, verbose=1):\n        super().__init__()\n        self.filepath = filepath\n        self.monitor = monitor\n        self.mode = mode\n        self.save_best_only = save_best_only\n        self.verbose = verbose\n        self.best = float('inf') if mode == 'min' else -float('inf')\n\n    def on_train_begin(self, logs=None):\n        self.model = self.trainer\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is None:\n            return\n\n        if (self.mode == 'min' and current < self.best) or \\\n           (self.mode == 'max' and current > self.best):\n            if self.verbose:\n                print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n            self.best = current\n            if self.save_best_only:\n                self.model.save_model(self.filepath)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:23:27.272620Z","iopub.execute_input":"2024-12-17T17:23:27.273431Z","iopub.status.idle":"2024-12-17T17:23:27.312863Z","shell.execute_reply.started":"2024-12-17T17:23:27.273391Z","shell.execute_reply":"2024-12-17T17:23:27.312060Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01,  # Increased from 2.68e-06\n    'device': 'gpu'\n\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED,\n    'tree_method': 'gpu_hist',\n\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU'\n\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:23:41.985513Z","iopub.execute_input":"2024-12-17T17:23:41.985871Z","iopub.status.idle":"2024-12-17T17:23:41.991603Z","shell.execute_reply.started":"2024-12-17T17:23:41.985841Z","shell.execute_reply":"2024-12-17T17:23:41.990672Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import lightgbm as lgb\nfrom catboost import CatBoostRegressor, CatBoostClassifier\nfrom xgboost import XGBRegressor\nfrom sklearn.ensemble import VotingRegressor\n\ntabnet = TabNetWrapper(**TabNet_Params)\nxgboost = XGBRegressor(**XGB_Params)\nlight = lgb.LGBMRegressor(**Params, verbose=-1, n_estimators=300, random_state=SEED)\ncat = CatBoostRegressor(**CatBoost_Params)\n\nvoting = voting_model = VotingRegressor(estimators=[\n    ('lightgbm', light),\n    ('xgboost', xgboost),\n    ('catboost', cat),\n    ('tabnet', tabnet)  # New:TabNet\n])\nSubmission, model = TrainML(voting, test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:23:54.597264Z","iopub.execute_input":"2024-12-17T17:23:54.597621Z","iopub.status.idle":"2024-12-17T17:26:34.931130Z","shell.execute_reply.started":"2024-12-17T17:23:54.597594Z","shell.execute_reply":"2024-12-17T17:26:34.930245Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission.to_csv('submission.csv', index=False)\nprint(Submission['sii'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:29:33.930379Z","iopub.execute_input":"2024-12-17T17:29:33.930769Z","iopub.status.idle":"2024-12-17T17:29:33.950610Z","shell.execute_reply.started":"2024-12-17T17:29:33.930739Z","shell.execute_reply":"2024-12-17T17:29:33.949629Z"}},"outputs":[],"execution_count":null}]}