{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":7453542,"sourceType":"datasetVersion","datasetId":921302}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":401.900596,"end_time":"2024-12-17T13:23:13.774044","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-12-17T13:16:31.873448","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"2c82fc36","cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nimport copy\nimport pickle\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nimport polars as pl\nimport polars.selectors as cs\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\n\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\n\nimport plotly.express as px\n\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\nSEED = 42\nn_splits = 5","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2024-12-17T13:16:34.332100Z","iopub.status.busy":"2024-12-17T13:16:34.331710Z","iopub.status.idle":"2024-12-17T13:16:54.042860Z","shell.execute_reply":"2024-12-17T13:16:54.041910Z"},"papermill":{"duration":19.729826,"end_time":"2024-12-17T13:16:54.044954","exception":false,"start_time":"2024-12-17T13:16:34.315128","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"7d030cd7","cell_type":"code","source":"import random\nimport torch\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\nseed_everything(100)","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:16:54.063323Z","iopub.status.busy":"2024-12-17T13:16:54.062677Z","iopub.status.idle":"2024-12-17T13:16:54.071418Z","shell.execute_reply":"2024-12-17T13:16:54.070646Z"},"papermill":{"duration":0.019412,"end_time":"2024-12-17T13:16:54.073099","exception":false,"start_time":"2024-12-17T13:16:54.053687","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"a561d9de","cell_type":"markdown","source":"# Define function","metadata":{"papermill":{"duration":0.008053,"end_time":"2024-12-17T13:16:54.089795","exception":false,"start_time":"2024-12-17T13:16:54.081742","status":"completed"},"tags":[]}},{"id":"5aab28b9","cell_type":"code","source":"class AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n\n        \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    \n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n                 \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:16:54.107870Z","iopub.status.busy":"2024-12-17T13:16:54.107149Z","iopub.status.idle":"2024-12-17T13:16:54.116035Z","shell.execute_reply":"2024-12-17T13:16:54.115235Z"},"papermill":{"duration":0.019525,"end_time":"2024-12-17T13:16:54.117590","exception":false,"start_time":"2024-12-17T13:16:54.098065","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"b14ae425","cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    stats, indexes = zip(*results)\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, X, y, test_data):\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    print('OPTIMIZED THRESHOLDS', KappaOPtimizer.x)\n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n    optimized_thresholds = KappaOPtimizer.x\n    return submission, oof_tuned, oof_non_rounded, y, optimized_thresholds\n\n","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:16:54.136007Z","iopub.status.busy":"2024-12-17T13:16:54.135737Z","iopub.status.idle":"2024-12-17T13:16:54.149519Z","shell.execute_reply":"2024-12-17T13:16:54.148683Z"},"papermill":{"duration":0.024923,"end_time":"2024-12-17T13:16:54.151111","exception":false,"start_time":"2024-12-17T13:16:54.126188","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"47e94917","cell_type":"markdown","source":"# Define features","metadata":{"papermill":{"duration":0.008173,"end_time":"2024-12-17T13:16:54.167588","exception":false,"start_time":"2024-12-17T13:16:54.159415","status":"completed"},"tags":[]}},{"id":"564b5b23","cell_type":"markdown","source":"## Normal features","metadata":{"papermill":{"duration":0.008097,"end_time":"2024-12-17T13:16:54.183987","exception":false,"start_time":"2024-12-17T13:16:54.175890","status":"completed"},"tags":[]}},{"id":"ccb2db13","cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntotal_features = list(test.columns)\ntotal_features.remove('id')\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:16:54.201732Z","iopub.status.busy":"2024-12-17T13:16:54.201400Z","iopub.status.idle":"2024-12-17T13:16:54.269234Z","shell.execute_reply":"2024-12-17T13:16:54.268609Z"},"papermill":{"duration":0.078719,"end_time":"2024-12-17T13:16:54.271027","exception":false,"start_time":"2024-12-17T13:16:54.192308","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"08689c69","cell_type":"code","source":"noseason_features = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW','BMI_PHR']","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:16:54.289082Z","iopub.status.busy":"2024-12-17T13:16:54.288852Z","iopub.status.idle":"2024-12-17T13:16:54.293747Z","shell.execute_reply":"2024-12-17T13:16:54.293026Z"},"papermill":{"duration":0.015448,"end_time":"2024-12-17T13:16:54.295277","exception":false,"start_time":"2024-12-17T13:16:54.279829","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"3db78dfe","cell_type":"markdown","source":"## Loading timeseries","metadata":{"papermill":{"duration":0.007861,"end_time":"2024-12-17T13:16:54.311389","exception":false,"start_time":"2024-12-17T13:16:54.303528","status":"completed"},"tags":[]}},{"id":"d111a129","cell_type":"code","source":"train_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:16:54.328556Z","iopub.status.busy":"2024-12-17T13:16:54.328292Z","iopub.status.idle":"2024-12-17T13:18:08.400769Z","shell.execute_reply":"2024-12-17T13:18:08.399818Z"},"papermill":{"duration":74.082862,"end_time":"2024-12-17T13:18:08.402345","exception":false,"start_time":"2024-12-17T13:16:54.319483","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"ed8f42f6","cell_type":"code","source":"df_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:18:08.451597Z","iopub.status.busy":"2024-12-17T13:18:08.451237Z","iopub.status.idle":"2024-12-17T13:18:08.456906Z","shell.execute_reply":"2024-12-17T13:18:08.456178Z"},"papermill":{"duration":0.03185,"end_time":"2024-12-17T13:18:08.458732","exception":false,"start_time":"2024-12-17T13:18:08.426882","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"da791df9","cell_type":"code","source":"df_total = pd.concat([df_train, df_test], axis=0, ignore_index=True)\ndf_total","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:18:08.511883Z","iopub.status.busy":"2024-12-17T13:18:08.511159Z","iopub.status.idle":"2024-12-17T13:18:08.600301Z","shell.execute_reply":"2024-12-17T13:18:08.599477Z"},"papermill":{"duration":0.117044,"end_time":"2024-12-17T13:18:08.602151","exception":false,"start_time":"2024-12-17T13:18:08.485107","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"a0b0cc2b","cell_type":"code","source":"# train_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=100, batch_size=32)\n# test_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=100, batch_size=32)\ntotal_ts_encoded = perform_autoencoder(df_total, encoding_dim=60, epochs=100, batch_size=32)","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:18:08.654991Z","iopub.status.busy":"2024-12-17T13:18:08.654712Z","iopub.status.idle":"2024-12-17T13:18:19.016848Z","shell.execute_reply":"2024-12-17T13:18:19.016024Z"},"papermill":{"duration":10.389102,"end_time":"2024-12-17T13:18:19.018746","exception":false,"start_time":"2024-12-17T13:18:08.629644","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"8b32ac83","cell_type":"code","source":"df_train.shape[1]","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:18:19.069895Z","iopub.status.busy":"2024-12-17T13:18:19.069170Z","iopub.status.idle":"2024-12-17T13:18:19.074681Z","shell.execute_reply":"2024-12-17T13:18:19.073888Z"},"papermill":{"duration":0.03241,"end_time":"2024-12-17T13:18:19.076211","exception":false,"start_time":"2024-12-17T13:18:19.043801","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"1a003211","cell_type":"code","source":"train_ts_encoded = total_ts_encoded.head(df_train.shape[0])\ntest_ts_encoded = total_ts_encoded.drop(train_ts_encoded.index, axis=0)\ntest_ts_encoded.reset_index(inplace=True, drop=True)","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:18:19.125975Z","iopub.status.busy":"2024-12-17T13:18:19.125712Z","iopub.status.idle":"2024-12-17T13:18:19.131241Z","shell.execute_reply":"2024-12-17T13:18:19.130565Z"},"papermill":{"duration":0.032358,"end_time":"2024-12-17T13:18:19.132997","exception":false,"start_time":"2024-12-17T13:18:19.100639","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"a641abcc","cell_type":"code","source":"test_ts_encoded","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:18:19.189254Z","iopub.status.busy":"2024-12-17T13:18:19.188481Z","iopub.status.idle":"2024-12-17T13:18:19.227264Z","shell.execute_reply":"2024-12-17T13:18:19.226347Z"},"papermill":{"duration":0.065933,"end_time":"2024-12-17T13:18:19.228874","exception":false,"start_time":"2024-12-17T13:18:19.162941","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"14c24971","cell_type":"code","source":"train_ts_encoded[\"id\"]=train_ts[\"id\"]\ntest_ts_encoded['id']=test_ts[\"id\"]","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:18:19.280228Z","iopub.status.busy":"2024-12-17T13:18:19.279618Z","iopub.status.idle":"2024-12-17T13:18:19.283886Z","shell.execute_reply":"2024-12-17T13:18:19.283200Z"},"papermill":{"duration":0.031095,"end_time":"2024-12-17T13:18:19.285290","exception":false,"start_time":"2024-12-17T13:18:19.254195","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"29705f69","cell_type":"markdown","source":"## Features timeseries","metadata":{"papermill":{"duration":0.024537,"end_time":"2024-12-17T13:18:19.338277","exception":false,"start_time":"2024-12-17T13:18:19.313740","status":"completed"},"tags":[]}},{"id":"b839e046","cell_type":"code","source":"time_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\ntime_encoded_cols = train_ts_encoded.columns.tolist()\ntime_encoded_cols.remove(\"id\")","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:18:19.388390Z","iopub.status.busy":"2024-12-17T13:18:19.388114Z","iopub.status.idle":"2024-12-17T13:18:19.391967Z","shell.execute_reply":"2024-12-17T13:18:19.391332Z"},"papermill":{"duration":0.030376,"end_time":"2024-12-17T13:18:19.393402","exception":false,"start_time":"2024-12-17T13:18:19.363026","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"276766b9","cell_type":"markdown","source":"# Submission 1","metadata":{"papermill":{"duration":0.036277,"end_time":"2024-12-17T13:18:19.456335","exception":false,"start_time":"2024-12-17T13:18:19.420058","status":"completed"},"tags":[]}},{"id":"fb9c557f","cell_type":"code","source":"# features_sub1 = total_features + time_series_cols\n\n# train_sub1 = pd.merge(train, train_ts, how=\"left\", on='id')\n# test_sub1 = pd.merge(test, test_ts, how=\"left\", on='id')\n\n# train_sub1 = train_sub1.drop('id', axis=1)\n# test_sub1 = test_sub1.drop('id', axis=1)   \n\n# train_sub1 = train_sub1.dropna(subset='sii')\n","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:18:19.518803Z","iopub.status.busy":"2024-12-17T13:18:19.518422Z","iopub.status.idle":"2024-12-17T13:18:19.522150Z","shell.execute_reply":"2024-12-17T13:18:19.521479Z"},"papermill":{"duration":0.031144,"end_time":"2024-12-17T13:18:19.523755","exception":false,"start_time":"2024-12-17T13:18:19.492611","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"32fbde28","cell_type":"code","source":"# train_sub1 = update(train_sub1)\n# test_sub1 = update(test_sub1)\n# for col in cat_c:\n#     mapping = create_mapping(col, train_sub1)\n#     mappingTe = create_mapping(col, test_sub1)\n    \n#     train_sub1[col] = train_sub1[col].replace(mapping).astype(int)\n#     test_sub1[col] = test_sub1[col].replace(mappingTe).astype(int)","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:18:19.611533Z","iopub.status.busy":"2024-12-17T13:18:19.611198Z","iopub.status.idle":"2024-12-17T13:18:19.614873Z","shell.execute_reply":"2024-12-17T13:18:19.614151Z"},"papermill":{"duration":0.067557,"end_time":"2024-12-17T13:18:19.616510","exception":false,"start_time":"2024-12-17T13:18:19.548953","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"f9dd1012","cell_type":"code","source":"# X_sub1 = train_sub1[features_sub1]\n# y_sub1 = train_sub1['sii']\n# test_sub1 = test_sub1[features_sub1]","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:18:19.668516Z","iopub.status.busy":"2024-12-17T13:18:19.668235Z","iopub.status.idle":"2024-12-17T13:18:19.671487Z","shell.execute_reply":"2024-12-17T13:18:19.670836Z"},"papermill":{"duration":0.031725,"end_time":"2024-12-17T13:18:19.673174","exception":false,"start_time":"2024-12-17T13:18:19.641449","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"548c74bf","cell_type":"code","source":"# SEED = 42\n# n_splits = 5\n\n# model = XGBRegressor(\n#     learning_rate=0.05,\n#     max_depth=6,\n#     n_estimators=200,\n#     subsample=0.8,\n#     colsample_bytree = 0.8,\n#     reg_alpha=1,\n#     reg_lambda=5,\n#     random_state=SEED\n# )\n\n# # we get out of fold predictions for further exploration\n# submission, y_pred, y_pred_non_rounded, y_true, optimized_thresholds = TrainML(model, X_sub1, y_sub1, test_sub1)","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:18:19.723244Z","iopub.status.busy":"2024-12-17T13:18:19.722999Z","iopub.status.idle":"2024-12-17T13:18:19.726609Z","shell.execute_reply":"2024-12-17T13:18:19.725907Z"},"papermill":{"duration":0.030505,"end_time":"2024-12-17T13:18:19.728158","exception":false,"start_time":"2024-12-17T13:18:19.697653","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"e035a488","cell_type":"markdown","source":"# Submission 2","metadata":{"papermill":{"duration":0.025724,"end_time":"2024-12-17T13:18:19.778999","exception":false,"start_time":"2024-12-17T13:18:19.753275","status":"completed"},"tags":[]}},{"id":"4716682c","cell_type":"code","source":"!pip -q install /kaggle/input/pytorchtabnet/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:18:19.830815Z","iopub.status.busy":"2024-12-17T13:18:19.830538Z","iopub.status.idle":"2024-12-17T13:19:00.669974Z","shell.execute_reply":"2024-12-17T13:19:00.668910Z"},"papermill":{"duration":40.867731,"end_time":"2024-12-17T13:19:00.671934","exception":false,"start_time":"2024-12-17T13:18:19.804203","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"c04050b7","cell_type":"code","source":"from pytorch_tabnet.tab_model import TabNetRegressor\n","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:19:00.723735Z","iopub.status.busy":"2024-12-17T13:19:00.723029Z","iopub.status.idle":"2024-12-17T13:19:00.739339Z","shell.execute_reply":"2024-12-17T13:19:00.738737Z"},"papermill":{"duration":0.043632,"end_time":"2024-12-17T13:19:00.740858","exception":false,"start_time":"2024-12-17T13:19:00.697226","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"05c328cd","cell_type":"code","source":"def feature_engineering(df):\n    season_cols = [col for col in df.columns if 'Season' in col]\n    df = df.drop(season_cols, axis=1) \n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    df['BMI_PHR'] = df['Physical-BMI'] * df['Physical-HeartRate']\n    \n    return df","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:19:00.792752Z","iopub.status.busy":"2024-12-17T13:19:00.792484Z","iopub.status.idle":"2024-12-17T13:19:00.799284Z","shell.execute_reply":"2024-12-17T13:19:00.798675Z"},"papermill":{"duration":0.033754,"end_time":"2024-12-17T13:19:00.800852","exception":false,"start_time":"2024-12-17T13:19:00.767098","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"d78f366c","cell_type":"code","source":"# notna_df = train.dropna(subset='sii', ignore_index=True)\n# na_df = train.drop(notna_df.index, axis=0)\n# na_df","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:19:00.850761Z","iopub.status.busy":"2024-12-17T13:19:00.850258Z","iopub.status.idle":"2024-12-17T13:19:00.853449Z","shell.execute_reply":"2024-12-17T13:19:00.852801Z"},"papermill":{"duration":0.029798,"end_time":"2024-12-17T13:19:00.854990","exception":false,"start_time":"2024-12-17T13:19:00.825192","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"09848d00","cell_type":"code","source":"train_sub2 = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\ntest_sub2 = pd.merge(test, test_ts_encoded, how=\"left\", on='id')\n\n# imputer = KNNImputer(n_neighbors=5)\n# numeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\n# imputed_data = imputer.fit_transform(train_sub2[numeric_cols])\n# train_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\n# train_imputed['sii'] = train_imputed['sii'].round().astype(int)\n\n# for col in train_sub2.columns:\n#     if col not in numeric_cols:\n#         train_imputed[col] = train_sub2[col]\n        \n# train_sub2 = train_imputed\n\ntrain_sub2 = feature_engineering(train_sub2)\ntrain_sub2 = train_sub2.dropna(subset='sii', ignore_index=True)\ntest_sub2 = feature_engineering(test_sub2)\n\ntrain_sub2 = train_sub2.drop('id', axis=1)\ntest_sub2  = test_sub2.drop('id', axis=1)   ","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:19:00.904970Z","iopub.status.busy":"2024-12-17T13:19:00.904704Z","iopub.status.idle":"2024-12-17T13:19:00.962959Z","shell.execute_reply":"2024-12-17T13:19:00.962023Z"},"papermill":{"duration":0.085262,"end_time":"2024-12-17T13:19:00.964800","exception":false,"start_time":"2024-12-17T13:19:00.879538","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"9a709ef6","cell_type":"code","source":"# train_sub2 = train_sub2.drop(na_df.index, axis=0)\n# train_sub2.reset_index(inplace=True, drop=True)\ntrain_sub2","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:19:01.015003Z","iopub.status.busy":"2024-12-17T13:19:01.014730Z","iopub.status.idle":"2024-12-17T13:19:01.123072Z","shell.execute_reply":"2024-12-17T13:19:01.122320Z"},"papermill":{"duration":0.136396,"end_time":"2024-12-17T13:19:01.125762","exception":false,"start_time":"2024-12-17T13:19:00.989366","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"e4044a76","cell_type":"code","source":"features_sub2 = noseason_features + time_encoded_cols\n\n# train_sub2 = pd.merge(train, train_ts, how=\"left\", on='id')\n# test_sub2 = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain_sub2 = train_sub2.dropna(subset='sii')\n","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:19:01.180344Z","iopub.status.busy":"2024-12-17T13:19:01.180093Z","iopub.status.idle":"2024-12-17T13:19:01.187315Z","shell.execute_reply":"2024-12-17T13:19:01.186750Z"},"papermill":{"duration":0.035711,"end_time":"2024-12-17T13:19:01.188808","exception":false,"start_time":"2024-12-17T13:19:01.153097","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"4cbf29ef","cell_type":"code","source":"if np.any(np.isinf(train_sub2)):\n    train_sub2 = train_sub2.replace([np.inf, -np.inf], np.nan)","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:19:01.242296Z","iopub.status.busy":"2024-12-17T13:19:01.242042Z","iopub.status.idle":"2024-12-17T13:19:01.246827Z","shell.execute_reply":"2024-12-17T13:19:01.246199Z"},"papermill":{"duration":0.032954,"end_time":"2024-12-17T13:19:01.248318","exception":false,"start_time":"2024-12-17T13:19:01.215364","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"23c7b117","cell_type":"code","source":"X_sub2 = train_sub2[features_sub2]\ny_sub2 = train_sub2['sii']\ntest_sub2 = test_sub2[features_sub2]","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:19:01.301774Z","iopub.status.busy":"2024-12-17T13:19:01.301521Z","iopub.status.idle":"2024-12-17T13:19:01.307176Z","shell.execute_reply":"2024-12-17T13:19:01.306389Z"},"papermill":{"duration":0.034524,"end_time":"2024-12-17T13:19:01.308821","exception":false,"start_time":"2024-12-17T13:19:01.274297","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"3166aa61","cell_type":"code","source":"y_sub2","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:19:01.363097Z","iopub.status.busy":"2024-12-17T13:19:01.362870Z","iopub.status.idle":"2024-12-17T13:19:01.369475Z","shell.execute_reply":"2024-12-17T13:19:01.368687Z"},"papermill":{"duration":0.035417,"end_time":"2024-12-17T13:19:01.371118","exception":false,"start_time":"2024-12-17T13:19:01.335701","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"4bea8ec6","cell_type":"code","source":"# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01,  # Increased from 2.68e-06\n    'device': 'cpu'\n\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED,\n    'tree_method': 'gpu_hist',\n\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU'\n\n}","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:19:01.427015Z","iopub.status.busy":"2024-12-17T13:19:01.426771Z","iopub.status.idle":"2024-12-17T13:19:01.432069Z","shell.execute_reply":"2024-12-17T13:19:01.431356Z"},"papermill":{"duration":0.035304,"end_time":"2024-12-17T13:19:01.433676","exception":false,"start_time":"2024-12-17T13:19:01.398372","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"7d7d0858","cell_type":"code","source":"# New: TabNet\n\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom pytorch_tabnet.callbacks import Callback\nimport os\nimport torch\nfrom pytorch_tabnet.callbacks import Callback\n\nclass TabNetWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, **kwargs):\n        self.model = TabNetRegressor(**kwargs)\n        self.kwargs = kwargs\n        self.imputer = SimpleImputer(strategy='median')\n        self.best_model_path = 'best_tabnet_model.pt'\n        \n    def fit(self, X, y):\n        # Handle missing values\n        X_imputed = self.imputer.fit_transform(X)\n        \n        if hasattr(y, 'values'):\n            y = y.values\n            \n        # Create internal validation set\n        X_train, X_valid, y_train, y_valid = train_test_split(\n            X_imputed, \n            y, \n            test_size=0.2,\n            random_state=42\n        )\n        \n        # Train TabNet model\n        history = self.model.fit(\n            X_train=X_train,\n            y_train=y_train.reshape(-1, 1),\n            eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n            eval_name=['valid'],\n            eval_metric=['mse'],\n            max_epochs=200,\n            patience=20,\n            batch_size=1024,\n            virtual_batch_size=128,\n            num_workers=0,\n            drop_last=False,\n            callbacks=[\n                TabNetPretrainedModelCheckpoint(\n                    filepath=self.best_model_path,\n                    monitor='valid_mse',\n                    mode='min',\n                    save_best_only=True,\n                    verbose=True\n                )\n            ]\n        )\n        \n        # Load the best model\n        if os.path.exists(self.best_model_path):\n            self.model.load_model(self.best_model_path)\n            os.remove(self.best_model_path)  # Remove temporary file\n        \n        return self\n    \n    def predict(self, X):\n        X_imputed = self.imputer.transform(X)\n        return self.model.predict(X_imputed).flatten()\n    \n    def __deepcopy__(self, memo):\n        # Add deepcopy support for scikit-learn\n        cls = self.__class__\n        result = cls.__new__(cls)\n        memo[id(self)] = result\n        for k, v in self.__dict__.items():\n            setattr(result, k, deepcopy(v, memo))\n        return result\n\n# TabNet hyperparameters\nTabNet_Params = {\n    'n_d': 64,              # Width of the decision prediction layer\n    'n_a': 64,              # Width of the attention embedding for each step\n    'n_steps': 5,           # Number of steps in the architecture\n    'gamma': 1.5,           # Coefficient for feature selection regularization\n    'n_independent': 2,     # Number of independent GLU layer in each GLU block\n    'n_shared': 2,          # Number of shared GLU layer in each GLU block\n    'lambda_sparse': 1e-4,  # Sparsity regularization\n    'optimizer_fn': torch.optim.Adam,\n    'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n    'mask_type': 'entmax',\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'verbose': 1,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n    def __init__(self, filepath, monitor='val_loss', mode='min', \n                 save_best_only=True, verbose=1):\n        super().__init__()  # Initialize parent class\n        self.filepath = filepath\n        self.monitor = monitor\n        self.mode = mode\n        self.save_best_only = save_best_only\n        self.verbose = verbose\n        self.best = float('inf') if mode == 'min' else -float('inf')\n        \n    def on_train_begin(self, logs=None):\n        self.model = self.trainer  # Use trainer itself as model\n        \n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is None:\n            return\n        \n        # Check if current metric is better than best\n        if (self.mode == 'min' and current < self.best) or \\\n           (self.mode == 'max' and current > self.best):\n            if self.verbose:\n                print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n            self.best = current\n            if self.save_best_only:\n                self.model.save_model(self.filepath)  # Save the entire model","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:19:01.490009Z","iopub.status.busy":"2024-12-17T13:19:01.489740Z","iopub.status.idle":"2024-12-17T13:19:01.504161Z","shell.execute_reply":"2024-12-17T13:19:01.503376Z"},"papermill":{"duration":0.045054,"end_time":"2024-12-17T13:19:01.505827","exception":false,"start_time":"2024-12-17T13:19:01.460773","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"337e3bb3","cell_type":"code","source":"def TrainML_sub2(model_class, X, y, test_data):\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n    \n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n        model = clone(model_class)\n        \n        imputer = KNNImputer(n_neighbors=5)\n        imputer.fit(X_train)\n        train_imputed = imputer.fit_transform(X_train)\n        X_train = pd.DataFrame(train_imputed, columns=X_train.columns)\n        val_imputed = imputer.fit_transform(X_val)\n        X_val = pd.DataFrame(val_imputed, columns=X_val.columns)\n        test_cpy = test_data.copy()\n        test_imputed = imputer.fit_transform(test_cpy)\n        test_cpy = pd.DataFrame(test_imputed, columns=test_cpy.columns)\n        \n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_cpy)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    print('OPTIMIZED THRESHOLDS', KappaOPtimizer.x)\n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n    optimized_thresholds = KappaOPtimizer.x\n    return submission, oof_tuned, oof_non_rounded, y, optimized_thresholds","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:19:01.564814Z","iopub.status.busy":"2024-12-17T13:19:01.564541Z","iopub.status.idle":"2024-12-17T13:19:01.575015Z","shell.execute_reply":"2024-12-17T13:19:01.574175Z"},"papermill":{"duration":0.041698,"end_time":"2024-12-17T13:19:01.576642","exception":false,"start_time":"2024-12-17T13:19:01.534944","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"5d2f9756","cell_type":"code","source":"# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\nTabNet_Model = TabNetWrapper(**TabNet_Params) \nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model),\n    ('tabnet', TabNet_Model)\n],weights=[4.0,4.0,5.0,4.0])\n","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:19:01.632034Z","iopub.status.busy":"2024-12-17T13:19:01.631328Z","iopub.status.idle":"2024-12-17T13:19:01.642373Z","shell.execute_reply":"2024-12-17T13:19:01.641785Z"},"papermill":{"duration":0.040205,"end_time":"2024-12-17T13:19:01.644095","exception":false,"start_time":"2024-12-17T13:19:01.603890","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"71c84873","cell_type":"code","source":"submission2, _, _, _, _= TrainML_sub2(voting_model, X_sub2, y_sub2, test_sub2)","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:19:01.698067Z","iopub.status.busy":"2024-12-17T13:19:01.697821Z","iopub.status.idle":"2024-12-17T13:20:18.362691Z","shell.execute_reply":"2024-12-17T13:20:18.361707Z"},"papermill":{"duration":76.693366,"end_time":"2024-12-17T13:20:18.364264","exception":false,"start_time":"2024-12-17T13:19:01.670898","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"8864caa5","cell_type":"code","source":"# submission2","metadata":{"collapsed":true,"execution":{"iopub.execute_input":"2024-12-17T13:20:18.420268Z","iopub.status.busy":"2024-12-17T13:20:18.419924Z","iopub.status.idle":"2024-12-17T13:20:18.423707Z","shell.execute_reply":"2024-12-17T13:20:18.422838Z"},"jupyter":{"outputs_hidden":true},"papermill":{"duration":0.033039,"end_time":"2024-12-17T13:20:18.425386","exception":false,"start_time":"2024-12-17T13:20:18.392347","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"73191e4e","cell_type":"code","source":"# !pip install shap","metadata":{"collapsed":true,"execution":{"iopub.execute_input":"2024-12-17T13:20:18.479022Z","iopub.status.busy":"2024-12-17T13:20:18.478766Z","iopub.status.idle":"2024-12-17T13:20:18.482120Z","shell.execute_reply":"2024-12-17T13:20:18.481403Z"},"jupyter":{"outputs_hidden":true},"papermill":{"duration":0.031862,"end_time":"2024-12-17T13:20:18.483666","exception":false,"start_time":"2024-12-17T13:20:18.451804","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"2486bc3a","cell_type":"code","source":"# import shap\n# explainer = shap.KernelExplainer(voting_model.predict,X_sub2)\n","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:20:18.539604Z","iopub.status.busy":"2024-12-17T13:20:18.539054Z","iopub.status.idle":"2024-12-17T13:20:18.542449Z","shell.execute_reply":"2024-12-17T13:20:18.541718Z"},"papermill":{"duration":0.033226,"end_time":"2024-12-17T13:20:18.544025","exception":false,"start_time":"2024-12-17T13:20:18.510799","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"7aef2648","cell_type":"markdown","source":"# Submission 3","metadata":{"papermill":{"duration":0.027091,"end_time":"2024-12-17T13:20:18.598697","exception":false,"start_time":"2024-12-17T13:20:18.571606","status":"completed"},"tags":[]}},{"id":"d1ef77dd","cell_type":"code","source":"train_sub3 = pd.merge(train, train_ts, how=\"left\", on='id')\ntest_sub3 = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain_sub3 = train_sub3.drop('id', axis=1)\ntest_sub3 = test_sub3.drop('id', axis=1) ","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:20:18.655596Z","iopub.status.busy":"2024-12-17T13:20:18.655036Z","iopub.status.idle":"2024-12-17T13:20:18.668164Z","shell.execute_reply":"2024-12-17T13:20:18.667574Z"},"papermill":{"duration":0.043692,"end_time":"2024-12-17T13:20:18.669647","exception":false,"start_time":"2024-12-17T13:20:18.625955","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"0fe1a552","cell_type":"code","source":"features_sub3 = total_features + time_series_cols","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:20:18.724682Z","iopub.status.busy":"2024-12-17T13:20:18.723966Z","iopub.status.idle":"2024-12-17T13:20:18.727508Z","shell.execute_reply":"2024-12-17T13:20:18.726771Z"},"papermill":{"duration":0.032782,"end_time":"2024-12-17T13:20:18.728983","exception":false,"start_time":"2024-12-17T13:20:18.696201","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"23f408fa","cell_type":"code","source":"train_sub3 = train_sub3.dropna(subset='sii')","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:20:18.783718Z","iopub.status.busy":"2024-12-17T13:20:18.783464Z","iopub.status.idle":"2024-12-17T13:20:18.789979Z","shell.execute_reply":"2024-12-17T13:20:18.789281Z"},"papermill":{"duration":0.0362,"end_time":"2024-12-17T13:20:18.791651","exception":false,"start_time":"2024-12-17T13:20:18.755451","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"10d17da7","cell_type":"code","source":"train_sub3 = update(train_sub3)\ntest_sub3 = update(test_sub3)\n\nfor col in cat_c:\n    mapping = create_mapping(col, train_sub3)\n    mappingTe = create_mapping(col, test_sub3)\n    \n    train_sub3[col] = train_sub3[col].replace(mapping).astype(int)\n    test_sub3[col] = test_sub3[col].replace(mappingTe).astype(int)\n","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:20:18.848304Z","iopub.status.busy":"2024-12-17T13:20:18.847644Z","iopub.status.idle":"2024-12-17T13:20:18.908271Z","shell.execute_reply":"2024-12-17T13:20:18.907641Z"},"papermill":{"duration":0.090631,"end_time":"2024-12-17T13:20:18.909984","exception":false,"start_time":"2024-12-17T13:20:18.819353","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"35dcdc80","cell_type":"code","source":"Params = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01  # Increased from 2.68e-06\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'cat_features': cat_c,\n    'verbose': 0,\n    'l2_leaf_reg': 10  # Increase this value\n}\n","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:20:18.964075Z","iopub.status.busy":"2024-12-17T13:20:18.963826Z","iopub.status.idle":"2024-12-17T13:20:18.968652Z","shell.execute_reply":"2024-12-17T13:20:18.967879Z"},"papermill":{"duration":0.03347,"end_time":"2024-12-17T13:20:18.970112","exception":false,"start_time":"2024-12-17T13:20:18.936642","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"43f0563d","cell_type":"code","source":"Light = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)\n])\n","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:20:19.025331Z","iopub.status.busy":"2024-12-17T13:20:19.024900Z","iopub.status.idle":"2024-12-17T13:20:19.029217Z","shell.execute_reply":"2024-12-17T13:20:19.028583Z"},"papermill":{"duration":0.033033,"end_time":"2024-12-17T13:20:19.030711","exception":false,"start_time":"2024-12-17T13:20:18.997678","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"3410878b","cell_type":"code","source":"X_sub3 = train_sub3[features_sub3]\ny_sub3 = train_sub3['sii']\ntest_sub3 = test_sub3[features_sub3]","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:20:19.084651Z","iopub.status.busy":"2024-12-17T13:20:19.084384Z","iopub.status.idle":"2024-12-17T13:20:19.090735Z","shell.execute_reply":"2024-12-17T13:20:19.089930Z"},"papermill":{"duration":0.035111,"end_time":"2024-12-17T13:20:19.092403","exception":false,"start_time":"2024-12-17T13:20:19.057292","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"1d789c78","cell_type":"code","source":"submission3, _, _, _, _= TrainML(voting_model, X_sub3, y_sub3, test_sub3)","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:20:19.149689Z","iopub.status.busy":"2024-12-17T13:20:19.149381Z","iopub.status.idle":"2024-12-17T13:21:09.008316Z","shell.execute_reply":"2024-12-17T13:21:09.007309Z"},"papermill":{"duration":49.889426,"end_time":"2024-12-17T13:21:09.010103","exception":false,"start_time":"2024-12-17T13:20:19.120677","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"11e6a864","cell_type":"markdown","source":"# Submission 4","metadata":{"papermill":{"duration":0.026521,"end_time":"2024-12-17T13:21:09.064665","exception":false,"start_time":"2024-12-17T13:21:09.038144","status":"completed"},"tags":[]}},{"id":"49e5fabc","cell_type":"code","source":"imputer = SimpleImputer(strategy='median')\n\nensemble = VotingRegressor(estimators=[\n    ('lgb', Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(random_state=SEED))])),\n    ('xgb', Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(random_state=SEED))])),\n    ('cat', Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(random_state=SEED, silent=True))])),\n    ('rf', Pipeline(steps=[('imputer', imputer), ('regressor', RandomForestRegressor(random_state=SEED))])),\n    ('gb', Pipeline(steps=[('imputer', imputer), ('regressor', GradientBoostingRegressor(random_state=SEED))]))\n])\n\n\nsubmission4, _, _, _, _= TrainML(ensemble, X_sub3, y_sub3, test_sub3)","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:21:09.119593Z","iopub.status.busy":"2024-12-17T13:21:09.119041Z","iopub.status.idle":"2024-12-17T13:23:10.266336Z","shell.execute_reply":"2024-12-17T13:23:10.265492Z"},"papermill":{"duration":121.176996,"end_time":"2024-12-17T13:23:10.268131","exception":false,"start_time":"2024-12-17T13:21:09.091135","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"0f57f1f3","cell_type":"markdown","source":"# Final","metadata":{"papermill":{"duration":0.028507,"end_time":"2024-12-17T13:23:10.325060","exception":false,"start_time":"2024-12-17T13:23:10.296553","status":"completed"},"tags":[]}},{"id":"6d29c843","cell_type":"code","source":"sub1 = submission2\nsub2 = submission3\nsub3 = submission4\n\nsub1 = sub1.sort_values(by='id').reset_index(drop=True)\nsub2 = sub2.sort_values(by='id').reset_index(drop=True)\nsub3 = sub3.sort_values(by='id').reset_index(drop=True)\n\ncombined = pd.DataFrame({\n    'id': sub1['id'],\n    'sii_1': sub1['sii'],\n    'sii_2': sub2['sii'],\n    'sii_3': sub3['sii']\n})\n\ndef majority_vote(row):\n    return row.mode()[0]\n\ncombined['final_sii'] = combined[['sii_1', 'sii_2', 'sii_3']].apply(majority_vote, axis=1)\n\nfinal_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\nfinal_submission.to_csv('submission.csv', index=False)\n\nprint(\"Majority voting completed and saved to 'Final_Submission.csv'\")","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:23:10.380882Z","iopub.status.busy":"2024-12-17T13:23:10.380373Z","iopub.status.idle":"2024-12-17T13:23:10.400012Z","shell.execute_reply":"2024-12-17T13:23:10.398986Z"},"papermill":{"duration":0.049454,"end_time":"2024-12-17T13:23:10.401722","exception":false,"start_time":"2024-12-17T13:23:10.352268","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"b5409660","cell_type":"code","source":"final_submission","metadata":{"execution":{"iopub.execute_input":"2024-12-17T13:23:10.460005Z","iopub.status.busy":"2024-12-17T13:23:10.459715Z","iopub.status.idle":"2024-12-17T13:23:10.467476Z","shell.execute_reply":"2024-12-17T13:23:10.466677Z"},"papermill":{"duration":0.037969,"end_time":"2024-12-17T13:23:10.469034","exception":false,"start_time":"2024-12-17T13:23:10.431065","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"b97388a3","cell_type":"code","source":"","metadata":{"papermill":{"duration":0.027147,"end_time":"2024-12-17T13:23:10.525351","exception":false,"start_time":"2024-12-17T13:23:10.498204","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}