{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":10178857,"sourceType":"datasetVersion","datasetId":6287320}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":171.504274,"end_time":"2024-12-18T08:35:12.428568","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-12-18T08:32:20.924294","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip -q install /kaggle/input/pytorch-tabnet-4-1-0-py3-none-any-whl/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:41:56.022141Z","iopub.execute_input":"2024-12-18T12:41:56.022707Z","iopub.status.idle":"2024-12-18T12:42:40.702379Z","shell.execute_reply.started":"2024-12-18T12:41:56.022627Z","shell.execute_reply":"2024-12-18T12:42:40.700645Z"},"papermill":{"duration":40.335871,"end_time":"2024-12-18T08:33:03.396173","exception":false,"start_time":"2024-12-18T08:32:23.060302","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Overview","metadata":{}},{"cell_type":"markdown","source":"I found a huge difference in performance between regressors and classifiers.\nRegressors handle continuous outputs and minimize errors, Classifiers aim to directly predict class labels.\nWhen using a regressor, the predicted continuous values (or probabilities) can be thresholded.","metadata":{}},{"cell_type":"markdown","source":"# Import libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\n\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score, mean_squared_error\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom sklearn.datasets import make_classification\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom pytorch_tabnet.tab_model import TabNetRegressor\nfrom sklearn.base import clone\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\nfrom sklearn.compose import ColumnTransformer\n\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.model_selection import train_test_split\nfrom pytorch_tabnet.callbacks import Callback\nimport os\nimport torch\nfrom pytorch_tabnet.callbacks import Callback\n","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:42:40.705276Z","iopub.execute_input":"2024-12-18T12:42:40.705716Z","iopub.status.idle":"2024-12-18T12:43:01.981459Z","shell.execute_reply.started":"2024-12-18T12:42:40.705664Z","shell.execute_reply":"2024-12-18T12:43:01.980260Z"},"papermill":{"duration":17.835296,"end_time":"2024-12-18T08:33:21.235607","exception":false,"start_time":"2024-12-18T08:33:03.400311","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42\nn_splits = 5","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:43:01.982863Z","iopub.execute_input":"2024-12-18T12:43:01.983814Z","iopub.status.idle":"2024-12-18T12:43:01.989536Z","shell.execute_reply.started":"2024-12-18T12:43:01.983775Z","shell.execute_reply":"2024-12-18T12:43:01.988167Z"},"papermill":{"duration":0.009741,"end_time":"2024-12-18T08:33:21.254173","exception":false,"start_time":"2024-12-18T08:33:21.244432","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Read time - series data with AutoEncoder","metadata":{}},{"cell_type":"code","source":"# 1. Processes a file containing time-series data\n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:43:01.992689Z","iopub.execute_input":"2024-12-18T12:43:01.993612Z","iopub.status.idle":"2024-12-18T12:43:02.005827Z","shell.execute_reply.started":"2024-12-18T12:43:01.993556Z","shell.execute_reply":"2024-12-18T12:43:02.004478Z"},"papermill":{"duration":0.009491,"end_time":"2024-12-18T08:33:21.268236","exception":false,"start_time":"2024-12-18T08:33:21.258745","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 2. Aggregates statistics from multiple files into a single DataFrame\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n\n    stats, indexes = zip(*results)\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:43:02.007433Z","iopub.execute_input":"2024-12-18T12:43:02.007984Z","iopub.status.idle":"2024-12-18T12:43:02.019499Z","shell.execute_reply.started":"2024-12-18T12:43:02.007943Z","shell.execute_reply":"2024-12-18T12:43:02.018261Z"},"papermill":{"duration":0.009396,"end_time":"2024-12-18T08:33:21.281255","exception":false,"start_time":"2024-12-18T08:33:21.271859","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 3. Defines a neural network-based autoencoder for dimensionality reduction and feature extraction.\n\nclass AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n        \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:43:02.020793Z","iopub.execute_input":"2024-12-18T12:43:02.021202Z","iopub.status.idle":"2024-12-18T12:43:02.042249Z","shell.execute_reply.started":"2024-12-18T12:43:02.021156Z","shell.execute_reply":"2024-12-18T12:43:02.040576Z"},"papermill":{"duration":0.009592,"end_time":"2024-12-18T08:33:21.294467","exception":false,"start_time":"2024-12-18T08:33:21.284875","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4. Trains the autoencoder and returns the encoded features\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}')\n                 \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded\n","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:43:02.043850Z","iopub.execute_input":"2024-12-18T12:43:02.045105Z","iopub.status.idle":"2024-12-18T12:43:02.057065Z","shell.execute_reply.started":"2024-12-18T12:43:02.045048Z","shell.execute_reply":"2024-12-18T12:43:02.055701Z"},"papermill":{"duration":0.012363,"end_time":"2024-12-18T08:33:21.310383","exception":false,"start_time":"2024-12-18T08:33:21.298020","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 5. Trains the autoencoder and returns the encoded features\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}')\n                 \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded\n","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:43:02.058623Z","iopub.execute_input":"2024-12-18T12:43:02.059122Z","iopub.status.idle":"2024-12-18T12:43:02.077593Z","shell.execute_reply.started":"2024-12-18T12:43:02.059073Z","shell.execute_reply":"2024-12-18T12:43:02.076270Z"},"papermill":{"duration":0.012317,"end_time":"2024-12-18T08:33:21.326368","exception":false,"start_time":"2024-12-18T08:33:21.314051","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:43:02.079117Z","iopub.execute_input":"2024-12-18T12:43:02.079500Z","iopub.status.idle":"2024-12-18T12:44:34.324088Z","shell.execute_reply.started":"2024-12-18T12:43:02.079462Z","shell.execute_reply":"2024-12-18T12:44:34.322759Z"},"papermill":{"duration":55.627029,"end_time":"2024-12-18T08:34:17.027439","exception":false,"start_time":"2024-12-18T08:33:21.400410","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop the 'id' column to focus on the features\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\n# Apply the autoencoder\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=100, batch_size=32)\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=100, batch_size=32)\n\n# Get the list of encoded column names\ntime_series_cols = train_ts_encoded.columns.tolist()\n\n# Add the 'id' column back\ntrain_ts_encoded[\"id\"]=train_ts[\"id\"]\ntest_ts_encoded['id']=test_ts[\"id\"]\n\n#  Assign back to the original variables\ntrain_ts=train_ts_encoded\ntest_ts=test_ts_encoded","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:44:34.328721Z","iopub.execute_input":"2024-12-18T12:44:34.329135Z","iopub.status.idle":"2024-12-18T12:44:51.366767Z","shell.execute_reply.started":"2024-12-18T12:44:34.329098Z","shell.execute_reply":"2024-12-18T12:44:51.364847Z"},"papermill":{"duration":8.171164,"end_time":"2024-12-18T08:34:25.281106","exception":false,"start_time":"2024-12-18T08:34:17.109942","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Read csv files","metadata":{}},{"cell_type":"code","source":"# read csv files\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:44:51.369555Z","iopub.execute_input":"2024-12-18T12:44:51.372258Z","iopub.status.idle":"2024-12-18T12:44:51.476730Z","shell.execute_reply.started":"2024-12-18T12:44:51.372208Z","shell.execute_reply":"2024-12-18T12:44:51.475227Z"},"papermill":{"duration":0.06651,"end_time":"2024-12-18T08:33:21.396386","exception":false,"start_time":"2024-12-18T08:33:21.329876","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Important features","metadata":{}},{"cell_type":"code","source":"# set features choosen in previous notebook by importances\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:44:51.478489Z","iopub.execute_input":"2024-12-18T12:44:51.478907Z","iopub.status.idle":"2024-12-18T12:44:51.486935Z","shell.execute_reply.started":"2024-12-18T12:44:51.478854Z","shell.execute_reply":"2024-12-18T12:44:51.485198Z"},"papermill":{"duration":0.02071,"end_time":"2024-12-18T08:34:17.096437","exception":false,"start_time":"2024-12-18T08:34:17.075727","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prepare dataset for training","metadata":{}},{"cell_type":"code","source":"# Prepare the time_series_cols \ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\n# Merge train and test datasets with their respective time series data\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\n# Drop 'id'\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)  \n\n# Update featuresCols\nfeaturesCols += time_series_cols\n\n# Filter columns and remove rows with NaN in 'sii'\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:44:51.488694Z","iopub.execute_input":"2024-12-18T12:44:51.489087Z","iopub.status.idle":"2024-12-18T12:44:51.545463Z","shell.execute_reply.started":"2024-12-18T12:44:51.489051Z","shell.execute_reply":"2024-12-18T12:44:51.544465Z"},"papermill":{"duration":0.046538,"end_time":"2024-12-18T08:34:25.342151","exception":false,"start_time":"2024-12-18T08:34:25.295613","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill Nan as 'missing' in categorical data\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:44:51.547030Z","iopub.execute_input":"2024-12-18T12:44:51.547539Z","iopub.status.idle":"2024-12-18T12:44:51.555274Z","shell.execute_reply.started":"2024-12-18T12:44:51.547475Z","shell.execute_reply":"2024-12-18T12:44:51.553639Z"},"papermill":{"duration":0.020843,"end_time":"2024-12-18T08:34:25.377377","exception":false,"start_time":"2024-12-18T08:34:25.356534","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = update(train)\ntest = update(test)","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:44:51.557125Z","iopub.execute_input":"2024-12-18T12:44:51.557686Z","iopub.status.idle":"2024-12-18T12:44:51.603547Z","shell.execute_reply.started":"2024-12-18T12:44:51.557640Z","shell.execute_reply":"2024-12-18T12:44:51.602216Z"},"papermill":{"duration":0.038147,"end_time":"2024-12-18T08:34:25.429928","exception":false,"start_time":"2024-12-18T08:34:25.391781","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# create mappings for categorical columns\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n\n# Quadratic Weighted Kappa function\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n# Evaluate predictions with thresholds\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T12:44:51.605213Z","iopub.execute_input":"2024-12-18T12:44:51.605619Z","iopub.status.idle":"2024-12-18T12:44:51.667376Z","shell.execute_reply.started":"2024-12-18T12:44:51.605581Z","shell.execute_reply":"2024-12-18T12:44:51.666273Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# Main model training function\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.6264773 , 0.89171596, 1.64], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    thresholds = KappaOPtimizer.x\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, thresholds)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    fold_weights = [1.25, 1.0, 1.0, 1.0, 1.0]\n    tpm = test_preds.dot(fold_weights) / np.sum(fold_weights)\n    tpTuned = threshold_Rounder(tpm, thresholds)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:44:51.668845Z","iopub.execute_input":"2024-12-18T12:44:51.669221Z","iopub.status.idle":"2024-12-18T12:44:51.682567Z","shell.execute_reply.started":"2024-12-18T12:44:51.669187Z","shell.execute_reply":"2024-12-18T12:44:51.681188Z"},"papermill":{"duration":0.058134,"end_time":"2024-12-18T08:34:25.528306","exception":false,"start_time":"2024-12-18T08:34:25.470172","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# First Simple Model","metadata":{}},{"cell_type":"code","source":"\nmodel = XGBRegressor(\n    learning_rate=0.05,\n    max_depth=6,\n    n_estimators=200,\n    subsample=0.8,\n    colsample_bytree = 0.8,\n    reg_alpha=1,\n    reg_lambda=5,\n    random_state=SEED\n)\n\n# we get out of fold predictions for further exploration\nsubmission = TrainML(model, test)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:44:51.684110Z","iopub.execute_input":"2024-12-18T12:44:51.684470Z","iopub.status.idle":"2024-12-18T12:45:12.746122Z","shell.execute_reply.started":"2024-12-18T12:44:51.684436Z","shell.execute_reply":"2024-12-18T12:45:12.744820Z"},"papermill":{"duration":12.100004,"end_time":"2024-12-18T08:34:37.642219","exception":false,"start_time":"2024-12-18T08:34:25.542215","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ensemble Model \n## using LightGBM, XGBoost, CatBoost","metadata":{}},{"cell_type":"code","source":"\n# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.045,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01  # Increased from 2.68e-06\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'cat_features': cat_c,\n    'verbose': 0,\n    'l2_leaf_reg': 10  # Increase this value\n}\n\n\n\n# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n#TabNet_Model = TabNetWrapper(**TabNet_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)\n    ]\n    )\n\n# Train the ensemble model\nSubmission2 = TrainML(voting_model, test)\n\n# Save submission\n#Submission2.to_csv('submission.csv', index=False)\nSubmission2","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:45:12.747360Z","iopub.execute_input":"2024-12-18T12:45:12.747710Z","iopub.status.idle":"2024-12-18T12:46:04.440403Z","shell.execute_reply.started":"2024-12-18T12:45:12.747676Z","shell.execute_reply":"2024-12-18T12:46:04.439322Z"},"papermill":{"duration":31.424689,"end_time":"2024-12-18T08:35:09.116025","exception":false,"start_time":"2024-12-18T08:34:37.691336","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# Save submission\nSubmission2.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-12-18T12:46:04.442081Z","iopub.execute_input":"2024-12-18T12:46:04.443067Z","iopub.status.idle":"2024-12-18T12:46:04.451681Z","shell.execute_reply.started":"2024-12-18T12:46:04.443010Z","shell.execute_reply":"2024-12-18T12:46:04.450641Z"},"papermill":{"duration":0.022644,"end_time":"2024-12-18T08:35:09.153075","exception":false,"start_time":"2024-12-18T08:35:09.130431","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n**Author:** Beata Faron  \n[LinkedIn](https://www.linkedin.com/in/beata-faron-24764832/) • [Kaggle](https://www.kaggle.com/beatafaron)\n\n*Data Scientist with a background in business, design, and machine learning. Focused on time series forecasting and real-world applications.*\n","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}