{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Imports and Set Up","metadata":{}},{"cell_type":"code","source":"!pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:33:00.313850Z","iopub.execute_input":"2024-12-11T00:33:00.314454Z","iopub.status.idle":"2024-12-11T00:33:09.775763Z","shell.execute_reply.started":"2024-12-11T00:33:00.314414Z","shell.execute_reply":"2024-12-11T00:33:09.774929Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install pytorch-tabnet","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:33:09.777706Z","iopub.execute_input":"2024-12-11T00:33:09.777992Z","iopub.status.idle":"2024-12-11T00:33:18.527972Z","shell.execute_reply.started":"2024-12-11T00:33:09.777963Z","shell.execute_reply":"2024-12-11T00:33:18.526866Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport random\nimport warnings\nfrom concurrent.futures import ThreadPoolExecutor\nfrom sklearn.model_selection import train_test_split\nfrom scipy.optimize import minimize\nimport seaborn as sns\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nfrom lightgbm import LGBMClassifier, LGBMRegressor\nfrom matplotlib import pyplot as plt\nfrom sklearn.base import clone\nfrom sklearn.ensemble import VotingClassifier, VotingRegressor, StackingClassifier, StackingRegressor\nfrom sklearn.impute import KNNImputer\nfrom sklearn.metrics import (accuracy_score, cohen_kappa_score,\n                             confusion_matrix, f1_score, mean_absolute_error,\n                             mean_squared_error, precision_score, recall_score,\n                             classification_report, make_scorer)\nfrom sklearn.model_selection import (GridSearchCV, KFold, RandomizedSearchCV,\n                                     cross_val_score, ParameterGrid)\nfrom sklearn.svm import SVC\nfrom sklearn.linear_model import LogisticRegression, Ridge, Lasso\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\nfrom tqdm import tqdm\nfrom xgboost import XGBClassifier, XGBRegressor\nfrom catboost import CatBoostClassifier, CatBoostRegressor\nfrom sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\nfrom imblearn.under_sampling import NearMiss\nfrom pytorch_tabnet.tab_model import TabNetClassifier, TabNetRegressor\n\nfrom torch.utils.data import DataLoader, TensorDataset\n\nimport torch.optim.lr_scheduler as lr_scheduler\nimport optuna\nfrom optuna import Trial","metadata":{"execution":{"iopub.status.busy":"2024-12-11T00:33:18.529367Z","iopub.execute_input":"2024-12-11T00:33:18.529703Z","iopub.status.idle":"2024-12-11T00:33:27.241880Z","shell.execute_reply.started":"2024-12-11T00:33:18.529673Z","shell.execute_reply":"2024-12-11T00:33:27.240924Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"warnings.filterwarnings('ignore')\npd.options.display.max_columns = None","metadata":{"execution":{"iopub.status.busy":"2024-12-11T00:33:27.244143Z","iopub.execute_input":"2024-12-11T00:33:27.245239Z","iopub.status.idle":"2024-12-11T00:33:27.249185Z","shell.execute_reply.started":"2024-12-11T00:33:27.245187Z","shell.execute_reply":"2024-12-11T00:33:27.248145Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed_value=2024):\n    random.seed(seed_value)\n    np.random.seed(seed_value)\n    torch.manual_seed(seed_value)\n    torch.cuda.manual_seed(seed_value)\n    torch.backends.cudnn.deterministic = True\n\nset_seed(2024)","metadata":{"execution":{"iopub.status.busy":"2024-12-11T00:33:27.250117Z","iopub.execute_input":"2024-12-11T00:33:27.250376Z","iopub.status.idle":"2024-12-11T00:33:27.264944Z","shell.execute_reply.started":"2024-12-11T00:33:27.250341Z","shell.execute_reply":"2024-12-11T00:33:27.264347Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Processing","metadata":{}},{"cell_type":"markdown","source":"### Load in Files","metadata":{}},{"cell_type":"code","source":"file_path = '/kaggle/input/child-mind-institute-problematic-internet-use'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:33:27.266017Z","iopub.execute_input":"2024-12-11T00:33:27.266353Z","iopub.status.idle":"2024-12-11T00:33:27.276145Z","shell.execute_reply.started":"2024-12-11T00:33:27.266317Z","shell.execute_reply":"2024-12-11T00:33:27.275427Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TRAIN_CSV = f'{file_path}/train.csv'\nTEST_CSV = f'{file_path}/test.csv'\nSAMPLE_SUBMISSION_CSV = f'{file_path}/sample_submission.csv'\nSERIES_TRAIN_DIR = f'{file_path}/series_train.parquet'\nSERIES_TEST_DIR = f'{file_path}/series_test.parquet'\n\n\ntrain_df = pd.read_csv(TRAIN_CSV)\ntest_df = pd.read_csv(TEST_CSV) # \nsample_submission_df = pd.read_csv(SAMPLE_SUBMISSION_CSV)\n\n# Drop all the PCIAT variables as they are not present in the test data\nfor col in train_df.columns:\n    if 'PCIAT' in col:\n        train_df.drop(col, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-12-11T00:33:27.277053Z","iopub.execute_input":"2024-12-11T00:33:27.277302Z","iopub.status.idle":"2024-12-11T00:33:27.384346Z","shell.execute_reply.started":"2024-12-11T00:33:27.277278Z","shell.execute_reply":"2024-12-11T00:33:27.383452Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to process individual time series files\n\n# Min length of the time series is 927 (hardcoded to speed this up)\ndef process_time_series(file_name, directory, max_len=927):\n    df = pd.read_parquet(os.path.join(directory, file_name, 'part-0.parquet'))\n    df['id'] = file_name.split('=')[1]\n    # Adjusted this so we get the full parquet - just concatenate all of them\n    return df[:max_len]","metadata":{"execution":{"iopub.status.busy":"2024-12-11T00:33:27.385436Z","iopub.execute_input":"2024-12-11T00:33:27.385706Z","iopub.status.idle":"2024-12-11T00:33:27.390331Z","shell.execute_reply.started":"2024-12-11T00:33:27.385681Z","shell.execute_reply":"2024-12-11T00:33:27.389449Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to load and aggregate time series data\ndef load_time_series_data(directory):\n    file_names = os.listdir(directory)\n\n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_time_series(fname, directory), file_names),\n                            total=len(file_names)))\n\n    return pd.concat(results, ignore_index=True) # This takes slightly longer but ok for now","metadata":{"execution":{"iopub.status.busy":"2024-12-11T00:33:27.391339Z","iopub.execute_input":"2024-12-11T00:33:27.391623Z","iopub.status.idle":"2024-12-11T00:33:27.401160Z","shell.execute_reply.started":"2024-12-11T00:33:27.391599Z","shell.execute_reply":"2024-12-11T00:33:27.400352Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_series_df = load_time_series_data(SERIES_TRAIN_DIR)\n# test_series_df = load_time_series_data(SERIES_TEST_DIR)","metadata":{"execution":{"iopub.status.busy":"2024-12-11T00:33:27.405114Z","iopub.execute_input":"2024-12-11T00:33:27.405360Z","iopub.status.idle":"2024-12-11T00:33:50.282830Z","shell.execute_reply.started":"2024-12-11T00:33:27.405336Z","shell.execute_reply":"2024-12-11T00:33:50.281796Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Replace with missing for the split and then reassign the values at the end\ntrain_df['sii'] = train_df['sii'].replace({np.nan: -1})\ntrain_df, test_df = train_test_split(train_df, test_size=0.2, random_state=2024, stratify=train_df['sii'])\ntest_series_df = train_series_df[train_series_df.id.isin(test_df.id)]\ntrain_series_df = train_series_df[train_series_df.id.isin(train_df.id)]\ntrain_df['sii'] = train_df['sii'].replace({-1: np.nan})\ntest_df['sii'] = test_df['sii'].replace({-1: np.nan})\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:33:50.284203Z","iopub.execute_input":"2024-12-11T00:33:50.284581Z","iopub.status.idle":"2024-12-11T00:33:50.580526Z","shell.execute_reply.started":"2024-12-11T00:33:50.284545Z","shell.execute_reply":"2024-12-11T00:33:50.579571Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Encoding of Time Series Data","metadata":{}},{"cell_type":"code","source":"# Encoder Class\nclass Encoder(nn.Module):\n    def __init__(self, input_size, hidden_size, dropout, seq_len, num_layers):\n        super(Encoder, self).__init__()\n        self.input_size = input_size\n        self.hidden_size = hidden_size\n        self.dropout = dropout\n        self.seq_len = seq_len\n        self.num_layers = num_layers\n\n        self.lstm_enc = nn.LSTM(input_size=input_size, num_layers=num_layers, hidden_size=hidden_size, dropout=dropout, batch_first=True)\n\n    def forward(self, x):\n        out, (last_h_state, last_c_state) = self.lstm_enc(x)\n        # Get the last hidden state\n        x_enc = last_h_state[-1, :, :]         \n        return x_enc, out\n\n\n# Decoder Class\nclass Decoder(nn.Module):\n    def __init__(self, input_size, hidden_size, dropout, seq_len, use_act, num_layers):\n        super(Decoder, self).__init__()\n        self.input_size = input_size\n        self.hidden_size = hidden_size\n        self.dropout = dropout\n        self.seq_len = seq_len\n        self.use_act = use_act  # Parameter to control the last sigmoid activation - depends on the normalization used.\n        self.act = nn.Sigmoid()\n\n        self.lstm_dec = nn.LSTM(input_size=hidden_size, num_layers=num_layers, hidden_size=hidden_size, dropout=dropout, batch_first=True)\n        self.fc = nn.Linear(hidden_size, input_size)\n\n    def forward(self, z):\n        z = z.unsqueeze(1).repeat(1, self.seq_len, 1)\n        dec_out, (hidden_state, cell_state) = self.lstm_dec(z)\n        dec_out = self.fc(dec_out)\n        if self.use_act:\n            dec_out = self.act(dec_out)\n\n        return dec_out, hidden_state\n\n\n# LSTM Auto-Encoder Class\nclass TimeSeriesAutoencoder(nn.Module):\n    def __init__(self, input_size, hidden_size, dropout_ratio, seq_len, num_layers, use_act=True):\n        super(TimeSeriesAutoencoder, self).__init__()\n        self.input_size = input_size\n        self.hidden_size = hidden_size\n        self.dropout_ratio = dropout_ratio\n        self.seq_len = seq_len\n\n        self.encoder = Encoder(input_size=input_size, num_layers=num_layers, hidden_size=hidden_size, dropout=dropout_ratio, seq_len=seq_len)\n        self.decoder = Decoder(input_size=input_size, num_layers=num_layers, hidden_size=hidden_size, dropout=dropout_ratio, seq_len=seq_len, use_act=use_act)\n\n    def forward(self, x, return_last_h=False, return_enc_out=False):\n        x_enc, enc_out = self.encoder(x)\n        x_dec, last_h = self.decoder(x_enc)\n\n        if return_last_h:\n            return x_dec, last_h\n        elif return_enc_out:\n            return x_dec, enc_out\n        return x_dec","metadata":{"execution":{"iopub.status.busy":"2024-12-11T00:33:50.641594Z","iopub.execute_input":"2024-12-11T00:33:50.641933Z","iopub.status.idle":"2024-12-11T00:33:50.664038Z","shell.execute_reply.started":"2024-12-11T00:33:50.641893Z","shell.execute_reply":"2024-12-11T00:33:50.663068Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def encodedDataPrep(train, test):\n    scaler = StandardScaler()\n    scaled_data_train = scaler.fit_transform(train.drop('id', axis=1))\n    scaled_data_test = scaler.transform(test.drop('id', axis=1))\n    return scaled_data_train, scaled_data_test\n\n# Sequence length is hardcoded at 927\ndef prepare_data(df):\n    # Convert the dataframe to a numpy array\n    ids = df['id'].unique()\n    num_samples = len(ids)\n    num_features = len(df.columns) - 2\n\n    data = np.array(df.drop(['id', 'step'], axis = 1).values)\n    print(data.shape)\n    # Convert the numpy array to a PyTorch tensor\n    reshaped_data = data.reshape(num_samples, 927, num_features)\n\n    data_tensor = torch.tensor(reshaped_data, dtype=torch.float32)\n\n\n    return data_tensor, ids\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:33:50.665649Z","iopub.execute_input":"2024-12-11T00:33:50.666117Z","iopub.status.idle":"2024-12-11T00:33:50.681418Z","shell.execute_reply.started":"2024-12-11T00:33:50.666073Z","shell.execute_reply":"2024-12-11T00:33:50.680433Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_encoded_features(df, lr=0.01, hidden_size=16, num_layers=1, device = 'mps', dropout_ratio=0.1, epochs=100, batch_size=32, sequence_length = 927):\n    # Reshape to df for easier column handling and id tracking\n    scaled_df = df.drop(['id', 'step'], axis=1)\n    incl_columns = list(df.columns)\n    incl_columns.remove('id')\n    print(incl_columns)\n    scaled_data_df = pd.DataFrame(scaled_df, columns=incl_columns)\n    scaled_data_df['id'] = df['id']\n    \n    tensor_data, ids = prepare_data(scaled_data_df)\n    input_size = tensor_data.shape[2]\n\n    autoencoder = TimeSeriesAutoencoder(input_size, hidden_size, dropout_ratio, sequence_length, num_layers=num_layers)\n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters(), lr=lr)\n\n    #device = torch.device('mps')\n    #autoencoder = autoencoder.to(device)\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"mps\" if torch.backends.mps.is_available() else \"cpu\")\n    autoencoder = autoencoder.to(device)\n\n    dataset = TensorDataset(tensor_data)\n    data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=False)\n    \n    # Step 5: Training Loop\n    autoencoder.train()\n    for epoch in range(epochs):\n        epoch_loss = 0\n        for batch in data_loader:\n            batch_data = batch[0].to(device)\n            optimizer.zero_grad()\n            outputs = autoencoder(batch_data)\n            loss = criterion(outputs, batch_data)\n            loss.backward()\n            optimizer.step()\n            epoch_loss += loss.item()\n\n        avg_epoch_loss = epoch_loss / len(data_loader)\n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Avg Loss: {avg_epoch_loss:.4f}')\n\n    with torch.no_grad():\n        tensor_data = tensor_data.to(device)\n        encoded_data, vals = autoencoder.encoder(tensor_data)  # Get encoded features\n\n    encoded_data = encoded_data.cpu().numpy()\n\n    encoded_df = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    encoded_df['id'] = ids\n    return encoded_df, autoencoder","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:33:50.682986Z","iopub.execute_input":"2024-12-11T00:33:50.683719Z","iopub.status.idle":"2024-12-11T00:33:50.696057Z","shell.execute_reply.started":"2024-12-11T00:33:50.683681Z","shell.execute_reply":"2024-12-11T00:33:50.695445Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def get_encoded_features(train_df, val_df, trial: Trial, device='mps', epochs=100, sequence_length=927):\n#     # Get hyperparameters from the trial object\n#     lr = 0.01\n#     # hidden_size = trial.suggest_int('hidden_size', 16, 20)  # Hidden layer size between 32 and 128\n#     hidden_size = 16\n#     num_layers = 1\n#     batch_size = 32\n#     dropout_ratio = 0.1\n#     # Prepare the data (train and val sets)\n#     train_tensor_data, train_ids = prepare_data(train_df)\n#     val_tensor_data, val_ids = prepare_data(val_df)\n    \n#     input_size = train_tensor_data.shape[2]\n\n#     # Build the autoencoder model with the hyperparameters suggested by Optuna\n#     autoencoder = TimeSeriesAutoencoder(input_size, hidden_size, dropout_ratio, sequence_length, num_layers=num_layers)\n#     criterion = nn.MSELoss()\n    \n#     # Use AdamW optimizer\n#     optimizer = optim.AdamW(autoencoder.parameters(), lr=lr, weight_decay=1e-5)\n\n#     device = torch.device(device)\n#     autoencoder = autoencoder.to(device)\n\n#     train_dataset = TensorDataset(train_tensor_data)\n#     val_dataset = TensorDataset(val_tensor_data)\n\n#     train_data_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False)\n#     val_data_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n    \n#     # Learning rate scheduler: Reduce the learning rate if the loss plateaus\n#     scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=5, factor=0.5, verbose=True)\n    \n#     # Training Loop\n#     autoencoder.train()\n#     for epoch in range(epochs):\n#         epoch_loss = 0\n#         for batch in train_data_loader:\n#             batch_data = batch[0].to(device)\n#             optimizer.zero_grad()\n#             outputs = autoencoder(batch_data)\n#             loss = criterion(outputs, batch_data)\n#             loss.backward()\n#             optimizer.step()\n#             epoch_loss += loss.item()\n\n#         avg_epoch_loss = epoch_loss / len(train_data_loader)\n        \n#         # Update the learning rate scheduler\n#         scheduler.step(avg_epoch_loss)\n        \n#         # Optionally, evaluate the model on the validation set every few epochs\n#         if (epoch + 1) % 10 == 0:\n#             val_loss = evaluate_model_on_val(val_data_loader, autoencoder, criterion, device)\n#             print(f\"Epoch [{epoch + 1}/{epochs}], Avg Train Loss: {avg_epoch_loss:.4f}, Val Loss: {val_loss:.4f}\")\n\n#     return avg_epoch_loss  # Return the final average training loss for the trial\n\n# def evaluate_model_on_val(val_data_loader, model, criterion, device):\n#     \"\"\"\n#     Evaluate the model on the validation set and return the loss.\n#     \"\"\"\n#     model.eval()  # Set the model to evaluation mode\n#     val_loss = 0\n#     with torch.no_grad():\n#         for batch in val_data_loader:\n#             batch_data = batch[0].to(device)\n#             outputs = model(batch_data)\n#             loss = criterion(outputs, batch_data)\n#             val_loss += loss.item()\n    \n#     return val_loss / len(val_data_loader)\n\n# def optimize_hyperparameters(df):\n#     # Split the dataset into training and validation sets\n#     unique_ids = df['id'].unique()\n\n#     # Split the unique ids into train and validation sets\n#     train_ids, val_ids = train_test_split(unique_ids, test_size=0.2, random_state=2024)\n\n#     # Now, use these ids to get the corresponding rows in the dataframe\n#     train_df = df[df['id'].isin(train_ids)]\n#     val_df = df[df['id'].isin(val_ids)]\n    \n#     # Define the Optuna study and objective function\n#     study = optuna.create_study(direction='minimize')  # We want to minimize the loss\n#     study.optimize(lambda trial: get_encoded_features(train_df, val_df, trial), n_trials=1)  # Number of trials\n\n#     # Print the best hyperparameters found\n#     print(f\"Best hyperparameters: {study.best_params}\")\n#     print(f\"Best loss: {study.best_value}\")\n    \n#     # Return the best model with the best hyperparameters\n#     best_model = study.best_model\n#     return best_model\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:33:50.697456Z","iopub.execute_input":"2024-12-11T00:33:50.697778Z","iopub.status.idle":"2024-12-11T00:33:50.711778Z","shell.execute_reply.started":"2024-12-11T00:33:50.697734Z","shell.execute_reply":"2024-12-11T00:33:50.711086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tensor_data_train, tensor_data_test = encodedDataPrep(train_series_df, test_series_df)\nnew_cols = train_series_df.columns\nnew_cols = new_cols.drop('id')\ndf_train = pd.DataFrame(tensor_data_train, columns=new_cols)\ndf_test = pd.DataFrame(tensor_data_test, columns=new_cols)\ntrain_series_df.reset_index(drop=True, inplace=True)\ntest_series_df.reset_index(drop=True, inplace=True)\ndf_train['id'] = train_series_df['id']\ndf_test['id'] = train_series_df['id']\n\ntrain_encoded, best_model = get_encoded_features(df_train)","metadata":{"execution":{"iopub.status.busy":"2024-12-11T00:33:50.713109Z","iopub.execute_input":"2024-12-11T00:33:50.713448Z","iopub.status.idle":"2024-12-11T00:34:11.688323Z","shell.execute_reply.started":"2024-12-11T00:33:50.713412Z","shell.execute_reply":"2024-12-11T00:34:11.687476Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def encode_test_data(autoencoder, test_df):\n    ids = test_df['id'].unique()\n    test_df = test_df.drop(['step', 'id'], axis=1).values.astype(np.float32)\n    test_tensor = torch.tensor(test_df, dtype=torch.float32)\n    test_tensor = test_tensor.reshape(-1, 927, test_df.shape[1])\n    #test_tensor = test_tensor.to('mps')\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"mps\" if torch.backends.mps.is_available() else \"cpu\")\n    test_tensor = test_tensor.to(device)\n    autoencoder = autoencoder.to(device)\n    print(test_tensor.shape)\n    with torch.no_grad():\n        encoded_data, vals = autoencoder.encoder(test_tensor)\n        encoded_data = encoded_data.cpu().numpy()\n    encoded_df = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    encoded_df['id'] = ids\n    return encoded_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:11.689607Z","iopub.execute_input":"2024-12-11T00:34:11.690462Z","iopub.status.idle":"2024-12-11T00:34:11.696985Z","shell.execute_reply.started":"2024-12-11T00:34:11.690413Z","shell.execute_reply":"2024-12-11T00:34:11.696140Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_encoded = encode_test_data(best_model, df_test)\n\n\ntrain_encoded['id'] = train_series_df['id']\ntest_encoded['id'] = test_series_df['id']\n\ntrain_df = train_df.merge(train_encoded, on='id', how='left')\ntest_df = test_df.merge(test_encoded, on='id', how='left')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:11.698057Z","iopub.execute_input":"2024-12-11T00:34:11.698327Z","iopub.status.idle":"2024-12-11T00:34:11.752567Z","shell.execute_reply.started":"2024-12-11T00:34:11.698301Z","shell.execute_reply":"2024-12-11T00:34:11.751722Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Imputation of Missing Numerical Values","metadata":{}},{"cell_type":"code","source":"# Imputing missing values using KNN imputer\n\ndef impute_missing_values(train_df, test_df, n_neighbors=5):\n    # Select only numeric columns\n    numeric_columns = train_df.select_dtypes(include=['float64', 'float32', 'int64']).columns\n\n    # Initialize the KNNImputer\n    imputer = KNNImputer(n_neighbors=n_neighbors)\n\n    # Fit on the training dataset and transform both datasets\n    train_imputed_array = imputer.fit_transform(train_df[numeric_columns])\n    test_imputed_array = imputer.transform(test_df[numeric_columns])\n\n    # Create DataFrames from the imputed arrays\n    train_imputed = pd.DataFrame(train_imputed_array, columns=numeric_columns, index=train_df.index)\n    test_imputed = pd.DataFrame(test_imputed_array, columns=numeric_columns, index=test_df.index)\n\n    # Replace numeric columns with imputed data\n    train_df[numeric_columns] = train_imputed\n    test_df[numeric_columns] = test_imputed\n\n    return train_df, test_df","metadata":{"execution":{"iopub.status.busy":"2024-12-11T00:34:11.753703Z","iopub.execute_input":"2024-12-11T00:34:11.753988Z","iopub.status.idle":"2024-12-11T00:34:11.760902Z","shell.execute_reply.started":"2024-12-11T00:34:11.753961Z","shell.execute_reply":"2024-12-11T00:34:11.760024Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_df.replace([np.inf, -np.inf], np.nan, inplace=True) # Debug why we have inf values\n\n# Apply imputation after splitting the dataset\ntrain_df, test_df = impute_missing_values(train_df, test_df)\n\n# Ensure target column 'sii' remains integer after imputation\ntrain_df['sii'] = train_df['sii'].round().astype(int)\ntest_df['sii'] = test_df['sii'].round().astype(int)\n\n# Imputation is needed in test set for some cases but not others to revisit\n\nprint(train_df.isna().sum())\nprint(test_df.isna().sum())","metadata":{"execution":{"iopub.status.busy":"2024-12-11T00:34:11.761821Z","iopub.execute_input":"2024-12-11T00:34:11.762073Z","iopub.status.idle":"2024-12-11T00:34:19.181105Z","shell.execute_reply.started":"2024-12-11T00:34:11.762049Z","shell.execute_reply":"2024-12-11T00:34:19.180139Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Categorical Processing","metadata":{}},{"cell_type":"code","source":"categorical_cols = list(train_df.select_dtypes(include=['object']).columns) #sii (outcome var) is categorical but we are encoding that differently\ncategorical_cols.remove('id')\nprint(categorical_cols)\n\ndef preprocess_categorical(df):\n    for col in categorical_cols:\n        df[col] = df[col].fillna('Missing').astype('category')\n    return df\n\ntrain_df = preprocess_categorical(train_df)\ntest_df = preprocess_categorical(test_df)\n\ntrain_df = pd.get_dummies(train_df, columns = categorical_cols, drop_first=True, dtype='int')\ntest_df = pd.get_dummies(test_df, columns = categorical_cols, drop_first=True, dtype='int')","metadata":{"execution":{"iopub.status.busy":"2024-12-11T00:34:19.182131Z","iopub.execute_input":"2024-12-11T00:34:19.182439Z","iopub.status.idle":"2024-12-11T00:34:19.227163Z","shell.execute_reply.started":"2024-12-11T00:34:19.182379Z","shell.execute_reply":"2024-12-11T00:34:19.226444Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = train_df.drop('id', axis=1)\ntest_df = test_df.drop('id', axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-12-11T00:34:19.228041Z","iopub.execute_input":"2024-12-11T00:34:19.228300Z","iopub.status.idle":"2024-12-11T00:34:19.234533Z","shell.execute_reply.started":"2024-12-11T00:34:19.228275Z","shell.execute_reply":"2024-12-11T00:34:19.233700Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the Quadratic Weighted Kappa metric\ndef quadratic_weighted_kappa(y_actual, y_predicted):\n    return cohen_kappa_score(y_actual, y_predicted, weights='quadratic')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:19.235589Z","iopub.execute_input":"2024-12-11T00:34:19.235901Z","iopub.status.idle":"2024-12-11T00:34:19.245445Z","shell.execute_reply.started":"2024-12-11T00:34:19.235866Z","shell.execute_reply":"2024-12-11T00:34:19.244669Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_cm(y_true, y_pred, title):\n    cm = confusion_matrix(y_true, y_pred)\n    plt.figure(figsize=(6, 5))\n    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=np.unique(y_test), yticklabels=np.unique(y_test))\n    plt.xlabel('Predicted')\n    plt.ylabel('True')\n    plt.title(f'{title} Confusion Matrix')\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:19.246335Z","iopub.execute_input":"2024-12-11T00:34:19.246651Z","iopub.status.idle":"2024-12-11T00:34:19.255201Z","shell.execute_reply.started":"2024-12-11T00:34:19.246627Z","shell.execute_reply":"2024-12-11T00:34:19.254578Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Resampling of Training Set","metadata":{}},{"cell_type":"code","source":"X = train_df.drop(columns='sii', axis=1)\ny = train_df['sii'].astype(int)\n\n# # Apply NearMiss for downsampling\n# nm = NearMiss(sampling_strategy='majority')  # You can change version to 2 or 3\n# # These are global variables for now\n# X_res, y_res = nm.fit_resample(X, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:19.260673Z","iopub.execute_input":"2024-12-11T00:34:19.260930Z","iopub.status.idle":"2024-12-11T00:34:19.267859Z","shell.execute_reply.started":"2024-12-11T00:34:19.260891Z","shell.execute_reply":"2024-12-11T00:34:19.267173Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Baseline - Random Forest Model","metadata":{}},{"cell_type":"code","source":"def train_and_evaluate(model, param_dist, n_iter=10, eval_metric=None):\n    kf = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)\n    search = RandomizedSearchCV(model, param_dist, n_iter=n_iter, cv=kf, n_jobs=-1, random_state=42, verbose=2, scoring=eval_metric)\n    search.fit(X, y) # The resampling did not help\n    best_model = search.best_estimator_\n    y_pred = best_model.predict(X)\n    print(f\"Best Parameters: {search.best_params_}\")\n    return best_model, y_pred, y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:19.268897Z","iopub.execute_input":"2024-12-11T00:34:19.269216Z","iopub.status.idle":"2024-12-11T00:34:19.277238Z","shell.execute_reply.started":"2024-12-11T00:34:19.269179Z","shell.execute_reply":"2024-12-11T00:34:19.276560Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rf_param_dist = {\n    'n_estimators': [100, 200, 300, 400],\n    'max_depth': [3, 4, 5, 6],\n    'min_samples_split': [2, 5],\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:19.278207Z","iopub.execute_input":"2024-12-11T00:34:19.278444Z","iopub.status.idle":"2024-12-11T00:34:19.289304Z","shell.execute_reply.started":"2024-12-11T00:34:19.278410Z","shell.execute_reply":"2024-12-11T00:34:19.288534Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Classifier","metadata":{}},{"cell_type":"code","source":"# Training and evaluation function\ndef train_and_evaluate_classifier(model, param_dist, eval_metric=None, n_iter=10):\n    best_model, y_pred, y = train_and_evaluate(model, param_dist, n_iter, eval_metric)\n    train_kappa = quadratic_weighted_kappa(y, y_pred)\n    train_accuracy = accuracy_score(y, y_pred)\n    train_f1 = f1_score(y, y_pred, average='weighted')\n    print(f\"Best Train QWK: {train_kappa:.4f}\")\n    print(f\"Best Train Accuracy: {train_accuracy:.4f}\")\n    print(f\"Best Train F1: {train_f1:.4f}\")\n    return best_model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:19.290444Z","iopub.execute_input":"2024-12-11T00:34:19.291210Z","iopub.status.idle":"2024-12-11T00:34:19.302098Z","shell.execute_reply.started":"2024-12-11T00:34:19.291172Z","shell.execute_reply":"2024-12-11T00:34:19.301380Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"RF = RandomForestClassifier(random_state=42)\nbest_model = train_and_evaluate_classifier(RF, rf_param_dist, n_iter=3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:19.303124Z","iopub.execute_input":"2024-12-11T00:34:19.303477Z","iopub.status.idle":"2024-12-11T00:34:26.388439Z","shell.execute_reply.started":"2024-12-11T00:34:19.303434Z","shell.execute_reply":"2024-12-11T00:34:26.387426Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test = test_df.drop('sii', axis=1)\ny_test = test_df['sii'].astype(int)\ny_test_pred = best_model.predict(X_test)\ntest_kappa = quadratic_weighted_kappa(y_test, y_test_pred)\ntest_accuracy = accuracy_score(y_test, y_test_pred)\ntest_f1 = f1_score(y_test, y_test_pred, average='weighted')\nprint(f\"Test QWK: {Fore.GREEN}{Style.BRIGHT}{test_kappa:.4f}{Style.RESET_ALL}\")\nprint(f\"Test Accuracy: {Fore.GREEN}{Style.BRIGHT}{test_accuracy:.4f}{Style.RESET_ALL}\")\nprint(f\"Test F1: {Fore.GREEN}{Style.BRIGHT}{test_f1:.4f}{Style.RESET_ALL}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:26.389898Z","iopub.execute_input":"2024-12-11T00:34:26.390181Z","iopub.status.idle":"2024-12-11T00:34:26.419259Z","shell.execute_reply.started":"2024-12-11T00:34:26.390151Z","shell.execute_reply":"2024-12-11T00:34:26.418479Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Regression","metadata":{}},{"cell_type":"code","source":"# Function to apply thresholds to continuous predictions\ndef apply_thresholds(predictions, thresholds):\n    return np.digitize(predictions, bins=thresholds)\n\n# Function to optimize thresholds to maximize QWK\ndef optimize_thresholds(y_true, predictions):\n    def loss_func(thresh):\n        # Ensure thresholds are sorted\n        thresh_sorted = np.sort(thresh)\n        preds = apply_thresholds(predictions, thresh_sorted)\n        return -quadratic_weighted_kappa(y_true, preds)\n    \n    initial_thresholds = [0.5, 1.5, 2.5]  # Initial guesses for thresholds\n    bounds = [(0, 3)] * 3  # Assuming classes are 0,1,2,3\n    result = minimize(loss_func, initial_thresholds, method='Nelder-Mead')\n    return result.x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:26.420457Z","iopub.execute_input":"2024-12-11T00:34:26.420819Z","iopub.status.idle":"2024-12-11T00:34:26.426509Z","shell.execute_reply.started":"2024-12-11T00:34:26.420779Z","shell.execute_reply":"2024-12-11T00:34:26.425725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_and_evaluate_regressor(model, param_dist, n_iter=10, eval_metric=None):\n    best_model, y_pred, y = train_and_evaluate(model, param_dist, n_iter, eval_metric)\n    thresholds = optimize_thresholds(y, y_pred)\n    y_pred = apply_thresholds(y_pred, thresholds)\n    train_kappa = quadratic_weighted_kappa(y, y_pred)\n    train_accuracy = accuracy_score(y, y_pred)\n    train_f1 = f1_score(y, y_pred, average='weighted')\n    print(f\"Best Train QWK: {train_kappa:.4f}\")\n    print(f\"Best Train Accuracy: {train_accuracy:.4f}\")\n    print(f\"Best Train F1: {train_f1:.4f}\")\n    return best_model, thresholds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:26.427789Z","iopub.execute_input":"2024-12-11T00:34:26.428359Z","iopub.status.idle":"2024-12-11T00:34:26.439630Z","shell.execute_reply.started":"2024-12-11T00:34:26.428321Z","shell.execute_reply":"2024-12-11T00:34:26.438868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"RF = RandomForestRegressor(random_state=42)\nbest_model, thresholds = train_and_evaluate_regressor(RF, rf_param_dist, n_iter=3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:26.440494Z","iopub.execute_input":"2024-12-11T00:34:26.440735Z","iopub.status.idle":"2024-12-11T00:34:52.996186Z","shell.execute_reply.started":"2024-12-11T00:34:26.440712Z","shell.execute_reply":"2024-12-11T00:34:52.995351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test = test_df.drop('sii', axis=1)\ny_test = test_df['sii'].astype(int)\ntest_X = X_test.values.astype(np.float32)\nraw_preds = best_model.predict(test_X)\ny_test_pred = apply_thresholds(raw_preds, thresholds)\ntest_kappa = quadratic_weighted_kappa(y_test, y_test_pred)\ntest_accuracy = accuracy_score(y_test, y_test_pred)\ntest_f1 = f1_score(y_test, y_test_pred, average='weighted')\n\nprint(f\"Best Test QWK: {test_kappa:.4f}\")\nprint(f\"Best Train Accuracy: {test_accuracy:.4f}\")\nprint(f\"Best Train F1: {test_f1:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:52.997371Z","iopub.execute_input":"2024-12-11T00:34:52.997675Z","iopub.status.idle":"2024-12-11T00:34:53.017482Z","shell.execute_reply.started":"2024-12-11T00:34:52.997646Z","shell.execute_reply":"2024-12-11T00:34:53.016757Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model 1 - Voting Classifier","metadata":{}},{"cell_type":"code","source":"# Training and evaluation function\ndef train_and_evaluate_classifier(model, param_dist, eval_metric=None, n_iter=10):\n    best_model, y_pred, y = train_and_evaluate(model, param_dist, n_iter, eval_metric)\n    train_kappa = quadratic_weighted_kappa(y, y_pred)\n    train_accuracy = accuracy_score(y, y_pred)\n    train_f1 = f1_score(y, y_pred, average='weighted')\n    print(f\"Best Train QWK: {train_kappa:.4f}\")\n    print(f\"Best Train Accuracy: {train_accuracy:.4f}\")\n    print(f\"Best Train F1: {train_f1:.4f}\")\n    return best_model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:53.018478Z","iopub.execute_input":"2024-12-11T00:34:53.018749Z","iopub.status.idle":"2024-12-11T00:34:53.023764Z","shell.execute_reply.started":"2024-12-11T00:34:53.018724Z","shell.execute_reply":"2024-12-11T00:34:53.022911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the classifiers\nrf = RandomForestClassifier(random_state=42)\nsvc = SVC(probability=True, random_state=42)\nxgb_model = XGBClassifier(random_state=42, objective='multi:softprob', num_class=4, verbosity=0, learning_rate=0.05)\nlgb_model = LGBMClassifier(random_state=42, objective='multiclass', num_class=4, verbose=-1, learning_rate=0.05)\ncatb_model = CatBoostClassifier(random_state=42, objective='MultiClass', verbose=False, learning_rate=0.05)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:53.024866Z","iopub.execute_input":"2024-12-11T00:34:53.025120Z","iopub.status.idle":"2024-12-11T00:34:53.037486Z","shell.execute_reply.started":"2024-12-11T00:34:53.025092Z","shell.execute_reply":"2024-12-11T00:34:53.036843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"param_dist = {\n            # 'rf__n_estimators': [100, 200, 300], 'rf__max_depth': [3], 'rf__min_samples_split': [2, 5, 10], 'rf__min_samples_leaf': [1], 'rf__max_features': ['auto', 'sqrt', 'log2'],\n            #   'svc__C': [0.1, 1, 10], 'svc__gamma': ['scale', 'auto'], 'svc__kernel': ['linear', 'rbf'],\n              'xgb__n_estimators': [200, 300], 'xgb__max_depth': [2, 5],\n              'lgb__n_estimators': [100, 200], 'lgb__num_leaves': [5, 10, 15],\n              'cat__iterations': [100, 200, 300], 'cat__depth': [2, 4, 5]\n              }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:53.038552Z","iopub.execute_input":"2024-12-11T00:34:53.038911Z","iopub.status.idle":"2024-12-11T00:34:53.052477Z","shell.execute_reply.started":"2024-12-11T00:34:53.038874Z","shell.execute_reply":"2024-12-11T00:34:53.051604Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Create an ensemble using VotingClassifier with soft voting\nensemble_model = VotingClassifier(\n    estimators=[('lgb', lgb_model), \n                ('xgb', xgb_model), \n                ('cat', catb_model)\n                # ('rf', rf) \n                # ,('svc', svc)\n                ],\n    voting='soft',\n    # weights=[4.0, 4.0, 5.0],\n    n_jobs=-1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:53.053606Z","iopub.execute_input":"2024-12-11T00:34:53.054095Z","iopub.status.idle":"2024-12-11T00:34:53.062635Z","shell.execute_reply.started":"2024-12-11T00:34:53.054066Z","shell.execute_reply":"2024-12-11T00:34:53.061904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_model = train_and_evaluate_classifier(ensemble_model, param_dist, n_iter=2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:34:53.063639Z","iopub.execute_input":"2024-12-11T00:34:53.063965Z","iopub.status.idle":"2024-12-11T00:35:30.142909Z","shell.execute_reply.started":"2024-12-11T00:34:53.063929Z","shell.execute_reply":"2024-12-11T00:35:30.141743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test = test_df.drop('sii', axis=1)\ny_test = test_df['sii'].astype(int)\ny_test_pred = best_model.predict(X_test)\ntest_kappa = quadratic_weighted_kappa(y_test, y_test_pred)\ntest_accuracy = accuracy_score(y_test, y_test_pred)\ntest_f1 = f1_score(y_test, y_test_pred, average='weighted')\nprint(f\"Test QWK: {Fore.GREEN}{Style.BRIGHT}{test_kappa:.4f}{Style.RESET_ALL}\")\nprint(f\"Test Accuracy: {Fore.GREEN}{Style.BRIGHT}{test_accuracy:.4f}{Style.RESET_ALL}\")\nprint(f\"Test F1: {Fore.GREEN}{Style.BRIGHT}{test_f1:.4f}{Style.RESET_ALL}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:35:30.144216Z","iopub.execute_input":"2024-12-11T00:35:30.144579Z","iopub.status.idle":"2024-12-11T00:35:30.194160Z","shell.execute_reply.started":"2024-12-11T00:35:30.144549Z","shell.execute_reply":"2024-12-11T00:35:30.192915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"make_cm(y_test, y_test_pred, \"Voting Classifier - Test\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:35:30.195590Z","iopub.execute_input":"2024-12-11T00:35:30.195999Z","iopub.status.idle":"2024-12-11T00:35:30.540064Z","shell.execute_reply.started":"2024-12-11T00:35:30.195955Z","shell.execute_reply":"2024-12-11T00:35:30.539205Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model 1.5 - Stacking Classifier","metadata":{}},{"cell_type":"code","source":"\n# Create an ensemble using VotingClassifier with soft voting\nensemble_model = StackingClassifier(\n    estimators=[('lgb', lgb_model), \n                ('xgb', xgb_model), \n                ('cat', catb_model)\n                # ('rf', rf) \n                # ,('svc', svc)\n                ],\n    # weights=[4.0, 4.0, 5.0],\n    n_jobs=-1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:35:30.541609Z","iopub.execute_input":"2024-12-11T00:35:30.541967Z","iopub.status.idle":"2024-12-11T00:35:30.546268Z","shell.execute_reply.started":"2024-12-11T00:35:30.541929Z","shell.execute_reply":"2024-12-11T00:35:30.545431Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_model = train_and_evaluate_classifier(ensemble_model, param_dist, n_iter=2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:35:30.547317Z","iopub.execute_input":"2024-12-11T00:35:30.547919Z","iopub.status.idle":"2024-12-11T00:38:41.844650Z","shell.execute_reply.started":"2024-12-11T00:35:30.547891Z","shell.execute_reply":"2024-12-11T00:38:41.843314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test = test_df.drop('sii', axis=1)\ny_test = test_df['sii'].astype(int)\ny_test_pred = best_model.predict(X_test)\ntest_kappa = quadratic_weighted_kappa(y_test, y_test_pred)\ntest_accuracy = accuracy_score(y_test, y_test_pred)\ntest_f1 = f1_score(y_test, y_test_pred, average='weighted')\nprint(f\"Test QWK: {Fore.GREEN}{Style.BRIGHT}{test_kappa:.4f}{Style.RESET_ALL}\")\nprint(f\"Test Accuracy: {Fore.GREEN}{Style.BRIGHT}{test_accuracy:.4f}{Style.RESET_ALL}\")\nprint(f\"Test F1: {Fore.GREEN}{Style.BRIGHT}{test_f1:.4f}{Style.RESET_ALL}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:38:41.846444Z","iopub.execute_input":"2024-12-11T00:38:41.846871Z","iopub.status.idle":"2024-12-11T00:38:41.902627Z","shell.execute_reply.started":"2024-12-11T00:38:41.846824Z","shell.execute_reply":"2024-12-11T00:38:41.901323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"make_cm(y_test, y_test_pred, \"Stacking Classifier - Test\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:38:41.904073Z","iopub.execute_input":"2024-12-11T00:38:41.904517Z","iopub.status.idle":"2024-12-11T00:38:42.437562Z","shell.execute_reply.started":"2024-12-11T00:38:41.904463Z","shell.execute_reply":"2024-12-11T00:38:42.436731Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model 2 - Voting Regressor","metadata":{}},{"cell_type":"code","source":"# Function to apply thresholds to continuous predictions\ndef apply_thresholds(predictions, thresholds):\n    return np.digitize(predictions, bins=thresholds)\n\n# Function to optimize thresholds to maximize QWK\ndef optimize_thresholds(y_true, predictions):\n    def loss_func(thresh):\n        # Ensure thresholds are sorted\n        thresh_sorted = np.sort(thresh)\n        preds = apply_thresholds(predictions, thresh_sorted)\n        return -quadratic_weighted_kappa(y_true, preds)\n    \n    initial_thresholds = [0.5, 1.5, 2.5]  # Initial guesses for thresholds\n    bounds = [(0, 3)] * 3  # Assuming classes are 0,1,2,3\n    result = minimize(loss_func, initial_thresholds, method='Nelder-Mead')\n    return result.x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:38:42.438925Z","iopub.execute_input":"2024-12-11T00:38:42.439591Z","iopub.status.idle":"2024-12-11T00:38:42.445575Z","shell.execute_reply.started":"2024-12-11T00:38:42.439549Z","shell.execute_reply":"2024-12-11T00:38:42.444584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_and_evaluate_regressor(model, param_dist, n_iter=10, eval_metric=None):\n    best_model, y_pred, y = train_and_evaluate(model, param_dist, n_iter, eval_metric)\n    thresholds = optimize_thresholds(y, y_pred)\n    y_pred = apply_thresholds(y_pred, thresholds)\n    train_kappa = quadratic_weighted_kappa(y, y_pred)\n    train_accuracy = accuracy_score(y, y_pred)\n    train_f1 = f1_score(y, y_pred, average='weighted')\n    print(f\"Best Train QWK: {train_kappa:.4f}\")\n    print(f\"Best Train Accuracy: {train_accuracy:.4f}\")\n    print(f\"Best Train F1: {train_f1:.4f}\")\n    return best_model, thresholds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:38:42.446453Z","iopub.execute_input":"2024-12-11T00:38:42.446693Z","iopub.status.idle":"2024-12-11T00:38:42.457163Z","shell.execute_reply.started":"2024-12-11T00:38:42.446669Z","shell.execute_reply":"2024-12-11T00:38:42.456322Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"svm = SVC(random_state=42)\nxgb_model = XGBRegressor(random_state=42, verbosity=0, learning_rate=0.05)\nlgb_model = LGBMRegressor(random_state=42, verbose=-1, learning_rate=0.05)\ncatb_model = CatBoostRegressor(random_state=42, verbose=False, learning_rate=0.05)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:38:42.458115Z","iopub.execute_input":"2024-12-11T00:38:42.458438Z","iopub.status.idle":"2024-12-11T00:38:42.472204Z","shell.execute_reply.started":"2024-12-11T00:38:42.458378Z","shell.execute_reply":"2024-12-11T00:38:42.471322Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use the same ones as above for consistency\n# param_dist = {\n#             # 'rf__n_estimators': [100, 200, 300], 'rf__max_depth': [3], 'rf__min_samples_split': [2, 5, 10], 'rf__min_samples_leaf': [1], 'rf__max_features': ['auto', 'sqrt', 'log2'],\n#             #   'svc__C': [0.1, 1, 10], 'svc__gamma': ['scale', 'auto'], 'svc__kernel': ['linear', 'rbf'],\n#               'xgb__n_estimators': [200, 300], 'xgb__max_depth': [2, 3],\n#               'lgb__n_estimators': [100, 200], 'lgb__num_leaves': [5, 10],\n#               'cat__iterations': [100, 200], 'cat__depth': [2, 4]\n#               }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:38:42.473362Z","iopub.execute_input":"2024-12-11T00:38:42.473708Z","iopub.status.idle":"2024-12-11T00:38:42.482898Z","shell.execute_reply.started":"2024-12-11T00:38:42.473656Z","shell.execute_reply":"2024-12-11T00:38:42.482103Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create an ensemble using VotingClassifier with soft voting\nensemble_model = VotingRegressor(\n    estimators=[('lgb', lgb_model), \n                ('xgb', xgb_model), \n                ('cat', catb_model)\n                # ('rf', rf) \n                # ,('svc', svc)\n                ],\n    # weights=[4.0, 4.0, 5.0],\n    n_jobs=-1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:38:42.483729Z","iopub.execute_input":"2024-12-11T00:38:42.483938Z","iopub.status.idle":"2024-12-11T00:38:42.493206Z","shell.execute_reply.started":"2024-12-11T00:38:42.483916Z","shell.execute_reply":"2024-12-11T00:38:42.492430Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_model, thresholds = train_and_evaluate_regressor(ensemble_model, param_dist, n_iter=2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:38:42.494124Z","iopub.execute_input":"2024-12-11T00:38:42.494351Z","iopub.status.idle":"2024-12-11T00:38:55.918724Z","shell.execute_reply.started":"2024-12-11T00:38:42.494328Z","shell.execute_reply":"2024-12-11T00:38:55.917751Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test = test_df.drop('sii', axis=1)\ny_test = test_df['sii'].astype(int)\ny_test_pred = best_model.predict(X_test)\ny_test_pred = apply_thresholds(y_test_pred, thresholds)\ntest_kappa = quadratic_weighted_kappa(y_test, y_test_pred)\ntest_accuracy = accuracy_score(y_test, y_test_pred)\ntest_f1 = f1_score(y_test, y_test_pred, average='weighted')\nprint(f\"Test QWK: {Fore.GREEN}{Style.BRIGHT}{test_kappa:.4f}{Style.RESET_ALL}\")\nprint(f\"Test Accuracy: {Fore.GREEN}{Style.BRIGHT}{test_accuracy:.4f}{Style.RESET_ALL}\")\nprint(f\"Test F1: {Fore.GREEN}{Style.BRIGHT}{test_f1:.4f}{Style.RESET_ALL}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:38:55.919863Z","iopub.execute_input":"2024-12-11T00:38:55.920164Z","iopub.status.idle":"2024-12-11T00:38:55.954820Z","shell.execute_reply.started":"2024-12-11T00:38:55.920135Z","shell.execute_reply":"2024-12-11T00:38:55.953776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"make_cm(y_test, y_test_pred, \"Voting Regressor - Test\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:38:55.955881Z","iopub.execute_input":"2024-12-11T00:38:55.956238Z","iopub.status.idle":"2024-12-11T00:38:56.229097Z","shell.execute_reply.started":"2024-12-11T00:38:55.956209Z","shell.execute_reply":"2024-12-11T00:38:56.228193Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = train_df.drop('sii', axis=1)\ny_train = train_df['sii'].astype(int)\ny_train_pred = best_model.predict(X_train)\ny_train_pred = apply_thresholds(y_train_pred, thresholds)\nmake_cm(y_train, y_train_pred, \"Voting Regressor - Train\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:38:56.230299Z","iopub.execute_input":"2024-12-11T00:38:56.230751Z","iopub.status.idle":"2024-12-11T00:38:56.617310Z","shell.execute_reply.started":"2024-12-11T00:38:56.230712Z","shell.execute_reply":"2024-12-11T00:38:56.616363Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model 2.5 - Stacking Regressor","metadata":{}},{"cell_type":"code","source":"# Create an ensemble using VotingClassifier with soft voting\nensemble_model = StackingRegressor(\n    estimators=[('lgb', lgb_model), \n                ('xgb', xgb_model), \n                ('cat', catb_model)\n                # ('rf', rf) \n                # ,('svc', svc)\n                ],\n    # weights=[4.0, 4.0, 5.0],\n    n_jobs=-1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:38:56.618544Z","iopub.execute_input":"2024-12-11T00:38:56.618863Z","iopub.status.idle":"2024-12-11T00:38:56.623328Z","shell.execute_reply.started":"2024-12-11T00:38:56.618835Z","shell.execute_reply":"2024-12-11T00:38:56.622440Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_model, thresholds = train_and_evaluate_regressor(ensemble_model, param_dist, n_iter=2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:38:56.624468Z","iopub.execute_input":"2024-12-11T00:38:56.624740Z","iopub.status.idle":"2024-12-11T00:40:02.962716Z","shell.execute_reply.started":"2024-12-11T00:38:56.624690Z","shell.execute_reply":"2024-12-11T00:40:02.961487Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test = test_df.drop('sii', axis=1)\ny_test = test_df['sii'].astype(int)\ny_test_pred = best_model.predict(X_test)\ny_test_pred = apply_thresholds(y_test_pred, thresholds)\ntest_kappa = quadratic_weighted_kappa(y_test, y_test_pred)\ntest_accuracy = accuracy_score(y_test, y_test_pred)\ntest_f1 = f1_score(y_test, y_test_pred, average='weighted')\nprint(f\"Test QWK: {Fore.GREEN}{Style.BRIGHT}{test_kappa:.4f}{Style.RESET_ALL}\")\nprint(f\"Test Accuracy: {Fore.GREEN}{Style.BRIGHT}{test_accuracy:.4f}{Style.RESET_ALL}\")\nprint(f\"Test F1: {Fore.GREEN}{Style.BRIGHT}{test_f1:.4f}{Style.RESET_ALL}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:40:02.963991Z","iopub.execute_input":"2024-12-11T00:40:02.964262Z","iopub.status.idle":"2024-12-11T00:40:03.012955Z","shell.execute_reply.started":"2024-12-11T00:40:02.964234Z","shell.execute_reply":"2024-12-11T00:40:03.012045Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model 3 - TabNet","metadata":{}},{"cell_type":"code","source":"\n# Function that instantiates a tabnet model.\ndef create_tabnet(n_d=32, n_steps=5, lr=0.02, gamma=1.5, \n                  n_independent=2, n_shared=2, lambda_sparse=1e-4, \n                  momentum=0.3, is_classifier=True):\n    if is_classifier:\n        tabnet_model = TabNetClassifier(\n            n_d=n_d,\n            n_a=n_d,\n            n_steps=n_steps,\n            gamma=gamma,\n            lambda_sparse=lambda_sparse,\n            seed=2024,\n            verbose=1,\n            device_name='cuda',\n            optimizer_fn=torch.optim.Adam,\n            scheduler_params = {\"gamma\": 0.95,\n                            \"step_size\": 20},\n            scheduler_fn=torch.optim.lr_scheduler.StepLR, epsilon=1e-15\n        )\n    else:\n        tabnet_model = TabNetRegressor(\n            n_d=n_d,\n            n_a=n_d,\n            n_steps=n_steps,\n            gamma=gamma,\n            lambda_sparse=lambda_sparse,\n            seed=2024,\n            verbose=1,\n            device_name='cuda',\n            optimizer_fn=torch.optim.Adam,\n            scheduler_params = {\"gamma\": 0.95,\n                            \"step_size\": 20},\n            scheduler_fn=torch.optim.lr_scheduler.StepLR, epsilon=1e-15\n        )\n    return tabnet_model\n                  \ndef getBestTabNetModel(is_classifier):\n    # Generate the parameter grid.\n    param_grid = dict(n_d = [64],\n                    n_a = [64],\n                    n_steps = [3],\n                    gamma = [1.3],\n                    lambda_sparse = [1e-2, 1e-3, 1e-4],\n                    momentum = [0.2],\n                    n_shared = [2],\n                    n_independent = [2],\n    )\n\n    grid = ParameterGrid(param_grid)\n\n    search_results = pd.DataFrame() \n    best_score = -np.inf\n    best_model = None\n    for params in tqdm(grid, desc=\"Hyperparameter Search\", ncols=100):\n        X_train, X_val, y_train, y_val = train_test_split(X.values.astype(np.float32), y.values.astype(np.int8), test_size=0.2, stratify=y)\n        if is_classifier == False:\n            y_train = y_train.reshape(-1, 1)\n            y_val = y_val.reshape(-1, 1)\n            eval_metric = ['rmse']\n        else:\n            eval_metric = ['logloss']\n        tqdm.write(f\"Before training - y_train shape: {str(y_train.shape)}, y_val shape: {str(y_val.shape)}\")\n        params['n_a'] = params['n_d'] # n_a=n_d always per the paper\n        tabnet = create_tabnet(is_classifier=is_classifier)\n        tabnet.set_params(**params)\n        tabnet.fit(X_train=X_train, y_train=y_train, eval_set=[(X_val, y_val)], eval_name=['validation'], eval_metric=eval_metric, max_epochs=200, patience=20,\n                batch_size=1024, virtual_batch_size=128)\n        if is_classifier:\n            y_pred = np.argmax(tabnet.predict_proba(X_val), axis=1)\n            score = quadratic_weighted_kappa(y_val, y_pred)  # Convert probabilities to class labels and calculate accuracy\n        else:\n            y_pred = tabnet.predict(X_val)\n            score = np.sqrt(mean_squared_error(y_val.astype(np.float32), y_pred))\n        if score > best_score:\n            best_model = tabnet\n            best_score = score\n    return best_model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:40:03.014273Z","iopub.execute_input":"2024-12-11T00:40:03.014667Z","iopub.status.idle":"2024-12-11T00:40:03.027386Z","shell.execute_reply.started":"2024-12-11T00:40:03.014626Z","shell.execute_reply":"2024-12-11T00:40:03.026469Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Classifier","metadata":{}},{"cell_type":"code","source":"best_model = getBestTabNetModel(is_classifier=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:40:03.028323Z","iopub.execute_input":"2024-12-11T00:40:03.028699Z","iopub.status.idle":"2024-12-11T00:40:31.663418Z","shell.execute_reply.started":"2024-12-11T00:40:03.028661Z","shell.execute_reply":"2024-12-11T00:40:31.662334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_X = X.values.astype(np.float32)\ntrain_pred = np.argmax(best_model.predict_proba(train_X), axis=1)\n\ntrain_kappa = quadratic_weighted_kappa(y, train_pred)\ntrain_accuracy = accuracy_score(y, train_pred)\ntrain_f1 = f1_score(y, train_pred, average='weighted')\n\nprint(f\"Best Train QWK: {train_kappa:.4f}\")\nprint(f\"Best Train Accuracy: {train_accuracy:.4f}\")\nprint(f\"Best Train F1: {train_f1:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:40:31.664591Z","iopub.execute_input":"2024-12-11T00:40:31.664893Z","iopub.status.idle":"2024-12-11T00:40:31.752984Z","shell.execute_reply.started":"2024-12-11T00:40:31.664864Z","shell.execute_reply":"2024-12-11T00:40:31.751983Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test = test_df.drop('sii', axis=1)\ny_test = test_df['sii'].astype(int)\ntest_X = X_test.values.astype(np.float32)\ny_test_pred = np.argmax(best_model.predict_proba(test_X), axis=1)\ntest_kappa = quadratic_weighted_kappa(y_test, y_test_pred)\ntest_accuracy = accuracy_score(y_test, y_test_pred)\ntest_f1 = f1_score(y_test, y_test_pred, average='weighted')\n\nprint(f\"Best Test QWK: {test_kappa:.4f}\")\nprint(f\"Best Train Accuracy: {test_accuracy:.4f}\")\nprint(f\"Best Train F1: {test_f1:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:40:31.754247Z","iopub.execute_input":"2024-12-11T00:40:31.754682Z","iopub.status.idle":"2024-12-11T00:40:31.789235Z","shell.execute_reply.started":"2024-12-11T00:40:31.754640Z","shell.execute_reply":"2024-12-11T00:40:31.788417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:40:31.790301Z","iopub.execute_input":"2024-12-11T00:40:31.790584Z","iopub.status.idle":"2024-12-11T00:40:31.797452Z","shell.execute_reply.started":"2024-12-11T00:40:31.790557Z","shell.execute_reply":"2024-12-11T00:40:31.796368Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Regressor","metadata":{}},{"cell_type":"code","source":"y.values.astype(np.int8).reshape(-1, 1).shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:40:31.798679Z","iopub.execute_input":"2024-12-11T00:40:31.799090Z","iopub.status.idle":"2024-12-11T00:40:31.810902Z","shell.execute_reply.started":"2024-12-11T00:40:31.799041Z","shell.execute_reply":"2024-12-11T00:40:31.809975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_model = getBestTabNetModel(is_classifier=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:40:31.812245Z","iopub.execute_input":"2024-12-11T00:40:31.812684Z","iopub.status.idle":"2024-12-11T00:41:00.850610Z","shell.execute_reply.started":"2024-12-11T00:40:31.812635Z","shell.execute_reply":"2024-12-11T00:41:00.849777Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_preds = best_model.predict(X.values.astype(np.float32))\nthresholds = optimize_thresholds(y.values.astype(np.float32), raw_preds)\ntrain_pred = apply_thresholds(raw_preds, thresholds)\n\ntrain_kappa = quadratic_weighted_kappa(y, train_pred)\ntrain_accuracy = accuracy_score(y, train_pred)\ntrain_f1 = f1_score(y, train_pred, average='weighted')\n\nprint(f\"Best Train QWK: {train_kappa:.4f}\")\nprint(f\"Best Train Accuracy: {train_accuracy:.4f}\")\nprint(f\"Best Train F1: {train_f1:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:41:00.851762Z","iopub.execute_input":"2024-12-11T00:41:00.852041Z","iopub.status.idle":"2024-12-11T00:41:01.624492Z","shell.execute_reply.started":"2024-12-11T00:41:00.852013Z","shell.execute_reply":"2024-12-11T00:41:01.623611Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test = test_df.drop('sii', axis=1)\ny_test = test_df['sii'].astype(int)\ntest_X = X_test.values.astype(np.float32)\nraw_preds = best_model.predict(test_X)\ny_test_pred = apply_thresholds(raw_preds, thresholds)\ntest_kappa = quadratic_weighted_kappa(y_test, y_test_pred)\ntest_accuracy = accuracy_score(y_test, y_test_pred)\ntest_f1 = f1_score(y_test, y_test_pred, average='weighted')\n\nprint(f\"Best Test QWK: {test_kappa:.4f}\")\nprint(f\"Best Train Accuracy: {test_accuracy:.4f}\")\nprint(f\"Best Train F1: {test_f1:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T00:41:01.625838Z","iopub.execute_input":"2024-12-11T00:41:01.626567Z","iopub.status.idle":"2024-12-11T00:41:01.661317Z","shell.execute_reply.started":"2024-12-11T00:41:01.626522Z","shell.execute_reply":"2024-12-11T00:41:01.660379Z"}},"outputs":[],"execution_count":null}]}