{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport gc\nimport random\n\nfrom tqdm import tqdm\nfrom concurrent.futures import ThreadPoolExecutor\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset\n\nfrom lightgbm import LGBMRegressor, LGBMClassifier\nfrom xgboost import XGBRegressor, XGBClassifier\nfrom catboost import CatBoostRegressor, CatBoostClassifier\n# from pytorch_tabnet.tab_model import TabNetRegressor, TabNetClassifier\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import VotingRegressor, VotingClassifier, RandomForestRegressor, RandomForestClassifier, GradientBoostingRegressor, GradientBoostingClassifier\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold, KFold\nfrom scipy.optimize import minimize\nfrom colorama import Fore, Style\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:05:52.161627Z","iopub.execute_input":"2024-12-10T11:05:52.162022Z","iopub.status.idle":"2024-12-10T11:05:59.469752Z","shell.execute_reply.started":"2024-12-10T11:05:52.161991Z","shell.execute_reply":"2024-12-10T11:05:59.468979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:06:13.710265Z","iopub.execute_input":"2024-12-10T11:06:13.711096Z","iopub.status.idle":"2024-12-10T11:06:13.748065Z","shell.execute_reply.started":"2024-12-10T11:06:13.711063Z","shell.execute_reply":"2024-12-10T11:06:13.747141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"seed = 42\ntorch.manual_seed(seed)\nnp.random.seed(seed)\nrandom.seed(seed)\ntorch.cuda.manual_seed(seed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:06:15.708828Z","iopub.execute_input":"2024-12-10T11:06:15.709147Z","iopub.status.idle":"2024-12-10T11:06:15.717730Z","shell.execute_reply.started":"2024-12-10T11:06:15.709122Z","shell.execute_reply":"2024-12-10T11:06:15.716856Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Time Series","metadata":{}},{"cell_type":"code","source":"train_ts_path = '/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet'\ntest_ts_path = '/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:06:18.534086Z","iopub.execute_input":"2024-12-10T11:06:18.534772Z","iopub.status.idle":"2024-12-10T11:06:18.538717Z","shell.execute_reply.started":"2024-12-10T11:06:18.534737Z","shell.execute_reply":"2024-12-10T11:06:18.537742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df = df[df['non-wear_flag'] != 1]\n    df = df[df['battery_voltage'] > 0]\n    df = df.set_index('step', drop=True)\n\n    df = df.drop(columns=['non-wear_flag', 'battery_voltage'])\n\n    return df, filename.split('=')[1]\n\ndef load_time_series(dirname):\n    ids = os.listdir(dirname)\n\n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n\n    dfs, indexes = zip(*results)\n\n    ts_dfs = {}\n\n    for i in range(len(indexes)):\n        ts_dfs[f\"id_{indexes[i]}\"] = dfs[i]\n\n    return ts_dfs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:06:21.057375Z","iopub.execute_input":"2024-12-10T11:06:21.058061Z","iopub.status.idle":"2024-12-10T11:06:21.064607Z","shell.execute_reply.started":"2024-12-10T11:06:21.058026Z","shell.execute_reply":"2024-12-10T11:06:21.063577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts = load_time_series(train_ts_path)\ntest_ts = load_time_series(test_ts_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:06:24.574001Z","iopub.execute_input":"2024-12-10T11:06:24.574359Z","iopub.status.idle":"2024-12-10T11:06:51.515000Z","shell.execute_reply.started":"2024-12-10T11:06:24.574328Z","shell.execute_reply":"2024-12-10T11:06:51.514032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def combine_movement(dfs):\n    mov_combined = []\n\n    for df in dfs.values():\n        mov_combined.append(torch.tensor(df[['X', 'Y', 'Z', 'enmo', 'anglez']].values).to(device))\n\n    mov_df = torch.cat(mov_combined, dim=0)\n\n    return mov_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:09:59.269198Z","iopub.execute_input":"2024-12-10T11:09:59.269841Z","iopub.status.idle":"2024-12-10T11:09:59.274572Z","shell.execute_reply.started":"2024-12-10T11:09:59.269802Z","shell.execute_reply":"2024-12-10T11:09:59.273702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:10:01.238632Z","iopub.execute_input":"2024-12-10T11:10:01.239485Z","iopub.status.idle":"2024-12-10T11:10:01.245663Z","shell.execute_reply.started":"2024-12-10T11:10:01.239423Z","shell.execute_reply":"2024-12-10T11:10:01.244790Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_autoencoder(df, autoencoder, optimizer, epochs=50, batch_size=32):\n    mean = df.mean(dim=0)\n    std = df.std(dim=0)\n\n    data_tensor = (df - mean) / std\n\n    criterion = nn.MSELoss()\n\n    autoencoder.train()\n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size].to(device)\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n\n        print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n    \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:10:04.700488Z","iopub.execute_input":"2024-12-10T11:10:04.700842Z","iopub.status.idle":"2024-12-10T11:10:04.706892Z","shell.execute_reply.started":"2024-12-10T11:10:04.700812Z","shell.execute_reply":"2024-12-10T11:10:04.705714Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts_mov = combine_movement(train_ts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:10:07.824665Z","iopub.execute_input":"2024-12-10T11:10:07.825437Z","iopub.status.idle":"2024-12-10T11:10:11.738419Z","shell.execute_reply.started":"2024-12-10T11:10:07.825382Z","shell.execute_reply":"2024-12-10T11:10:11.737659Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mov_autoencoder_train = AutoEncoder(5, 1).to(device)\nmov_optimizer_train = optim.Adam(mov_autoencoder_train.parameters())\ntrain_autoencoder(train_ts_mov, mov_autoencoder_train, mov_optimizer_train, epochs=10, batch_size=8192)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:10:14.703193Z","iopub.execute_input":"2024-12-10T11:10:14.703608Z","iopub.status.idle":"2024-12-10T11:18:33.719186Z","shell.execute_reply.started":"2024-12-10T11:10:14.703572Z","shell.execute_reply":"2024-12-10T11:18:33.718267Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:22:49.948344Z","iopub.execute_input":"2024-12-10T11:22:49.949579Z","iopub.status.idle":"2024-12-10T11:22:49.960103Z","shell.execute_reply.started":"2024-12-10T11:22:49.949544Z","shell.execute_reply":"2024-12-10T11:22:49.959070Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def encode_movement(dfs, autoencoder):\n    for df in dfs.values():\n        df_mov = df[['X', 'Y', 'Z', 'enmo', 'anglez']]\n        scaler = StandardScaler()\n        df_mov_scaled = scaler.fit_transform(df_mov)\n        df_mov_encoded = autoencoder.encoder(torch.FloatTensor(df_mov_scaled).to(device)).cpu().detach().numpy()\n        df['movement'] = df_mov_encoded\n        df.drop(columns=['X', 'Y', 'Z', 'enmo', 'anglez'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:22:53.130321Z","iopub.execute_input":"2024-12-10T11:22:53.131137Z","iopub.status.idle":"2024-12-10T11:22:53.135877Z","shell.execute_reply.started":"2024-12-10T11:22:53.131101Z","shell.execute_reply":"2024-12-10T11:22:53.135004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mov_autoencoder_train.eval()\nencode_movement(train_ts, mov_autoencoder_train)\nencode_movement(test_ts, mov_autoencoder_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:22:55.499787Z","iopub.execute_input":"2024-12-10T11:22:55.500137Z","iopub.status.idle":"2024-12-10T11:23:13.780865Z","shell.execute_reply.started":"2024-12-10T11:22:55.500105Z","shell.execute_reply":"2024-12-10T11:23:13.780129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def merge_date(dfs):\n    dfs_new = dfs.copy()\n    for id, df in dfs_new.items():\n        df['time_of_day'] = df['time_of_day'] / 360000000000\n\n        day_filtered = df[(df['time_of_day'] >= 6) & (df['time_of_day'] <= 18)].copy()\n        day_filtered['group'] = (abs(day_filtered['time_of_day'].diff() > (5 / 3600))).cumsum()\n        day_group = day_filtered.groupby('group').agg({\n            'relative_date_PCIAT': 'first',\n            'weekday': 'first',\n            'quarter': 'first',\n            'movement': 'mean',\n            'light': 'mean',\n            'time_of_day': 'mean',\n        }).reset_index(drop=True)\n        day_avg = day_group[['relative_date_PCIAT', 'quarter', 'weekday', 'movement', 'light']]\n        day_avg.columns = ['relative_date_PCIAT', 'quarter', 'weekday', 'day_mov', 'day_light']\n        day_avg = day_avg.groupby(['relative_date_PCIAT', 'quarter', 'weekday'], as_index=False).agg({\n            'day_mov': 'mean',\n            'day_light': 'mean',\n        })\n\n        eve_filtered = df[(df['time_of_day'] > 18) & (df['time_of_day'] <= 22)].copy()\n        eve_filtered['group'] = (abs(eve_filtered['time_of_day'].diff() > (5 / 3600))).cumsum()\n        eve_group = eve_filtered.groupby('group').agg({\n            'relative_date_PCIAT': 'first',\n            'weekday': 'first',\n            'quarter': 'first',\n            'movement': 'mean',\n            'light': 'mean',\n            'time_of_day': 'mean',\n        }).reset_index(drop=True)\n        eve_avg = eve_group[['relative_date_PCIAT', 'quarter', 'weekday', 'movement', 'light']]\n        eve_avg.columns = ['relative_date_PCIAT', 'quarter', 'weekday', 'eve_mov', 'eve_light']\n        eve_avg = eve_avg.groupby(['relative_date_PCIAT', 'quarter', 'weekday'], as_index=False).agg({\n            'eve_mov': 'mean',\n            'eve_light': 'mean',\n        })\n\n        night_filtered = df[((df['time_of_day'] >= 0) & (df['time_of_day'] < 6)) | ((df['time_of_day'] > 22) & (df['time_of_day'] <= 24))].copy()\n        night_filtered['group'] = (abs(night_filtered['time_of_day'].diff() > (5 / 3600))).cumsum()\n        night_group = night_filtered.groupby('group').agg({\n            'relative_date_PCIAT': 'first',\n            'weekday': 'first',\n            'quarter': 'first',\n            'movement': 'mean',\n            'light': 'mean',\n            'time_of_day': 'mean',\n        }).reset_index(drop=True)\n        night_avg = night_group[['relative_date_PCIAT', 'quarter', 'weekday', 'movement', 'light']]\n        night_avg.columns = ['relative_date_PCIAT', 'quarter', 'weekday', 'night_mov', 'night_light']\n        night_avg = night_avg.groupby(['relative_date_PCIAT', 'quarter', 'weekday'], as_index=False).agg({\n            'night_mov': 'mean',\n            'night_light': 'mean',\n        })\n\n        final_result = pd.merge(day_avg, eve_avg, on=['relative_date_PCIAT', 'quarter', 'weekday'], how='left')\n        final_result = pd.merge(final_result, night_avg, on=['relative_date_PCIAT', 'quarter', 'weekday'], how='left')\n\n        dfs_new[id] = final_result\n\n    return dfs_new","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:24:29.795037Z","iopub.execute_input":"2024-12-10T11:24:29.795394Z","iopub.status.idle":"2024-12-10T11:24:29.807848Z","shell.execute_reply.started":"2024-12-10T11:24:29.795363Z","shell.execute_reply":"2024-12-10T11:24:29.806708Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts_new = merge_date(train_ts)\ntest_ts_new = merge_date(test_ts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:24:33.889766Z","iopub.execute_input":"2024-12-10T11:24:33.890083Z","iopub.status.idle":"2024-12-10T11:25:03.810177Z","shell.execute_reply.started":"2024-12-10T11:24:33.890057Z","shell.execute_reply":"2024-12-10T11:25:03.809475Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del train_ts\ngc.collect()\ndel test_ts\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:25:22.370958Z","iopub.execute_input":"2024-12-10T11:25:22.371784Z","iopub.status.idle":"2024-12-10T11:25:22.839524Z","shell.execute_reply.started":"2024-12-10T11:25:22.371751Z","shell.execute_reply":"2024-12-10T11:25:22.838704Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Merge","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:25:26.281917Z","iopub.execute_input":"2024-12-10T11:25:26.282264Z","iopub.status.idle":"2024-12-10T11:25:26.344517Z","shell.execute_reply.started":"2024-12-10T11:25:26.282233Z","shell.execute_reply":"2024-12-10T11:25:26.343495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def merge_ts(dfs):\n    dfs_new = dfs.copy()\n    describes = []\n    ids = []\n    for id, df in dfs_new.items():\n        describe = df.describe().values.reshape(-1)\n        describes.append(describe)\n        new_id = id.replace('id_', '')\n        ids.append(new_id)\n\n    stats = pd.DataFrame(describes, columns=[f\"stat_{i}\" for i in range(len(describes[0]))])\n    stats['id'] = ids\n\n    return stats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:25:28.592450Z","iopub.execute_input":"2024-12-10T11:25:28.593023Z","iopub.status.idle":"2024-12-10T11:25:28.598705Z","shell.execute_reply.started":"2024-12-10T11:25:28.592990Z","shell.execute_reply":"2024-12-10T11:25:28.597615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts_final = merge_ts(train_ts_new)\ntest_ts_final = merge_ts(test_ts_new)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:25:31.027085Z","iopub.execute_input":"2024-12-10T11:25:31.027860Z","iopub.status.idle":"2024-12-10T11:25:42.372837Z","shell.execute_reply.started":"2024-12-10T11:25:31.027827Z","shell.execute_reply":"2024-12-10T11:25:42.372060Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.drop(columns=[f'PCIAT-PCIAT_{i:02}' for i in range(1, 21)])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:26:16.420107Z","iopub.execute_input":"2024-12-10T11:26:16.420454Z","iopub.status.idle":"2024-12-10T11:26:16.430206Z","shell.execute_reply.started":"2024-12-10T11:26:16.420417Z","shell.execute_reply":"2024-12-10T11:26:16.429233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.drop(columns=['PCIAT-Season'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:26:18.299915Z","iopub.execute_input":"2024-12-10T11:26:18.300760Z","iopub.status.idle":"2024-12-10T11:26:18.306840Z","shell.execute_reply.started":"2024-12-10T11:26:18.300725Z","shell.execute_reply":"2024-12-10T11:26:18.305894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"season_mapping = {\n    'Winter': 1,\n    'Spring': 2,\n    'Summer': 3,\n    'Fall': 4\n}\n\nfor col in train.columns:\n    if 'Season' in col:\n        train[col] = train[col].replace(season_mapping)\n\nfor col in test.columns:\n    if 'Season' in col:\n        test[col] = test[col].replace(season_mapping)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:26:22.488604Z","iopub.execute_input":"2024-12-10T11:26:22.489161Z","iopub.status.idle":"2024-12-10T11:26:22.522097Z","shell.execute_reply.started":"2024-12-10T11:26:22.489108Z","shell.execute_reply":"2024-12-10T11:26:22.521285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def perform_autoencoder(df, autoencoder, optimizer, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    criterion = nn.MSELoss()\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n                 \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:26:25.114241Z","iopub.execute_input":"2024-12-10T11:26:25.114872Z","iopub.status.idle":"2024-12-10T11:26:25.121298Z","shell.execute_reply.started":"2024-12-10T11:26:25.114838Z","shell.execute_reply":"2024-12-10T11:26:25.120443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ts_cols = train_ts_final.columns.tolist()\nts_cols.remove(\"id\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:26:28.419807Z","iopub.execute_input":"2024-12-10T11:26:28.420474Z","iopub.status.idle":"2024-12-10T11:26:28.424418Z","shell.execute_reply.started":"2024-12-10T11:26:28.420441Z","shell.execute_reply":"2024-12-10T11:26:28.423516Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = train_ts_final.drop('id', axis=1)\ndf_test = test_ts_final.drop('id', axis=1)\n\nimputer = KNNImputer(n_neighbors=5)\ndf_train_imputed = imputer.fit_transform(df_train)\n\nimputer = KNNImputer(n_neighbors=5)\ndf_test_imputed = imputer.fit_transform(df_test)\n\nts_train_encoder = AutoEncoder(72, 45)\nts_train_optimizer = optim.Adam(ts_train_encoder.parameters())\ntrain_ts_encoded = perform_autoencoder(df_train_imputed, ts_train_encoder, ts_train_optimizer, epochs=100, batch_size=32)\n\nts_test_encoder = AutoEncoder(72, 45)\nts_test_optimizer = optim.Adam(ts_test_encoder.parameters())\ntest_ts_encoded = perform_autoencoder(df_test_imputed, ts_test_encoder, ts_test_optimizer, epochs=100, batch_size=32)\n\nts_cols_encoded = train_ts_encoded.columns.tolist()\ntrain_ts_encoded[\"id\"] = train_ts_final[\"id\"]\ntest_ts_encoded['id'] = test_ts_final[\"id\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:26:30.830440Z","iopub.execute_input":"2024-12-10T11:26:30.831235Z","iopub.status.idle":"2024-12-10T11:26:39.275809Z","shell.execute_reply.started":"2024-12-10T11:26:30.831200Z","shell.execute_reply":"2024-12-10T11:26:39.274928Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train1 = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\ntest1 = pd.merge(test, test_ts_encoded, how=\"left\", on='id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:26:46.461242Z","iopub.execute_input":"2024-12-10T11:26:46.461577Z","iopub.status.idle":"2024-12-10T11:26:46.478059Z","shell.execute_reply.started":"2024-12-10T11:26:46.461548Z","shell.execute_reply":"2024-12-10T11:26:46.477097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_engineering(df):\n    season_cols = [col for col in df.columns if 'Season' in col]\n    df = df.drop(season_cols, axis=1) \n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:26:48.914993Z","iopub.execute_input":"2024-12-10T11:26:48.915567Z","iopub.status.idle":"2024-12-10T11:26:48.922443Z","shell.execute_reply.started":"2024-12-10T11:26:48.915533Z","shell.execute_reply":"2024-12-10T11:26:48.921443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imputer = KNNImputer(n_neighbors=5)\nnumeric_cols = train1.select_dtypes(include=['float64', 'int64']).columns\nimputed_data = imputer.fit_transform(train1[numeric_cols])\ntrain_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\ntrain_imputed['sii'] = train_imputed['sii'].round().astype(int)\nfor col in train1.columns:\n    if col not in numeric_cols:\n        train_imputed[col] = train1[col]\n\ntrain1 = train_imputed\ntest1 = test1.copy()\n\ntrain1 = feature_engineering(train1)\ntrain1 = train1.dropna(thresh=10, axis=0)\ntest1 = feature_engineering(test1)\n\ntrain1 = train1.drop('id', axis=1)\ntest1 = test1.drop('id', axis=1) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:26:51.675599Z","iopub.execute_input":"2024-12-10T11:26:51.675970Z","iopub.status.idle":"2024-12-10T11:26:57.841321Z","shell.execute_reply.started":"2024-12-10T11:26:51.675941Z","shell.execute_reply":"2024-12-10T11:26:57.840370Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featuresColsTrain = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'PCIAT-PCIAT_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW']\n\nfeaturesColsTrain += ts_cols_encoded\n    \ntrain1 = train1[featuresColsTrain]\ntrain1 = train1.dropna(subset='sii')\n\nfeaturesColsTest = [x for x in featuresColsTrain if x not in ['PCIAT-PCIAT_Total', 'sii']]\ntest1 = test1[featuresColsTest]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:27:58.564314Z","iopub.execute_input":"2024-12-10T11:27:58.564777Z","iopub.status.idle":"2024-12-10T11:27:58.581412Z","shell.execute_reply.started":"2024-12-10T11:27:58.564733Z","shell.execute_reply":"2024-12-10T11:27:58.580329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if np.any(np.isinf(train1)):\n    train1 = train1.replace([np.inf, -np.inf], np.nan)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:28:02.036803Z","iopub.execute_input":"2024-12-10T11:28:02.037594Z","iopub.status.idle":"2024-12-10T11:28:02.047693Z","shell.execute_reply.started":"2024-12-10T11:28:02.037561Z","shell.execute_reply":"2024-12-10T11:28:02.046922Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"n_splits = 5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:28:04.396063Z","iopub.execute_input":"2024-12-10T11:28:04.396990Z","iopub.status.idle":"2024-12-10T11:28:04.400899Z","shell.execute_reply.started":"2024-12-10T11:28:04.396954Z","shell.execute_reply":"2024-12-10T11:28:04.399891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def TrainML(model_class, train, test_data):\n    X = train.drop(['sii', 'PCIAT-PCIAT_Total'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n\n    return tpTuned","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:28:07.532803Z","iopub.execute_input":"2024-12-10T11:28:07.533129Z","iopub.status.idle":"2024-12-10T11:28:07.542947Z","shell.execute_reply.started":"2024-12-10T11:28:07.533102Z","shell.execute_reply":"2024-12-10T11:28:07.541929Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Tuning","metadata":{}},{"cell_type":"code","source":"LGB_Params_R = {\n    'learning_rate': 0.07121634122263829,\n    'max_depth': 13,\n    'num_leaves': 260,\n    'min_data_in_leaf': 40,\n    'feature_fraction': 0.6294390743753487,\n    'bagging_fraction': 0.4620672349666255,\n    'bagging_freq': 4,\n    'lambda_l1': 8.274276610342241,  # Increased from 6.59\n    'lambda_l2': 0.4262585435596645,  # Increased from 2.68e-06\n    'device': 'cpu'\n\n}\n\nXGB_Params_R = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': seed,\n    'tree_method': 'gpu_hist',\n\n}\n\n\nCatBoost_Params_R = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': seed,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU'\n\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:35:43.058989Z","iopub.execute_input":"2024-12-10T11:35:43.059362Z","iopub.status.idle":"2024-12-10T11:35:43.065766Z","shell.execute_reply.started":"2024-12-10T11:35:43.059329Z","shell.execute_reply":"2024-12-10T11:35:43.064881Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Result","metadata":{}},{"cell_type":"code","source":"LGB_ML = LGBMRegressor(**LGB_Params_R, random_state=seed, verbose=-1, n_estimators=300)\nXGB_ML = XGBRegressor(**XGB_Params_R)\nCatBoost_ML = CatBoostRegressor(**CatBoost_Params_R)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:35:47.032567Z","iopub.execute_input":"2024-12-10T11:35:47.033182Z","iopub.status.idle":"2024-12-10T11:35:47.039900Z","shell.execute_reply.started":"2024-12-10T11:35:47.033147Z","shell.execute_reply":"2024-12-10T11:35:47.039013Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"voting_ML = VotingRegressor(estimators=[\n    ('lightgbm', LGB_ML),\n    ('xgboost', XGB_ML),\n    ('catboost', CatBoost_ML),\n], weights=[4.0,4.0,5.0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:35:49.734307Z","iopub.execute_input":"2024-12-10T11:35:49.734947Z","iopub.status.idle":"2024-12-10T11:35:49.739060Z","shell.execute_reply.started":"2024-12-10T11:35:49.734913Z","shell.execute_reply":"2024-12-10T11:35:49.738152Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ML1 = TrainML(voting_ML, train1, test1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:35:52.605088Z","iopub.execute_input":"2024-12-10T11:35:52.605441Z","iopub.status.idle":"2024-12-10T11:36:18.843986Z","shell.execute_reply.started":"2024-12-10T11:35:52.605402Z","shell.execute_reply":"2024-12-10T11:36:18.843186Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission1 = pd.DataFrame({\n    'id': sample['id'],\n    'sii': ML1\n})\n\nSubmission1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:36:38.827250Z","iopub.execute_input":"2024-12-10T11:36:38.827967Z","iopub.status.idle":"2024-12-10T11:36:38.839595Z","shell.execute_reply.started":"2024-12-10T11:36:38.827932Z","shell.execute_reply":"2024-12-10T11:36:38.838790Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Round 2","metadata":{}},{"cell_type":"code","source":"train2 = pd.merge(train, train_ts_final, how=\"left\", on='id')\ntest2 = pd.merge(test, test_ts_final, how=\"left\", on='id')\n\ntrain2 = train2.drop('id', axis=1)\ntest2 = test2.drop('id', axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:36:55.576605Z","iopub.execute_input":"2024-12-10T11:36:55.577249Z","iopub.status.idle":"2024-12-10T11:36:55.593092Z","shell.execute_reply.started":"2024-12-10T11:36:55.577214Z","shell.execute_reply":"2024-12-10T11:36:55.592206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featuresColsTrain = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'PCIAT-PCIAT_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\nfeaturesColsTrain += ts_cols\n\ntrain2 = train2[featuresColsTrain]\ntrain2 = train2.dropna(subset='sii')\n\nseason_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\n\ndef season_fill(df):\n    global season_c\n    for c in season_c: \n        df[c] = df[c].fillna(0)\n        \n    return df\n\ntrain2 = season_fill(train2)\ntest2 = season_fill(test2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:37:03.981149Z","iopub.execute_input":"2024-12-10T11:37:03.982075Z","iopub.status.idle":"2024-12-10T11:37:04.002506Z","shell.execute_reply.started":"2024-12-10T11:37:03.982028Z","shell.execute_reply":"2024-12-10T11:37:04.001455Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"LGB_Params_R = {\n    'learning_rate': 0.03161667182533057,\n    'max_depth': 11,\n    'num_leaves': 357,\n    'min_data_in_leaf': 32,\n    'feature_fraction': 0.7858471193300243,\n    'bagging_fraction': 0.47927763242231847,\n    'bagging_freq': 6,\n    'lambda_l1': 7.674389480143931,  # Increased from 6.59\n    'lambda_l2': 0.19722158884072064,  # Increased from 2.68e-06\n    'device': 'cpu'\n\n}\n\nXGB_Params_R = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': seed,\n    'tree_method': 'gpu_hist',\n\n}\n\n\nCatBoost_Params_R = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': seed,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU'\n\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:41:35.218186Z","iopub.execute_input":"2024-12-10T11:41:35.218568Z","iopub.status.idle":"2024-12-10T11:41:35.224580Z","shell.execute_reply.started":"2024-12-10T11:41:35.218535Z","shell.execute_reply":"2024-12-10T11:41:35.223633Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"LGB_ML = LGBMRegressor(**LGB_Params_R, random_state=seed, verbose=-1, n_estimators=300)\nXGB_ML = XGBRegressor(**XGB_Params_R)\nCatBoost_ML = CatBoostRegressor(**CatBoost_Params_R)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:41:39.369015Z","iopub.execute_input":"2024-12-10T11:41:39.369708Z","iopub.status.idle":"2024-12-10T11:41:39.374115Z","shell.execute_reply.started":"2024-12-10T11:41:39.369642Z","shell.execute_reply":"2024-12-10T11:41:39.373172Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"voting_ML = VotingRegressor(estimators=[\n    ('lightgbm', LGB_ML),\n    ('xgboost', XGB_ML),\n    ('catboost', CatBoost_ML),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:41:41.590562Z","iopub.execute_input":"2024-12-10T11:41:41.590917Z","iopub.status.idle":"2024-12-10T11:41:41.595384Z","shell.execute_reply.started":"2024-12-10T11:41:41.590888Z","shell.execute_reply":"2024-12-10T11:41:41.594438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ML2 = TrainML(voting_ML, train2, test2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:41:49.278262Z","iopub.execute_input":"2024-12-10T11:41:49.279067Z","iopub.status.idle":"2024-12-10T11:42:00.786407Z","shell.execute_reply.started":"2024-12-10T11:41:49.279032Z","shell.execute_reply":"2024-12-10T11:42:00.785495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission2 = pd.DataFrame({\n    'id': sample['id'],\n    'sii': ML2\n})\n\nSubmission2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:42:09.416827Z","iopub.execute_input":"2024-12-10T11:42:09.417594Z","iopub.status.idle":"2024-12-10T11:42:09.426444Z","shell.execute_reply.started":"2024-12-10T11:42:09.417562Z","shell.execute_reply":"2024-12-10T11:42:09.425537Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Round 3","metadata":{}},{"cell_type":"code","source":"train3 = train2.copy()\ntest3 = test2.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:42:16.589548Z","iopub.execute_input":"2024-12-10T11:42:16.589923Z","iopub.status.idle":"2024-12-10T11:42:16.597590Z","shell.execute_reply.started":"2024-12-10T11:42:16.589892Z","shell.execute_reply":"2024-12-10T11:42:16.596850Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imputer = SimpleImputer(strategy='median')\n\nensemble_ML = VotingRegressor(estimators=[\n    ('lgb', Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(random_state=seed))])),\n    ('xgb', Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(random_state=seed))])),\n    ('cat', Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(random_state=seed, silent=True))])),\n    ('rf', Pipeline(steps=[('imputer', imputer), ('regressor', RandomForestRegressor(random_state=seed))])),\n    ('gb', Pipeline(steps=[('imputer', imputer), ('regressor', GradientBoostingRegressor(random_state=seed))]))\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:42:19.012774Z","iopub.execute_input":"2024-12-10T11:42:19.013679Z","iopub.status.idle":"2024-12-10T11:42:19.019661Z","shell.execute_reply.started":"2024-12-10T11:42:19.013623Z","shell.execute_reply":"2024-12-10T11:42:19.018916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ML3 = TrainML(ensemble_ML, train3, test3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:42:21.482551Z","iopub.execute_input":"2024-12-10T11:42:21.482930Z","iopub.status.idle":"2024-12-10T11:43:57.548687Z","shell.execute_reply.started":"2024-12-10T11:42:21.482899Z","shell.execute_reply":"2024-12-10T11:43:57.547843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission3 = pd.DataFrame({\n    'id': sample['id'],\n    'sii': ML3\n})\n\nSubmission3","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:44:14.959194Z","iopub.execute_input":"2024-12-10T11:44:14.959917Z","iopub.status.idle":"2024-12-10T11:44:14.969336Z","shell.execute_reply.started":"2024-12-10T11:44:14.959883Z","shell.execute_reply":"2024-12-10T11:44:14.968409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub1 = Submission1\nsub2 = Submission2\nsub3 = Submission3\n\nsub1 = sub1.sort_values(by='id').reset_index(drop=True)\nsub2 = sub2.sort_values(by='id').reset_index(drop=True)\nsub3 = sub3.sort_values(by='id').reset_index(drop=True)\n\ncombined = pd.DataFrame({\n    'id': sub1['id'],\n    'sii_1': sub1['sii'],\n    'sii_2': sub2['sii'],\n    'sii_3': sub3['sii']\n})\n\ndef majority_vote(row):\n    return row.mode()[0]\n\ncombined['final_sii'] = combined[['sii_1', 'sii_2', 'sii_3']].apply(majority_vote, axis=1)\n\nfinal_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\nfinal_submission.to_csv('submission.csv', index=False)\n\nfinal_submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T11:44:28.221004Z","iopub.execute_input":"2024-12-10T11:44:28.221759Z","iopub.status.idle":"2024-12-10T11:44:28.246198Z","shell.execute_reply.started":"2024-12-10T11:44:28.221724Z","shell.execute_reply":"2024-12-10T11:44:28.245162Z"}},"outputs":[],"execution_count":null}]}