{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":7453542,"sourceType":"datasetVersion","datasetId":921302}],"dockerImageVersionId":30762,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":673.251127,"end_time":"2024-12-12T20:25:29.287784","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-12-12T20:14:16.036657","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip -q install /kaggle/input/pytorchtabnet/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pytorch_tabnet.tab_model import TabNetRegressor\nimport torch","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nimport polars as pl\nimport polars.selectors as cs\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\n\n\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None","metadata":{"papermill":{"duration":23.622575,"end_time":"2024-12-12T20:14:42.367516","exception":false,"start_time":"2024-12-12T20:14:18.744941","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\nseed_everything(2024)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_labels = ['None', 'Mild', 'Moderate', 'Severe']","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"season_dtype = pl.Enum(['Spring', 'Summer', 'Fall', 'Winter'])\n\ntrain = (\n    pl.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n    .with_columns(pl.col('^.*Season$').cast(season_dtype))\n)\n\ntest = (\n    pl.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n    .with_columns(pl.col('^.*Season$').cast(season_dtype))\n)\n\n\ntrain\ntest","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"For a supervised learning, we need the target value, but some (sii) are missing. So we only use the part with valid target value(sii).","metadata":{}},{"cell_type":"code","source":"supervised_usable = (\n\n    train\n\n    .filter(pl.col('sii').is_not_null())\n\n)\n\n\n\nmissing_count = (\n\n    supervised_usable\n\n    .null_count()\n\n    .transpose(include_header=True,\n\n               header_name='feature',\n\n               column_names=['null_count'])\n\n    .sort('null_count', descending=True)\n\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n\n)\n\nplt.figure(figsize=(6, 15))\n\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\n\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\n\nplt.barh(np.arange(len(missing_count)), \n\n         1 - missing_count.get_column('null_ratio'),\n\n         left=missing_count.get_column('null_ratio'),\n\n         color='darkseagreen', label='available')\n\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\n\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\n\nplt.xlim(0, 1)\n\nplt.legend()\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train.select(pl.col('PCIAT-PCIAT_Total').is_null() == pl.col('sii').is_null()).to_series().mean())\n\n\n\n(train\n\n .select(pl.col('PCIAT-PCIAT_Total'))\n\n .group_by(train.get_column('sii'))\n\n .agg(pl.col('PCIAT-PCIAT_Total').min().alias('PCIAT-PCIAT_Total min'),\n\n      pl.col('PCIAT-PCIAT_Total').max().alias('PCIAT-PCIAT_Total max'),\n\n      pl.col('PCIAT-PCIAT_Total').len().alias('count'))\n\n .sort('sii')\n\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Columns missing in test:')\n\nprint([f for f in train.columns if f not in test.columns])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Demographics","metadata":{"papermill":{"duration":0.020504,"end_time":"2024-12-12T20:16:40.18136","exception":false,"start_time":"2024-12-12T20:16:40.160856","status":"completed"},"tags":[]}},{"cell_type":"code","source":"vc = train.get_column('Basic_Demos-Enroll_Season').value_counts()\n\nplt.pie(vc.get_column('count'), labels=vc.get_column('Basic_Demos-Enroll_Season'))\n\nplt.title('Season of enrollment')\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vc = train.get_column('Basic_Demos-Sex').value_counts()\n\nplt.pie(vc.get_column('count'), labels=['boys', 'girls'])\n\nplt.title('Sex of participant')\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, axs = plt.subplots(2, 1, sharex=True)\n\nfor sex in range(2):\n\n    ax = axs.ravel()[sex]\n\n    vc = train.filter(pl.col('Basic_Demos-Sex') == sex).get_column('Basic_Demos-Age').value_counts()\n\n    ax.bar(vc.get_column('Basic_Demos-Age'),\n\n           vc.get_column('count'),\n\n           color=['lightblue', 'coral'][sex],\n\n           label=['boys', 'girls'][sex])\n\n    ax.xaxis.set_major_locator(MaxNLocator(integer=True))\n\n    ax.set_ylabel('count')\n\n    ax.legend()\n\nplt.suptitle('Age distribution')\n\naxs.ravel()[1].set_xlabel('years')\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, axs = plt.subplots(2, 1, sharex=True, sharey=True)\n\nfor sex in range(2):\n\n    ax = axs.ravel()[sex]\n\n    vc = train.filter(pl.col('Basic_Demos-Sex') == sex).get_column('sii').value_counts()\n\n    ax.bar(vc.get_column('sii'),\n\n           vc.get_column('count') / vc.get_column('count').sum(),\n\n           color=['lightblue', 'coral'][sex],\n\n           label=['boys', 'girls'][sex])\n\n    ax.set_xticks(np.arange(4), target_labels)\n\n    ax.yaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\n\n    ax.set_ylabel('count')\n\n    ax.legend()\n\nplt.suptitle('Target distribution')\n\naxs.ravel()[1].set_xlabel('Severity Impairment Index (sii)')\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Now we look at correlations","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(14, 12))\n\ncorr_matrix = supervised_usable.select([\n\n    'PCIAT-PCIAT_Total', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-BMI', \n\n    'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n\n    'Physical-Diastolic_BP', 'Physical-Systolic_BP', 'Physical-HeartRate',\n\n    'PreInt_EduHx-computerinternet_hoursday', 'SDS-SDS_Total_T', 'PAQ_A-PAQ_A_Total',\n\n    'PAQ_C-PAQ_C_Total', 'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins','Fitness_Endurance-Time_Sec',\n\n    'FGC-FGC_CU', 'FGC-FGC_GSND','FGC-FGC_GSD','FGC-FGC_PU','FGC-FGC_SRL','FGC-FGC_SRR','FGC-FGC_TL','BIA-BIA_Activity_Level_num', \n\n    'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n\n    'BIA-BIA_FFMI','BIA-BIA_FMI', 'BIA-BIA_Fat','BIA-BIA_Frame_num','BIA-BIA_ICW','BIA-BIA_LDM','BIA-BIA_LST',\n\n    'BIA-BIA_SMM','BIA-BIA_TBW'\n\n    # Add other relevant columns\n\n]).to_pandas().corr()\n\n\n\nsii_corr = corr_matrix['PCIAT-PCIAT_Total'].drop('PCIAT-PCIAT_Total')\n\nfiltered_corr = sii_corr[(sii_corr > 0.1) | (sii_corr < -0.1)]\n\n\n\nprint(filtered_corr)\n\n\n\nplt.figure(figsize=(8, 6))\n\nfiltered_corr.sort_values().plot(kind='barh', color='coral')\n\nplt.title('Features with Correlation > 0.1 or < -0.1 with PCIAT-PCIAT_Total')\n\nplt.xlabel('Correlation coefficient')\n\nplt.ylabel('Features')\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Actigraphy (time series)","metadata":{"papermill":{"duration":0.020535,"end_time":"2024-12-12T20:16:40.222671","exception":false,"start_time":"2024-12-12T20:16:40.202136","status":"completed"},"tags":[]}},{"cell_type":"code","source":"actigraphy = pl.read_parquet('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id=0417c91e/part-0.parquet')\n\nactigraphy","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_actigraphy(id, only_one_week=False, small=False):\n\n    actigraphy = pl.read_parquet(f'/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id={id}/part-0.parquet')\n\n    day = actigraphy.get_column('relative_date_PCIAT') + actigraphy.get_column('time_of_day') / 86400e9\n\n    sample = train.filter(pl.col('id') == id)\n\n    age = sample.get_column('Basic_Demos-Age').item()\n\n    sex = ['boy', 'girl'][sample.get_column('Basic_Demos-Sex').item()]\n\n    actigraphy = (\n\n        actigraphy\n\n        .with_columns(\n\n            (day.diff() * 86400).alias('diff_seconds'),\n\n            (np.sqrt(np.square(pl.col('X')) + np.square(pl.col('Y')) + np.square(pl.col('Z'))).alias('norm'))\n\n        )\n\n    )\n\n\n\n    if only_one_week:\n\n        start = np.ceil(day.min())\n\n        mask = (start <= day.to_numpy()) & (day.to_numpy() <= start + 7*3)\n\n        mask &= ~ actigraphy.get_column('non-wear_flag').cast(bool).to_numpy()\n\n    else:\n\n        mask = np.full(len(day), True)\n\n        \n\n    if small:\n\n        timelines = [\n\n            ('enmo', 'forestgreen'),\n\n            ('light', 'orange'),\n\n        ]\n\n    else:\n\n        timelines = [\n\n            ('X', 'm'),\n\n            ('Y', 'm'),\n\n            ('Z', 'm'),\n\n#             ('norm', 'c'),\n\n            ('enmo', 'forestgreen'),\n\n            ('anglez', 'lightblue'),\n\n            ('light', 'orange'),\n\n            ('non-wear_flag', 'chocolate')\n\n    #         ('diff_seconds', 'k'),\n\n        ]\n\n        \n\n    _, axs = plt.subplots(len(timelines), 1, sharex=True, figsize=(12, len(timelines) * 1.1 + 0.5))\n\n    for ax, (feature, color) in zip(axs, timelines):\n\n        ax.set_facecolor('#eeeeee')\n\n        ax.scatter(day.to_numpy()[mask],\n\n                   actigraphy.get_column(feature).to_numpy()[mask],\n\n                   color=color, label=feature, s=1)\n\n        ax.legend(loc='upper left', facecolor='#eeeeee')\n\n        if feature == 'diff_seconds':\n\n            ax.set_ylim(-0.5, 20.5)\n\n    axs[-1].set_xlabel('day')\n\n    axs[-1].xaxis.set_major_locator(MaxNLocator(integer=True))\n\n    plt.tight_layout()\n\n    axs[0].set_title(f'id={id}, {sex}, age={age}')\n\n    plt.show()\n\n\n\nanalyze_actigraphy('0417c91e', only_one_week=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42\n\nn_splits = 5","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering\n\n\n\n- **Feature Selection**: The dataset contains features related to physical characteristics (e.g., BMI, Height, Weight), behavioral aspects (e.g., internet usage), and fitness data (e.g., endurance time). \n\n- **Categorical Feature Encoding**: Categorical features are mapped to numerical values using custom mappings for each unique category within the dataset. This ensures compatibility with machine learning algorithms that require numerical input.\n\n- **Time Series Aggregation**: Time series statistics (e.g., mean, standard deviation) from the actigraphy data are computed and merged into the main dataset to create additional features for model training.\n","metadata":{}},{"cell_type":"code","source":"def process_file(filename, dirname):\n\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n\n    df.drop('step', axis=1, inplace=True)\n\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\n\n\ndef load_time_series(dirname) -> pd.DataFrame:\n\n    ids = os.listdir(dirname)\n\n    \n\n    with ThreadPoolExecutor() as executor:\n\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n\n    \n\n    stats, indexes = zip(*results)\n\n    \n\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n\n    df['id'] = indexes\n\n    return df\n\nfrom torch.utils.data import Dataset,DataLoader\nfrom sklearn.preprocessing import StandardScaler\n\nclass CustomDataset(Dataset):\n    \n    def __init__(self, dataframe):\n        # Apply StandardScaler to the input features\n        self.scaler = StandardScaler()\n        if 'sii' in dataframe.columns:\n            self.train = True\n            features = dataframe.drop(['id', 'sii'], axis=1)  # Drop ID and target column\n            \n\n            self.targets = dataframe['sii'].values  # Keep target values (sii)\n        else:\n            self.train = False\n            features = dataframe.drop(['id'],axis=1)\n            \n        self.scaled_data = self.scaler.fit_transform(features)  # Scale features\n            \n\n    def __len__(self):\n        return len(self.scaled_data)\n    \n    def __getitem__(self, idx):\n        # Return the scaled input features and target value\n        if self.train:\n            return torch.tensor(self.scaled_data[idx], dtype=torch.float32), torch.tensor(self.targets[idx], dtype=torch.long)  # Ensure targets are long for classification\n        else:\n            return torch.tensor(self.scaled_data[idx], dtype=torch.float32)\n            \nclass LSTMEncoder(nn.Module):\n    \n    def __init__(self, input_size, hidden_size, latent_dim, num_classes, num_layers=1):\n        super(LSTMEncoder, self).__init__()\n        # LSTM layer\n        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)\n        # Fully connected layers for latent dimension and classification\n        self.fc_latent = nn.Linear(hidden_size, latent_dim)  # Latent space\n        self.fc_class = nn.Linear(latent_dim, num_classes)   # Classifier layer for output\n        \n    def forward(self, x):\n        _, (hn, _) = self.lstm(x)  # Get hidden state from LSTM (hn)\n        latent = self.fc_latent(hn[-1])  # Compress the last hidden state to latent space\n        out = self.fc_class(latent)  # Classify using the latent space\n        return latent, out\n    \nclass VAE(nn.Module):\n    def __init__(self, input_channels, seq_length, latent_dim, dropout_prob=0.4):\n        super(VAE, self).__init__()\n        \n        # Encoder: deeper Conv1d layers with BatchNorm1d\n        self.encoder = nn.Sequential(\n            nn.Conv1d(input_channels, 16, kernel_size=3, stride=2, padding=1),\n            nn.BatchNorm1d(16),\n            nn.ReLU(0.2),\n            nn.Dropout(dropout_prob),\n            \n            nn.Conv1d(16, 32, kernel_size=3, stride=2, padding=1),\n            nn.BatchNorm1d(32),\n            nn.ReLU(0.2),\n            nn.Dropout(dropout_prob),\n            \n            nn.Conv1d(32, 64, kernel_size=3, stride=2, padding=1),\n            nn.BatchNorm1d(64),\n            nn.ReLU(0.2),\n            nn.Flatten()\n        )\n        \n        # Calculate size after Conv1d layers\n        conv_output_size = (seq_length // 8) * 64\n        \n        self.fc_mu = nn.Linear(conv_output_size, latent_dim)\n        self.fc_logvar = nn.Linear(conv_output_size, latent_dim)\n        \n        # Decoder: Fully connected and ConvTranspose1d layers\n        self.decoder_input = nn.Linear(latent_dim, conv_output_size)\n        self.decoder = nn.Sequential(\n            nn.Unflatten(1, (64, seq_length // 8)),\n            nn.ConvTranspose1d(64, 32, kernel_size=3, stride=2, padding=1, output_padding=1),\n            nn.BatchNorm1d(32),\n            nn.ReLU(0.2),\n            nn.Dropout(dropout_prob),\n            \n            nn.ConvTranspose1d(32, 16, kernel_size=3, stride=2, padding=1, output_padding=1),\n            nn.BatchNorm1d(16),\n            nn.ReLU(0.2),\n            nn.Dropout(dropout_prob),\n            \n            nn.ConvTranspose1d(16, input_channels, kernel_size=3, stride=2, padding=1, output_padding=1),\n            nn.Sigmoid()\n        )\n    \n    def reparameterize(self, mu, logvar):\n        std = torch.exp(0.5 * logvar)\n        eps = torch.randn_like(std)\n        return mu + eps * std\n    \n    def forward(self, x):\n        encoded = self.encoder(x)\n        mu = self.fc_mu(encoded)\n        logvar = self.fc_logvar(encoded)\n        z = self.reparameterize(mu, logvar)\n        decoded = self.decoder(self.decoder_input(z))\n        return decoded, mu, logvar\n\ndef reconstruction_loss(original, reconstructed, alpha=0.5, beta=0.5):\n    \"\"\"\n    定义重建损失，MSE + 余弦相似度的组合\n    \"\"\"\n    # Mean Squared Error\n    mse_loss = F.mse_loss(reconstructed, original)\n    \n    # Cosine Similarity -> Cosine Distance\n    cosine_sim = F.cosine_similarity(reconstructed, original, dim=-1)  # 按最后一维计算相似度\n    cosine_dist = 1 - cosine_sim.mean()  # 余弦距离（越小越相似）\n    \n    # 总损失：MSE + Cosine Distance\n    total_loss = alpha * mse_loss + beta * cosine_dist\n    return total_loss\n    \ndef perform_autoencoder(df, latent_dim=64, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled).unsqueeze(1)\n\n    input_channels = data_tensor.shape[1]\n    seq_length = data_tensor.shape[2]\n\n    autoencoder = VAE(input_channels, seq_length, latent_dim)\n    optimizer = optim.AdamW(autoencoder.parameters(), lr=5e-3, weight_decay=1e-4)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5)\n    criterion = nn.MSELoss()  # Reconstruction loss\n    \n    # 权重初始化函数\n    def init_weights(m):\n        if isinstance(m, (nn.Conv1d, nn.ConvTranspose1d, nn.Linear)):\n            nn.init.xavier_uniform_(m.weight)  # 使用Xavier初始化\n            if m.bias is not None:\n                nn.init.constant_(m.bias, 0.01)  # 偏置设置为0.01\n    \n    # Apply weight initialization\n    autoencoder.apply(init_weights)\n\n    for epoch in range(epochs):\n        epoch_loss = 0\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed, mu, logvar = autoencoder(batch)\n            \n            # 重建损失：MSE + Cosine Distance\n            loss = reconstruction_loss(batch, reconstructed)\n            \n            # 加入KL散度和稀疏性正则化\n            kl_divergence = -0.5 * torch.mean(1 + logvar - mu.pow(2) - logvar.exp())\n            l1_regularization = sum(param.abs().sum() for param in autoencoder.encoder.parameters())\n            \n            total_loss = loss + kl_divergence + 1e-4 * l1_regularization\n            total_loss.backward()\n            optimizer.step()\n            epoch_loss += total_loss.item()\n        \n        scheduler.step(epoch_loss)\n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {epoch_loss / len(data_tensor):.4f}')    \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n\n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    return df_encoded\n\ndef feature_engineering(df):\n    season_cols = [col for col in df.columns if 'Season' in col]\n    df = df.drop(season_cols, axis=1) \n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    \n    return df\n\n\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"###############################################################################\ntemp_df = train[['id','sii']]\n\ntime_series_df_with_target = pd.merge(train_ts,temp_df,how=\"left\",on='id')\n\ntemp_df = test[['id']]\n\ntime_series_df_without_target = pd.merge(test_ts,temp_df,how=\"left\",on='id')\n\ntrain_dataset = CustomDataset(time_series_df_with_target)\ntrain_dataloader = DataLoader(train_dataset,batch_size=32,shuffle=True)\n\ntest_dataset = CustomDataset(time_series_df_without_target)\ntest_dataloader = DataLoader(test_dataset,batch_size=1,shuffle=True)\n\n# Example parameters\ninput_size = df_train.shape[1]  # Number of features\nhidden_size = 64  # Number of hidden units in LSTM\nlatent_dim = 64  # Latent space dimension\nnum_classes = len(train['sii'].unique())  # Number of target classes\nnum_layers = 2  # Number of LSTM layers\n\n# Model initialization\nmodel = LSTMEncoder(input_size, hidden_size, latent_dim, num_classes, num_layers)\n\n# Define loss function and optimizer\ncriterion = nn.CrossEntropyLoss()  # For multi-class classification\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n\n# Training loop\nepochs = 30  # Number of epochs to train\n\nfor epoch in range(epochs):\n    total_loss = 0\n    for batch_data, batch_targets in train_dataloader:\n        optimizer.zero_grad()\n        # Forward pass: latent representation and predictions\n        latent, predictions = model(batch_data.unsqueeze(1))\n        # Compute the loss\n        loss = criterion(predictions, batch_targets)\n        total_loss+=loss.item()\n        loss.backward()  # Backpropagate the error\n        optimizer.step()  # Update the weights\n\n    print(f\"Epoch [{epoch+1}], Loss: {total_loss/32}\")\n\n###############################################################################\nfinal_list = []\n\n# After training, you can use the encoder to extract both the latent dimensions and class predictions\nwith torch.no_grad():\n    for batch_data, batch_targets in train_dataloader:\n        batch_data = batch_data.unsqueeze(1)  # Add sequence length dim if needed\n        latent, predictions = model(batch_data)\n        final_list.append(latent)\n\n# Step 1: Concatenate all the latent representations into a single tensor\n# Assuming each 'latent' is of shape (batch_size, latent_dim)\nall_latents = torch.cat(final_list, dim=0)  # Concatenate along the first dimension (batch dimension)\n\n# Step 2: Convert the concatenated tensor to a NumPy array\nlatent_array = all_latents.numpy()  # Convert to NumPy array\n\n# Step 3: Create a DataFrame from the NumPy array\nnum_latent_dims = latent_array.shape[1]  # Get the number of latent dimensions\ntrain_latent = pd.DataFrame(latent_array, columns=[f'enc_{i + 1}' for i in range(num_latent_dims)])\n\ntrain_latent['id'] = time_series_df_with_target['id']\n\ntrain = pd.merge(train, train_latent, how=\"left\", on='id')\n###############################################################################\nfinal_list = []\n\n# After training, you can use the encoder to extract both the latent dimensions and class predictions\nwith torch.no_grad():\n    for batch_data in test_dataloader:\n        batch_data = batch_data.unsqueeze(1)  # Add sequence length dim if needed\n        latent, predictions = model(batch_data)\n        final_list.append(latent)\n\n# Step 1: Concatenate all the latent representations into a single tensor\n# Assuming each 'latent' is of shape (batch_size, latent_dim)\nall_latents = torch.cat(final_list, dim=0)  # Concatenate along the first dimension (batch dimension)\n\n# Step 2: Convert the concatenated tensor to a NumPy array\nlatent_array = all_latents.numpy()  # Convert to NumPy array\n\n# Step 3: Create a DataFrame from the NumPy array\nnum_latent_dims = latent_array.shape[1]  # Get the number of latent dimensions\ntest_latent = pd.DataFrame(latent_array, columns=[f'enc_{i + 1}' for i in range(num_latent_dims)])\n\ntest_latent['id'] = time_series_df_without_target['id']\ntest = pd.merge(test, test_latent, how=\"left\", on='id')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts_encoded = perform_autoencoder(df_train, latent_dim=75, epochs=100, batch_size=32)\ntest_ts_encoded = perform_autoencoder(df_test, latent_dim=75, epochs=100, batch_size=32)\n\ntime_series_cols = train_ts_encoded.columns.tolist()\ntrain_ts_encoded[\"id\"]=train_ts[\"id\"]\ntest_ts_encoded['id']=test_ts[\"id\"]\n\ntrain = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\ntest = pd.merge(test, test_ts_encoded, how=\"left\", on='id')\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imputer = KNNImputer(n_neighbors=5)\n\nnumeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\n\nimputed_data = imputer.fit_transform(train[numeric_cols])\n\ntrain_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\n\ntrain_imputed['sii'] = train_imputed['sii'].round().astype(int)\n\nfor col in train.columns:\n\n    if col not in numeric_cols:\n\n        train_imputed[col] = train[col]\n\n        \n\ntrain = train_imputed\n\n\n\ntrain = feature_engineering(train)\n\ntrain = train.dropna(thresh=10, axis=0)\n\ntest = feature_engineering(test)\n\n\n\ntrain = train.drop('id', axis=1)\n\ntest  = test .drop('id', axis=1)   \n\n\n\n\n\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n\n                'CGAS-CGAS_Score', 'Physical-BMI',\n\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n\n                'Fitness_Endurance-Max_Stage',\n\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n\n                'SDS-SDS_Total_T',\n\n                'PreInt_EduHx-computerinternet_hoursday', 'sii', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW']\n\n\n\nfeaturesCols += time_series_cols\n\n\n\ntrain = train[featuresCols]\n\ntrain = train.dropna(subset='sii')\n\n\n\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n\n                'CGAS-CGAS_Score', 'Physical-BMI',\n\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n\n                'Fitness_Endurance-Max_Stage',\n\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n\n                'SDS-SDS_Total_T',\n\n                'PreInt_EduHx-computerinternet_hoursday', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW']\n\n\n\nfeaturesCols += time_series_cols\n\ntest = test[featuresCols]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if np.any(np.isinf(train)):\n    train = train.replace([np.inf, -np.inf], np.nan)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Training and Evaluation\n\n- **Model Types**: Various models are used, including:\n  - **LightGBM**: A gradient-boosting framework known for its speed and efficiency with large datasets.\n  - **XGBoost**: Another powerful gradient-boosting model used for structured data.\n  - **CatBoost**: Optimized for categorical features without the need for extensive preprocessing.\n  - **Voting Regressor**: An ensemble model that combines the predictions of LightGBM, XGBoost, and CatBoost for better accuracy.\n- **Cross-Validation**: Stratified K-Folds cross-validation is employed to split the data into training and validation sets, ensuring balanced class distribution in each fold.\n- **Quadratic Weighted Kappa (QWK)**: The performance of the models is evaluated using QWK, which measures the agreement between predicted and actual values, taking into account the ordinal nature of the target variable.\n- **Threshold Optimization**: The `minimize` function from `scipy.optimize` is used to fine-tune decision thresholds that map continuous predictions to discrete categories (None, Mild, Moderate, Severe).\n","metadata":{"papermill":{"duration":0.035922,"end_time":"2024-12-12T20:18:48.348136","exception":false,"start_time":"2024-12-12T20:18:48.312214","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n\n        test_preds[:, fold] = model.predict(test_data)\n\n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n\n\n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n\n    return submission","metadata":{"papermill":{"duration":0.047855,"end_time":"2024-12-12T20:18:48.432006","exception":false,"start_time":"2024-12-12T20:18:48.384151","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n# Hyperparameter Tuning\n\n- **LightGBM Parameters**: Hyperparameters such as `learning_rate`, `max_depth`, `num_leaves`, and `feature_fraction` are tuned to improve the performance of the LightGBM model. These parameters control the complexity of the model and its ability to generalize to new data.\n- **XGBoost and CatBoost Parameters**: Similar tuning is applied for XGBoost and CatBoost, adjusting parameters such as `n_estimators`, `max_depth`, `learning_rate`, `subsample`, and `regularization` terms (`reg_alpha`, `reg_lambda`). These help in controlling overfitting and ensuring the model's robustness.","metadata":{"papermill":{"duration":0.035531,"end_time":"2024-12-12T20:18:48.503391","exception":false,"start_time":"2024-12-12T20:18:48.46786","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01,  # Increased from 2.68e-06\n    'device': 'cpu'\n\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED,\n    'tree_method': 'gpu_hist',\n\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU'\n\n}","metadata":{"papermill":{"duration":0.042924,"end_time":"2024-12-12T20:18:48.581996","exception":false,"start_time":"2024-12-12T20:18:48.539072","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# New: TabNet\n\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom pytorch_tabnet.callbacks import Callback\nimport os\nimport torch\nfrom pytorch_tabnet.callbacks import Callback\n\nclass TabNetWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, **kwargs):\n        self.model = TabNetRegressor(**kwargs)\n        self.kwargs = kwargs\n        self.imputer = SimpleImputer(strategy='median')\n        self.best_model_path = 'best_tabnet_model.pt'\n        \n    def fit(self, X, y):\n        # Handle missing values\n        X_imputed = self.imputer.fit_transform(X)\n        \n        if hasattr(y, 'values'):\n            y = y.values\n            \n        # Create internal validation set\n        X_train, X_valid, y_train, y_valid = train_test_split(\n            X_imputed, \n            y, \n            test_size=0.2,\n            random_state=42\n        )\n        \n        # Train TabNet model\n        history = self.model.fit(\n            X_train=X_train,\n            y_train=y_train.reshape(-1, 1),\n            eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n            eval_name=['valid'],\n            eval_metric=['mse'],\n            max_epochs=500,\n            patience=50,\n            batch_size=1024,\n            virtual_batch_size=128,\n            num_workers=0,\n            drop_last=False,\n            callbacks=[\n                TabNetPretrainedModelCheckpoint(\n                    filepath=self.best_model_path,\n                    monitor='valid_mse',\n                    mode='min',\n                    save_best_only=True,\n                    verbose=True\n                )\n            ]\n        )\n        \n        # Load the best model\n        if os.path.exists(self.best_model_path):\n            self.model.load_model(self.best_model_path)\n            os.remove(self.best_model_path)  # Remove temporary file\n        \n        return self\n    \n    def predict(self, X):\n        X_imputed = self.imputer.transform(X)\n        return self.model.predict(X_imputed).flatten()\n    \n    def __deepcopy__(self, memo):\n        # Add deepcopy support for scikit-learn\n        cls = self.__class__\n        result = cls.__new__(cls)\n        memo[id(self)] = result\n        for k, v in self.__dict__.items():\n            setattr(result, k, deepcopy(v, memo))\n        return result\n\n# TabNet hyperparameters\nTabNet_Params = {\n    'n_d': 64,              # Width of the decision prediction layer\n    'n_a': 64,              # Width of the attention embedding for each step\n    'n_steps': 5,           # Number of steps in the architecture\n    'gamma': 1.5,           # Coefficient for feature selection regularization\n    'n_independent': 2,     # Number of independent GLU layer in each GLU block\n    'n_shared': 2,          # Number of shared GLU layer in each GLU block\n    'lambda_sparse': 1e-4,  # Sparsity regularization\n    'optimizer_fn': torch.optim.Adam,\n    'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n    'mask_type': 'entmax',\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'verbose': 1,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n    def __init__(self, filepath, monitor='val_loss', mode='min', \n                 save_best_only=True, verbose=1):\n        super().__init__()  # Initialize parent class\n        self.filepath = filepath\n        self.monitor = monitor\n        self.mode = mode\n        self.save_best_only = save_best_only\n        self.verbose = verbose\n        self.best = float('inf') if mode == 'min' else -float('inf')\n        \n    def on_train_begin(self, logs=None):\n        self.model = self.trainer  # Use trainer itself as model\n        \n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is None:\n            return\n        \n        # Check if current metric is better than best\n        if (self.mode == 'min' and current < self.best) or \\\n           (self.mode == 'max' and current > self.best):\n            if self.verbose:\n                print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n            self.best = current\n            if self.save_best_only:\n                self.model.save_model(self.filepath)  # Save the entire model","metadata":{"papermill":{"duration":0.051218,"end_time":"2024-12-12T20:18:48.669191","exception":false,"start_time":"2024-12-12T20:18:48.617973","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ensemble Learning and Submission Preparation\n\n- **Ensemble Learning**: The model uses a **Voting Regressor**, which combines the predictions from LightGBM, XGBoost, and CatBoost. This approach is beneficial as it leverages the strengths of multiple models, reducing overfitting and improving overall model performance.\n- **Out-of-Fold (OOF) Predictions**: During cross-validation, out-of-fold predictions are generated for the training set, which helps in model evaluation without data leakage.\n- **Kappa Optimizer**: The Kappa Optimizer ensures that the predicted values are as close to the actual values as possible by adjusting the thresholds used to convert raw model outputs into class labels.\n- **Test Set Predictions**: After the model is trained and thresholds are optimized, the test dataset is processed, and predictions are generated using the ensemble model. These predictions are converted into the appropriate format for submission.\n- **Submission File Creation**: The predictions are saved in a CSV file following the required format for submission (e.g., for a Kaggle competition), which includes columns like `id` and `sii` (Severity Impairment Index).","metadata":{"papermill":{"duration":0.035821,"end_time":"2024-12-12T20:18:48.741796","exception":false,"start_time":"2024-12-12T20:18:48.705975","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Final Results and Performance Metrics\n\n- **Train and Validation Scores**: After training across multiple folds, the mean Quadratic Weighted Kappa (QWK) score is calculated for both the training and validation datasets, providing an indicator of model performance. \n- **Optimized QWK Score**: The final optimized QWK score after threshold tuning is displayed, showcasing the model's ability to predict the severity levels effectively.\n- **Test Predictions**: The test set predictions are evaluated, and a breakdown of the predicted severity levels (None, Mild, Moderate, Severe) is shown, along with their respective counts.","metadata":{"papermill":{"duration":0.038161,"end_time":"2024-12-12T20:18:48.817502","exception":false,"start_time":"2024-12-12T20:18:48.779341","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\nTabNet_Model = TabNetWrapper(**TabNet_Params) # New","metadata":{"papermill":{"duration":0.050985,"end_time":"2024-12-12T20:18:48.906901","exception":false,"start_time":"2024-12-12T20:18:48.855916","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"voting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model),\n    ('tabnet', TabNet_Model)\n],weights=[4.0,4.0,5.0,4.0])\n\nSubmission1 = TrainML(voting_model, test)\n\nSubmission1","metadata":{"papermill":{"duration":74.74773,"end_time":"2024-12-12T20:20:03.763394","exception":false,"start_time":"2024-12-12T20:18:49.015664","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"```\n],weights=[5.0,4.0,4.0,4.0])\nMean Train QWK --> 0.7424\nMean Validation QWK ---> 0.4735\n----> || Optimized QWK SCORE ::  0.533\n\n```","metadata":{"papermill":{"duration":0.037036,"end_time":"2024-12-12T20:20:03.838467","exception":false,"start_time":"2024-12-12T20:20:03.801431","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n\n\ndef process_file(filename, dirname):\n\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n\n    df.drop('step', axis=1, inplace=True)\n\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\n\n\ndef load_time_series(dirname) -> pd.DataFrame:\n\n    ids = os.listdir(dirname)\n\n    \n\n    with ThreadPoolExecutor() as executor:\n\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n\n    \n\n    stats, indexes = zip(*results)\n\n    \n\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n\n    df['id'] = indexes\n\n    return df\n\n        \n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\n\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\n\n\ntime_series_cols = train_ts.columns.tolist()\n\ntime_series_cols.remove(\"id\")\n\n\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\n\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\n\n\ntrain = train.drop('id', axis=1)\n\ntest = test.drop('id', axis=1)   \n\n\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\n\n\nfeaturesCols += time_series_cols\n\n\n\ntrain = train[featuresCols]\n\ntrain = train.dropna(subset='sii')\n\n\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n\n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n\n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\n\n\ndef update(df):\n\n    global cat_c\n\n    for c in cat_c: \n\n        df[c] = df[c].fillna('Missing')\n\n        df[c] = df[c].astype('category')\n\n    return df\n\n        \n\ntrain = update(train)\n\ntest = update(test)\n\n\n\ndef create_mapping(column, dataset):\n\n    unique_values = dataset[column].unique()\n\n    return {value: idx for idx, value in enumerate(unique_values)}\n\n\n\nfor col in cat_c:\n\n    mapping = create_mapping(col, train)\n\n    mappingTe = create_mapping(col, test)\n\n    \n\n    train[col] = train[col].replace(mapping).astype(int)\n\n    test[col] = test[col].replace(mappingTe).astype(int)\n\n\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n\n    return np.where(oof_non_rounded < thresholds[0], 0,\n\n                    np.where(oof_non_rounded < thresholds[1], 1,\n\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\n\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\n\n\ndef TrainML(model_class, test_data):\n\n    X = train.drop(['sii'], axis=1)\n\n    y = train['sii']\n\n\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n\n    \n\n    train_S = []\n\n    test_S = []\n\n    \n\n    oof_non_rounded = np.zeros(len(y), dtype=float) \n\n    oof_rounded = np.zeros(len(y), dtype=int) \n\n    test_preds = np.zeros((len(test_data), n_splits))\n\n\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n\n\n        model = clone(model_class)\n\n        model.fit(X_train, y_train)\n\n\n\n        y_train_pred = model.predict(X_train)\n\n        y_val_pred = model.predict(X_val)\n\n\n\n        oof_non_rounded[test_idx] = y_val_pred\n\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n\n\n        train_S.append(train_kappa)\n\n        test_S.append(val_kappa)\n\n        \n\n        test_preds[:, fold] = model.predict(test_data)\n\n        \n\n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n\n        clear_output(wait=True)\n\n\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n\n                              method='Nelder-Mead')\n\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n\n    \n\n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n\n\n    tpm = test_preds.mean(axis=1)\n\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n\n    \n\n    submission = pd.DataFrame({\n\n        'id': sample['id'],\n\n        'sii': tpTuned\n\n    })\n\n\n\n    return submission\n\n\n\n# Model parameters for LightGBM\n\nParams = {\n\n    'learning_rate': 0.046,\n\n    'max_depth': 12,\n\n    'num_leaves': 478,\n\n    'min_data_in_leaf': 13,\n\n    'feature_fraction': 0.893,\n\n    'bagging_fraction': 0.784,\n\n    'bagging_freq': 4,\n\n    'lambda_l1': 10,  # Increased from 6.59\n\n    'lambda_l2': 0.01  # Increased from 2.68e-06\n\n}\n\n\n\n\n\n# XGBoost parameters\n\nXGB_Params = {\n\n    'learning_rate': 0.05,\n\n    'max_depth': 6,\n\n    'n_estimators': 200,\n\n    'subsample': 0.8,\n\n    'colsample_bytree': 0.8,\n\n    'reg_alpha': 1,  # Increased from 0.1\n\n    'reg_lambda': 5,  # Increased from 1\n\n    'random_state': SEED\n\n}\n\n\n\n\n\nCatBoost_Params = {\n\n    'learning_rate': 0.05,\n\n    'depth': 6,\n\n    'iterations': 200,\n\n    'random_seed': SEED,\n\n    'cat_features': cat_c,\n\n    'verbose': 0,\n\n    'l2_leaf_reg': 10  # Increase this value\n\n}\n\n\n\n# Create model instances\n\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\n\nXGB_Model = XGBRegressor(**XGB_Params)\n\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\n\n\n# Combine models using Voting Regressor\n\nvoting_model = VotingRegressor(estimators=[\n\n    ('lightgbm', Light),\n\n    ('xgboost', XGB_Model),\n\n    ('catboost', CatBoost_Model),\n\n    #('tabnet', TabNet_Model)\n\n])\n\n\n\n# Train the ensemble model\n\nSubmission2 = TrainML(voting_model, test)\n\n\n\n# Save submission\n\n#Submission2.to_csv('submission.csv', index=False)\n\nSubmission2","metadata":{"papermill":{"duration":125.974049,"end_time":"2024-12-12T20:22:09.922074","exception":false,"start_time":"2024-12-12T20:20:03.948025","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\n\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n\n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n\n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\n\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\n\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\n\n\ntime_series_cols = train_ts.columns.tolist()\n\ntime_series_cols.remove(\"id\")\n\n\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\n\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\n\n\ntrain = train.drop('id', axis=1)\n\ntest = test.drop('id', axis=1)\n\n\n\nfeaturesCols += time_series_cols\n\n\n\ntrain = train[featuresCols]\n\ntrain = train.dropna(subset='sii')\n\n\n\ndef update(df):\n\n    global cat_c\n\n    for c in cat_c: \n\n        df[c] = df[c].fillna('Missing')\n\n        df[c] = df[c].astype('category')\n\n    return df\n\n\n\ntrain = update(train)\n\ntest = update(test)\n\n\n\ndef create_mapping(column, dataset):\n\n    unique_values = dataset[column].unique()\n\n    return {value: idx for idx, value in enumerate(unique_values)}\n\n\n\nfor col in cat_c:\n\n    mapping = create_mapping(col, train)\n\n    mappingTe = create_mapping(col, test)\n\n    \n\n    train[col] = train[col].replace(mapping).astype(int)\n\n    test[col] = test[col].replace(mappingTe).astype(int)\n\n\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n\n    return np.where(oof_non_rounded < thresholds[0], 0,\n\n                    np.where(oof_non_rounded < thresholds[1], 1,\n\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\n\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\n\n\ndef TrainML(model_class, test_data):\n\n    X = train.drop(['sii'], axis=1)\n\n    y = train['sii']\n\n\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n\n    \n\n    train_S = []\n\n    test_S = []\n\n    \n\n    oof_non_rounded = np.zeros(len(y), dtype=float) \n\n    oof_rounded = np.zeros(len(y), dtype=int) \n\n    test_preds = np.zeros((len(test_data), n_splits))\n\n\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n\n\n        model = clone(model_class)\n\n        model.fit(X_train, y_train)\n\n\n\n        y_train_pred = model.predict(X_train)\n\n        y_val_pred = model.predict(X_val)\n\n\n\n        oof_non_rounded[test_idx] = y_val_pred\n\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n\n\n        train_S.append(train_kappa)\n\n        test_S.append(val_kappa)\n\n        \n\n        test_preds[:, fold] = model.predict(test_data)\n\n        \n\n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n\n        clear_output(wait=True)\n\n\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n\n                              method='Nelder-Mead')\n\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n\n    \n\n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n\n\n    tpm = test_preds.mean(axis=1)\n\n    tp_rounded = threshold_Rounder(tpm, KappaOPtimizer.x)\n\n\n\n    return tp_rounded\n\n\n\nimputer = SimpleImputer(strategy='median')\n\n\n\nensemble = VotingRegressor(estimators=[\n\n    ('lgb', Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(random_state=SEED))])),\n\n    ('xgb', Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(random_state=SEED))])),\n\n    ('cat', Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(random_state=SEED, silent=True))])),\n\n    ('rf', Pipeline(steps=[('imputer', imputer), ('regressor', RandomForestRegressor(random_state=SEED))])),\n\n    ('gb', Pipeline(steps=[('imputer', imputer), ('regressor', GradientBoostingRegressor(random_state=SEED))])),\n    \n    #('tabnet', Pipeline(steps=[('imputer', imputer), ('regressor', TabNetWrapper(**TabNet_Params))]))\n])\n\n\n\nSubmission3 = TrainML(ensemble, test)\n\nSubmission3 = pd.DataFrame({\n\n    'id': sample['id'],\n\n    'sii': Submission3\n\n})\n\n\n\nSubmission3","metadata":{"papermill":{"duration":195.852934,"end_time":"2024-12-12T20:25:25.901091","exception":false,"start_time":"2024-12-12T20:22:10.048157","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub1 = Submission1\n\nsub2 = Submission2\n\nsub3 = Submission3\n\n\n\nsub1 = sub1.sort_values(by='id').reset_index(drop=True)\n\nsub2 = sub2.sort_values(by='id').reset_index(drop=True)\n\nsub3 = sub3.sort_values(by='id').reset_index(drop=True)\n\n\n\ncombined = pd.DataFrame({\n\n    'id': sub1['id'],\n\n    'sii_1': sub1['sii'],\n\n    'sii_2': sub2['sii'],\n\n    'sii_3': sub3['sii']\n\n})\n\n\n\ndef majority_vote(row):\n\n    return row.mode()[0]\n\n\n\n#combined['final_sii'] = combined[['sii_1', 'sii_2', 'sii_3']].apply(majority_vote, axis=1)\n\ncombined['final_sii'] = 0.5*combined['sii_1'] + 0.25*combined['sii_2'] + 0.25*combined['sii_3']\n# combined[['sii_1', 'sii_2', 'sii_3']].apply(majority_vote, axis=1)\n# \nfinal_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\nfinal_submission[\"sii\"] = final_submission[\"sii\"].round().astype(int)\n\n\nfinal_submission.to_csv('submission.csv', index=False)\n\n\n\nprint(\"Majority voting completed and saved to 'Final_Submission.csv'\")","metadata":{"papermill":{"duration":0.064111,"end_time":"2024-12-12T20:25:26.078163","exception":false,"start_time":"2024-12-12T20:25:26.014052","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_submission","metadata":{"papermill":{"duration":0.049067,"end_time":"2024-12-12T20:25:26.165082","exception":false,"start_time":"2024-12-12T20:25:26.116015","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}