{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **0. Before you start**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pandas.api.types import CategoricalDtype\nimport os\nimport re\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nimport polars as pl\nimport polars.selectors as cs\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\n\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor, LGBMClassifier\nfrom xgboost import XGBRegressor, XGBClassifier\nfrom catboost import CatBoostRegressor, CatBoostClassifier\nfrom sklearn.ensemble import VotingRegressor, VotingClassifier, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.ensemble import VotingClassifier, RandomForestClassifier, GradientBoostingClassifier\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nfrom joblib import Parallel, delayed\n\n\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T13:35:41.386490Z","iopub.execute_input":"2024-11-30T13:35:41.387153Z","iopub.status.idle":"2024-11-30T13:36:00.395142Z","shell.execute_reply.started":"2024-11-30T13:35:41.387118Z","shell.execute_reply":"2024-11-30T13:36:00.394251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Make use of a GPU or MPS (Apple) if one is available.  (see module 3.2)\nimport torch\nhas_mps = torch.backends.mps.is_built()\ndevice = \"mps\" if has_mps else \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(f\"Using device: {device}\")","metadata":{"execution":{"iopub.status.busy":"2024-11-30T13:36:00.396698Z","iopub.execute_input":"2024-11-30T13:36:00.397312Z","iopub.status.idle":"2024-11-30T13:36:00.433559Z","shell.execute_reply.started":"2024-11-30T13:36:00.397283Z","shell.execute_reply":"2024-11-30T13:36:00.432585Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **1. Data Preprocess method**","metadata":{}},{"cell_type":"markdown","source":"## **1.1 Tabular data preprocess method**","metadata":{}},{"cell_type":"code","source":"def table_preprocess(train_0, submit_0):\n    diff_columns = set(train_0.columns) - set(submit_0.columns)\n    diff_columns.remove('sii')\n    train_0 = train_0.drop(columns=diff_columns)\n    \n    def object_to_one_hot(df, column_name):\n        dummies = pd.get_dummies(df[column_name], prefix=column_name)\n        df = pd.concat([df, dummies], axis=1)\n        df = df.drop(column_name, axis=1)\n    \n        return df\n    \n    object_columns = train_0.select_dtypes(include=['object']).columns\n    object_columns = object_columns.drop('id')\n    \n    train_1 = object_to_one_hot(train_0, object_columns)\n    submit_1 = object_to_one_hot(submit_0, object_columns)\n    \n    \n    diff_columns_2 = set(train_1.columns) - set(submit_1.columns)\n    diff_columns_2.remove('sii')\n    #in submit file, some columns are missing\n    for col in diff_columns_2:\n        submit_1[col] = 0\n    \n    #Transfer boolean to int\n    is_boolean_columns = train_1.select_dtypes(include=['boolean']).columns\n    for column in is_boolean_columns:\n        train_1[column] = train_1[column].map({True: 1, False : 0})\n        submit_1[column] = submit_1[column].map({True: 1, False : 0})\n    return train_1, submit_1","metadata":{"execution":{"iopub.status.busy":"2024-11-30T13:36:00.434891Z","iopub.execute_input":"2024-11-30T13:36:00.435629Z","iopub.status.idle":"2024-11-30T13:36:00.449151Z","shell.execute_reply.started":"2024-11-30T13:36:00.435589Z","shell.execute_reply":"2024-11-30T13:36:00.448365Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **1.2 Autoencoder method**","metadata":{}},{"cell_type":"code","source":"# import numpy as np\n# import pandas as pd\n# from sklearn.impute import SimpleImputer, KNNImputer\n# from sklearn.preprocessing import StandardScaler\n# from torch import nn, optim\n# import torch\n# from torch.utils.data import DataLoader, TensorDataset\n\n# def autoencoder_process(df_train, df_submit, encoding_dim=50, num_epochs=100, batch_size=2, learning_rate=0.01):\n#     # Step 1: Impute missing values (fit on df_train, transform both df_train and df_submit)\n#     # imputer = SimpleImputer(strategy='mean')\n#     # filled_train_data = imputer.fit_transform(df_train)\n#     # filled_submit_data = imputer.transform(df_submit)\n\n#     # imputer = KNNImputer(n_neighbors=5)\n#     # filled_train_data = imputer.fit_transform(df_train)\n#     # filled_submit_data = imputer.transform(df_submit)\n\n#     filled_train_data = df_train\n#     filled_submit_data = df_submit\n\n#     # Step 2: Standardize the data (fit on df_train, transform both df_train and df_submit)\n#     scaler = StandardScaler()\n#     scaled_train_data = scaler.fit_transform(filled_train_data)\n#     scaled_submit_data = scaler.transform(filled_submit_data)\n\n#     # Step 3: Convert the training data to PyTorch tensors\n#     device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n#     data_tensor_train = torch.tensor(scaled_train_data, dtype=torch.float32).to(device)\n#     dataset_train = TensorDataset(data_tensor_train)\n#     dataloader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=True)\n\n#     # Step 4: Define the Autoencoder model\n#     class Autoencoder(nn.Module):\n#         def __init__(self, input_dim, encoding_dim):\n#             super(Autoencoder, self).__init__()\n#             self.encoder = nn.Sequential(\n#                 nn.Linear(input_dim, encoding_dim*3),\n#                 nn.ReLU(),\n#                 nn.Linear(encoding_dim*3, encoding_dim*2),\n#                 nn.ReLU(),\n#                 nn.Linear(encoding_dim*2, encoding_dim)\n#             )\n#             self.decoder = nn.Sequential(\n#                 nn.Linear(encoding_dim, input_dim*2),\n#                 nn.ReLU(),\n#                 nn.Linear(input_dim*2, input_dim*3),\n#                 nn.ReLU(),\n#                 nn.Linear(input_dim*3, input_dim),\n#                 nn.Sigmoid()\n#             )\n\n#         def forward(self, x):\n#             encoded = self.encoder(x)\n#             decoded = self.decoder(encoded)\n#             return decoded\n\n#     # Step 5: Initialize the model, loss function, and optimizer\n#     input_dim = scaled_train_data.shape[1]\n#     autoencoder = Autoencoder(input_dim, encoding_dim).to(device)\n#     criterion = nn.MSELoss()\n#     optimizer = optim.Adam(autoencoder.parameters(), lr=learning_rate)\n\n#     # Step 6: Train the autoencoder\n#     for epoch in range(num_epochs):\n#         for data_batch in dataloader_train:\n#             batch = data_batch[0].to(device)\n#             # Forward pass\n#             reconstructed = autoencoder(batch)\n#             loss = criterion(reconstructed, batch)\n#             # Backward pass\n#             optimizer.zero_grad()\n#             loss.backward()\n#             optimizer.step()\n\n#         if (epoch + 1) % 10 == 0:\n#             print(f'Epoch [{epoch + 1}/{num_epochs}], Loss: {loss.item():.4f}')\n\n#     # Step 7: Use the trained autoencoder for reconstruction on df_train and df_submit\n#     with torch.no_grad():\n#         data_tensor_train = torch.tensor(scaled_train_data, dtype=torch.float32).to(device)\n#         reconstructed_train_data = autoencoder(data_tensor_train).cpu()\n#         reconstructed_train_data = scaler.inverse_transform(reconstructed_train_data.numpy())\n#         reconstructed_train_df = pd.DataFrame(reconstructed_train_data, columns=df_train.columns)\n\n#         data_tensor_submit = torch.tensor(scaled_submit_data, dtype=torch.float32).to(device)\n#         reconstructed_submit_data = autoencoder(data_tensor_submit).cpu()\n#         reconstructed_submit_data = scaler.inverse_transform(reconstructed_submit_data.numpy())\n#         reconstructed_submit_df = pd.DataFrame(reconstructed_submit_data, columns=df_submit.columns)\n\n#     return reconstructed_train_df, reconstructed_submit_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T13:36:00.451059Z","iopub.execute_input":"2024-11-30T13:36:00.451360Z","iopub.status.idle":"2024-11-30T13:36:00.462352Z","shell.execute_reply.started":"2024-11-30T13:36:00.451335Z","shell.execute_reply":"2024-11-30T13:36:00.461502Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"New autoencoder method!!","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import Dataset,DataLoader\nfrom sklearn.preprocessing import StandardScaler\nimport torch.nn.functional as F\n\nclass CustomDataset(Dataset):\n    \n    def __init__(self, dataframe):\n        # Apply StandardScaler to the input features\n        self.scaler = StandardScaler()\n        if 'sii' in dataframe.columns:\n            self.train = True\n            features = dataframe.drop(['id', 'sii'], axis=1)  # Drop ID and target column\n            \n\n            self.targets = dataframe['sii'].values  # Keep target values (sii)\n        else:\n            self.train = False\n            features = dataframe.drop(['id'],axis=1)\n            \n        self.scaled_data = self.scaler.fit_transform(features)  # Scale features\n            \n\n    def __len__(self):\n        return len(self.scaled_data)\n    \n    def __getitem__(self, idx):\n        # Return the scaled input features and target value\n        if self.train:\n            return torch.tensor(self.scaled_data[idx], dtype=torch.float32), torch.tensor(self.targets[idx], dtype=torch.long)  # Ensure targets are long for classification\n        else:\n            return torch.tensor(self.scaled_data[idx], dtype=torch.float32)\n            \nclass LSTMEncoder(nn.Module):\n    \n    def __init__(self, input_size, hidden_size, latent_dim, num_classes, num_layers=1):\n        super(LSTMEncoder, self).__init__()\n        # LSTM layer\n        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)\n        # Fully connected layers for latent dimension and classification\n        self.fc_latent = nn.Linear(hidden_size, latent_dim)  # Latent space\n        self.fc_class = nn.Linear(latent_dim, num_classes)   # Classifier layer for output\n        \n    def forward(self, x):\n        _, (hn, _) = self.lstm(x)  # Get hidden state from LSTM (hn)\n        latent = self.fc_latent(hn[-1])  # Compress the last hidden state to latent space\n        out = self.fc_class(latent)  # Classify using the latent space\n        return latent, out\n    \nclass VAE(nn.Module):\n    def __init__(self, input_channels, seq_length, latent_dim, dropout_prob=0.4):\n        super(VAE, self).__init__()\n        \n        # Encoder: deeper Conv1d layers with BatchNorm1d\n        self.encoder = nn.Sequential(\n            nn.Conv1d(input_channels, 16, kernel_size=3, stride=2, padding=1),\n            nn.BatchNorm1d(16),\n            nn.ReLU(0.2),\n            nn.Dropout(dropout_prob),\n            \n            nn.Conv1d(16, 32, kernel_size=3, stride=2, padding=1),\n            nn.BatchNorm1d(32),\n            nn.ReLU(0.2),\n            nn.Dropout(dropout_prob),\n            \n            nn.Conv1d(32, 64, kernel_size=3, stride=2, padding=1),\n            nn.BatchNorm1d(64),\n            nn.ReLU(0.2),\n            nn.Flatten()\n        )\n        \n        # Calculate size after Conv1d layers\n        conv_output_size = (seq_length // 8) * 64\n        \n        self.fc_mu = nn.Linear(conv_output_size, latent_dim)\n        self.fc_logvar = nn.Linear(conv_output_size, latent_dim)\n        \n        # Decoder: Fully connected and ConvTranspose1d layers\n        self.decoder_input = nn.Linear(latent_dim, conv_output_size)\n        self.decoder = nn.Sequential(\n            nn.Unflatten(1, (64, seq_length // 8)),\n            nn.ConvTranspose1d(64, 32, kernel_size=3, stride=2, padding=1, output_padding=1),\n            nn.BatchNorm1d(32),\n            nn.ReLU(0.2),\n            nn.Dropout(dropout_prob),\n            \n            nn.ConvTranspose1d(32, 16, kernel_size=3, stride=2, padding=1, output_padding=1),\n            nn.BatchNorm1d(16),\n            nn.ReLU(0.2),\n            nn.Dropout(dropout_prob),\n            \n            nn.ConvTranspose1d(16, input_channels, kernel_size=3, stride=2, padding=1, output_padding=1),\n            nn.Sigmoid()\n        )\n    \n    def reparameterize(self, mu, logvar):\n        std = torch.exp(0.5 * logvar)\n        eps = torch.randn_like(std)\n        return mu + eps * std\n    \n    def forward(self, x):\n        encoded = self.encoder(x)\n        mu = self.fc_mu(encoded)\n        logvar = self.fc_logvar(encoded)\n        z = self.reparameterize(mu, logvar)\n        decoded = self.decoder(self.decoder_input(z))\n        return decoded, mu, logvar\n\ndef reconstruction_loss(original, reconstructed, alpha=0.65, beta=0.35):\n    \"\"\"\n    定义重建损失，MSE + 余弦相似度的组合\n    \"\"\"\n    # Mean Squared Error\n    mse_loss = F.mse_loss(reconstructed, original)\n    \n    # Cosine Similarity -> Cosine Distance\n    cosine_sim = F.cosine_similarity(reconstructed, original, dim=-1)  # 按最后一维计算相似度\n    cosine_dist = 1 - cosine_sim.mean()  # 余弦距离（越小越相似）\n    \n    # 总损失：MSE + Cosine Distance\n    total_loss = alpha * mse_loss + beta * cosine_dist\n    return total_loss\n    \ndef perform_autoencoder(df, latent_dim=64, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled).unsqueeze(1)\n\n    input_channels = data_tensor.shape[1]\n    seq_length = data_tensor.shape[2]\n\n    autoencoder = VAE(input_channels, seq_length, latent_dim)\n    optimizer = optim.AdamW(autoencoder.parameters(), lr=5e-3, weight_decay=1e-4)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5)\n    criterion = nn.MSELoss()  # Reconstruction loss\n    \n    # 权重初始化函数\n    def init_weights(m):\n        if isinstance(m, (nn.Conv1d, nn.ConvTranspose1d, nn.Linear)):\n            nn.init.xavier_uniform_(m.weight)  # 使用Xavier初始化\n            if m.bias is not None:\n                nn.init.constant_(m.bias, 0.01)  # 偏置设置为0.01\n    \n    # Apply weight initialization\n    autoencoder.apply(init_weights)\n\n    for epoch in range(epochs):\n        epoch_loss = 0\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed, mu, logvar = autoencoder(batch)\n            \n            # 重建损失：MSE + Cosine Distance\n            loss = reconstruction_loss(batch, reconstructed)\n            \n            # 加入KL散度和稀疏性正则化\n            kl_divergence = -0.5 * torch.mean(1 + logvar - mu.pow(2) - logvar.exp())\n            l1_regularization = sum(param.abs().sum() for param in autoencoder.encoder.parameters())\n            \n            total_loss = loss + kl_divergence + 1e-4 * l1_regularization\n            total_loss.backward()\n            optimizer.step()\n            epoch_loss += total_loss.item()\n        \n        scheduler.step(epoch_loss)\n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {epoch_loss / len(data_tensor):.4f}')    \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n\n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    return df_encoded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T13:36:00.463657Z","iopub.execute_input":"2024-11-30T13:36:00.463908Z","iopub.status.idle":"2024-11-30T13:36:00.487574Z","shell.execute_reply.started":"2024-11-30T13:36:00.463885Z","shell.execute_reply":"2024-11-30T13:36:00.486681Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **1.3 Feature creating(still working)**","metadata":{}},{"cell_type":"code","source":"def feature_engineering(df):\n    season_cols = [col for col in df.columns if 'Season' in col]\n    df = df.drop(season_cols, axis=1) \n\n    # Existing features\n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n\n    # New features\n#     df['BMI_Age_Ratio'] = df['Physical-BMI'] / df['Basic_Demos-Age']\n#     df['Weight_Height_Ratio'] = df['Physical-Weight'] / df['Physical-Height']\n#     df['HeartRate_Age_Ratio'] = df['Physical-HeartRate'] / df['Basic_Demos-Age']\n#     df['Endurance_Age_Ratio'] = df['Fitness_Endurance-Time_Mins'] / df['Basic_Demos-Age']\n#     df['Waist_Height_Ratio'] = df['Physical-Waist_Circumference'] / df['Physical-Height']\n#     df['FFM_Height_Ratio'] = df['BIA-BIA_FFM'] / df['Physical-Height']\n#     df['DEE_Weight_Ratio'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n#     df['HeartRate_BP_Ratio'] = df['Physical-HeartRate'] / df['Physical-Systolic_BP']\n#     df['Internet_Physical_Activity_Ratio'] = df['PreInt_EduHx-computerinternet_hoursday'] / df['PAQ_A-PAQ_A_Total']\n#     df['BMI_SDS_Raw_Multiplication'] = df['Physical-BMI'] * df['SDS-SDS_Total_Raw']\n#     df['SMM_Weight_Ratio'] = df['BIA-BIA_SMM'] / df['Physical-Weight']\n#     df['Fat_Age_Ratio'] = df['BIA-BIA_Fat'] / df['Basic_Demos-Age']\n#     df['Hydration_Age_Ratio'] = df['Hydration_Status'] / df['Basic_Demos-Age']\n#     df['ICW_TBW_Ratio'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n#     df['Activity_Endurance_Multiplication'] = df['BIA-BIA_Activity_Level_num'] * df['Fitness_Endurance-Max_Stage']\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T13:36:00.488530Z","iopub.execute_input":"2024-11-30T13:36:00.488771Z","iopub.status.idle":"2024-11-30T13:36:00.499418Z","shell.execute_reply.started":"2024-11-30T13:36:00.488737Z","shell.execute_reply":"2024-11-30T13:36:00.498684Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **1.4 Actigraphy data preprocess method**","metadata":{}},{"cell_type":"code","source":"def classify_row(row):\n    if row['is_weekend']:\n        return 'weekend'\n    else:\n        return 'other'\n\ndef preprocess_actigraphy(data, id): #id has to be a string\n    data['hour_in_day'] = (data['time_of_day'] // 3_600_000_000_000).astype(int)\n    data['hour_in_day'] = data['hour_in_day']\n    data['is_night'] = np.where((data['hour_in_day'] > 8) & (data['hour_in_day'] < 23), 0, 1)\n    data = data.dropna(subset=['non-wear_flag'], axis=0)\n    data = data[data['non-wear_flag'] == 0]\n    data['is_weekend'] = data['weekday'] >= 5\n    data['category'] = data.apply(classify_row, axis=1)\n    data = data.groupby(['hour_in_day', 'category']).mean().reset_index()\n    data['id'] = id\n    \n    columns_to_process = ['enmo', 'anglez', 'light', 'battery_voltage']  \n    for col in columns_to_process:\n        data[f'{col}_mean'] = data[col].mean()\n        data[f'{col}_std'] = data[col].std()\n        data[f'{col}_min'] = data[col].min()\n        data[f'{col}_max'] = data[col].max()\n        data[f'{col}_25th'] = data[col].quantile(0.25)\n        data[f'{col}_50th'] = data[col].median()\n        data[f'{col}_75th'] = data[col].quantile(0.75)\n        data[f'{col}_range'] = data[col].max() - data[col].min()\n        data[f'{col}_skew'] = data[col].skew()\n        data[f'{col}_kurtosis'] = data[col].kurtosis()\n    return data\n    \n\ndef actigraphy_features_engineer(data): #DataFrame\n    data['light_per_hour'] = data['light'] / data['hour_in_day']\n    data['anglez_per_hour'] = data['anglez'] / data['hour_in_day']\n    data['enmo_per_hour'] = data['enmo'] / data['hour_in_day']\n    data = data[['enmo_per_hour','anglez_per_hour', 'light_per_hour', 'is_weekend', 'quarter']]\n    average_values = data.mean()\n    average_values_df = data.mean().to_frame().T\n    average_values_df['id'] = id\n    return average_values_df\n\ndef load_test_actigraphy():\n    def process_file(data_name):\n        id_value = data_name[3:11]\n        data = pd.read_parquet(f'/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet/{data_name}')\n        return preprocess_actigraphy(data, id_value)\n    \n    # Get list of files to process\n    data_files = os.listdir('/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet')\n    \n    # Use joblib to process files in parallel\n    final_datas = Parallel(n_jobs=-1)(delayed(process_file)(data_name) for data_name in tqdm(data_files))\n    \n    # Combine all DataFrames in the list into a single DataFrame\n    return pd.concat(final_datas, ignore_index=True)\n\ndef load_train_actigraphy():\n    def process_file(data_name):\n        id_value = data_name[3:11]\n        data = pd.read_parquet(f'/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/{data_name}')\n        # return preprocess_actigraphy(data, id_value)\n        return preprocess_actigraphy(data, id_value)\n    \n    # Get list of files to process\n    data_files = os.listdir('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet')\n    \n    # Use joblib to process files in parallel\n    final_datas = Parallel(n_jobs=-1)(delayed(process_file)(data_name) for data_name in tqdm(data_files))\n    \n    # Combine all DataFrames in the list into a single DataFrame\n    return pd.concat(final_datas, ignore_index=True)\n\ndef remove_overmissing_col(df, missing_threshold = 0.5):\n    for col in df.columns:\n        missing_ratio = df[col].isnull().mean()\n        if missing_ratio > missing_threshold:\n            # Columns with more than 50% of missing values are deleted\n            df = df.drop(columns=[col], inplace=True)\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T13:36:00.500513Z","iopub.execute_input":"2024-11-30T13:36:00.500801Z","iopub.status.idle":"2024-11-30T13:36:00.514064Z","shell.execute_reply.started":"2024-11-30T13:36:00.500776Z","shell.execute_reply":"2024-11-30T13:36:00.513439Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **2. Data preparation**","metadata":{}},{"cell_type":"markdown","source":"## **2.1 v1(abandon)**","metadata":{}},{"cell_type":"code","source":"# train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n# test = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n# sample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n# train_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\n# test_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\n# df_train = train_ts.drop('id', axis=1)\n# df_test = test_ts.drop('id', axis=1)\n\n# df_train = perform_autoencoder_with_mask(df_train, encoding_dim=60, epochs=100, batch_size=32)\n# df_test = perform_autoencoder_with_mask(df_test, encoding_dim=60, epochs=100, batch_size=32)\n\n# train_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=100, batch_size=32)\n# test_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=100, batch_size=32)\n\n# time_series_cols = train_ts_encoded.columns.tolist()\n# train_ts_encoded[\"id\"]=train_ts[\"id\"]\n# test_ts_encoded['id']=test_ts[\"id\"]\n\n# train = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\n# test = pd.merge(test, test_ts_encoded, how=\"left\", on='id')\n\n# imputer = KNNImputer(n_neighbors=5)\n# numeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\n# imputed_data = imputer.fit_transform(train[numeric_cols])\n# train_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\n# train_imputed['sii'] = train_imputed['sii'].round().astype(int)\n# for col in train.columns:\n#     if col not in numeric_cols:\n#         train_imputed[col] = train[col]\n        \n# train = train_imputed\n\n# train = feature_engineering(train)\n# train = train.dropna(thresh=10, axis=0)\n# test = feature_engineering(test)\n\n# train = train.drop('id', axis=1)\n# test  = test.drop('id', axis=1)   \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T13:36:00.514970Z","iopub.execute_input":"2024-11-30T13:36:00.515212Z","iopub.status.idle":"2024-11-30T13:36:00.524293Z","shell.execute_reply.started":"2024-11-30T13:36:00.515166Z","shell.execute_reply":"2024-11-30T13:36:00.523607Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **2.2 v2(new logic)**","metadata":{}},{"cell_type":"markdown","source":"## **2.3 v3(working on this one)**","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\", na_values=['NA', '?'])\ndf_submit = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\", na_values=['NA', '?'])\ndf_train = df_train.dropna(subset = ['sii'])\ndf_submit_ts = load_test_actigraphy()\ndf_train_ts = load_train_actigraphy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T13:36:00.525070Z","iopub.execute_input":"2024-11-30T13:36:00.525301Z","iopub.status.idle":"2024-11-30T13:46:35.641278Z","shell.execute_reply.started":"2024-11-30T13:36:00.525278Z","shell.execute_reply":"2024-11-30T13:46:35.640151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_2, df_submit_2 = table_preprocess(df_train, df_submit)\n\ndf_train_ts_2 = df_train_ts.drop(columns = ['id', 'category'], axis=1)\ndf_submit_ts_2 = df_submit_ts.drop(columns = ['id', 'category'], axis=1)\ndf_train_ts_2 = remove_overmissing_col(df_train_ts_2)\ndf_submit_ts_2 = df_submit_ts_2[df_train_ts_2.columns]\n\nprint('autoencoder started')\ndf_train_ts_2 = perform_autoencoder(df_train_ts_2)\ndf_submit_ts_2 = perform_autoencoder(df_submit_ts_2)\nprint('autoencoder finished')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T13:46:35.645238Z","iopub.execute_input":"2024-11-30T13:46:35.645533Z","iopub.status.idle":"2024-11-30T13:58:19.678502Z","shell.execute_reply.started":"2024-11-30T13:46:35.645503Z","shell.execute_reply":"2024-11-30T13:58:19.677606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\ntime_series_cols = df_submit_ts_2.columns.tolist()\ndf_train_ts_2[\"id\"]=df_train_ts[\"id\"]\ndf_submit_ts_2['id']=df_submit_ts[\"id\"]\n\ntrain = pd.merge(df_train, df_train_ts_2, how=\"left\", on='id')\nsubmit = pd.merge(df_submit, df_submit_ts_2, how=\"left\", on='id')\nprint('Checking point 1/3')\n\n\n# imputer = KNNImputer(n_neighbors=5)\n# numeric_cols = train.select_dtypes(include=['float64', 'float32', 'int64']).columns\n# if np.any(np.isinf(train[numeric_cols])):\n#     train[numeric_cols] = train[numeric_cols].replace([np.inf, -np.inf], np.nan)\n# imputed_data = imputer.fit_transform(train[numeric_cols])\n# train_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\n# for col in train.columns:\n#     if col not in numeric_cols:\n#         train_imputed[col] = train[col]\n# train = train_imputed\nprint('Checking point 2/3')\n\n# numeric_cols = submit.select_dtypes(include=['float64', 'float32', 'int64']).columns\n# if np.any(np.isinf(submit[numeric_cols])):\n#     submit[numeric_cols] = submit[numeric_cols].replace([np.inf, -np.inf], np.nan)\n# imputed_data = imputer.fit_transform(submit[numeric_cols])\n# submit_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\n# for col in submit.columns:\n#     if col not in numeric_cols:\n#         submit_imputed[col] = submit[col]\n# submit = submit_imputed\nprint('Checking point 3/3')\n\ntrain = feature_engineering(train)\ntrain = train.dropna(thresh=10, axis=0)\nsubmit = feature_engineering(submit)\nprint('done')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:22:16.111091Z","iopub.execute_input":"2024-11-30T14:22:16.111469Z","iopub.status.idle":"2024-11-30T14:22:16.483129Z","shell.execute_reply.started":"2024-11-30T14:22:16.111438Z","shell.execute_reply":"2024-11-30T14:22:16.482218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#have sii\n# featuresCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n#                 'CGAS-CGAS_Score', 'Physical-BMI',\n#                 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n#                 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n#                 'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n#                 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND', 'FGC-FGC_GSND_Zone',\n#                 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU', 'FGC-FGC_PU_Zone',\n#                 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR', 'FGC-FGC_SRR_Zone',\n#                 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-BIA_Activity_Level_num',\n#                 'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW',\n#                 'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n#                 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW',\n#                 'PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T',\n#                 'PreInt_EduHx-computerinternet_hoursday', 'sii', 'BMI_Age', 'Internet_Hours_Age',\n#                 'BMI_Internet_Hours', 'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR',\n#                 'BFP_DEE', 'BMR_Weight', 'DEE_Weight', 'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status',\n#                 'ICW_TBW', 'BMI_Age_Ratio', 'Weight_Height_Ratio', 'HeartRate_Age_Ratio', \n#                 'Endurance_Age_Ratio', 'Waist_Height_Ratio', 'FFM_Height_Ratio', \n#                 'DEE_Weight_Ratio', 'HeartRate_BP_Ratio', 'Internet_Physical_Activity_Ratio',\n#                 'BMI_SDS_Raw_Multiplication', 'SMM_Weight_Ratio', 'Fat_Age_Ratio', \n#                 'Hydration_Age_Ratio', 'ICW_TBW_Ratio', 'Activity_Endurance_Multiplication']\n\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii','BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\n\n# if np.any(np.isinf(train)):\n#     train = train.replace([np.inf, -np.inf], np.nan)\n#     imputer = KNNImputer(n_neighbors=5)\n#     numeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\n#     imputed_data = imputer.fit_transform(train[featuresCols])\n#     train = pd.DataFrame(imputed_data, columns=featuresCols)\n\n# train = train.dropna(subset='sii')\n\n# no 'sii'\n# featuresCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n#                 'CGAS-CGAS_Score', 'Physical-BMI',\n#                 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n#                 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n#                 'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n#                 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND', 'FGC-FGC_GSND_Zone',\n#                 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU', 'FGC-FGC_PU_Zone',\n#                 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR', 'FGC-FGC_SRR_Zone',\n#                 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-BIA_Activity_Level_num',\n#                 'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW',\n#                 'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n#                 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW',\n#                 'PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T',\n#                 'PreInt_EduHx-computerinternet_hoursday', 'BMI_Age', 'Internet_Hours_Age',\n#                 'BMI_Internet_Hours', 'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR',\n#                 'BFP_DEE', 'BMR_Weight', 'DEE_Weight', 'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status',\n#                 'ICW_TBW', 'BMI_Age_Ratio', 'Weight_Height_Ratio', 'HeartRate_Age_Ratio', \n#                 'Endurance_Age_Ratio', 'Waist_Height_Ratio', 'FFM_Height_Ratio', \n#                 'DEE_Weight_Ratio', 'HeartRate_BP_Ratio', 'Internet_Physical_Activity_Ratio',\n#                 'BMI_SDS_Raw_Multiplication', 'SMM_Weight_Ratio', 'Fat_Age_Ratio', \n#                 'Hydration_Age_Ratio', 'ICW_TBW_Ratio', 'Activity_Endurance_Multiplication']\n\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW', 'id']\n\nfeaturesCols += time_series_cols\nsubmit = submit[featuresCols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:22:24.527672Z","iopub.execute_input":"2024-11-30T14:22:24.527972Z","iopub.status.idle":"2024-11-30T14:22:24.574082Z","shell.execute_reply.started":"2024-11-30T14:22:24.527947Z","shell.execute_reply":"2024-11-30T14:22:24.573054Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.to_csv(\"train_ae_done.csv\")\nsubmit.to_csv(\"submit_ae_done.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:23:50.484351Z","iopub.execute_input":"2024-11-30T14:23:50.484940Z","iopub.status.idle":"2024-11-30T14:24:01.085333Z","shell.execute_reply.started":"2024-11-30T14:23:50.484909Z","shell.execute_reply":"2024-11-30T14:24:01.084414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.to_csv('train01.csv')\nsubmit.to_csv('submit01.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.947270Z","iopub.status.idle":"2024-11-30T14:17:20.947700Z","shell.execute_reply.started":"2024-11-30T14:17:20.947474Z","shell.execute_reply":"2024-11-30T14:17:20.947496Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **3. Train the model**","metadata":{}},{"cell_type":"markdown","source":"## **3.1 AutoML**","metadata":{}},{"cell_type":"markdown","source":"### **3.1.1 AutoML: Model**","metadata":{}},{"cell_type":"code","source":"# import tempfile\n# from autogluon.tabular import TabularPredictor\n# from sklearn.preprocessing import StandardScaler\n\n\n# target = 'sii'\n# # train = train.drop('id', axis=1)\n# train.replace([np.inf, -np.inf], np.nan, inplace=True)\n# train.fillna(train.median(), inplace=True)\n\n# scaler = StandardScaler()\n# scaled_data = scaler.fit_transform(train)\n# scaled_df = pd.DataFrame(scaled_data, columns=train.columns)\n\n# # Create a temporary directory to avoid saving models to a permanent folder\n# temp_model_path = tempfile.mkdtemp()\n\n# # Train AutoGluon model with the desired adjustments\n# predictor = TabularPredictor(label=target, eval_metric='root_mean_squared_error',problem_type='regression', path=temp_model_path).fit(\n#     scaled_df,\n#     presets='best_quality',  # Using the best quality preset for higher accuracy\n#     time_limit=300,  # Control the training time (in seconds)\n#     num_bag_folds=10,  # Enable ensembling by using bagging with 5 folds\n#     num_stack_levels=2,  # Enable stacking to further improve model performance\n#     # ag_args_fit={'num_gpus': 1},# Set to True if using GPU for model training\n#     verbosity=3  # Suppress logging output (0: most silent, 4: most detailed)\n# )\n\n\n# leaderboard = predictor.leaderboard(silent=True)\n# leaderboard.to_csv('leaderboard_2.csv', index=False)\n# leaderboard.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.949147Z","iopub.status.idle":"2024-11-30T14:17:20.949595Z","shell.execute_reply.started":"2024-11-30T14:17:20.949373Z","shell.execute_reply":"2024-11-30T14:17:20.949395Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### **3.1.2 AutoML: Send the prediction**","metadata":{}},{"cell_type":"code","source":"sample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n# save_id = pd.DataFrame(data=submit['id'].values.reshape(-1, 1), columns=['id'])\ntest_data_processed = pd.DataFrame(data=submit.drop(columns = ['id'], axis=1))\ntest_data_processed.replace([np.inf, -np.inf], np.nan, inplace=True)\ntest_data_processed.fillna(train.median(), inplace=True)\n\nscaler = StandardScaler()\nscaled_data = scaler.fit_transform(test_data_processed)\nscaled_df = pd.DataFrame(scaled_data, columns=test_data_processed.columns)\n\n\n\n\n# Use AutoGluon to predict class labels (0 or 1) for 'depression'\ny_test_pred = predictor.predict(scaled_df)\ny_test_pred = y_test_pred.round().astype(int)\ny_test_pred = np.maximum(y_test_pred, 0)\n\n#Ensure alinged correctly\n# X_test = X_test.reset_index(drop=True)\n# save_id = save_id.reset_index(drop=True)\ny_test_pred = pd.Series(y_test_pred).reset_index(drop=True)\n\n# Create the submission file\nsub2 = pd.DataFrame({\n    'id': sample['id'],\n    'sii': y_test_pred\n})\n\n# Save the submission file\nsub2.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.950699Z","iopub.status.idle":"2024-11-30T14:17:20.951119Z","shell.execute_reply.started":"2024-11-30T14:17:20.950899Z","shell.execute_reply":"2024-11-30T14:17:20.950921Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **3.2 CatBooster**","metadata":{}},{"cell_type":"markdown","source":"### **3.2.1 Cat: Model**","metadata":{}},{"cell_type":"code","source":"import catboost as cb\nfrom catboost import CatBoostRegressor\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split,StratifiedShuffleSplit \nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import StandardScaler\n\n\ntrain = train.drop('id', axis=1)\n# Split data into features and target\nX = train.drop('sii', axis=1)\ny = train['sii']\n\n# Split data into training and validation sets\n# X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=42)\nstratified_split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\nfor train_index, valid_index in stratified_split.split(X, y):\n    X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\n    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n\n# Initialize CatBoost Regressor\ncat_model = CatBoostRegressor(\n    iterations=1035,\n    learning_rate=0.05391374159446765,\n    depth=10,\n    l2_leaf_reg=0.0018395898086979588,\n    loss_function='RMSE',\n    verbose=100,  # Prints training progress every 100 iterations\n    task_type=\"GPU\"\n)\n\n# Train the model\ncat_model.fit(X_train, y_train, eval_set=(X_valid, y_valid), early_stopping_rounds=50)\n\n# Make predictions\ny_pred = cat_model.predict(X_valid)\n\n# Calculate RMSE\nrmse = np.sqrt(mean_squared_error(y_valid, y_pred))\nprint(f'RMSE: {rmse}')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.952896Z","iopub.status.idle":"2024-11-30T14:17:20.953351Z","shell.execute_reply.started":"2024-11-30T14:17:20.953096Z","shell.execute_reply":"2024-11-30T14:17:20.953119Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### **3.2.2 Cat: Send the prediction**","metadata":{}},{"cell_type":"code","source":"save_id = pd.DataFrame(data = submit['id'].values, columns = ['id'])\ntest_data_processed = pd.DataFrame(data=submit.drop(columns = ['id'], axis=1))\n\ny_test_pred = cat_model.predict(test_data_processed)\ny_test_pred = y_test_pred.round().astype(int)\ny_test_pred = np.maximum(y_test_pred, 0)\n\n\n#Ensure alinged correctly\n# X_test = X_test.reset_index(drop=True)\nsave_id = save_id.reset_index(drop=True)\ny_test_pred = pd.Series(y_test_pred).reset_index(drop=True)\n\n# Create the submission file\nsub_cat = pd.DataFrame({\n    'id': save_id['id'],\n    'sii': y_test_pred\n})\n\n# Save the submission file\nsub_cat.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.954735Z","iopub.status.idle":"2024-11-30T14:17:20.955010Z","shell.execute_reply.started":"2024-11-30T14:17:20.954878Z","shell.execute_reply":"2024-11-30T14:17:20.954892Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### **3.2.3 Optuna for Cat**","metadata":{}},{"cell_type":"code","source":"# import catboost as cb\n# from catboost import CatBoostRegressor\n# import pandas as pd\n# import numpy as np\n# from sklearn.model_selection import train_test_split, StratifiedShuffleSplit\n# from sklearn.metrics import mean_squared_error\n# import optuna\n\n# # Split data into features and target\n# X = train.drop(columns=['sii'])  # Replace 'target' with your target column name\n# y = train['sii']\n\n# stratified_split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n# for train_index, valid_index in stratified_split.split(X, y):\n#     X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\n#     y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n\n# # Define objective function for Optuna\n# def objective(trial):\n#     # Suggest hyperparameters\n#     iterations = trial.suggest_int('iterations', 500, 2000)\n#     learning_rate = trial.suggest_loguniform('learning_rate', 0.01, 0.3)\n#     depth = trial.suggest_int('depth', 4, 10)\n#     l2_leaf_reg = trial.suggest_loguniform('l2_leaf_reg', 1e-3, 10.0)\n\n#     # Initialize CatBoost Regressor with suggested hyperparameters\n#     catboost_model = CatBoostRegressor(\n#         iterations=iterations,\n#         learning_rate=learning_rate,\n#         depth=depth,\n#         l2_leaf_reg=l2_leaf_reg,\n#         loss_function='RMSE',\n#         verbose=0,\n#         task_type=\"GPU\"\n#     )\n\n#     # Train the model\n#     catboost_model.fit(X_train, y_train, eval_set=(X_valid, y_valid), early_stopping_rounds=50, verbose=0)\n\n#     # Make predictions\n#     y_pred = catboost_model.predict(X_valid)\n\n#     # Calculate RMSE\n#     rmse = np.sqrt(mean_squared_error(y_valid, y_pred))\n#     return rmse\n\n# # Run Optuna optimization\n# study = optuna.create_study(direction='minimize')\n# study.optimize(objective, n_trials=35)\n# print(\"Best Parameters: \", study.best_params)\n# print(\"Best RMSE: \", study.best_value)\n\n# # Train the final model with the best hyperparameters\n# best_params = study.best_params\n# catboost_model = CatBoostRegressor(\n#     iterations=best_params['iterations'],\n#     learning_rate=best_params['learning_rate'],\n#     depth=best_params['depth'],\n#     l2_leaf_reg=best_params['l2_leaf_reg'],\n#     loss_function='RMSE',\n#     verbose=100\n# )\n\n# # Train the model\n# catboost_model.fit(X_train, y_train, eval_set=(X_valid, y_valid), early_stopping_rounds=50)\n\n# # Make predictions\n# y_pred = catboost_model.predict(X_valid)\n\n# # Calculate RMSE\n# rmse = np.sqrt(mean_squared_error(y_valid, y_pred))\n# print(f'RMSE: {rmse}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.957596Z","iopub.status.idle":"2024-11-30T14:17:20.957903Z","shell.execute_reply.started":"2024-11-30T14:17:20.957760Z","shell.execute_reply":"2024-11-30T14:17:20.957776Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### **3.2.4 Features Permutation**","metadata":{}},{"cell_type":"code","source":"# from sklearn.metrics import mean_absolute_error\n# import numpy as np\n# import pandas as pd\n# from tqdm import tqdm\n\n# def permutation_importance(model, X_val, y_val, n_repeats=10):\n#     # Convert X_val to DataFrame if it's a numpy array\n#     if isinstance(X_val, np.ndarray):\n#         X_val = pd.DataFrame(X_val)\n\n#     # Calculate the baseline score\n#     baseline_score = mean_absolute_error(y_val, model.predict(X_val))\n#     feature_importances = {}\n\n#     # Iterate over each feature in the validation set with a progress bar\n#     for col in tqdm(X_val.columns, desc=\"Calculating feature importances\"):\n#         scores = []\n#         for _ in range(n_repeats):\n#             # Shuffle the feature values\n#             X_val_shuffled = X_val.copy()\n#             X_val_shuffled[col] = np.random.permutation(X_val_shuffled[col])\n\n#             # Calculate the score with the shuffled feature\n#             shuffled_score = mean_absolute_error(y_val, model.predict(X_val_shuffled))\n#             scores.append(shuffled_score - baseline_score)\n\n#         # Average the score increase across all repeats\n#         feature_importances[col] = np.mean(scores)\n\n#     # Sort features by importance\n#     feature_importances = pd.Series(feature_importances).sort_values(ascending=False)\n#     return feature_importances","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.959327Z","iopub.status.idle":"2024-11-30T14:17:20.959636Z","shell.execute_reply.started":"2024-11-30T14:17:20.959490Z","shell.execute_reply":"2024-11-30T14:17:20.959505Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Calculate permutation feature importance\n# importances = permutation_importance(cat_model, X_valid, y_valid)\n# print(importances)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.960762Z","iopub.status.idle":"2024-11-30T14:17:20.961028Z","shell.execute_reply.started":"2024-11-30T14:17:20.960895Z","shell.execute_reply":"2024-11-30T14:17:20.960909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# importance_threshold = 0.001  # This can be adjusted based on your needs\n\n# # Get the list of important feature indices above the threshold\n# important_feature_indices = importances[importances > importance_threshold].index.tolist()\n# temp = X[important_feature_indices]\n# display(temp.dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.962345Z","iopub.status.idle":"2024-11-30T14:17:20.962646Z","shell.execute_reply.started":"2024-11-30T14:17:20.962499Z","shell.execute_reply":"2024-11-30T14:17:20.962515Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **3.3 Ensemble model**","metadata":{}},{"cell_type":"code","source":"# Model parameters for LightGBM\n\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01,  # Increased from 2.68e-06\n    'device': 'gpu'\n}\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED,\n    'tree_method': 'gpu_hist',\n}\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU'\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.964148Z","iopub.status.idle":"2024-11-30T14:17:20.964612Z","shell.execute_reply.started":"2024-11-30T14:17:20.964371Z","shell.execute_reply":"2024-11-30T14:17:20.964393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\n\nXGB_Model = XGBRegressor(**XGB_Params)\n\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.966171Z","iopub.status.idle":"2024-11-30T14:17:20.966575Z","shell.execute_reply.started":"2024-11-30T14:17:20.966397Z","shell.execute_reply":"2024-11-30T14:17:20.966420Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# voting_model = VotingRegressor(estimators=[\n#     ('lightgbm', Light),\n#     ('xgboost', XGB_Model),\n#     ('catboost', CatBoost_Model)\n# ],weights=[4.0,4.0,5.0])\n\n# Submission1 = TrainML(voting_model, test)\n\n# Submission1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.968811Z","iopub.status.idle":"2024-11-30T14:17:20.969119Z","shell.execute_reply.started":"2024-11-30T14:17:20.968975Z","shell.execute_reply":"2024-11-30T14:17:20.968990Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_test_pred = voting_model.predict(test_data_processed)\ny_test_pred = y_test_pred.round().astype(int)\ny_test_pred = np.maximum(y_test_pred, 0)\n\n\n#Ensure alinged correctly\n# X_test = X_test.reset_index(drop=True)\nsave_id = save_id.reset_index(drop=True)\ny_test_pred = pd.Series(y_test_pred).reset_index(drop=True)\n\n# Create the submission file\nsub_cat = pd.DataFrame({\n    'id': save_id['id'],\n    'sii': y_test_pred\n})\n\n# Save the submission file\nsub_cat.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.970659Z","iopub.status.idle":"2024-11-30T14:17:20.970968Z","shell.execute_reply.started":"2024-11-30T14:17:20.970822Z","shell.execute_reply":"2024-11-30T14:17:20.970838Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **4. R.I.P. code**","metadata":{}},{"cell_type":"code","source":"# def quadratic_weighted_kappa(y_true, y_pred):\n#     return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n# def threshold_Rounder(oof_non_rounded, thresholds):\n#     return np.where(oof_non_rounded < thresholds[0], 0,\n#                     np.where(oof_non_rounded < thresholds[1], 1,\n#                              np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\n# def evaluate_predictions(thresholds, y_true, oof_non_rounded):\n#     rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n#     return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.971832Z","iopub.status.idle":"2024-11-30T14:17:20.972097Z","shell.execute_reply.started":"2024-11-30T14:17:20.971964Z","shell.execute_reply":"2024-11-30T14:17:20.971978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from sklearn.impute import SimpleImputer\n# from sklearn.metrics import accuracy_score, f1_score\n# from sklearn.preprocessing import LabelEncoder\n# from scipy.optimize import minimize\n\n# def TrainML(model_class, test_data):\n#     X = train.drop(['sii'], axis=1)\n#     y = train['sii']\n\n#     SKF = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n    \n#     train_S = []\n#     test_S = []\n    \n#     oof_non_rounded = np.zeros(len(y), dtype=float) \n#     oof_rounded = np.zeros(len(y), dtype=int) \n#     test_preds = np.zeros((len(test_data), 5))\n\n#     for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=5)):\n#         X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n#         y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n#         model = clone(model_class)\n#         model.fit(X_train, y_train)\n\n#         y_train_pred = model.predict(X_train)\n#         y_val_pred = model.predict(X_val)\n\n#         oof_non_rounded[test_idx] = y_val_pred\n#         y_val_pred_rounded = y_val_pred.round(0).astype(int)\n#         oof_rounded[test_idx] = y_val_pred_rounded\n\n#         train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n#         val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n#         train_S.append(train_kappa)\n#         test_S.append(val_kappa)\n        \n#         test_preds[:, fold] = model.predict(test_data)\n        \n#         print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n#         clear_output(wait=True)\n\n#     print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n#     print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n#     KappaOPtimizer = minimize(evaluate_predictions,\n#                               x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n#                               method='Nelder-Mead')\n#     assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n#     oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n#     tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n#     print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n#     tpm = test_preds.mean(axis=1)\n#     tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n#     submission = pd.DataFrame({\n#         'id': sample['id'],\n#         'sii': tpTuned\n#     })\n\n#     return submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-30T14:17:20.973363Z","iopub.status.idle":"2024-11-30T14:17:20.973787Z","shell.execute_reply.started":"2024-11-30T14:17:20.973567Z","shell.execute_reply":"2024-11-30T14:17:20.973588Z"}},"outputs":[],"execution_count":null}]}