{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\nimport torch\nfrom torch import nn\n\nSEED = 42\nn_splits = 5","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-19T08:17:56.496751Z","iopub.execute_input":"2024-10-19T08:17:56.497712Z","iopub.status.idle":"2024-10-19T08:18:16.747225Z","shell.execute_reply.started":"2024-10-19T08:17:56.497671Z","shell.execute_reply":"2024-10-19T08:18:16.746296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:18:16.748793Z","iopub.execute_input":"2024-10-19T08:18:16.749417Z","iopub.status.idle":"2024-10-19T08:18:16.756937Z","shell.execute_reply.started":"2024-10-19T08:18:16.749380Z","shell.execute_reply":"2024-10-19T08:18:16.755818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:18:43.132536Z","iopub.execute_input":"2024-10-19T08:18:43.133534Z","iopub.status.idle":"2024-10-19T08:20:10.160202Z","shell.execute_reply.started":"2024-10-19T08:18:43.133490Z","shell.execute_reply":"2024-10-19T08:20:10.159152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_df = train[['id','sii']]\n\ntime_series_df_with_target = pd.merge(train_ts,temp_df,how=\"left\",on='id')\n\ntemp_df = test[['id']]\n\ntime_series_df_without_target = pd.merge(test_ts,temp_df,how=\"left\",on='id')","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:20:10.162431Z","iopub.execute_input":"2024-10-19T08:20:10.163249Z","iopub.status.idle":"2024-10-19T08:20:10.186672Z","shell.execute_reply.started":"2024-10-19T08:20:10.163198Z","shell.execute_reply":"2024-10-19T08:20:10.185285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset,DataLoader\nfrom sklearn.preprocessing import StandardScaler\n\nclass CustomDataset(Dataset):\n    \n    def __init__(self, dataframe):\n        # Apply StandardScaler to the input features\n        self.scaler = StandardScaler()\n        if 'sii' in dataframe.columns:\n            self.train = True\n            features = dataframe.drop(['id', 'sii'], axis=1)  # Drop ID and target column\n            \n\n            self.targets = dataframe['sii'].values  # Keep target values (sii)\n        else:\n            self.train = False\n            features = dataframe.drop(['id'],axis=1)\n            \n        self.scaled_data = self.scaler.fit_transform(features)  # Scale features\n            \n\n    def __len__(self):\n        return len(self.scaled_data)\n    \n    def __getitem__(self, idx):\n        # Return the scaled input features and target value\n        if self.train:\n            return torch.tensor(self.scaled_data[idx], dtype=torch.float32), torch.tensor(self.targets[idx], dtype=torch.long)  # Ensure targets are long for classification\n        else:\n            return torch.tensor(self.scaled_data[idx], dtype=torch.float32)\n        \ntrain_dataset = CustomDataset(time_series_df_with_target)\ntrain_dataloader = DataLoader(train_dataset,batch_size=32,shuffle=True)\n\ntest_dataset = CustomDataset(time_series_df_without_target)\ntest_dataloader = DataLoader(test_dataset,batch_size=1,shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:20:10.188056Z","iopub.execute_input":"2024-10-19T08:20:10.188402Z","iopub.status.idle":"2024-10-19T08:20:10.213055Z","shell.execute_reply.started":"2024-10-19T08:20:10.188367Z","shell.execute_reply":"2024-10-19T08:20:10.211883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LSTMEncoder(nn.Module):\n    \n    def __init__(self, input_size, hidden_size, latent_dim, num_classes, num_layers=1):\n        super(LSTMEncoder, self).__init__()\n        # LSTM layer\n        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)\n        # Fully connected layers for latent dimension and classification\n        self.fc_latent = nn.Linear(hidden_size, latent_dim)  # Latent space\n        self.fc_class = nn.Linear(latent_dim, num_classes)   # Classifier layer for output\n        \n    def forward(self, x):\n        _, (hn, _) = self.lstm(x)  # Get hidden state from LSTM (hn)\n        latent = self.fc_latent(hn[-1])  # Compress the last hidden state to latent space\n        out = self.fc_class(latent)  # Classify using the latent space\n        return latent, out","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:20:10.215761Z","iopub.execute_input":"2024-10-19T08:20:10.216804Z","iopub.status.idle":"2024-10-19T08:20:10.224122Z","shell.execute_reply.started":"2024-10-19T08:20:10.216757Z","shell.execute_reply":"2024-10-19T08:20:10.222738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example parameters\ninput_size = time_series_df_with_target.shape[1] - 2  # Number of features (excluding the ID column)\nhidden_size = 64  # Number of hidden units in LSTM\nlatent_dim = 30  # Number of dimensions to encode into\nnum_classes = len(time_series_df_with_target['sii'].unique())  # Number of target classes\nnum_layers = 2  # LSTM layers\n\n\nmodel = LSTMEncoder(input_size, hidden_size, latent_dim, num_classes)\n\n# Define a loss function and optimizer\ncriterion = nn.CrossEntropyLoss()  # For multi-class classification\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:20:10.225775Z","iopub.execute_input":"2024-10-19T08:20:10.226527Z","iopub.status.idle":"2024-10-19T08:20:11.311828Z","shell.execute_reply.started":"2024-10-19T08:20:10.226477Z","shell.execute_reply":"2024-10-19T08:20:11.310923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training loop\nepochs = 40  # Number of epochs to train\n\nfor epoch in range(epochs):\n    total_loss = 0\n    for batch_data, batch_targets in train_dataloader:\n        optimizer.zero_grad()\n        # Forward pass: latent representation and predictions\n        latent, predictions = model(batch_data.unsqueeze(1))\n        # Compute the loss\n        loss = criterion(predictions, batch_targets)\n        total_loss+=loss.item()\n        loss.backward()  # Backpropagate the error\n        optimizer.step()  # Update the weights\n\n    print(f\"Epoch [{epoch+1}], Loss: {total_loss/32}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:20:11.313140Z","iopub.execute_input":"2024-10-19T08:20:11.313869Z","iopub.status.idle":"2024-10-19T08:20:15.382735Z","shell.execute_reply.started":"2024-10-19T08:20:11.313831Z","shell.execute_reply":"2024-10-19T08:20:15.381108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\nfinal_list = []\n\n# After training, you can use the encoder to extract both the latent dimensions and class predictions\nwith torch.no_grad():\n    for batch_data, batch_targets in train_dataloader:\n        batch_data = batch_data.unsqueeze(1)  # Add sequence length dim if needed\n        latent, predictions = model(batch_data)\n        final_list.append(latent)\n\n# Step 1: Concatenate all the latent representations into a single tensor\n# Assuming each 'latent' is of shape (batch_size, latent_dim)\nall_latents = torch.cat(final_list, dim=0)  # Concatenate along the first dimension (batch dimension)\n\n# Step 2: Convert the concatenated tensor to a NumPy array\nlatent_array = all_latents.numpy()  # Convert to NumPy array\n\n# Step 3: Create a DataFrame from the NumPy array\nnum_latent_dims = latent_array.shape[1]  # Get the number of latent dimensions\ntrain_latent = pd.DataFrame(latent_array, columns=[f'enc_{i + 1}' for i in range(num_latent_dims)])\n","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:20:52.616896Z","iopub.execute_input":"2024-10-19T08:20:52.619281Z","iopub.status.idle":"2024-10-19T08:20:52.682889Z","shell.execute_reply.started":"2024-10-19T08:20:52.619234Z","shell.execute_reply":"2024-10-19T08:20:52.681840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_latent['id'] = time_series_df_with_target['id']","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:20:56.965702Z","iopub.execute_input":"2024-10-19T08:20:56.966170Z","iopub.status.idle":"2024-10-19T08:20:56.975025Z","shell.execute_reply.started":"2024-10-19T08:20:56.966120Z","shell.execute_reply":"2024-10-19T08:20:56.973892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.merge(train, train_latent, how=\"left\", on='id')","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:20:57.789129Z","iopub.execute_input":"2024-10-19T08:20:57.789942Z","iopub.status.idle":"2024-10-19T08:20:57.810073Z","shell.execute_reply.started":"2024-10-19T08:20:57.789875Z","shell.execute_reply":"2024-10-19T08:20:57.808996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer, KNNImputer\n\nimputer = KNNImputer(n_neighbors=5)\nnumeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\nimputed_data = imputer.fit_transform(train[numeric_cols])\ntrain_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\ntrain_imputed['sii'] = train_imputed['sii'].round().astype(int)\nfor col in train.columns:\n    if col not in numeric_cols:\n        train_imputed[col] = train[col]\n        \ntrain = train_imputed\n","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:22:59.732329Z","iopub.execute_input":"2024-10-19T08:22:59.733226Z","iopub.status.idle":"2024-10-19T08:23:07.467054Z","shell.execute_reply.started":"2024-10-19T08:22:59.733182Z","shell.execute_reply":"2024-10-19T08:23:07.466016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:23:15.512758Z","iopub.execute_input":"2024-10-19T08:23:15.513351Z","iopub.status.idle":"2024-10-19T08:23:16.306986Z","shell.execute_reply.started":"2024-10-19T08:23:15.513303Z","shell.execute_reply":"2024-10-19T08:23:16.305868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_list = []\n\n# After training, you can use the encoder to extract both the latent dimensions and class predictions\nwith torch.no_grad():\n    for batch_data in test_dataloader:\n        batch_data = batch_data.unsqueeze(1)  # Add sequence length dim if needed\n        latent, predictions = model(batch_data)\n        final_list.append(latent)\n\n# Step 1: Concatenate all the latent representations into a single tensor\n# Assuming each 'latent' is of shape (batch_size, latent_dim)\nall_latents = torch.cat(final_list, dim=0)  # Concatenate along the first dimension (batch dimension)\n\n# Step 2: Convert the concatenated tensor to a NumPy array\nlatent_array = all_latents.numpy()  # Convert to NumPy array\n\n# Step 3: Create a DataFrame from the NumPy array\nnum_latent_dims = latent_array.shape[1]  # Get the number of latent dimensions\ntest_latent = pd.DataFrame(latent_array, columns=[f'enc_{i + 1}' for i in range(num_latent_dims)])","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:23:53.836173Z","iopub.execute_input":"2024-10-19T08:23:53.836709Z","iopub.status.idle":"2024-10-19T08:23:53.853386Z","shell.execute_reply.started":"2024-10-19T08:23:53.836662Z","shell.execute_reply":"2024-10-19T08:23:53.851987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_latent['id'] = time_series_df_without_target['id']\ntest = pd.merge(test, test_latent, how=\"left\", on='id')","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:23:54.907335Z","iopub.execute_input":"2024-10-19T08:23:54.907847Z","iopub.status.idle":"2024-10-19T08:23:54.918615Z","shell.execute_reply.started":"2024-10-19T08:23:54.907800Z","shell.execute_reply":"2024-10-19T08:23:54.917154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"featuresCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\nfeaturesCols += ['enc_1', 'enc_2', 'enc_3', 'enc_4', 'enc_5', 'enc_6', 'enc_7', 'enc_8',\n       'enc_9', 'enc_10', 'enc_11', 'enc_12', 'enc_13', 'enc_14', 'enc_15',\n       'enc_16', 'enc_17', 'enc_18', 'enc_19', 'enc_20','enc_21','enc_22','enc_23','enc_24','enc_25','enc_26','enc_27','enc_28','enc_29','enc_30']\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\ntest  = test.drop('id',axis=1)\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:24:03.010384Z","iopub.execute_input":"2024-10-19T08:24:03.010876Z","iopub.status.idle":"2024-10-19T08:24:03.031853Z","shell.execute_reply.started":"2024-10-19T08:24:03.010838Z","shell.execute_reply":"2024-10-19T08:24:03.030846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        \ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:24:04.410101Z","iopub.execute_input":"2024-10-19T08:24:04.410527Z","iopub.status.idle":"2024-10-19T08:24:04.484358Z","shell.execute_reply.started":"2024-10-19T08:24:04.410487Z","shell.execute_reply":"2024-10-19T08:24:04.483332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:24:05.356969Z","iopub.execute_input":"2024-10-19T08:24:05.357952Z","iopub.status.idle":"2024-10-19T08:24:05.369877Z","shell.execute_reply.started":"2024-10-19T08:24:05.357879Z","shell.execute_reply":"2024-10-19T08:24:05.368610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Params = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01  # Increased from 2.68e-06\n}\n\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED,\n    'tree_method': 'exact'\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'verbose': 0,\n    'l2_leaf_reg': 10  # Increase this value\n}\n\n# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)\n])","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:24:25.012651Z","iopub.execute_input":"2024-10-19T08:24:25.013208Z","iopub.status.idle":"2024-10-19T08:24:25.026453Z","shell.execute_reply.started":"2024-10-19T08:24:25.013161Z","shell.execute_reply":"2024-10-19T08:24:25.024932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Submission1 = TrainML(voting_model, test)\n\n# Save submission\nSubmission1.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-19T08:24:34.466223Z","iopub.execute_input":"2024-10-19T08:24:34.466700Z","iopub.status.idle":"2024-10-19T08:25:21.299015Z","shell.execute_reply.started":"2024-10-19T08:24:34.466662Z","shell.execute_reply":"2024-10-19T08:25:21.297795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}