{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":7453542,"sourceType":"datasetVersion","datasetId":921302}],"dockerImageVersionId":30805,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 0. Import library","metadata":{}},{"cell_type":"code","source":"!pip -q install /kaggle/input/pytorchtabnet/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:58:11.593449Z","iopub.execute_input":"2024-12-04T16:58:11.594323Z","iopub.status.idle":"2024-12-04T16:58:20.258248Z","shell.execute_reply.started":"2024-12-04T16:58:11.594288Z","shell.execute_reply":"2024-12-04T16:58:20.257002Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone, BaseEstimator, RegressorMixin\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score, mean_squared_error\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.decomposition import PCA\nfrom sklearn.datasets import make_classification\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nfrom pytorch_tabnet.tab_model import TabNetRegressor\nfrom pytorch_tabnet.callbacks import Callback","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:58:20.260477Z","iopub.execute_input":"2024-12-04T16:58:20.260801Z","iopub.status.idle":"2024-12-04T16:58:20.270997Z","shell.execute_reply.started":"2024-12-04T16:58:20.260771Z","shell.execute_reply":"2024-12-04T16:58:20.270030Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Data Processing","metadata":{}},{"cell_type":"markdown","source":"## 1.1 AutoEncoder","metadata":{}},{"cell_type":"code","source":"# AutoEncoder using for Time Series\nclass AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n        \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:58:20.272050Z","iopub.execute_input":"2024-12-04T16:58:20.272283Z","iopub.status.idle":"2024-12-04T16:58:20.288497Z","shell.execute_reply.started":"2024-12-04T16:58:20.272260Z","shell.execute_reply":"2024-12-04T16:58:20.287601Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    \n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n                 \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:58:20.290623Z","iopub.execute_input":"2024-12-04T16:58:20.290943Z","iopub.status.idle":"2024-12-04T16:58:20.303667Z","shell.execute_reply.started":"2024-12-04T16:58:20.290917Z","shell.execute_reply":"2024-12-04T16:58:20.302896Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:58:20.304726Z","iopub.execute_input":"2024-12-04T16:58:20.305018Z","iopub.status.idle":"2024-12-04T16:58:20.317605Z","shell.execute_reply.started":"2024-12-04T16:58:20.304992Z","shell.execute_reply":"2024-12-04T16:58:20.316899Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1.2 Data Loader","metadata":{}},{"cell_type":"code","source":"# Bỏ hết các mùa\ndef feature_engineering(df):\n    season_cols = [col for col in df.columns if 'Season' in col]\n    df = df.drop(season_cols, axis=1)\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:58:20.318633Z","iopub.execute_input":"2024-12-04T16:58:20.318923Z","iopub.status.idle":"2024-12-04T16:58:20.331555Z","shell.execute_reply.started":"2024-12-04T16:58:20.318897Z","shell.execute_reply":"2024-12-04T16:58:20.330804Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=100, batch_size=32)\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=100, batch_size=32)\n\ntime_series_cols = train_ts_encoded.columns.tolist()\ntrain_ts_encoded[\"id\"]=train_ts[\"id\"]\ntest_ts_encoded['id']=test_ts[\"id\"]\n\ntrain = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\ntest = pd.merge(test, test_ts_encoded, how=\"left\", on='id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:58:20.332625Z","iopub.execute_input":"2024-12-04T16:58:20.332912Z","iopub.status.idle":"2024-12-04T16:59:45.129133Z","shell.execute_reply.started":"2024-12-04T16:58:20.332886Z","shell.execute_reply":"2024-12-04T16:59:45.128114Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill tất cả data thiếu features với KNN (kể cả sii)\nimputer = KNNImputer(n_neighbors=5)\nnumeric_cols_train = train.select_dtypes(include=['int32', 'int64', 'float64', 'int64']).columns\nnumeric_cols_test = test.select_dtypes(include=['int32', 'int64', 'float64', 'int64']).columns\n\nimputed_data_train = imputer.fit_transform(train[numeric_cols_train])\nimputed_data_test = imputer.fit_transform(test[numeric_cols_test])\n\ntrain_imputed = pd.DataFrame(imputed_data_train, columns=numeric_cols_train)\ntest_imputed = pd.DataFrame(imputed_data_test, columns=numeric_cols_test)\n\ntrain_imputed['sii'] = train_imputed['sii'].round().astype(int)\nfor col in train.columns:\n    if col not in numeric_cols_train:\n        train_imputed[col] = train[col]\nfor col in test.columns:\n    if col not in numeric_cols_test:\n        test_imputed[col] = test[col]\n\n## Xử lý dữ liệu sau khi fill\ntrain = train_imputed\ntest = test_imputed\ntrain = feature_engineering(train)\ntest = feature_engineering(test)\n\n## Loại bỏ cột id ở cả 2 tập\ntrain.drop(columns=['id'], inplace=True)\ntest.drop(columns=['id'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:59:45.130297Z","iopub.execute_input":"2024-12-04T16:59:45.130575Z","iopub.status.idle":"2024-12-04T16:59:52.014178Z","shell.execute_reply.started":"2024-12-04T16:59:45.130547Z","shell.execute_reply":"2024-12-04T16:59:52.013203Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Để sau này khớp, hiện tại vẫn train thừa 1 cột sii là label\ncommon_columns = train.columns.intersection(test.columns)\n# Khớp số feature giữa train và test, loại bỏ PICAT\n# train = train[common_columns]\ntrain.shape, test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:59:52.015816Z","iopub.execute_input":"2024-12-04T16:59:52.016100Z","iopub.status.idle":"2024-12-04T16:59:52.022445Z","shell.execute_reply.started":"2024-12-04T16:59:52.016074Z","shell.execute_reply":"2024-12-04T16:59:52.021575Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Model Define and training function","metadata":{}},{"cell_type":"markdown","source":"## 2.1 Training function","metadata":{}},{"cell_type":"code","source":"## Hyperparameters\nN_SPLITS = 10\nSEED = 42","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:59:52.025866Z","iopub.execute_input":"2024-12-04T16:59:52.026193Z","iopub.status.idle":"2024-12-04T16:59:52.032337Z","shell.execute_reply.started":"2024-12-04T16:59:52.026145Z","shell.execute_reply":"2024-12-04T16:59:52.031626Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Useful function\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n    \ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:59:52.033274Z","iopub.execute_input":"2024-12-04T16:59:52.033520Z","iopub.status.idle":"2024-12-04T16:59:52.044718Z","shell.execute_reply.started":"2024-12-04T16:59:52.033495Z","shell.execute_reply":"2024-12-04T16:59:52.043822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Train and get predict function\ndef train_predict(model, train_data, test_data):\n    # Khớp giữ liệu input của train và test\n    X = train_data[common_columns]\n    y = train_data['sii']\n\n    # Định nghĩa K-Fold\n    SKF = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\n\n    train_his = []\n    val_his = []\n\n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), N_SPLITS))\n\n    for fold, (train_index, val_index) in enumerate(tqdm(SKF.split(X, y), desc=\"Train progress\", total = N_SPLITS)):\n        # Xác định dữ liệu của fold\n        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n        # Train\n        model_ = clone(model) # tạo bản sao làm mới tại mỗi fold -> độc lập\n        model_.fit(X_train, y_train)\n\n        # Tính toán sai số\n        y_train_pred = model_.predict(X_train)\n        y_val_pred = model_.predict(X_val)\n\n        oof_non_rounded[val_index] = y_val_pred\n        y_train_pred_rounded = y_train_pred.round(0).astype(int)\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[val_index] = y_val_pred_rounded\n\n        # Kiểm tra sai số giữa mô hình đánh giá và thực tế\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred_rounded)\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_his.append(train_kappa)\n        val_his.append(val_kappa)\n\n        test_preds[:, fold] = model_.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n\n    print(f\"Mean Train QWK --> {np.mean(train_his):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(val_his):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:59:52.046099Z","iopub.execute_input":"2024-12-04T16:59:52.046620Z","iopub.status.idle":"2024-12-04T16:59:52.058202Z","shell.execute_reply.started":"2024-12-04T16:59:52.046592Z","shell.execute_reply":"2024-12-04T16:59:52.057325Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.2 Model Define","metadata":{}},{"cell_type":"code","source":"# Parameter for 3 model\nLightGBM_Params = {\n    'random_state': SEED, \n    'verbose':-1,\n    'n_estimators': 300,\n    'learning_rate': 0.05,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.8,\n    'bagging_fraction': 0.78,\n    'bagging_freq': 4,\n    'lambda_l1': 10,\n    'lambda_l2': 0.01,\n    'device': 'cpu',\n}\n\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,\n    'reg_lambda': 5,\n    'random_state': 42,\n    'tree_method': 'gpu_hist'\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': 42,\n    'verbose': 0,\n    'l2_leaf_reg': 10,\n    'task_type': 'GPU',\n}\n\n# Create model instances\nLightGBM_Model = LGBMRegressor(**LightGBM_Params)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', LightGBM_Model),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)\n], weights=[4.0,4.0,5.0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:59:52.059264Z","iopub.execute_input":"2024-12-04T16:59:52.059534Z","iopub.status.idle":"2024-12-04T16:59:52.072787Z","shell.execute_reply.started":"2024-12-04T16:59:52.059508Z","shell.execute_reply":"2024-12-04T16:59:52.072027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Submission1 = train_predict(voting_model, train, test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:59:52.073995Z","iopub.execute_input":"2024-12-04T16:59:52.074270Z","iopub.status.idle":"2024-12-04T16:59:52.083332Z","shell.execute_reply.started":"2024-12-04T16:59:52.074244Z","shell.execute_reply":"2024-12-04T16:59:52.082401Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Submission1.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:59:52.084647Z","iopub.execute_input":"2024-12-04T16:59:52.085038Z","iopub.status.idle":"2024-12-04T16:59:52.093886Z","shell.execute_reply.started":"2024-12-04T16:59:52.084998Z","shell.execute_reply":"2024-12-04T16:59:52.092884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Submission1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:59:52.095136Z","iopub.execute_input":"2024-12-04T16:59:52.095467Z","iopub.status.idle":"2024-12-04T16:59:52.104518Z","shell.execute_reply.started":"2024-12-04T16:59:52.095440Z","shell.execute_reply":"2024-12-04T16:59:52.103752Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.3 TabNet Model","metadata":{}},{"cell_type":"code","source":"class TabNetModel(BaseEstimator, RegressorMixin):\n    def __init__(self, **kwargs):\n        self.model = TabNetRegressor(**kwargs, verbose=False)\n        self.kwargs = kwargs\n        self.imputer = SimpleImputer(strategy='median')\n        self.best_model_path = 'best_model.pt'\n\n\n    def fit(self, X, y):\n        X_imputed = self.imputer.fit_transform(X)\n\n        if hasattr(y, 'values'):\n            y = y.values\n\n        X_train, X_val, y_train, y_val = train_test_split(X_imputed, y, test_size=0.2, random_state=SEED)\n\n        history = self.model.fit(\n            X_train = X_train,\n            y_train = y_train.reshape(-1, 1),\n            eval_set = [(X_val, y_val.reshape(-1, 1))],\n            eval_name = ['valid'],\n            eval_metric = ['mse'],\n            max_epochs = 200,\n            patience = 20,\n            batch_size = 1024,\n            virtual_batch_size = 128,\n            num_workers = 0,\n            drop_last = False,\n            callbacks = [\n                TabNetPretrainedModelCheckpoint(\n                    filepath = self.best_model_path,\n                    monitor = 'valid_mse',\n                    mode = 'min',\n                    save_best_only = True,\n                    verbose = True\n                )\n            ]\n        )\n\n        if os.path.exists(self.best_model_path):\n            self.model.load_model(self.best_model_path)\n            os.remove(self.best_model_path)\n\n        return self\n\n\n    def predict(self, X):\n        X_imputed = self.imputer.transform(X)\n        return self.model.predict(X_imputed).flatten()\n\n    def __deepcopy__(self, memo):\n        cls = self.__class__\n        result = self.__new__(cls)\n        memo[id(self)] = result\n        for k, v in self.__dict__.items():\n            setattr(result, k, deepcopy(v, memo))\n\n        return result\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n    def __init__(self, filepath, monitor='val_loss', mode='min', save_best_only=True, verbose=1):\n        super().__init__()\n        self.filepath = filepath\n        self.monitor = monitor\n        self.mode = mode\n        self.save_best_only = save_best_only\n        self.verbose = verbose\n        self.best = float('inf') if mode == 'min' else -float('inf')\n        \n    def on_train_begin(self, logs=None):\n        self.model = self.trainer\n        \n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is None:\n            return\n        \n        if (self.mode == 'min' and current < self.best) or (self.mode == 'max' and current > self.best):\n            self.best = current\n            if self.save_best_only:\n                self.model.save_model(self.filepath)\n                clear_output(wait=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:59:52.105638Z","iopub.execute_input":"2024-12-04T16:59:52.105913Z","iopub.status.idle":"2024-12-04T16:59:52.118936Z","shell.execute_reply.started":"2024-12-04T16:59:52.105875Z","shell.execute_reply":"2024-12-04T16:59:52.118025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Parameter for 3 model\nLightGBM_Params = {\n    'random_state': SEED, \n    'verbose':-1,\n    'n_estimators': 300,\n    'learning_rate': 0.05,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.8,\n    'bagging_fraction': 0.78,\n    'bagging_freq': 4,\n    'lambda_l1': 10,\n    'lambda_l2': 0.01,\n    'device': 'cpu',\n}\n\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,\n    'reg_lambda': 5,\n    'random_state': 42,\n    'tree_method': 'gpu_hist'\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': 42,\n    'verbose': 0,\n    'l2_leaf_reg': 10,\n    'task_type': 'GPU',\n}\n\nTabNet_Params = {\n    'n_d': 64,              # Width of the decision prediction layer\n    'n_a': 64,              # Width of the attention embedding for each step\n    'n_steps': 5,           # Number of steps in the architecture\n    'gamma': 1.5,           # Coefficient for feature selection regularization\n    'n_independent': 2,     # Number of independent GLU layer in each GLU block\n    'n_shared': 2,          # Number of shared GLU layer in each GLU block\n    'lambda_sparse': 1e-4,  # Sparsity regularization\n    'optimizer_fn': torch.optim.Adam,\n    'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n    'mask_type': 'entmax',\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\n# Create model instances\nLightGBM_Model = LGBMRegressor(**LightGBM_Params)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\nTabNet_Model = TabNetModel(**TabNet_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', LightGBM_Model),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model),\n    ('tabnet', TabNet_Model)\n], weights=[4.0,4.0,5.0,4.0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:59:52.120113Z","iopub.execute_input":"2024-12-04T16:59:52.120373Z","iopub.status.idle":"2024-12-04T16:59:52.136683Z","shell.execute_reply.started":"2024-12-04T16:59:52.120348Z","shell.execute_reply":"2024-12-04T16:59:52.135942Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission1 = train_predict(voting_model, train, test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T16:59:52.137967Z","iopub.execute_input":"2024-12-04T16:59:52.138331Z","iopub.status.idle":"2024-12-04T17:03:09.861315Z","shell.execute_reply.started":"2024-12-04T16:59:52.138305Z","shell.execute_reply":"2024-12-04T17:03:09.860352Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Model 2","metadata":{}},{"cell_type":"markdown","source":"## 3.1 Reload Data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)   ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:03:09.863144Z","iopub.execute_input":"2024-12-04T17:03:09.863746Z","iopub.status.idle":"2024-12-04T17:04:22.939522Z","shell.execute_reply.started":"2024-12-04T17:03:09.863701Z","shell.execute_reply":"2024-12-04T17:04:22.938828Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:04:22.940450Z","iopub.execute_input":"2024-12-04T17:04:22.940681Z","iopub.status.idle":"2024-12-04T17:04:23.086843Z","shell.execute_reply.started":"2024-12-04T17:04:22.940658Z","shell.execute_reply":"2024-12-04T17:04:23.085968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"common_columns = train.columns.intersection(test.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:04:23.088099Z","iopub.execute_input":"2024-12-04T17:04:23.088437Z","iopub.status.idle":"2024-12-04T17:04:23.093886Z","shell.execute_reply.started":"2024-12-04T17:04:23.088400Z","shell.execute_reply":"2024-12-04T17:04:23.092991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        \ntrain = update(train)\ntest = update(test)\ntrain = train.dropna(subset='sii')\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:04:23.095237Z","iopub.execute_input":"2024-12-04T17:04:23.095930Z","iopub.status.idle":"2024-12-04T17:04:23.163597Z","shell.execute_reply.started":"2024-12-04T17:04:23.095890Z","shell.execute_reply":"2024-12-04T17:04:23.162901Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.2 Define model","metadata":{}},{"cell_type":"code","source":"# Parameter for 3 model\nLightGBM_Params = {\n    'random_state': SEED, \n    'verbose':-1,\n    'n_estimators': 300,\n    'learning_rate': 0.05,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.8,\n    'bagging_fraction': 0.78,\n    'bagging_freq': 4,\n    'lambda_l1': 10,\n    'lambda_l2': 0.01,\n    'device': 'cpu',\n}\n\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,\n    'reg_lambda': 5,\n    'random_state': 42,\n    'tree_method': 'gpu_hist'\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': 42,\n    'verbose': 0,\n    'l2_leaf_reg': 10,\n    'task_type': 'GPU',\n}\n\nTabNet_Params = {\n    'n_d': 64,              # Width of the decision prediction layer\n    'n_a': 64,              # Width of the attention embedding for each step\n    'n_steps': 5,           # Number of steps in the architecture\n    'gamma': 1.5,           # Coefficient for feature selection regularization\n    'n_independent': 2,     # Number of independent GLU layer in each GLU block\n    'n_shared': 2,          # Number of shared GLU layer in each GLU block\n    'lambda_sparse': 1e-4,  # Sparsity regularization\n    'optimizer_fn': torch.optim.Adam,\n    'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n    'mask_type': 'entmax',\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\n# Create model instances\nLightGBM_Model = LGBMRegressor(**LightGBM_Params)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n# TabNet_Model = TabNetModel(**TabNet_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', LightGBM_Model),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)\n], weights=[4.0,4.0,5.0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:04:23.164708Z","iopub.execute_input":"2024-12-04T17:04:23.165164Z","iopub.status.idle":"2024-12-04T17:04:23.174548Z","shell.execute_reply.started":"2024-12-04T17:04:23.165131Z","shell.execute_reply":"2024-12-04T17:04:23.173616Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:04:23.175667Z","iopub.execute_input":"2024-12-04T17:04:23.176214Z","iopub.status.idle":"2024-12-04T17:04:23.341482Z","shell.execute_reply.started":"2024-12-04T17:04:23.176185Z","shell.execute_reply":"2024-12-04T17:04:23.340614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission2 = train_predict(voting_model, train, test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:04:23.342686Z","iopub.execute_input":"2024-12-04T17:04:23.343047Z","iopub.status.idle":"2024-12-04T17:04:55.629680Z","shell.execute_reply.started":"2024-12-04T17:04:23.343008Z","shell.execute_reply":"2024-12-04T17:04:55.628728Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Final Submit - Voting","metadata":{}},{"cell_type":"code","source":"sub1 = Submission1\nsub2 = Submission2\n\nsub1 = sub1.sort_values(by='id').reset_index(drop=True)\nsub2 = sub2.sort_values(by='id').reset_index(drop=True)\n\ncombined = pd.DataFrame({\n    'id': sub1['id'],\n    'sii_1': sub1['sii'],\n    'sii_2': sub2['sii']\n})\n\ndef majority_vote(row):\n    return row.mode()[0]\n\ncombined['final_sii'] = combined[['sii_1', 'sii_2']].apply(majority_vote, axis=1)\n\nfinal_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\nfinal_submission.to_csv('submission.csv', index=False)\n\nprint(\"Majority voting completed and saved to 'Final_Submission.csv'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:05:34.072035Z","iopub.execute_input":"2024-12-04T17:05:34.072727Z","iopub.status.idle":"2024-12-04T17:05:34.089158Z","shell.execute_reply.started":"2024-12-04T17:05:34.072695Z","shell.execute_reply":"2024-12-04T17:05:34.088344Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:05:38.400976Z","iopub.execute_input":"2024-12-04T17:05:38.401303Z","iopub.status.idle":"2024-12-04T17:05:38.410689Z","shell.execute_reply.started":"2024-12-04T17:05:38.401276Z","shell.execute_reply":"2024-12-04T17:05:38.409475Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}