{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":10248047,"sourceType":"datasetVersion","datasetId":6338367}],"dockerImageVersionId":30762,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Prepare data","metadata":{}},{"cell_type":"markdown","source":"## 1.1 Import Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport polars as pl\nimport pandas as pd\nimport datetime as dt\nfrom colorama import Fore, Style\nfrom scipy.optimize import minimize\nfrom tqdm import tqdm\nimport missingno as msno\nimport matplotlib.pyplot as plt\nfrom sklearn.decomposition import PCA\nfrom IPython.display import clear_output\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nfrom concurrent.futures import ThreadPoolExecutor\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.base import clone\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nimport torch.optim as optim\n\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.metrics import cohen_kappa_score\nimport lightgbm as lgb\nfrom sklearn.metrics import accuracy_score, mean_squared_error","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:24:09.039069Z","iopub.execute_input":"2024-12-19T17:24:09.040001Z","iopub.status.idle":"2024-12-19T17:24:09.046616Z","shell.execute_reply.started":"2024-12-19T17:24:09.039962Z","shell.execute_reply":"2024-12-19T17:24:09.045687Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1.2 Config Path","metadata":{}},{"cell_type":"code","source":"# parquet_path_data\ntrain_parquet_path = \"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\"\ntest_parquet_path = \"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\"\n\n# dictionary_path_data\ndictionary_path = \"/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv\"\n\n###\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:24:09.048165Z","iopub.execute_input":"2024-12-19T17:24:09.048414Z","iopub.status.idle":"2024-12-19T17:24:09.060501Z","shell.execute_reply.started":"2024-12-19T17:24:09.048390Z","shell.execute_reply":"2024-12-19T17:24:09.059669Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# read parquet file\ndef read_parquet_file(filename, path):\n    df = pd.read_parquet(os.path.join(path, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\n# read parquet folder\ndef read_parquet_folder(path) -> pd.DataFrame:\n    id_path = os.listdir(path)\n    with ThreadPoolExecutor() as executor:\n        dfs = list(tqdm(executor.map(lambda filename: read_parquet_file(filename, path), id_path), total=len(id_path)))\n    features, indexes = zip(*dfs)\n    df = pd.DataFrame(features, columns=[f\"Features_{i}\" for i in range(len(features[0]))])\n    df['id'] = indexes\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:24:09.061570Z","iopub.execute_input":"2024-12-19T17:24:09.062428Z","iopub.status.idle":"2024-12-19T17:24:09.073101Z","shell.execute_reply.started":"2024-12-19T17:24:09.062388Z","shell.execute_reply":"2024-12-19T17:24:09.072367Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1.3 read data","metadata":{}},{"cell_type":"code","source":"# csv_data\ntrain_csv_data = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\")\ntest_csv_data = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\")\n\n# parquet_data\ntrain_parquet_data = read_parquet_folder(train_parquet_path)\ntest_parquet_data = read_parquet_folder(test_parquet_path)\n","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:24:09.074932Z","iopub.execute_input":"2024-12-19T17:24:09.075225Z","iopub.status.idle":"2024-12-19T17:25:18.112945Z","shell.execute_reply.started":"2024-12-19T17:24:09.075192Z","shell.execute_reply":"2024-12-19T17:25:18.112063Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1.4 View Data","metadata":{}},{"cell_type":"markdown","source":"### CSV feature, shape","metadata":{}},{"cell_type":"code","source":"train_csv_features = train_csv_data.columns.tolist()\nprint(\"Train features shape:\",train_csv_data.shape)\nprint(\"Train features are: \",train_csv_features)\n\ntest_csv_features = test_csv_data.columns.tolist()\nprint(\"Test features shape:\",test_csv_data.shape)\nprint(\"Test features are: \",test_csv_features)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:18.113960Z","iopub.execute_input":"2024-12-19T17:25:18.114232Z","iopub.status.idle":"2024-12-19T17:25:18.119815Z","shell.execute_reply.started":"2024-12-19T17:25:18.114205Z","shell.execute_reply":"2024-12-19T17:25:18.118904Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_csv_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:18.121539Z","iopub.execute_input":"2024-12-19T17:25:18.121792Z","iopub.status.idle":"2024-12-19T17:25:18.155865Z","shell.execute_reply.started":"2024-12-19T17:25:18.121766Z","shell.execute_reply":"2024-12-19T17:25:18.155008Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_csv_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:18.156879Z","iopub.execute_input":"2024-12-19T17:25:18.157121Z","iopub.status.idle":"2024-12-19T17:25:18.179497Z","shell.execute_reply.started":"2024-12-19T17:25:18.157096Z","shell.execute_reply":"2024-12-19T17:25:18.178767Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Parquet features, shape","metadata":{}},{"cell_type":"code","source":"train_parquet_features = train_parquet_data.columns.tolist()\nprint(\"Train features shape:\",train_parquet_data.shape)\nprint(\"Train features are: \",train_parquet_features)\n\ntest_parquet_features = test_parquet_data.columns.tolist()\nprint(\"Test features shape:\",test_parquet_data.shape)\nprint(\"Test features are: \",test_parquet_features)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:18.180479Z","iopub.execute_input":"2024-12-19T17:25:18.180777Z","iopub.status.idle":"2024-12-19T17:25:18.189480Z","shell.execute_reply.started":"2024-12-19T17:25:18.180734Z","shell.execute_reply":"2024-12-19T17:25:18.188495Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_parquet_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:18.190611Z","iopub.execute_input":"2024-12-19T17:25:18.191407Z","iopub.status.idle":"2024-12-19T17:25:18.220496Z","shell.execute_reply.started":"2024-12-19T17:25:18.191367Z","shell.execute_reply":"2024-12-19T17:25:18.219432Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_parquet_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:18.221582Z","iopub.execute_input":"2024-12-19T17:25:18.221814Z","iopub.status.idle":"2024-12-19T17:25:18.242428Z","shell.execute_reply.started":"2024-12-19T17:25:18.221775Z","shell.execute_reply":"2024-12-19T17:25:18.241697Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Pre-processing","metadata":{}},{"cell_type":"markdown","source":"### Check value function","metadata":{}},{"cell_type":"code","source":"def thong_ke(df):\n    thong_ke = pd.DataFrame({\n        'Số ô trống': df.isnull().sum(),\n        'Số ô có giá trị': df.notnull().sum(),\n        'Số ô có giá trị 0': (df == 0).sum()  # Tính số ô có giá trị bằng 0\n    })\n    print(thong_ke.to_string())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:18.244747Z","iopub.execute_input":"2024-12-19T17:25:18.245221Z","iopub.status.idle":"2024-12-19T17:25:18.252889Z","shell.execute_reply.started":"2024-12-19T17:25:18.245194Z","shell.execute_reply":"2024-12-19T17:25:18.252124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"thong_ke(train_parquet_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:18.253775Z","iopub.execute_input":"2024-12-19T17:25:18.254123Z","iopub.status.idle":"2024-12-19T17:25:18.269586Z","shell.execute_reply.started":"2024-12-19T17:25:18.254080Z","shell.execute_reply":"2024-12-19T17:25:18.268786Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.1 Process time series data ","metadata":{}},{"cell_type":"markdown","source":"### drop id","metadata":{}},{"cell_type":"code","source":"ts_train_data = train_parquet_data.drop('id', axis=1)\nts_test_data  = test_parquet_data.drop('id', axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:18.270560Z","iopub.execute_input":"2024-12-19T17:25:18.270803Z","iopub.status.idle":"2024-12-19T17:25:18.276721Z","shell.execute_reply.started":"2024-12-19T17:25:18.270780Z","shell.execute_reply":"2024-12-19T17:25:18.275889Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Implement AE","metadata":{}},{"cell_type":"code","source":"class AutoEncoder(nn.Module):\n    # init layer\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n\n    # forward \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n\n# implement\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    # scaler\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    # input\n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    # loss config\n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n                 \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_Feature_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:18.277791Z","iopub.execute_input":"2024-12-19T17:25:18.278274Z","iopub.status.idle":"2024-12-19T17:25:18.289293Z","shell.execute_reply.started":"2024-12-19T17:25:18.278249Z","shell.execute_reply":"2024-12-19T17:25:18.288463Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ts_train_encoded = perform_autoencoder(ts_train_data, encoding_dim=60, epochs=100, batch_size=32)\nts_test_encoded = perform_autoencoder(ts_test_data, encoding_dim=60, epochs=100, batch_size=32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:18.290395Z","iopub.execute_input":"2024-12-19T17:25:18.290651Z","iopub.status.idle":"2024-12-19T17:25:27.888264Z","shell.execute_reply.started":"2024-12-19T17:25:18.290627Z","shell.execute_reply":"2024-12-19T17:25:27.887409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ts_train_encoded[\"id\"]=train_parquet_data[\"id\"]\nts_test_encoded['id']=test_parquet_data[\"id\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:27.889392Z","iopub.execute_input":"2024-12-19T17:25:27.889693Z","iopub.status.idle":"2024-12-19T17:25:27.895201Z","shell.execute_reply.started":"2024-12-19T17:25:27.889663Z","shell.execute_reply":"2024-12-19T17:25:27.894297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ts_train_encoded.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:27.896277Z","iopub.execute_input":"2024-12-19T17:25:27.896525Z","iopub.status.idle":"2024-12-19T17:25:27.923428Z","shell.execute_reply.started":"2024-12-19T17:25:27.896498Z","shell.execute_reply":"2024-12-19T17:25:27.922585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ts_test_encoded.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:27.924376Z","iopub.execute_input":"2024-12-19T17:25:27.924626Z","iopub.status.idle":"2024-12-19T17:25:27.946577Z","shell.execute_reply.started":"2024-12-19T17:25:27.924602Z","shell.execute_reply":"2024-12-19T17:25:27.945723Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.2 Merge Data","metadata":{}},{"cell_type":"code","source":"train = pd.merge(train_csv_data, ts_train_encoded, on='id', how='left')\ntest = pd.merge(test_csv_data, ts_test_encoded, on='id', how='left')","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:27.947567Z","iopub.execute_input":"2024-12-19T17:25:27.947794Z","iopub.status.idle":"2024-12-19T17:25:27.964581Z","shell.execute_reply.started":"2024-12-19T17:25:27.947771Z","shell.execute_reply":"2024-12-19T17:25:27.963821Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:27.965541Z","iopub.execute_input":"2024-12-19T17:25:27.965892Z","iopub.status.idle":"2024-12-19T17:25:27.985285Z","shell.execute_reply.started":"2024-12-19T17:25:27.965862Z","shell.execute_reply":"2024-12-19T17:25:27.984427Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:27.986281Z","iopub.execute_input":"2024-12-19T17:25:27.986626Z","iopub.status.idle":"2024-12-19T17:25:28.007788Z","shell.execute_reply.started":"2024-12-19T17:25:27.986592Z","shell.execute_reply":"2024-12-19T17:25:28.006888Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Train_shape: \", train.shape)\nprint(\"Test_shape: \", test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:28.008943Z","iopub.execute_input":"2024-12-19T17:25:28.009686Z","iopub.status.idle":"2024-12-19T17:25:28.017964Z","shell.execute_reply.started":"2024-12-19T17:25:28.009651Z","shell.execute_reply":"2024-12-19T17:25:28.017055Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Có 3960 mẫu dữ liệu được thu thập ở train.csv\n- Có 142 Features có ở trong train\n- Có 20 mẫu dữ liệu ở test và có 119 features ở test.csv","metadata":{}},{"cell_type":"markdown","source":"## 2.3 Fill-Nan Strategy","metadata":{}},{"cell_type":"markdown","source":"### Ở train.csv, với mỗi feature, có bao nhiêu ô trống và bao nhiêu ô có giá trị","metadata":{}},{"cell_type":"code","source":"thong_ke(train)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:28.019115Z","iopub.execute_input":"2024-12-19T17:25:28.019741Z","iopub.status.idle":"2024-12-19T17:25:28.042121Z","shell.execute_reply.started":"2024-12-19T17:25:28.019703Z","shell.execute_reply":"2024-12-19T17:25:28.041274Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fill knn\nimputer = KNNImputer(n_neighbors=5)\n\ndef fill_df(df):\n    \n    # 1. Các ô dạng digital_type\n    #--------------------------------------------------------------------------------------\n    num_cols = df.select_dtypes(include=np.number).columns\n    \n    # Chiến lược xử lí giá trị nan\n    # a. fill knn\n    data_array = df[num_cols].values\n    imputed_array = imputer.fit_transform(data_array)\n    df[num_cols] = imputed_array\n\n    # b. fill median\n    # df[num_cols] = df[num_cols].fillna(df[num_cols].median())  # chọn bằng median\n\n    # Process target feature\n    if 'sii' in num_cols:\n        df['sii'] = df['sii'].round().astype(int)\n    \n    \n    # 2. Các ô dạng string\n    #---------------------------------------------------------------------------------------\n    obj_cols = df.select_dtypes(include='object').columns\n    # Chiến lược xử lí giá trị nan \n    for col in obj_cols:\n        df[col] = df[col].fillna('Missing')  # Fill bằng Missing :), đã thử fill bằng frequently\n\n    #--------------------------------------------------------------------------------------\n    # Tránh rò rỉ dữ liệu\n    pd.set_option('future.no_silent_downcasting', True)\n        \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:28.043250Z","iopub.execute_input":"2024-12-19T17:25:28.043841Z","iopub.status.idle":"2024-12-19T17:25:28.050416Z","shell.execute_reply.started":"2024-12-19T17:25:28.043803Z","shell.execute_reply":"2024-12-19T17:25:28.049611Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Drop target feature if you want","metadata":{}},{"cell_type":"code","source":"# Drop\n# train = train.dropna(subset=['sii'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:28.051582Z","iopub.execute_input":"2024-12-19T17:25:28.052120Z","iopub.status.idle":"2024-12-19T17:25:28.060077Z","shell.execute_reply.started":"2024-12-19T17:25:28.052082Z","shell.execute_reply":"2024-12-19T17:25:28.059248Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Apply","metadata":{}},{"cell_type":"markdown","source":"### Dữ liệu thiếu nhiều quá: cụ thể là có ít hơn 10 feature trên tổng số 142 thì delete","metadata":{}},{"cell_type":"code","source":"train = train.dropna(thresh=10, axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:28.065731Z","iopub.execute_input":"2024-12-19T17:25:28.066249Z","iopub.status.idle":"2024-12-19T17:25:28.075649Z","shell.execute_reply.started":"2024-12-19T17:25:28.066214Z","shell.execute_reply":"2024-12-19T17:25:28.074757Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = fill_df(train)\ntest = fill_df(test)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:28.076783Z","iopub.execute_input":"2024-12-19T17:25:28.077090Z","iopub.status.idle":"2024-12-19T17:25:37.217556Z","shell.execute_reply.started":"2024-12-19T17:25:28.077054Z","shell.execute_reply":"2024-12-19T17:25:37.216793Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:37.218656Z","iopub.execute_input":"2024-12-19T17:25:37.219017Z","iopub.status.idle":"2024-12-19T17:25:37.243903Z","shell.execute_reply.started":"2024-12-19T17:25:37.218982Z","shell.execute_reply":"2024-12-19T17:25:37.242795Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:37.244939Z","iopub.execute_input":"2024-12-19T17:25:37.245214Z","iopub.status.idle":"2024-12-19T17:25:37.268713Z","shell.execute_reply.started":"2024-12-19T17:25:37.245188Z","shell.execute_reply":"2024-12-19T17:25:37.267783Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"thong_ke(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:37.269812Z","iopub.execute_input":"2024-12-19T17:25:37.270119Z","iopub.status.idle":"2024-12-19T17:25:37.303448Z","shell.execute_reply.started":"2024-12-19T17:25:37.270093Z","shell.execute_reply":"2024-12-19T17:25:37.302520Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.4 Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"### Chú ý phép chia cho 0 là vô nghĩa","metadata":{}},{"cell_type":"code","source":"(train['Physical-Weight'] == 0).sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:37.304955Z","iopub.execute_input":"2024-12-19T17:25:37.305475Z","iopub.status.idle":"2024-12-19T17:25:37.311730Z","shell.execute_reply.started":"2024-12-19T17:25:37.305431Z","shell.execute_reply":"2024-12-19T17:25:37.310741Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def add_more_feature(df):\n    # Tính toán các đặc trưng mới\n    features = {\n        'BMI_Age': df['Physical-BMI'] * df['Basic_Demos-Age'],\n        'Internet_Hours_Age': df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age'],\n        'BMI_Internet_Hours': df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday'],\n        'BFP_BMI': np.where(df['BIA-BIA_BMI'] != 0, df['BIA-BIA_Fat'] / df['BIA-BIA_BMI'], df['BIA-BIA_BMI']),\n        'FFMI_BFP': np.where(df['BIA-BIA_Fat'] != 0, df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat'], df['BIA-BIA_Fat']),\n        'FMI_BFP': np.where(df['BIA-BIA_Fat'] != 0, df['BIA-BIA_FMI'] / df['BIA-BIA_Fat'], df['BIA-BIA_Fat']),\n        'LST_TBW': np.where(df['BIA-BIA_TBW'] != 0, df['BIA-BIA_LST'] / df['BIA-BIA_TBW'], df['BIA-BIA_TBW']),\n        'BFP_BMR': df['BIA-BIA_Fat'] * df['BIA-BIA_BMR'],\n        'BFP_DEE': df['BIA-BIA_Fat'] * df['BIA-BIA_DEE'],\n        'BMR_Weight': np.where(df['Physical-Weight'] != 0, df['BIA-BIA_BMR'] / df['Physical-Weight'], df['Physical-Weight']),\n        'DEE_Weight': np.where(df['Physical-Weight'] != 0, df['BIA-BIA_DEE'] / df['Physical-Weight'], df['Physical-Weight']),\n        'SMM_Height': np.where(df['Physical-Height'] != 0, df['BIA-BIA_SMM'] / df['Physical-Height'], df['Physical-Height']),\n        'Muscle_to_Fat': np.where(df['BIA-BIA_FMI'] != 0, df['BIA-BIA_SMM'] / df['BIA-BIA_FMI'], df['BIA-BIA_FMI']),\n        'Hydration_Status': np.where(df['Physical-Weight'] != 0, df['BIA-BIA_TBW'] / df['Physical-Weight'], df['Physical-Weight']),\n        'ICW_TBW': np.where(df['BIA-BIA_TBW'] != 0, df['BIA-BIA_ICW'] / df['BIA-BIA_TBW'], df['BIA-BIA_TBW']),\n        'BMI_PHR': df['Physical-BMI'] * df['Physical-HeartRate']\n    }\n    \n    # Tạo DataFrame mới với các đặc trưng mới\n    features_df = pd.DataFrame(features)\n    \n    # Kết hợp DataFrame ban đầu với DataFrame mới\n    df = pd.concat([df, features_df], axis=1)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.313158Z","iopub.execute_input":"2024-12-19T17:25:37.313889Z","iopub.status.idle":"2024-12-19T17:25:37.323888Z","shell.execute_reply.started":"2024-12-19T17:25:37.313828Z","shell.execute_reply":"2024-12-19T17:25:37.323180Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = add_more_feature(train)\ntest = add_more_feature(test)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.325067Z","iopub.execute_input":"2024-12-19T17:25:37.325401Z","iopub.status.idle":"2024-12-19T17:25:37.350529Z","shell.execute_reply.started":"2024-12-19T17:25:37.325364Z","shell.execute_reply":"2024-12-19T17:25:37.349855Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"thong_ke(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:37.351618Z","iopub.execute_input":"2024-12-19T17:25:37.351997Z","iopub.status.idle":"2024-12-19T17:25:37.370785Z","shell.execute_reply.started":"2024-12-19T17:25:37.351959Z","shell.execute_reply":"2024-12-19T17:25:37.369922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"thong_ke(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:37.371891Z","iopub.execute_input":"2024-12-19T17:25:37.372172Z","iopub.status.idle":"2024-12-19T17:25:37.381435Z","shell.execute_reply.started":"2024-12-19T17:25:37.372146Z","shell.execute_reply":"2024-12-19T17:25:37.380393Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.5 Feature selection","metadata":{}},{"cell_type":"markdown","source":"### Drop features low coefficient","metadata":{}},{"cell_type":"code","source":"useless_features = [\n]","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.382577Z","iopub.execute_input":"2024-12-19T17:25:37.382938Z","iopub.status.idle":"2024-12-19T17:25:37.389551Z","shell.execute_reply.started":"2024-12-19T17:25:37.382905Z","shell.execute_reply":"2024-12-19T17:25:37.388681Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = train.drop(columns=useless_features)\nX_test = test.drop(columns=useless_features)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.390623Z","iopub.execute_input":"2024-12-19T17:25:37.390926Z","iopub.status.idle":"2024-12-19T17:25:37.400703Z","shell.execute_reply.started":"2024-12-19T17:25:37.390889Z","shell.execute_reply":"2024-12-19T17:25:37.399840Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Drop features not in test","metadata":{}},{"cell_type":"code","source":"drop_columns = [\n    \"PCIAT-Season\",\n    \"PCIAT-PCIAT_01\",\n    \"PCIAT-PCIAT_02\",\n    \"PCIAT-PCIAT_03\",\n    \"PCIAT-PCIAT_04\",\n    \"PCIAT-PCIAT_05\",\n    \"PCIAT-PCIAT_06\",\n    \"PCIAT-PCIAT_07\",\n    \"PCIAT-PCIAT_08\",\n    \"PCIAT-PCIAT_09\",\n    \"PCIAT-PCIAT_10\",\n    \"PCIAT-PCIAT_11\",\n    \"PCIAT-PCIAT_12\",\n    \"PCIAT-PCIAT_13\",\n    \"PCIAT-PCIAT_14\",\n    \"PCIAT-PCIAT_15\",\n    \"PCIAT-PCIAT_16\",    \n    \"PCIAT-PCIAT_17\",\n    \"PCIAT-PCIAT_18\",\n    \"PCIAT-PCIAT_19\",\n    \"PCIAT-PCIAT_20\",\n    \"PCIAT-PCIAT_Total\",\n    \"sii\",\n    \"id\",\n]\n\ntarget_column = \"sii\"\nid_column = \"id\"","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.401780Z","iopub.execute_input":"2024-12-19T17:25:37.402134Z","iopub.status.idle":"2024-12-19T17:25:37.408183Z","shell.execute_reply.started":"2024-12-19T17:25:37.402102Z","shell.execute_reply":"2024-12-19T17:25:37.407368Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train = X_train[target_column]\nX_train = X_train.drop(columns=drop_columns)\nid_test = X_test[id_column]\nX_test = X_test.drop(columns = id_column)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:37.409097Z","iopub.execute_input":"2024-12-19T17:25:37.409358Z","iopub.status.idle":"2024-12-19T17:25:37.420947Z","shell.execute_reply.started":"2024-12-19T17:25:37.409333Z","shell.execute_reply":"2024-12-19T17:25:37.420058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:37.422074Z","iopub.execute_input":"2024-12-19T17:25:37.422363Z","iopub.status.idle":"2024-12-19T17:25:37.445274Z","shell.execute_reply.started":"2024-12-19T17:25:37.422333Z","shell.execute_reply":"2024-12-19T17:25:37.444397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:37.446352Z","iopub.execute_input":"2024-12-19T17:25:37.446618Z","iopub.status.idle":"2024-12-19T17:25:37.469114Z","shell.execute_reply.started":"2024-12-19T17:25:37.446592Z","shell.execute_reply":"2024-12-19T17:25:37.468237Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.6 Encode obj_columns","metadata":{}},{"cell_type":"code","source":"thong_ke(X_train)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.470209Z","iopub.execute_input":"2024-12-19T17:25:37.470488Z","iopub.status.idle":"2024-12-19T17:25:37.492059Z","shell.execute_reply.started":"2024-12-19T17:25:37.470462Z","shell.execute_reply":"2024-12-19T17:25:37.491193Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"thong_ke(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.493180Z","iopub.execute_input":"2024-12-19T17:25:37.493459Z","iopub.status.idle":"2024-12-19T17:25:37.502510Z","shell.execute_reply.started":"2024-12-19T17:25:37.493433Z","shell.execute_reply":"2024-12-19T17:25:37.501618Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"obj_cols = X_train.select_dtypes(include=['object']).columns.tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:37.503657Z","iopub.execute_input":"2024-12-19T17:25:37.504025Z","iopub.status.idle":"2024-12-19T17:25:37.512632Z","shell.execute_reply.started":"2024-12-19T17:25:37.503988Z","shell.execute_reply":"2024-12-19T17:25:37.511857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"obj_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:37.513693Z","iopub.execute_input":"2024-12-19T17:25:37.514164Z","iopub.status.idle":"2024-12-19T17:25:37.523937Z","shell.execute_reply.started":"2024-12-19T17:25:37.514123Z","shell.execute_reply":"2024-12-19T17:25:37.523069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\ncolumns_to_check = [\n    'Basic_Demos-Enroll_Season',\n    'CGAS-Season',\n    'Physical-Season',\n    'Fitness_Endurance-Season',\n    'FGC-Season',\n    'BIA-Season',\n    'PAQ_A-Season',\n    'PAQ_C-Season',\n    'SDS-Season',\n    'PreInt_EduHx-Season'\n]\n'''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:37.525093Z","iopub.execute_input":"2024-12-19T17:25:37.525458Z","iopub.status.idle":"2024-12-19T17:25:37.535207Z","shell.execute_reply.started":"2024-12-19T17:25:37.525419Z","shell.execute_reply":"2024-12-19T17:25:37.534290Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nfor col in columns_to_check:\n    unique_values = X_train[col].unique()\n    print(f\"Giá trị duy nhất trong cột '{col}': {unique_values}\")\n'''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:37.536446Z","iopub.execute_input":"2024-12-19T17:25:37.536748Z","iopub.status.idle":"2024-12-19T17:25:37.544271Z","shell.execute_reply.started":"2024-12-19T17:25:37.536721Z","shell.execute_reply":"2024-12-19T17:25:37.543560Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Nhận xét: Các cột mùa chỉ có 4 giá trị: Spring, Winter, Fall, Summer. Các giá trị này đã được fill bằng missing\n### Có thể chọn drop luôn\n### Dễ thấy label encoder là phương pháp tối ưu trong trường hợp này","metadata":{}},{"cell_type":"code","source":"# Bước 1: Tự động xác định các cột chuỗi\nobj_cols = X_train.select_dtypes(include=['object']).columns.tolist()\n\n# Hàm tạo ánh xạ\ndef create_mapping(column, df):\n    unique_values = df[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\n# Bước 2: Tạo ánh xạ từ DataFrame train cho từng cột\nmappings = {col: create_mapping(col, X_train) for col in obj_cols}\n\n# Bước 3: Ánh xạ cho cả hai DataFrame\ndef map_column(df, column, mapping):\n    df[column] = df[column].replace(mapping)  # Thay thế theo ánh xạ\n    df[column] = df[column].astype(int)  # Chuyển đổi thành kiểu int\n\n# Ánh xạ cho DataFrame X_train và X_test\nfor col in obj_cols:\n    map_column(X_train, col, mappings[col])\n    if col in test.columns:\n        map_column(X_test, col, mappings[col])\n","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.545678Z","iopub.execute_input":"2024-12-19T17:25:37.546041Z","iopub.status.idle":"2024-12-19T17:25:37.579327Z","shell.execute_reply.started":"2024-12-19T17:25:37.546003Z","shell.execute_reply":"2024-12-19T17:25:37.578561Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.580361Z","iopub.execute_input":"2024-12-19T17:25:37.580630Z","iopub.status.idle":"2024-12-19T17:25:37.601245Z","shell.execute_reply.started":"2024-12-19T17:25:37.580603Z","shell.execute_reply":"2024-12-19T17:25:37.600284Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.602393Z","iopub.execute_input":"2024-12-19T17:25:37.602683Z","iopub.status.idle":"2024-12-19T17:25:37.611252Z","shell.execute_reply.started":"2024-12-19T17:25:37.602656Z","shell.execute_reply":"2024-12-19T17:25:37.610500Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.612394Z","iopub.execute_input":"2024-12-19T17:25:37.612760Z","iopub.status.idle":"2024-12-19T17:25:37.635944Z","shell.execute_reply.started":"2024-12-19T17:25:37.612721Z","shell.execute_reply":"2024-12-19T17:25:37.634866Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test.shape","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.637024Z","iopub.execute_input":"2024-12-19T17:25:37.637345Z","iopub.status.idle":"2024-12-19T17:25:37.645572Z","shell.execute_reply.started":"2024-12-19T17:25:37.637318Z","shell.execute_reply":"2024-12-19T17:25:37.644686Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.646748Z","iopub.execute_input":"2024-12-19T17:25:37.647117Z","iopub.status.idle":"2024-12-19T17:25:37.656934Z","shell.execute_reply.started":"2024-12-19T17:25:37.647079Z","shell.execute_reply":"2024-12-19T17:25:37.656021Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.7 Handle imbalaced data","metadata":{}},{"cell_type":"code","source":"smote = SMOTE()","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.657829Z","iopub.execute_input":"2024-12-19T17:25:37.658107Z","iopub.status.idle":"2024-12-19T17:25:37.667519Z","shell.execute_reply.started":"2024-12-19T17:25:37.658082Z","shell.execute_reply":"2024-12-19T17:25:37.666722Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# X_train, y_train = smote.fit_resample(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.668671Z","iopub.execute_input":"2024-12-19T17:25:37.668966Z","iopub.status.idle":"2024-12-19T17:25:37.676627Z","shell.execute_reply.started":"2024-12-19T17:25:37.668940Z","shell.execute_reply":"2024-12-19T17:25:37.675905Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"thong_ke(X_train)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.677694Z","iopub.execute_input":"2024-12-19T17:25:37.677990Z","iopub.status.idle":"2024-12-19T17:25:37.695812Z","shell.execute_reply.started":"2024-12-19T17:25:37.677963Z","shell.execute_reply":"2024-12-19T17:25:37.695012Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"thong_ke(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.696993Z","iopub.execute_input":"2024-12-19T17:25:37.697315Z","iopub.status.idle":"2024-12-19T17:25:37.707018Z","shell.execute_reply.started":"2024-12-19T17:25:37.697267Z","shell.execute_reply":"2024-12-19T17:25:37.706042Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.8. PCA","metadata":{}},{"cell_type":"code","source":"# # Bước 1: Chuẩn hóa dữ liệu\n# scaler = StandardScaler()\n# X_train = scaler.fit_transform(X_train)\n# X_test = scaler.transform(X_test)\n\n# # Bước 2: Áp dụng PCA với n_components = 0.95\n# pca = PCA(n_components=0.95)  # Giữ lại 95% phương sai\n# X_train = pca.fit_transform(X_train)\n# X_train = pd.DataFrame(\n#     X_train, columns = [f'PC{i+1}' for i in range (X_train.shape[1])]\n# )\n\n# X_test = pca.transform(X_test)\n# X_test = pd.DataFrame(\n#     X_test, columns = [f'PC{i+1}' for i in range (X_train.shape[1])]\n# )\n","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.708149Z","iopub.execute_input":"2024-12-19T17:25:37.708452Z","iopub.status.idle":"2024-12-19T17:25:37.718263Z","shell.execute_reply.started":"2024-12-19T17:25:37.708425Z","shell.execute_reply":"2024-12-19T17:25:37.717409Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.719345Z","iopub.execute_input":"2024-12-19T17:25:37.719607Z","iopub.status.idle":"2024-12-19T17:25:37.729210Z","shell.execute_reply.started":"2024-12-19T17:25:37.719582Z","shell.execute_reply":"2024-12-19T17:25:37.728455Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test.shape","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:25:37.730201Z","iopub.execute_input":"2024-12-19T17:25:37.730498Z","iopub.status.idle":"2024-12-19T17:25:37.743589Z","shell.execute_reply.started":"2024-12-19T17:25:37.730473Z","shell.execute_reply":"2024-12-19T17:25:37.742927Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Model training & Evaluation","metadata":{}},{"cell_type":"markdown","source":"## 3.1 Evaluation method","metadata":{}},{"cell_type":"code","source":"!pip -q install /kaggle/input/pytorchtabnet/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:25:37.744523Z","iopub.execute_input":"2024-12-19T17:25:37.744784Z","iopub.status.idle":"2024-12-19T17:26:18.903789Z","shell.execute_reply.started":"2024-12-19T17:25:37.744760Z","shell.execute_reply":"2024-12-19T17:26:18.902693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:26:18.905378Z","iopub.execute_input":"2024-12-19T17:26:18.905744Z","iopub.status.idle":"2024-12-19T17:26:18.912120Z","shell.execute_reply.started":"2024-12-19T17:26:18.905714Z","shell.execute_reply":"2024-12-19T17:26:18.911149Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.2 Training","metadata":{}},{"cell_type":"code","source":"from pytorch_tabnet.tab_model import TabNetRegressor\nimport torch\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom pytorch_tabnet.callbacks import Callback\nimport os\nimport torch\nfrom pytorch_tabnet.callbacks import Callback\n\nclass TabNetWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, **kwargs):\n        self.model = TabNetRegressor(**kwargs)\n        self.kwargs = kwargs\n        self.imputer = SimpleImputer(strategy='median')\n        self.best_model_path = 'best_tabnet_model.pt'\n        \n    def fit(self, X, y):\n        # Handle missing values\n        X_imputed = self.imputer.fit_transform(X)\n        \n        if hasattr(y, 'values'):\n            y = y.values\n            \n        # Create internal validation set\n        X_train_1, X_valid, y_train_1, y_valid = train_test_split(\n            X_imputed, \n            y, \n            test_size=0.2,\n            random_state=42\n        )\n        \n        # Train TabNet model\n        history = self.model.fit(\n            X_train=X_train_1,\n            y_train=y_train_1.reshape(-1, 1),\n            eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n            eval_name=['valid'],\n            eval_metric=['mse'],\n            max_epochs=200,\n            patience=20,\n            batch_size=1024,\n            virtual_batch_size=128,\n            num_workers=0,\n            drop_last=False,\n            callbacks=[\n                TabNetPretrainedModelCheckpoint(\n                    filepath=self.best_model_path,\n                    monitor='valid_mse',\n                    mode='min',\n                    save_best_only=True,\n                    verbose=True\n                )\n            ]\n        )\n        # Load the best model\n        if os.path.exists(self.best_model_path):\n            self.model.load_model(self.best_model_path)\n            os.remove(self.best_model_path)  # Remove temporary file\n        \n        return self\n    \n    def predict(self, X):\n        X_imputed = self.imputer.transform(X)\n        return self.model.predict(X_imputed).flatten()\n    \n    def __deepcopy__(self, memo):\n        # Add deepcopy support for scikit-learn\n        cls = self.__class__\n        result = cls.__new__(cls)\n        memo[id(self)] = result\n        for k, v in self.__dict__.items():\n            setattr(result, k, deepcopy(v, memo))\n        return result\n        \nclass TabNetPretrainedModelCheckpoint(Callback):\n    def __init__(self, filepath, monitor='val_loss', mode='min', \n                 save_best_only=True, verbose=1):\n        super().__init__()  # Initialize parent class\n        self.filepath = filepath\n        self.monitor = monitor\n        self.mode = mode\n        self.save_best_only = save_best_only\n        self.verbose = verbose\n        self.best = float('inf') if mode == 'min' else -float('inf')\n        \n    def on_train_begin(self, logs=None):\n        self.model = self.trainer  # Use trainer itself as model\n        \n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is None:\n            return\n        \n        # Check if current metric is better than best\n        if (self.mode == 'min' and current < self.best) or \\\n           (self.mode == 'max' and current > self.best):\n            if self.verbose:\n                print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n            self.best = current\n            if self.save_best_only:\n                self.model.save_model(self.filepath)  # Save the entire model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:26:18.913622Z","iopub.execute_input":"2024-12-19T17:26:18.913905Z","iopub.status.idle":"2024-12-19T17:26:18.933198Z","shell.execute_reply.started":"2024-12-19T17:26:18.913868Z","shell.execute_reply":"2024-12-19T17:26:18.932342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def TrainML(model_class,X,y,test_data, n_splits = 5):\n    \n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train_1, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train_1, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train_1, y_train_1)\n\n        y_train_pred = model.predict(X_train_1)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n        train_kappa = quadratic_weighted_kappa(y_train_1, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        # clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    submission = pd.DataFrame({\n        'id': id_test,\n        'sii': tpTuned\n    })\n\n    return submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:26:18.934388Z","iopub.execute_input":"2024-12-19T17:26:18.934995Z","iopub.status.idle":"2024-12-19T17:26:18.953608Z","shell.execute_reply.started":"2024-12-19T17:26:18.934969Z","shell.execute_reply":"2024-12-19T17:26:18.952899Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.3 Model Finetuning","metadata":{}},{"cell_type":"code","source":"LGBM_Params = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'random_state' : 42,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01,  # Increased from 2.68e-06\n    'verbose' : -1,\n    'device': 'cpu'\n\n}\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': 42,\n    'tree_method': 'hist',  # Use 'hist' instead of 'gpu_hist'\n    'device': 'cuda',       # Specify CUDA for GPU training\n\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': 42,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU'\n\n}\n\n# TabNet hyperparameters\nTabNet_Params = {\n    'n_d': 64,              # Width of the decision prediction layer\n    'n_a': 64,              # Width of the attention embedding for each step\n    'n_steps': 5,           # Number of steps in the architecture\n    'gamma': 1.5,           # Coefficient for feature selection regularization\n    'n_independent': 2,     # Number of independent GLU layer in each GLU block\n    'n_shared': 2,          # Number of shared GLU layer in each GLU block\n    'lambda_sparse': 1e-4,  # Sparsity regularization\n    'optimizer_fn': torch.optim.Adam,\n    'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n    'mask_type': 'entmax',\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'verbose': 1,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:26:18.954532Z","iopub.execute_input":"2024-12-19T17:26:18.954771Z","iopub.status.idle":"2024-12-19T17:26:18.970735Z","shell.execute_reply.started":"2024-12-19T17:26:18.954749Z","shell.execute_reply":"2024-12-19T17:26:18.970045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Light = LGBMRegressor(**LGBM_Params, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\nTabNet_Model = TabNetWrapper(**TabNet_Params)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:26:18.971784Z","iopub.execute_input":"2024-12-19T17:26:18.972077Z","iopub.status.idle":"2024-12-19T17:26:18.986433Z","shell.execute_reply.started":"2024-12-19T17:26:18.972038Z","shell.execute_reply":"2024-12-19T17:26:18.985541Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.4 Ensamble training","metadata":{}},{"cell_type":"code","source":"voting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model),\n    ('tabnet', TabNet_Model)\n],weights=[5.0,5.0,5.0,2.0])\n\nsubmission = TrainML(voting_model,X_train,y_train,X_test) # default n_spilit = 5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T17:26:18.987556Z","iopub.execute_input":"2024-12-19T17:26:18.988098Z","iopub.status.idle":"2024-12-19T17:27:33.124221Z","shell.execute_reply.started":"2024-12-19T17:26:18.988061Z","shell.execute_reply":"2024-12-19T17:27:33.123343Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.5 Submission","metadata":{}},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:27:33.125622Z","iopub.execute_input":"2024-12-19T17:27:33.125956Z","iopub.status.idle":"2024-12-19T17:27:33.135652Z","shell.execute_reply.started":"2024-12-19T17:27:33.125926Z","shell.execute_reply":"2024-12-19T17:27:33.134653Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Xuất ra file CSV\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-12-19T17:27:33.136417Z","iopub.execute_input":"2024-12-19T17:27:33.136690Z","iopub.status.idle":"2024-12-19T17:27:33.145787Z","shell.execute_reply.started":"2024-12-19T17:27:33.136662Z","shell.execute_reply":"2024-12-19T17:27:33.144991Z"},"trusted":true},"outputs":[],"execution_count":null}]}