{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":7453542,"sourceType":"datasetVersion","datasetId":921302},{"sourceId":10161923,"sourceType":"datasetVersion","datasetId":6275040},{"sourceId":10162319,"sourceType":"datasetVersion","datasetId":6275333}],"dockerImageVersionId":30776,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Phát triển từ code gốc:\n- Giữ nguyên tiền xử lý, logic, feature selection, cấu trúc model\n- Thêm mô hình TabTransformer\n- Ensemble mô hình TabTransformer với LightGBM, XGBoost, CatBoost (và các model khác nếu có)\n- Đưa ra kết quả cuối cùng\n\nDựa trên notebook gốc: \nhttps://www.kaggle.com/code/honganzhu/cmi-piu-competition?scriptVersionId=201912528 Version44 LB0.492\n\nThực hiện:\n- Giữ nguyên luồng xử lý dữ liệu, feature engineering\n- Thêm model TabTransformer\n- Ensemble các model để có kết quả tốt nhất","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --no-deps /kaggle/input/pytorch/einops-0.8.0-py3-none-any.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T19:21:26.277579Z","iopub.execute_input":"2024-12-10T19:21:26.278307Z","iopub.status.idle":"2024-12-10T19:21:28.67942Z","shell.execute_reply.started":"2024-12-10T19:21:26.278279Z","shell.execute_reply":"2024-12-10T19:21:28.678363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install --no-index --no-deps /kaggle/input/pytorchtransformer/tab_transformer_pytorch-0.3.0-py3-none-any.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T19:21:28.681594Z","iopub.execute_input":"2024-12-10T19:21:28.681992Z","iopub.status.idle":"2024-12-10T19:21:30.643838Z","shell.execute_reply.started":"2024-12-10T19:21:28.681951Z","shell.execute_reply":"2024-12-10T19:21:30.642695Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip -q install /kaggle/input/pytorchtabnet/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T19:21:30.645251Z","iopub.execute_input":"2024-12-10T19:21:30.645528Z","iopub.status.idle":"2024-12-10T19:22:11.519592Z","shell.execute_reply.started":"2024-12-10T19:21:30.645502Z","shell.execute_reply":"2024-12-10T19:22:11.518405Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pytorch_tabnet.tab_model import TabNetRegressor\nimport torch\n\nimport numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nimport polars as pl\nimport polars.selectors as cs\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\n\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None","metadata":{"_uuid":"24133267-6049-4da2-8165-9e8ce674c800","_cell_guid":"276c533d-9a71-4174-af89-694e68e6776a","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-10T19:22:11.52195Z","iopub.execute_input":"2024-12-10T19:22:11.522236Z","iopub.status.idle":"2024-12-10T19:22:31.8442Z","shell.execute_reply.started":"2024-12-10T19:22:11.522209Z","shell.execute_reply":"2024-12-10T19:22:31.843267Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_labels = ['None', 'Mild', 'Moderate', 'Severe']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T19:22:31.845276Z","iopub.execute_input":"2024-12-10T19:22:31.845869Z","iopub.status.idle":"2024-12-10T19:22:31.850009Z","shell.execute_reply.started":"2024-12-10T19:22:31.845841Z","shell.execute_reply":"2024-12-10T19:22:31.849018Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"season_dtype = pl.Enum(['Spring', 'Summer', 'Fall', 'Winter'])\n\ntrain = (\n    pl.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n    .with_columns(pl.col('^.*Season$').cast(season_dtype))\n)\n\ntest = (\n    pl.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n    .with_columns(pl.col('^.*Season$').cast(season_dtype))\n)\n\ntrain\ntest","metadata":{"_uuid":"ca6da450-2a5f-431c-9c2b-e161f9b61841","_cell_guid":"fba1fd88-ca4b-41b5-bd44-91140f411df3","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-10T19:22:31.851233Z","iopub.execute_input":"2024-12-10T19:22:31.851577Z","iopub.status.idle":"2024-12-10T19:22:32.070254Z","shell.execute_reply.started":"2024-12-10T19:22:31.85154Z","shell.execute_reply":"2024-12-10T19:22:32.069391Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"For a supervised learning, we need the target value, but some (sii) are missing. So we only use the part with valid target value(sii).","metadata":{"_uuid":"f551da78-276b-4da8-b751-56f30af7a221","_cell_guid":"b0469392-4bd7-457e-9a03-2b0e3586f9ab","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"supervised_usable = (\n    train\n    .filter(pl.col('sii').is_not_null())\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\nplt.figure(figsize=(6, 15))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)),\n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"_uuid":"04304dc4-6e5e-4b87-a32a-2e64e17dd661","_cell_guid":"7f738cd2-5399-447e-afac-69d5648529d1","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-10T19:22:32.071352Z","iopub.execute_input":"2024-12-10T19:22:32.071624Z","iopub.status.idle":"2024-12-10T19:22:33.115603Z","shell.execute_reply.started":"2024-12-10T19:22:32.071598Z","shell.execute_reply":"2024-12-10T19:22:33.11463Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train.select(pl.col('PCIAT-PCIAT_Total').is_null() == pl.col('sii').is_null()).to_series().mean())\n\n(train\n .select(pl.col('PCIAT-PCIAT_Total'))\n .group_by(train.get_column('sii'))\n .agg(pl.col('PCIAT-PCIAT_Total').min().alias('PCIAT-PCIAT_Total min'),\n      pl.col('PCIAT-PCIAT_Total').max().alias('PCIAT-PCIAT_Total max'),\n      pl.col('PCIAT-PCIAT_Total').len().alias('count'))\n .sort('sii')\n)","metadata":{"_uuid":"d7c37ef5-1569-4005-ae86-6b985ea30e02","_cell_guid":"50f8ca6e-5fad-42ea-99e9-13d2e0ced103","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-10T19:22:33.116619Z","iopub.execute_input":"2024-12-10T19:22:33.116947Z","iopub.status.idle":"2024-12-10T19:22:33.151955Z","shell.execute_reply.started":"2024-12-10T19:22:33.116915Z","shell.execute_reply":"2024-12-10T19:22:33.151155Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Insight:**\n\nThis dataset is imbalanced. Half of the samples are in class 0, while very few in class 3.","metadata":{"_uuid":"d2467f5e-3143-40fa-84e4-8198ae7a652c","_cell_guid":"a47e2854-16f7-46ff-9eae-55f9bda80c90","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"print('Columns missing in test:')\nprint([f for f in train.columns if f not in test.columns])","metadata":{"_uuid":"febd3bb2-8d18-43f9-9ce9-e4f3a626c5e2","_cell_guid":"b219c634-d074-4a1e-a7e0-0a920a9817b0","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-10T19:22:33.152961Z","iopub.execute_input":"2024-12-10T19:22:33.1532Z","iopub.status.idle":"2024-12-10T19:22:33.157684Z","shell.execute_reply.started":"2024-12-10T19:22:33.153177Z","shell.execute_reply":"2024-12-10T19:22:33.156663Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Demographics","metadata":{"_uuid":"555df222-9a93-4898-84b6-c638b2ec4514","_cell_guid":"c4be6ffc-eb6e-44d8-a75b-873975810241","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"Now we look at some basic demographics.","metadata":{"_uuid":"27670bba-4613-4c1c-a0f8-5e1aed8a4ddc","_cell_guid":"cb87962b-30a4-42d1-9daa-8be5f8cd83e2","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"vc = train.get_column('Basic_Demos-Enroll_Season').value_counts()\nplt.pie(vc.get_column('count'), labels=vc.get_column('Basic_Demos-Enroll_Season'))\nplt.title('Season of enrollment')\nplt.show()","metadata":{"_uuid":"13510604-a738-4c18-810c-0c907b143db2","_cell_guid":"e3e48d12-5894-4071-bc59-9a2fe10d7338","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-10T19:22:33.160287Z","iopub.execute_input":"2024-12-10T19:22:33.160543Z","iopub.status.idle":"2024-12-10T19:22:33.278067Z","shell.execute_reply.started":"2024-12-10T19:22:33.16052Z","shell.execute_reply":"2024-12-10T19:22:33.276915Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vc = train.get_column('Basic_Demos-Sex').value_counts()\nplt.pie(vc.get_column('count'), labels=['boys', 'girls'])\nplt.title('Sex of participant')\nplt.show()","metadata":{"_uuid":"0326dc27-1803-44dd-9708-16cbd75c9320","_cell_guid":"9ef81109-b9b2-4935-b562-4f71aa3d4951","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-10T19:22:33.279639Z","iopub.execute_input":"2024-12-10T19:22:33.280714Z","iopub.status.idle":"2024-12-10T19:22:33.420036Z","shell.execute_reply.started":"2024-12-10T19:22:33.280658Z","shell.execute_reply":"2024-12-10T19:22:33.418915Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, axs = plt.subplots(2, 1, sharex=True)\nfor sex in range(2):\n    ax = axs.ravel()[sex]\n    vc = train.filter(pl.col('Basic_Demos-Sex') == sex).get_column('Basic_Demos-Age').value_counts()\n    ax.bar(vc.get_column('Basic_Demos-Age'),\n           vc.get_column('count'),\n           color=['lightblue', 'coral'][sex],\n           label=['boys', 'girls'][sex])\n    ax.xaxis.set_major_locator(MaxNLocator(integer=True))\n    ax.set_ylabel('count')\n    ax.legend()\nplt.suptitle('Age distribution')\naxs.ravel()[1].set_xlabel('years')\nplt.show()","metadata":{"_uuid":"0b071874-6d03-48ff-a210-6d8d6c3967dd","_cell_guid":"98caace0-d38f-4e51-8a49-1240e2b30ec0","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-10T19:22:33.42147Z","iopub.execute_input":"2024-12-10T19:22:33.42191Z","iopub.status.idle":"2024-12-10T19:22:33.876816Z","shell.execute_reply.started":"2024-12-10T19:22:33.421848Z","shell.execute_reply":"2024-12-10T19:22:33.87595Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, axs = plt.subplots(2, 1, sharex=True, sharey=True)\nfor sex in range(2):\n    ax = axs.ravel()[sex]\n    vc = train.filter(pl.col('Basic_Demos-Sex') == sex).get_column('sii').value_counts()\n    ax.bar(vc.get_column('sii'),\n           vc.get_column('count') / vc.get_column('count').sum(),\n           color=['lightblue', 'coral'][sex],\n           label=['boys', 'girls'][sex])\n    ax.set_xticks(np.arange(4), target_labels)\n    ax.yaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\n    ax.set_ylabel('count')\n    ax.legend()\nplt.suptitle('Target distribution')\naxs.ravel()[1].set_xlabel('Severity Impairment Index (sii)')\nplt.show()","metadata":{"_uuid":"e0636684-4745-4c5f-82be-548d025aeba1","_cell_guid":"cd61f7a5-7723-4499-aa31-2a895a0c7720","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-10T19:22:33.877914Z","iopub.execute_input":"2024-12-10T19:22:33.878213Z","iopub.status.idle":"2024-12-10T19:22:34.110327Z","shell.execute_reply.started":"2024-12-10T19:22:33.878186Z","shell.execute_reply":"2024-12-10T19:22:34.109379Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Now we look at correlations","metadata":{"_uuid":"d80b5ea2-9668-45f0-86d7-0a743946bbb2","_cell_guid":"ad01294b-d2b2-4162-855f-4f1105e1ddc7","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"plt.figure(figsize=(14, 12))\ncorr_matrix = supervised_usable.select([\n    'PCIAT-PCIAT_Total', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-BMI', \n    'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n    'Physical-Diastolic_BP', 'Physical-Systolic_BP', 'Physical-HeartRate',\n    'PreInt_EduHx-computerinternet_hoursday', 'SDS-SDS_Total_T', 'PAQ_A-PAQ_A_Total',\n    'PAQ_C-PAQ_C_Total', 'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins','Fitness_Endurance-Time_Sec',\n    'FGC-FGC_CU', 'FGC-FGC_GSND','FGC-FGC_GSD','FGC-FGC_PU','FGC-FGC_SRL','FGC-FGC_SRR','FGC-FGC_TL','BIA-BIA_Activity_Level_num', \n    'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n    'BIA-BIA_FFMI','BIA-BIA_FMI', 'BIA-BIA_Fat','BIA-BIA_Frame_num','BIA-BIA_ICW','BIA-BIA_LDM','BIA-BIA_LST',\n    'BIA-BIA_SMM','BIA-BIA_TBW'\n    # Add other relevant columns\n]).to_pandas().corr()\n\nsii_corr = corr_matrix['PCIAT-PCIAT_Total'].drop('PCIAT-PCIAT_Total')\nfiltered_corr = sii_corr[(sii_corr > 0.1) | (sii_corr < -0.1)]\n\nprint(filtered_corr)\n\nplt.figure(figsize=(8, 6))\nfiltered_corr.sort_values().plot(kind='barh', color='coral')\nplt.title('Features with Correlation > 0.1 or < -0.1 with PCIAT-PCIAT_Total')\nplt.xlabel('Correlation coefficient')\nplt.ylabel('Features')\nplt.show()","metadata":{"_uuid":"056e607f-b6ed-4450-bfdc-fb75933a7c27","_cell_guid":"5b1af16d-7e9a-4780-b1c3-167f7ccc1c20","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-10T19:22:34.111531Z","iopub.execute_input":"2024-12-10T19:22:34.111926Z","iopub.status.idle":"2024-12-10T19:22:34.472633Z","shell.execute_reply.started":"2024-12-10T19:22:34.111859Z","shell.execute_reply":"2024-12-10T19:22:34.471709Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Actigraphy (time series)","metadata":{"_uuid":"909ab45a-ac6f-45bf-8cc6-08fd56b930c2","_cell_guid":"fb66a87c-5760-4776-912f-d8faaf9696bb","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"actigraphy = pl.read_parquet('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id=0417c91e/part-0.parquet')\nactigraphy","metadata":{"_uuid":"090b7eac-d1da-44bc-be5b-7dc9ebb59262","_cell_guid":"f8b1473b-ba69-4e55-a488-c8f9c00a2927","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-10T19:22:34.47373Z","iopub.execute_input":"2024-12-10T19:22:34.474012Z","iopub.status.idle":"2024-12-10T19:22:34.613921Z","shell.execute_reply.started":"2024-12-10T19:22:34.473985Z","shell.execute_reply":"2024-12-10T19:22:34.613058Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_actigraphy(id, only_one_week=False, small=False):\n    actigraphy = pl.read_parquet(f'/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id={id}/part-0.parquet')\n    day = actigraphy.get_column('relative_date_PCIAT') + actigraphy.get_column('time_of_day') / 86400e9\n    sample = train.filter(pl.col('id') == id)\n    age = sample.get_column('Basic_Demos-Age').item()\n    sex = ['boy', 'girl'][sample.get_column('Basic_Demos-Sex').item()]\n    actigraphy = (\n        actigraphy\n        .with_columns(\n            (day.diff() * 86400).alias('diff_seconds'),\n            (np.sqrt(np.square(pl.col('X')) + np.square(pl.col('Y')) + np.square(pl.col('Z'))).alias('norm'))\n        )\n    )\n\n    if only_one_week:\n        start = np.ceil(day.min())\n        mask = (start <= day.to_numpy()) & (day.to_numpy() <= start + 7*3)\n        mask &= ~ actigraphy.get_column('non-wear_flag').cast(bool).to_numpy()\n    else:\n        mask = np.full(len(day), True)\n        \n    if small:\n        timelines = [\n            ('enmo', 'forestgreen'),\n            ('light', 'orange'),\n        ]\n    else:\n        timelines = [\n            ('X', 'm'),\n            ('Y', 'm'),\n            ('Z', 'm'),\n#             ('norm', 'c'),\n            ('enmo', 'forestgreen'),\n            ('anglez', 'lightblue'),\n            ('light', 'orange'),\n            ('non-wear_flag', 'chocolate')\n    #         ('diff_seconds', 'k'),\n        ]\n        \n    _, axs = plt.subplots(len(timelines), 1, sharex=True, figsize=(12, len(timelines) * 1.1 + 0.5))\n    for ax, (feature, color) in zip(axs, timelines):\n        ax.set_facecolor('#eeeeee')\n        ax.scatter(day.to_numpy()[mask],\n                   actigraphy.get_column(feature).to_numpy()[mask],\n                   color=color, label=feature, s=1)\n        ax.legend(loc='upper left', facecolor='#eeeeee')\n        if feature == 'diff_seconds':\n            ax.set_ylim(-0.5, 20.5)\n    axs[-1].set_xlabel('day')\n    axs[-1].xaxis.set_major_locator(MaxNLocator(integer=True))\n    plt.tight_layout()\n    axs[0].set_title(f'id={id}, {sex}, age={age}')\n    plt.show()\n\nanalyze_actigraphy('0417c91e', only_one_week=False)","metadata":{"_uuid":"e3a4d643-5902-47f9-b835-1ba4276fca34","_cell_guid":"a2e3c687-f82c-4505-81ae-547006388dc2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-10T19:22:34.61504Z","iopub.execute_input":"2024-12-10T19:22:34.615336Z","iopub.status.idle":"2024-12-10T19:22:36.949341Z","shell.execute_reply.started":"2024-12-10T19:22:34.615308Z","shell.execute_reply":"2024-12-10T19:22:36.948471Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42\nn_splits = 5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T19:22:36.95034Z","iopub.execute_input":"2024-12-10T19:22:36.950581Z","iopub.status.idle":"2024-12-10T19:22:36.954485Z","shell.execute_reply.started":"2024-12-10T19:22:36.950558Z","shell.execute_reply":"2024-12-10T19:22:36.95356Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering\n\n- **Feature Selection**: The dataset contains features related to physical characteristics (e.g., BMI, Height, Weight), behavioral aspects (e.g., internet usage), and fitness data (e.g., endurance time). \n- **Categorical Feature Encoding**: Categorical features are mapped to numerical values using custom mappings for each unique category within the dataset. This ensures compatibility with machine learning algorithms that require numerical input.\n- **Time Series Aggregation**: Time series statistics (e.g., mean, standard deviation) from the actigraphy data are computed and merged into the main dataset to create additional features for model training.","metadata":{"_uuid":"dce1cfee-99d7-4ea3-beb5-7241ede5248c","_cell_guid":"125c0266-3259-4f8c-8b79-dcf1a359a855","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n\n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n\n    stats, indexes = zip(*results)\n\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n\nclass AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n\n    data_tensor = torch.FloatTensor(df_scaled)\n\n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n\n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n\n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n\n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n\n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n\n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n\n    return df_encoded\n\ndef feature_engineering(df):\n    season_cols = [col for col in df.columns if 'Season' in col]\n    df = df.drop(season_cols, axis=1)\n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n\n    return df","metadata":{"_uuid":"c9f6d170-0fa0-4bf2-9282-74cede06d9cc","_cell_guid":"1450c156-efb7-42ca-a773-4e5b6bf43420","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-10T19:22:36.955649Z","iopub.execute_input":"2024-12-10T19:22:36.956445Z","iopub.status.idle":"2024-12-10T19:22:36.973151Z","shell.execute_reply.started":"2024-12-10T19:22:36.9564Z","shell.execute_reply":"2024-12-10T19:22:36.972301Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=100, batch_size=32)\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=100, batch_size=32)\n\ntime_series_cols = train_ts_encoded.columns.tolist()\ntrain_ts_encoded[\"id\"]=train_ts[\"id\"]\ntest_ts_encoded['id']=test_ts[\"id\"]\n\ntrain = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\ntest = pd.merge(test, test_ts_encoded, how=\"left\", on='id')\n\nimputer = KNNImputer(n_neighbors=5)\nnumeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\nimputed_data = imputer.fit_transform(train[numeric_cols])\ntrain_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\ntrain_imputed['sii'] = train_imputed['sii'].round().astype(int)\nfor col in train.columns:\n    if col not in numeric_cols:\n        train_imputed[col] = train[col]\n\ntrain = train_imputed\n\ntrain = feature_engineering(train)\ntrain = train.dropna(thresh=10, axis=0)\ntest = feature_engineering(test)\n\ntrain = train.drop('id', axis=1)\ntest  = test .drop('id', axis=1)\n\n# Không thay đổi featureCols\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW']\n\nfeaturesCols += time_series_cols\ntest = test[featuresCols]\n\n#add\ncat_c = [c for c in cat_c if c in train.columns]\n\ndef update(df):\n    global cat_c\n    for c in cat_c:\n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n\ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n\n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T19:26:25.289569Z","iopub.execute_input":"2024-12-10T19:26:25.289993Z","iopub.status.idle":"2024-12-10T19:27:53.702414Z","shell.execute_reply.started":"2024-12-10T19:26:25.28996Z","shell.execute_reply":"2024-12-10T19:27:53.701438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if np.any(np.isinf(train)):\n    train = train.replace([np.inf, -np.inf], np.nan)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n\n    train_S = []\n    test_S = []\n\n    oof_non_rounded = np.zeros(len(y), dtype=float)\n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n\n        test_preds[:, fold] = model.predict(test_data)\n\n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded),\n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n\n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n\n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T19:29:34.885831Z","iopub.execute_input":"2024-12-10T19:29:34.886219Z","iopub.status.idle":"2024-12-10T19:29:34.899442Z","shell.execute_reply.started":"2024-12-10T19:29:34.886187Z","shell.execute_reply":"2024-12-10T19:29:34.898764Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n# Hyperparameter Tuning\n\n- **LightGBM Parameters**: Hyperparameters such as `learning_rate`, `max_depth`, `num_leaves`, and `feature_fraction` are tuned to improve the performance of the LightGBM model. These parameters control the complexity of the model and its ability to generalize to new data.\n- **XGBoost and CatBoost Parameters**: Similar tuning is applied for XGBoost and CatBoost, adjusting parameters such as `n_estimators`, `max_depth`, `learning_rate`, `subsample`, and `regularization` terms (`reg_alpha`, `reg_lambda`). These help in controlling overfitting and ensuring the model's robustness.","metadata":{"_uuid":"581e28d7-00db-4903-847c-3aa980e6a955","_cell_guid":"482b5440-1259-4193-8b90-4bb2c8db6ab6","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"Params = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,\n    'lambda_l2': 0.01\n}\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,\n    'reg_lambda': 5,\n    'random_state': SEED\n}\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'cat_features': [c for c in train.columns if 'Season' in c], #just as original\n    'verbose': 0,\n    'l2_leaf_reg': 10\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T19:29:34.90068Z","iopub.execute_input":"2024-12-10T19:29:34.900958Z","iopub.status.idle":"2024-12-10T19:29:34.913679Z","shell.execute_reply.started":"2024-12-10T19:29:34.900933Z","shell.execute_reply":"2024-12-10T19:29:34.912992Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, RegressorMixin\nfrom pytorch_tabnet.callbacks import Callback\nimport os\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n    def __init__(self, filepath, monitor='val_loss', mode='min',\n                 save_best_only=True, verbose=1):\n        super().__init__()\n        self.filepath = filepath\n        self.monitor = monitor\n        self.mode = mode\n        self.save_best_only = save_best_only\n        self.verbose = verbose\n        self.best = float('inf') if mode == 'min' else -float('inf')\n\n    def on_train_begin(self, logs=None):\n        self.model = self.trainer\n\n    def on_epoch_end(self, epoch, logs=None):\n        pass\n\nTabNet_Params = {\n    'n_d': 64,\n    'n_a': 64,\n    'n_steps': 5,\n    'gamma': 1.5,\n    'n_independent': 2,\n    'n_shared': 2,\n    'lambda_sparse': 1e-4,\n    'optimizer_fn': torch.optim.Adam,\n    'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n    'mask_type': 'entmax',\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'verbose': 1,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\nclass TabNetWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, **kwargs):\n        self.model = TabNetRegressor(**kwargs)\n        self.kwargs = kwargs\n        self.imputer = SimpleImputer(strategy='median')\n\n    def fit(self, X, y):\n        X_imputed = self.imputer.fit_transform(X)\n        if hasattr(y, 'values'):\n            y = y.values\n        from sklearn.model_selection import train_test_split\n        X_train, X_valid, y_train, y_valid = train_test_split(X_imputed, y, test_size=0.2, random_state=42)\n\n        self.model.fit(\n            X_train=X_train,\n            y_train=y_train.reshape(-1, 1),\n            eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n            eval_name=['valid'],\n            eval_metric=['mse'],\n            max_epochs=500,\n            patience=50,\n            batch_size=1024,\n            virtual_batch_size=128,\n            num_workers=0,\n            drop_last=False\n        )\n        return self\n\n    def predict(self, X):\n        X_imputed = self.imputer.transform(X)\n        return self.model.predict(X_imputed).flatten()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T19:29:34.914661Z","iopub.execute_input":"2024-12-10T19:29:34.914933Z","iopub.status.idle":"2024-12-10T19:29:34.929457Z","shell.execute_reply.started":"2024-12-10T19:29:34.914897Z","shell.execute_reply":"2024-12-10T19:29:34.928682Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#TabTransformer","metadata":{}},{"cell_type":"code","source":"from tab_transformer_pytorch import TabTransformer\n\nclass TabTransformerWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, dim=32, depth=6, heads=8, attn_dropout=0.1, ff_dropout=0.1, max_epochs=10, lr=1e-3):\n        self.dim = dim\n        self.depth = depth\n        self.heads = heads\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.max_epochs = max_epochs\n        self.lr = lr\n        self.imputer = SimpleImputer(strategy='median')\n        \n    def fit(self, X, y):\n        # Ánh xạ cột phân loại như trong code gốc\n        # Ở đây ta đã có cat_c để biết cột nào là category\n        # Ta giả sử cat_c đã được định nghĩa ở phần trên\n        global cat_c\n        if hasattr(y, 'values'):\n            y = y.values\n        X_imputed = self.imputer.fit_transform(X)\n        \n        from sklearn.model_selection import train_test_split\n        X_train, X_valid, y_train, y_valid = train_test_split(X_imputed, y, test_size=0.2, random_state=42)\n\n        # Tìm cột cat và cont\n        all_cols = X.columns.tolist()\n        categorical_columns = [c for c in all_cols if c in cat_c]\n        continuous_columns = [c for c in all_cols if (c not in cat_c) and (c != 'sii')]\n\n        # Chuyển sang tensor\n        X_train_categ = torch.tensor(X_train[:, [all_cols.index(c) for c in categorical_columns]], dtype=torch.long)\n        X_train_cont = torch.tensor(X_train[:, [all_cols.index(c) for c in continuous_columns]], dtype=torch.float32)\n        y_train_t = torch.tensor(y_train, dtype=torch.long)\n\n        X_valid_categ = torch.tensor(X_valid[:, [all_cols.index(c) for c in categorical_columns]], dtype=torch.long)\n        X_valid_cont = torch.tensor(X_valid[:, [all_cols.index(c) for c in continuous_columns]], dtype=torch.float32)\n        y_valid_t = torch.tensor(y_valid, dtype=torch.long)\n\n        categories = [int(X[c].nunique()) for c in categorical_columns]\n        num_continuous = len(continuous_columns)\n\n        # Khởi tạo TabTransformer\n        self.model = TabTransformer(\n            categories=categories,\n            num_continuous=num_continuous,\n            dim=self.dim,\n            dim_out=4,\n            depth=self.depth,\n            heads=self.heads,\n            attn_dropout=self.attn_dropout,\n            ff_dropout=self.ff_dropout\n        )\n\n        criterion = nn.CrossEntropyLoss()\n        optimizer = optim.Adam(self.model.parameters(), lr=self.lr)\n\n        train_dataset = torch.utils.data.TensorDataset(X_train_categ, X_train_cont, y_train_t)\n        train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=64, shuffle=True)\n\n        self.model.train()\n        for epoch in range(self.max_epochs):\n            total_loss = 0\n            for batch_categ, batch_cont, batch_labels in train_loader:\n                optimizer.zero_grad()\n                outputs = self.model(batch_categ, batch_cont)\n                loss = criterion(outputs, batch_labels)\n                loss.backward()\n                optimizer.step()\n                total_loss += loss.item()\n\n        self.model.eval()\n        return self\n\n    def predict(self, X):\n        self.model.eval()\n        X_imputed = self.imputer.transform(X)\n\n        global cat_c\n        all_cols = X.columns.tolist()\n        categorical_columns = [c for c in all_cols if c in cat_c]\n        continuous_columns = [c for c in all_cols if (c not in cat_c) and (c != 'sii')]\n\n        X_categ = torch.tensor(X_imputed[:, [all_cols.index(c) for c in categorical_columns]], dtype=torch.long)\n        X_cont = torch.tensor(X_imputed[:, [all_cols.index(c) for c in continuous_columns]], dtype=torch.float32)\n\n        with torch.no_grad():\n            logits = self.model(X_categ, X_cont)\n            probs = torch.softmax(logits, dim=1)\n            class_values = torch.tensor([0,1,2,3], dtype=torch.float32)\n            preds_cont = (probs * class_values).sum(dim=1).numpy()\n        return preds_cont","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T19:29:35.04294Z","iopub.execute_input":"2024-12-10T19:29:35.043175Z","iopub.status.idle":"2024-12-10T19:29:35.061775Z","shell.execute_reply.started":"2024-12-10T19:29:35.043152Z","shell.execute_reply":"2024-12-10T19:29:35.061015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tạo model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\nTabNet_Model = TabNetWrapper(**TabNet_Params) # TabNet như cũ\nTabTransformer_Model = TabTransformerWrapper() # Mới thêm TabTransformer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T19:29:35.063378Z","iopub.execute_input":"2024-12-10T19:29:35.063639Z","iopub.status.idle":"2024-12-10T19:29:35.085293Z","shell.execute_reply.started":"2024-12-10T19:29:35.063609Z","shell.execute_reply":"2024-12-10T19:29:35.084671Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ensemble các model cũ + TabNet + TabTransformer\nensemble = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model),\n    ('tabnet', TabNet_Model),\n    ('tabtransformer', TabTransformer_Model) # Thêm TabTransformer\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T19:29:35.086153Z","iopub.execute_input":"2024-12-10T19:29:35.086401Z","iopub.status.idle":"2024-12-10T19:29:35.091647Z","shell.execute_reply.started":"2024-12-10T19:29:35.086369Z","shell.execute_reply":"2024-12-10T19:29:35.090851Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission_final = TrainML(ensemble, test)\n\nSubmission_final.to_csv('submission.csv', index=False)\nprint(\"Submission saved to 'submission.csv'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T19:29:35.092538Z","iopub.execute_input":"2024-12-10T19:29:35.092804Z"}},"outputs":[],"execution_count":null}]}