{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":7453542,"sourceType":"datasetVersion","datasetId":921302},{"sourceId":10161923,"sourceType":"datasetVersion","datasetId":6275040},{"sourceId":10162319,"sourceType":"datasetVersion","datasetId":6275333}],"dockerImageVersionId":30776,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Phát triển từ code gốc:\n- Giữ nguyên tiền xử lý, logic, feature selection, cấu trúc model\n- Thêm mô hình TabTransformer\n- Ensemble mô hình TabTransformer với LightGBM, XGBoost, CatBoost (và các model khác nếu có)\n- Đưa ra kết quả cuối cùng\n\nDựa trên notebook gốc: \nhttps://www.kaggle.com/code/honganzhu/cmi-piu-competition?scriptVersionId=201912528 Version44 LB0.492\n\nThực hiện:\n- Giữ nguyên luồng xử lý dữ liệu, feature engineering\n- Thêm model TabTransformer\n- Ensemble các model để có kết quả tốt nhất","metadata":{"_uuid":"47b16f4e-f31d-49fc-abbd-01a4b5439882","_cell_guid":"67bcd539-aedf-4852-a50a-1cec76f34a51","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"!pip install --no-index --no-deps /kaggle/input/pytorch/einops-0.8.0-py3-none-any.whl","metadata":{"_uuid":"e86a577d-8f63-42ff-a2d7-8b7ba541e265","_cell_guid":"5614a0a9-18ad-4f8d-8db1-3f486b2d9959","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:08.355109Z","iopub.execute_input":"2024-12-11T04:40:08.355602Z","iopub.status.idle":"2024-12-11T04:40:10.224Z","shell.execute_reply.started":"2024-12-11T04:40:08.355553Z","shell.execute_reply":"2024-12-11T04:40:10.222799Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install --no-index --no-deps /kaggle/input/pytorchtransformer/tab_transformer_pytorch-0.3.0-py3-none-any.whl","metadata":{"_uuid":"333f2377-fdd9-4d48-ac49-7769f6e63ec4","_cell_guid":"bb849395-c74a-4cbc-bad7-06c2ed6d135f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:10.226004Z","iopub.execute_input":"2024-12-11T04:40:10.226334Z","iopub.status.idle":"2024-12-11T04:40:12.088689Z","shell.execute_reply.started":"2024-12-11T04:40:10.226304Z","shell.execute_reply":"2024-12-11T04:40:12.087532Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip -q install /kaggle/input/pytorchtabnet/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"_uuid":"5fb9bc3b-6faf-45da-963a-09197c276ef6","_cell_guid":"8248732f-b721-4f25-9514-c8b6a47ea13d","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:12.090521Z","iopub.execute_input":"2024-12-11T04:40:12.090832Z","iopub.status.idle":"2024-12-11T04:40:52.416056Z","shell.execute_reply.started":"2024-12-11T04:40:12.090804Z","shell.execute_reply":"2024-12-11T04:40:52.414819Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pytorch_tabnet.tab_model import TabNetRegressor\nimport torch\n\nimport numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nimport polars as pl\nimport polars.selectors as cs\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\n\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None","metadata":{"_uuid":"4c09fc98-11dc-43c0-a4b1-e088e3eae133","_cell_guid":"6a5de975-34bc-44a1-9b12-98b8cdcf3814","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:52.417761Z","iopub.execute_input":"2024-12-11T04:40:52.418072Z","iopub.status.idle":"2024-12-11T04:40:52.426863Z","shell.execute_reply.started":"2024-12-11T04:40:52.418043Z","shell.execute_reply":"2024-12-11T04:40:52.425969Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_labels = ['None', 'Mild', 'Moderate', 'Severe']","metadata":{"_uuid":"7e963de2-9005-4dda-96a0-f596ed5b393d","_cell_guid":"b288efa2-0eb6-455c-967b-cfac3f662238","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:52.429852Z","iopub.execute_input":"2024-12-11T04:40:52.430098Z","iopub.status.idle":"2024-12-11T04:40:52.439726Z","shell.execute_reply.started":"2024-12-11T04:40:52.430076Z","shell.execute_reply":"2024-12-11T04:40:52.439039Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"season_dtype = pl.Enum(['Spring', 'Summer', 'Fall', 'Winter'])\n\ntrain = (\n    pl.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n    .with_columns(pl.col('^.*Season$').cast(season_dtype))\n)\n\ntest = (\n    pl.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n    .with_columns(pl.col('^.*Season$').cast(season_dtype))\n)\n\ntrain\ntest","metadata":{"_uuid":"215ad187-7782-4382-b224-66c4297de1a5","_cell_guid":"71d94435-c40e-4997-a3e5-e2fc12c0daaf","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:52.441013Z","iopub.execute_input":"2024-12-11T04:40:52.441896Z","iopub.status.idle":"2024-12-11T04:40:52.474262Z","shell.execute_reply.started":"2024-12-11T04:40:52.441867Z","shell.execute_reply":"2024-12-11T04:40:52.473542Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"For a supervised learning, we need the target value, but some (sii) are missing. So we only use the part with valid target value(sii).","metadata":{"_uuid":"2c843336-ebc2-4469-b6a7-82d00b79794b","_cell_guid":"9d70a82f-bcec-4cb1-8998-fbb909942114","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"supervised_usable = (\n    train\n    .filter(pl.col('sii').is_not_null())\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\nplt.figure(figsize=(6, 15))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)),\n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"_uuid":"5e60a90d-2144-4d75-ba6f-fb9d1082a1b7","_cell_guid":"eeae5e67-3a62-4eda-841e-e95522bc38ce","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:52.475416Z","iopub.execute_input":"2024-12-11T04:40:52.475747Z","iopub.status.idle":"2024-12-11T04:40:53.497538Z","shell.execute_reply.started":"2024-12-11T04:40:52.475713Z","shell.execute_reply":"2024-12-11T04:40:53.496643Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train.select(pl.col('PCIAT-PCIAT_Total').is_null() == pl.col('sii').is_null()).to_series().mean())\n\n(train\n .select(pl.col('PCIAT-PCIAT_Total'))\n .group_by(train.get_column('sii'))\n .agg(pl.col('PCIAT-PCIAT_Total').min().alias('PCIAT-PCIAT_Total min'),\n      pl.col('PCIAT-PCIAT_Total').max().alias('PCIAT-PCIAT_Total max'),\n      pl.col('PCIAT-PCIAT_Total').len().alias('count'))\n .sort('sii')\n)","metadata":{"_uuid":"bfebd249-564e-4f9a-bf40-ef98eac64711","_cell_guid":"7a6dc281-313a-493f-9cb2-0b9441179568","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:53.498872Z","iopub.execute_input":"2024-12-11T04:40:53.499504Z","iopub.status.idle":"2024-12-11T04:40:53.511495Z","shell.execute_reply.started":"2024-12-11T04:40:53.499464Z","shell.execute_reply":"2024-12-11T04:40:53.510431Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Insight:**\n\nThis dataset is imbalanced. Half of the samples are in class 0, while very few in class 3.","metadata":{"_uuid":"467d361c-0113-4f12-be44-7a0712d7b89d","_cell_guid":"919b21cc-2a72-492b-acd4-857c507ea83b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"print('Columns missing in test:')\nprint([f for f in train.columns if f not in test.columns])","metadata":{"_uuid":"2ce9799e-9228-4573-a679-506eff2b9fd1","_cell_guid":"0fcab376-600a-4de4-b7d1-45458bcaefd1","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:53.512734Z","iopub.execute_input":"2024-12-11T04:40:53.513537Z","iopub.status.idle":"2024-12-11T04:40:53.520346Z","shell.execute_reply.started":"2024-12-11T04:40:53.513494Z","shell.execute_reply":"2024-12-11T04:40:53.519472Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Demographics","metadata":{"_uuid":"7ae41309-8d15-4519-8a8d-1c5be04c3447","_cell_guid":"c5c07bf3-5837-425a-ac22-c72b00569202","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"Now we look at some basic demographics.","metadata":{"_uuid":"bf75ffc9-92a7-48ba-953b-a400f0ae1a8a","_cell_guid":"c30fc1c1-f096-4675-89b3-686b0e14f22a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"vc = train.get_column('Basic_Demos-Enroll_Season').value_counts()\nplt.pie(vc.get_column('count'), labels=vc.get_column('Basic_Demos-Enroll_Season'))\nplt.title('Season of enrollment')\nplt.show()","metadata":{"_uuid":"403353ac-2c32-4da8-b923-1befc55888fe","_cell_guid":"2295f6f8-44ae-4c73-ae64-1ea30efbe267","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:53.52168Z","iopub.execute_input":"2024-12-11T04:40:53.522308Z","iopub.status.idle":"2024-12-11T04:40:53.615907Z","shell.execute_reply.started":"2024-12-11T04:40:53.52225Z","shell.execute_reply":"2024-12-11T04:40:53.614963Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vc = train.get_column('Basic_Demos-Sex').value_counts()\nplt.pie(vc.get_column('count'), labels=['boys', 'girls'])\nplt.title('Sex of participant')\nplt.show()","metadata":{"_uuid":"15e5654f-81f7-4fd0-a3e0-25d543ea26de","_cell_guid":"41fa4ed3-4626-4ade-abcd-dfad11160589","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:53.617267Z","iopub.execute_input":"2024-12-11T04:40:53.617739Z","iopub.status.idle":"2024-12-11T04:40:53.768719Z","shell.execute_reply.started":"2024-12-11T04:40:53.617686Z","shell.execute_reply":"2024-12-11T04:40:53.767462Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, axs = plt.subplots(2, 1, sharex=True)\nfor sex in range(2):\n    ax = axs.ravel()[sex]\n    vc = train.filter(pl.col('Basic_Demos-Sex') == sex).get_column('Basic_Demos-Age').value_counts()\n    ax.bar(vc.get_column('Basic_Demos-Age'),\n           vc.get_column('count'),\n           color=['lightblue', 'coral'][sex],\n           label=['boys', 'girls'][sex])\n    ax.xaxis.set_major_locator(MaxNLocator(integer=True))\n    ax.set_ylabel('count')\n    ax.legend()\nplt.suptitle('Age distribution')\naxs.ravel()[1].set_xlabel('years')\nplt.show()","metadata":{"_uuid":"65c98576-aa56-4024-a08b-46268838563e","_cell_guid":"178537d8-5ce7-4b15-836f-7f728adaf174","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:53.771883Z","iopub.execute_input":"2024-12-11T04:40:53.772344Z","iopub.status.idle":"2024-12-11T04:40:54.207398Z","shell.execute_reply.started":"2024-12-11T04:40:53.772278Z","shell.execute_reply":"2024-12-11T04:40:54.206544Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, axs = plt.subplots(2, 1, sharex=True, sharey=True)\nfor sex in range(2):\n    ax = axs.ravel()[sex]\n    vc = train.filter(pl.col('Basic_Demos-Sex') == sex).get_column('sii').value_counts()\n    ax.bar(vc.get_column('sii'),\n           vc.get_column('count') / vc.get_column('count').sum(),\n           color=['lightblue', 'coral'][sex],\n           label=['boys', 'girls'][sex])\n    ax.set_xticks(np.arange(4), target_labels)\n    ax.yaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\n    ax.set_ylabel('count')\n    ax.legend()\nplt.suptitle('Target distribution')\naxs.ravel()[1].set_xlabel('Severity Impairment Index (sii)')\nplt.show()","metadata":{"_uuid":"6cab61fb-f8f1-4834-97dd-6ff01dc54349","_cell_guid":"5982ef2b-2884-43f3-bc16-f0d0daade57e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:54.208422Z","iopub.execute_input":"2024-12-11T04:40:54.208653Z","iopub.status.idle":"2024-12-11T04:40:54.5037Z","shell.execute_reply.started":"2024-12-11T04:40:54.20863Z","shell.execute_reply":"2024-12-11T04:40:54.502952Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Now we look at correlations","metadata":{"_uuid":"ee7aaa69-ec29-41b1-9dc1-4dd472b49e23","_cell_guid":"75472fc1-c9f5-45d0-becd-0a604e82c170","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"plt.figure(figsize=(14, 12))\ncorr_matrix = supervised_usable.select([\n    'PCIAT-PCIAT_Total', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-BMI', \n    'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n    'Physical-Diastolic_BP', 'Physical-Systolic_BP', 'Physical-HeartRate',\n    'PreInt_EduHx-computerinternet_hoursday', 'SDS-SDS_Total_T', 'PAQ_A-PAQ_A_Total',\n    'PAQ_C-PAQ_C_Total', 'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins','Fitness_Endurance-Time_Sec',\n    'FGC-FGC_CU', 'FGC-FGC_GSND','FGC-FGC_GSD','FGC-FGC_PU','FGC-FGC_SRL','FGC-FGC_SRR','FGC-FGC_TL','BIA-BIA_Activity_Level_num', \n    'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n    'BIA-BIA_FFMI','BIA-BIA_FMI', 'BIA-BIA_Fat','BIA-BIA_Frame_num','BIA-BIA_ICW','BIA-BIA_LDM','BIA-BIA_LST',\n    'BIA-BIA_SMM','BIA-BIA_TBW'\n    # Add other relevant columns\n]).to_pandas().corr()\n\nsii_corr = corr_matrix['PCIAT-PCIAT_Total'].drop('PCIAT-PCIAT_Total')\nfiltered_corr = sii_corr[(sii_corr > 0.1) | (sii_corr < -0.1)]\n\nprint(filtered_corr)\n\nplt.figure(figsize=(8, 6))\nfiltered_corr.sort_values().plot(kind='barh', color='coral')\nplt.title('Features with Correlation > 0.1 or < -0.1 with PCIAT-PCIAT_Total')\nplt.xlabel('Correlation coefficient')\nplt.ylabel('Features')\nplt.show()","metadata":{"_uuid":"d19c7555-53b8-47c8-a531-77a6c249238c","_cell_guid":"2764c2b1-3267-4b36-84fe-484d6cdcf549","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:54.507682Z","iopub.execute_input":"2024-12-11T04:40:54.507952Z","iopub.status.idle":"2024-12-11T04:40:54.849481Z","shell.execute_reply.started":"2024-12-11T04:40:54.507926Z","shell.execute_reply":"2024-12-11T04:40:54.84841Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Actigraphy (time series)","metadata":{"_uuid":"2e331a63-f150-4328-a625-f160c2eb2c28","_cell_guid":"e12a4df9-87c3-4e0f-9207-096d2bd1a231","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"actigraphy = pl.read_parquet('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id=0417c91e/part-0.parquet')\nactigraphy","metadata":{"_uuid":"2be2316f-c7f8-4684-8171-93dd76025245","_cell_guid":"94f31488-3677-4b50-ad8a-38df10e42825","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:54.850757Z","iopub.execute_input":"2024-12-11T04:40:54.851118Z","iopub.status.idle":"2024-12-11T04:40:54.874114Z","shell.execute_reply.started":"2024-12-11T04:40:54.851078Z","shell.execute_reply":"2024-12-11T04:40:54.873447Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_actigraphy(id, only_one_week=False, small=False):\n    actigraphy = pl.read_parquet(f'/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id={id}/part-0.parquet')\n    day = actigraphy.get_column('relative_date_PCIAT') + actigraphy.get_column('time_of_day') / 86400e9\n    sample = train.filter(pl.col('id') == id)\n    age = sample.get_column('Basic_Demos-Age').item()\n    sex = ['boy', 'girl'][sample.get_column('Basic_Demos-Sex').item()]\n    actigraphy = (\n        actigraphy\n        .with_columns(\n            (day.diff() * 86400).alias('diff_seconds'),\n            (np.sqrt(np.square(pl.col('X')) + np.square(pl.col('Y')) + np.square(pl.col('Z'))).alias('norm'))\n        )\n    )\n\n    if only_one_week:\n        start = np.ceil(day.min())\n        mask = (start <= day.to_numpy()) & (day.to_numpy() <= start + 7*3)\n        mask &= ~ actigraphy.get_column('non-wear_flag').cast(bool).to_numpy()\n    else:\n        mask = np.full(len(day), True)\n        \n    if small:\n        timelines = [\n            ('enmo', 'forestgreen'),\n            ('light', 'orange'),\n        ]\n    else:\n        timelines = [\n            ('X', 'm'),\n            ('Y', 'm'),\n            ('Z', 'm'),\n#             ('norm', 'c'),\n            ('enmo', 'forestgreen'),\n            ('anglez', 'lightblue'),\n            ('light', 'orange'),\n            ('non-wear_flag', 'chocolate')\n    #         ('diff_seconds', 'k'),\n        ]\n        \n    _, axs = plt.subplots(len(timelines), 1, sharex=True, figsize=(12, len(timelines) * 1.1 + 0.5))\n    for ax, (feature, color) in zip(axs, timelines):\n        ax.set_facecolor('#eeeeee')\n        ax.scatter(day.to_numpy()[mask],\n                   actigraphy.get_column(feature).to_numpy()[mask],\n                   color=color, label=feature, s=1)\n        ax.legend(loc='upper left', facecolor='#eeeeee')\n        if feature == 'diff_seconds':\n            ax.set_ylim(-0.5, 20.5)\n    axs[-1].set_xlabel('day')\n    axs[-1].xaxis.set_major_locator(MaxNLocator(integer=True))\n    plt.tight_layout()\n    axs[0].set_title(f'id={id}, {sex}, age={age}')\n    plt.show()\n\nanalyze_actigraphy('0417c91e', only_one_week=False)","metadata":{"_uuid":"f9bdd332-306e-4d0d-9d1e-b81a99b8a155","_cell_guid":"368ed7f8-75d7-4b50-b357-f4b23076528c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:54.875339Z","iopub.execute_input":"2024-12-11T04:40:54.875596Z","iopub.status.idle":"2024-12-11T04:40:56.923516Z","shell.execute_reply.started":"2024-12-11T04:40:54.875571Z","shell.execute_reply":"2024-12-11T04:40:56.92264Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42\nn_splits = 5","metadata":{"_uuid":"40753ee8-07ba-44d2-95e6-7c2f40343f94","_cell_guid":"9955b8f8-a7db-4269-9f6d-c2680f4dcf36","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:56.924618Z","iopub.execute_input":"2024-12-11T04:40:56.924883Z","iopub.status.idle":"2024-12-11T04:40:56.928908Z","shell.execute_reply.started":"2024-12-11T04:40:56.924856Z","shell.execute_reply":"2024-12-11T04:40:56.928067Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering\n\n- **Feature Selection**: The dataset contains features related to physical characteristics (e.g., BMI, Height, Weight), behavioral aspects (e.g., internet usage), and fitness data (e.g., endurance time). \n- **Categorical Feature Encoding**: Categorical features are mapped to numerical values using custom mappings for each unique category within the dataset. This ensures compatibility with machine learning algorithms that require numerical input.\n- **Time Series Aggregation**: Time series statistics (e.g., mean, standard deviation) from the actigraphy data are computed and merged into the main dataset to create additional features for model training.","metadata":{"_uuid":"421049ea-36ce-4f59-9f94-73eb46de1afa","_cell_guid":"70865880-c964-4ce7-bc49-60fe72931cf7","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n\n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n\n    stats, indexes = zip(*results)\n\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n\nclass AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n\n    data_tensor = torch.FloatTensor(df_scaled)\n\n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n\n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n\n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n\n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n\n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n\n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n\n    return df_encoded\n\ndef feature_engineering(df):\n    season_cols = [col for col in df.columns if 'Season' in col]\n    df = df.drop(season_cols, axis=1)\n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n\n    return df","metadata":{"_uuid":"c2b7a236-f805-46df-a31b-dc49caf1a473","_cell_guid":"36ef2c77-c5ef-4c59-ac0e-08c2f624ad05","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:56.930855Z","iopub.execute_input":"2024-12-11T04:40:56.931494Z","iopub.status.idle":"2024-12-11T04:40:56.946208Z","shell.execute_reply.started":"2024-12-11T04:40:56.931465Z","shell.execute_reply":"2024-12-11T04:40:56.945334Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=100, batch_size=32)\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=100, batch_size=32)\n\ntime_series_cols = train_ts_encoded.columns.tolist()\ntrain_ts_encoded[\"id\"]=train_ts[\"id\"]\ntest_ts_encoded['id']=test_ts[\"id\"]\n\ntrain = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\ntest = pd.merge(test, test_ts_encoded, how=\"left\", on='id')\n\nimputer = KNNImputer(n_neighbors=5)\nnumeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\nimputed_data = imputer.fit_transform(train[numeric_cols])\ntrain_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\ntrain_imputed['sii'] = train_imputed['sii'].round().astype(int)\nfor col in train.columns:\n    if col not in numeric_cols:\n        train_imputed[col] = train[col]\n\ntrain = train_imputed\n\ntrain = feature_engineering(train)\ntrain = train.dropna(thresh=10, axis=0)\ntest = feature_engineering(test)\n\ntrain = train.drop('id', axis=1)\ntest  = test .drop('id', axis=1)\n\n# Không thay đổi featureCols\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW']\n\nfeaturesCols += time_series_cols\ntest = test[featuresCols]\n\ncat_c = train.select_dtypes(include=['object', 'category']).columns.tolist()\n\n\n#add\ncat_c = [c for c in cat_c if c in train.columns]\n\ndef update(df):\n    global cat_c\n    for c in cat_c:\n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n\ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n\n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)","metadata":{"_uuid":"5cf46a53-9db6-46c3-809f-b3bf9a66ba07","_cell_guid":"48fd857d-deab-4603-b28b-09f5338022b2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T04:40:56.947424Z","iopub.execute_input":"2024-12-11T04:40:56.947789Z","iopub.status.idle":"2024-12-11T04:42:27.473878Z","shell.execute_reply.started":"2024-12-11T04:40:56.947764Z","shell.execute_reply":"2024-12-11T04:42:27.473101Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if np.any(np.isinf(train)):\n    train = train.replace([np.inf, -np.inf], np.nan)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    # Đảm bảo X và test_data là DataFrame\n    X = pd.DataFrame(X)\n    test_data = pd.DataFrame(test_data)\n\n    # Đảm bảo categorical columns được xử lý đúng\n    global cat_c\n    for c in cat_c:\n        if c in X.columns:\n            X[c] = X[c].astype('category')\n        if c in test_data.columns:\n            test_data[c] = test_data[c].astype('category')\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n\n    train_S = []\n    test_S = []\n\n    oof_non_rounded = np.zeros(len(y), dtype=float)\n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx].copy(), X.iloc[test_idx].copy()\n        y_train, y_val = y.iloc[train_idx].copy(), y.iloc[test_idx].copy()\n\n        try:\n            model = clone(model_class)\n            model.fit(X_train, y_train)\n\n            y_train_pred = model.predict(X_train)\n            y_val_pred = model.predict(X_val)\n\n            oof_non_rounded[test_idx] = y_val_pred\n            y_val_pred_rounded = y_val_pred.round(0).astype(int)\n\n            train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n            val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n            train_S.append(train_kappa)\n            test_S.append(val_kappa)\n\n            test_preds[:, fold] = model.predict(test_data)\n\n            print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n            clear_output(wait=True)\n        except Exception as e:\n            print(f\"Error in fold {fold+1}: {str(e)}\")\n            print(f\"X_train shape: {X_train.shape}\")\n            print(f\"Categorical columns: {[c for c in cat_c if c in X_train.columns]}\")\n            print(f\"Continuous columns: {[c for c in X_train.columns if c not in cat_c]}\")\n            raise e\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded),\n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n\n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n\n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission, tKappa","metadata":{"_uuid":"13900884-490d-4998-8890-b6e7b50bf7e1","_cell_guid":"a5309eab-eb7e-4e70-ac48-1efe39749fb5","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T05:23:23.419775Z","iopub.execute_input":"2024-12-11T05:23:23.42012Z","iopub.status.idle":"2024-12-11T05:23:23.436035Z","shell.execute_reply.started":"2024-12-11T05:23:23.420092Z","shell.execute_reply":"2024-12-11T05:23:23.435261Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n# Hyperparameter Tuning\n\n- **LightGBM Parameters**: Hyperparameters such as `learning_rate`, `max_depth`, `num_leaves`, and `feature_fraction` are tuned to improve the performance of the LightGBM model. These parameters control the complexity of the model and its ability to generalize to new data.\n- **XGBoost and CatBoost Parameters**: Similar tuning is applied for XGBoost and CatBoost, adjusting parameters such as `n_estimators`, `max_depth`, `learning_rate`, `subsample`, and `regularization` terms (`reg_alpha`, `reg_lambda`). These help in controlling overfitting and ensuring the model's robustness.","metadata":{"_uuid":"8750051f-7256-4096-8908-3bef3122aa32","_cell_guid":"2c0308ea-cdce-4e88-a86e-8ece7b05e58b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"Params = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,\n    'lambda_l2': 0.01\n}\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,\n    'reg_lambda': 5,\n    'random_state': SEED\n}\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'cat_features': [c for c in train.columns if 'Season' in c], #just as original\n    'verbose': 0,\n    'l2_leaf_reg': 10\n}","metadata":{"_uuid":"5d605139-cb77-412a-a28f-46d1fb42a896","_cell_guid":"04fabfb2-7e98-4de5-a04b-989f9cc80491","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T05:23:24.331Z","iopub.execute_input":"2024-12-11T05:23:24.331844Z","iopub.status.idle":"2024-12-11T05:23:24.3372Z","shell.execute_reply.started":"2024-12-11T05:23:24.331809Z","shell.execute_reply":"2024-12-11T05:23:24.336215Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, RegressorMixin\nfrom pytorch_tabnet.callbacks import Callback\nimport os\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n    def __init__(self, filepath, monitor='val_loss', mode='min',\n                 save_best_only=True, verbose=1):\n        super().__init__()\n        self.filepath = filepath\n        self.monitor = monitor\n        self.mode = mode\n        self.save_best_only = save_best_only\n        self.verbose = verbose\n        self.best = float('inf') if mode == 'min' else -float('inf')\n\n    def on_train_begin(self, logs=None):\n        self.model = self.trainer\n\n    def on_epoch_end(self, epoch, logs=None):\n        pass\n\nTabNet_Params = {\n    'n_d': 64,\n    'n_a': 64,\n    'n_steps': 5,\n    'gamma': 1.5,\n    'n_independent': 2,\n    'n_shared': 2,\n    'lambda_sparse': 1e-4,\n    'optimizer_fn': torch.optim.Adam,\n    'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n    'mask_type': 'entmax',\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'verbose': 1,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\nclass TabNetWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, **kwargs):\n        self.model = TabNetRegressor(**kwargs)\n        self.kwargs = kwargs\n        self.imputer = SimpleImputer(strategy='median')\n\n    def fit(self, X, y):\n        X_imputed = self.imputer.fit_transform(X)\n        if hasattr(y, 'values'):\n            y = y.values\n        from sklearn.model_selection import train_test_split\n        X_train, X_valid, y_train, y_valid = train_test_split(X_imputed, y, test_size=0.2, random_state=42)\n\n        self.model.fit(\n            X_train=X_train,\n            y_train=y_train.reshape(-1, 1),\n            eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n            eval_name=['valid'],\n            eval_metric=['mse'],\n            max_epochs=500,\n            patience=50,\n            batch_size=1024,\n            virtual_batch_size=128,\n            num_workers=0,\n            drop_last=False\n        )\n        return self\n\n    def predict(self, X):\n        X_imputed = self.imputer.transform(X)\n        return self.model.predict(X_imputed).flatten()","metadata":{"_uuid":"daa8fdb0-e3eb-474d-9a04-756dafda0654","_cell_guid":"1c0bb472-0935-461c-8aa4-144be243375a","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T05:23:24.819522Z","iopub.execute_input":"2024-12-11T05:23:24.82042Z","iopub.status.idle":"2024-12-11T05:23:24.83087Z","shell.execute_reply.started":"2024-12-11T05:23:24.820383Z","shell.execute_reply":"2024-12-11T05:23:24.829915Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#TabTransformer","metadata":{"_uuid":"6ee056c2-0c1d-4219-a4f1-f474e5f7c345","_cell_guid":"e3e49d46-a4e6-44de-ac86-a4d2a7819f38","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass SAINT(nn.Module):\n    def __init__(self, categories, num_continuous, dim, depth, heads, dim_out=1, dropout=0.1, ff_dropout=0.1):\n        super().__init__()\n        \n        # Embedding cho categorical features\n        self.cat_embeds = nn.ModuleList([nn.Embedding(c, dim) for c in categories])\n        \n        # Linear layer cho continuous features\n        self.cont_embed = nn.Linear(num_continuous, dim)\n        \n        # Transformer layers\n        self.transformer = nn.ModuleList([])\n        for _ in range(depth):\n            self.transformer.append(nn.ModuleList([\n                nn.LayerNorm(dim),\n                nn.MultiheadAttention(dim, heads, dropout=dropout),\n                nn.LayerNorm(dim),\n                nn.Sequential(\n                    nn.Linear(dim, dim * 4),\n                    nn.ReLU(),\n                    nn.Dropout(ff_dropout),\n                    nn.Linear(dim * 4, dim),\n                    nn.Dropout(ff_dropout)\n                )\n            ]))\n        \n        # Output layers\n        self.norm = nn.LayerNorm(dim)\n        self.fc = nn.Linear(dim, dim_out)\n        \n    def forward(self, x_cat, x_cont):\n        # Process categorical features\n        if len(self.cat_embeds) > 0:\n            x_cat = [embed(x_cat[:, i]) for i, embed in enumerate(self.cat_embeds)]\n            x_cat = torch.stack(x_cat, dim=1)\n        else:\n            x_cat = torch.zeros((x_cont.size(0), 1, self.dim), device=x_cont.device)\n        \n        # Process continuous features\n        x_cont = self.cont_embed(x_cont).unsqueeze(1)\n        \n        # Combine features\n        x = torch.cat((x_cat, x_cont), dim=1)\n        \n        # Apply transformer layers\n        for norm1, attn, norm2, ff in self.transformer:\n            x = norm1(x)\n            x = x + attn(x, x, x)[0]\n            x = norm2(x)\n            x = x + ff(x)\n        \n        # Output processing\n        x = self.norm(x)\n        x = x.mean(dim=1)\n        x = self.fc(x)\n        \n        return x","metadata":{"_uuid":"5b905e94-533a-4d40-a063-d979abc46c6b","_cell_guid":"bc676bec-7a65-4493-b77f-987a94276400","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T05:29:40.012668Z","iopub.execute_input":"2024-12-11T05:29:40.013032Z","iopub.status.idle":"2024-12-11T05:29:40.023465Z","shell.execute_reply.started":"2024-12-11T05:29:40.012998Z","shell.execute_reply":"2024-12-11T05:29:40.022377Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport numpy as np\n\nclass SAINTWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, dim=32, depth=6, heads=8, dropout=0.1, ff_dropout=0.1, max_epochs=10, lr=1e-3):\n        self.dim = dim\n        self.depth = depth\n        self.heads = heads\n        self.dropout = dropout\n        self.ff_dropout = ff_dropout\n        self.max_epochs = max_epochs\n        self.lr = lr\n        self.imputer = SimpleImputer(strategy='median')\n        \n    def fit(self, X, y):\n        if hasattr(y, 'values'):\n            y = y.values\n        X_imputed = self.imputer.fit_transform(X)\n        \n        # Chuyển đổi dữ liệu thành tensors\n        X_tensor = torch.tensor(X_imputed, dtype=torch.float32)\n        y_tensor = torch.tensor(y, dtype=torch.long)\n        \n        # Tạo một dummy categorical tensor\n        X_cat = torch.zeros((X_imputed.shape[0], 1), dtype=torch.long)\n        \n        # Khởi tạo model\n        self.model = SAINT(\n            categories=[1],  # Một category duy nhất cho dummy feature\n            num_continuous=X_imputed.shape[1],\n            dim=self.dim,\n            depth=self.depth,\n            heads=self.heads,\n            dim_out=4,\n            dropout=self.dropout,\n            ff_dropout=self.ff_dropout\n        )\n        \n        # Training setup\n        criterion = nn.CrossEntropyLoss()\n        optimizer = optim.Adam(self.model.parameters(), lr=self.lr)\n        \n        # DataLoader\n        dataset = torch.utils.data.TensorDataset(X_cat, X_tensor, y_tensor)\n        loader = torch.utils.data.DataLoader(dataset, batch_size=64, shuffle=True)\n        \n        # Training loop\n        self.model.train()\n        for epoch in range(self.max_epochs):\n            for cat, cont, labels in loader:\n                optimizer.zero_grad()\n                \n                # Forward pass\n                outputs = self.model(cat, cont)\n                \n                # Compute loss\n                loss = criterion(outputs, labels)\n                \n                # Backward pass\n                loss.backward()\n                optimizer.step()\n                \n        return self\n    \n    def predict(self, X):\n        # Chuyển sang evaluation mode\n        self.model.eval()\n        \n        # Xử lý input\n        X_imputed = self.imputer.transform(X)\n        X_tensor = torch.tensor(X_imputed, dtype=torch.float32)\n        X_cat = torch.zeros((X_imputed.shape[0], 1), dtype=torch.long)\n        \n        # Dự đoán\n        with torch.no_grad():\n            outputs = self.model(X_cat, X_tensor)\n            probabilities = torch.softmax(outputs, dim=1)\n            \n            # Tính weighted average cho regression\n            weights = torch.tensor([0, 1, 2, 3], dtype=torch.float32)\n            predictions = (probabilities * weights.unsqueeze(0)).sum(dim=1)\n            \n        return predictions.numpy()","metadata":{"_uuid":"370656a3-4635-4b86-ba0a-02d9a58480e0","_cell_guid":"92609574-56b6-43d5-8965-d8560072b57e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-11T05:29:40.288991Z","iopub.execute_input":"2024-12-11T05:29:40.289368Z","iopub.status.idle":"2024-12-11T05:29:40.301857Z","shell.execute_reply.started":"2024-12-11T05:29:40.289333Z","shell.execute_reply":"2024-12-11T05:29:40.300828Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell để thử nghiệm các params khác nhau\nimport itertools\nfrom tqdm import tqdm\n\n# Định nghĩa các params muốn thử\nparam_combinations = {\n    'dim': [64],\n    'depth': [8],\n    'heads': [4],\n    'dropout': [0.1],  # Đổi từ attn_dropout thành dropout cho SAINT\n    'ff_dropout': [0.1],\n    'max_epochs': [20],\n    'lr': [1e-3]\n}\n\n# Tạo tất cả combinations có thể\nkeys = param_combinations.keys()\nvalues = param_combinations.values()\nall_combinations = [dict(zip(keys, v)) for v in itertools.product(*values)]\n\n# Lưu kết quả\nresults = []\n\n# Thử từng combination\nfor params in tqdm(all_combinations, desc=\"Testing SAINT parameters\"):\n    try:\n        # Tạo model instances với params mới cho SAINT\n        Light = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\n        XGB_Model = XGBRegressor(**XGB_Params)\n        CatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n        TabNet_Model = TabNetWrapper(**TabNet_Params)\n        SAINT_Model = SAINTWrapper(**params)  # Sử dụng SAINT thay vì TabTransformer\n\n        # Ensemble các model\n        ensemble = VotingRegressor(estimators=[\n            ('lightgbm', Light),\n            ('xgboost', XGB_Model),\n            ('catboost', CatBoost_Model),\n            ('tabnet', TabNet_Model),\n            ('saint', SAINT_Model)  # Đổi tên thành saint\n        ])\n\n        # Train và lấy kết quả\n        Submission_final, tKappa = TrainML(ensemble, test)\n        \n        # Lưu kết quả\n        results.append({\n            **params,\n            'tKappa': tKappa\n        })\n        \n        print(f\"\\nParameters: {params}\")\n        print(f\"tKappa Score: {tKappa}\")\n        \n    except Exception as e:\n        print(f\"Error with params {params}: {str(e)}\")\n        continue\n\n# Sắp xếp kết quả theo tKappa\nresults_df = pd.DataFrame(results)\nresults_df = results_df.sort_values('tKappa', ascending=False)\n\nprint(\"\\nBest Parameters:\")\nprint(results_df.head())\n\n# Lưu kết quả ra file\nresults_df.to_csv('saint_parameter_search_results.csv', index=False)  # Đổi tên file\n\n# Sử dụng best params để tạo submission cuối cùng\nbest_params = results_df.iloc[0].drop('tKappa').to_dict()\nprint(\"\\nTraining final model with best parameters:\", best_params)\n\n# Tạo model instances với best params\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\nTabNet_Model = TabNetWrapper(**TabNet_Params)\nSAINT_Model = SAINTWrapper(**best_params)  # Sử dụng SAINT với best params\n\n# Ensemble các model\nfinal_ensemble = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model),\n    ('tabnet', TabNet_Model),\n    ('saint', SAINT_Model)  # Đổi tên thành saint\n])\n\n# Train và tạo submission cuối cùng\nfinal_submission, final_tKappa = TrainML(final_ensemble, test)\nfinal_submission.to_csv('submission.csv', index=False)  # Đổi tên file\nprint(f\"\\nFinal tKappa Score: {final_tKappa}\")","metadata":{"_uuid":"7653f22c-395f-4e9c-b4d5-183110759a73","_cell_guid":"f3b262f8-e55f-47b0-96a0-6c8e68e1bddf","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-11T05:29:40.772127Z","iopub.execute_input":"2024-12-11T05:29:40.773112Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"f455657f-ec2d-4765-a817-815c66503d1c","_cell_guid":"57d40bde-33f6-4d7a-9208-577940b683b4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"0bfaf777-72d3-430a-af34-e3801991712f","_cell_guid":"4ab9978b-d7a2-471c-98c2-907fa5833c29","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}