{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":7453542,"sourceType":"datasetVersion","datasetId":921302}],"dockerImageVersionId":30776,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"ℹ️ Info¶¶\n\n* **forked original great work kernels**\n    * https://www.kaggle.com/code/ichigoe/lb0-494-with-tabnet\n\n* **My upd(2024/11/08)**\n    * As a discussion issue for the original kernel.There was a problem that reproducibility was not guaranteed and LB was unstable.\n    * Therefore, Seed fixes such as Torch and LGB have been modified to ensure reproducibility.\n    * Below is the result of measuring LB with this kernel.\n        * [Version1.LB]0.492\n        * [Version2.LB]0.492\n        * [Version3.LB]0.492\n\n\n\n","metadata":{}},{"cell_type":"markdown","source":"---\n---","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Based\n- https://www.kaggle.com/code/honganzhu/cmi-piu-competition?scriptVersionId=201912528 Version44 LB0.492","metadata":{}},{"cell_type":"markdown","source":"》# Description of Imported Libraries\n\n- **NumPy (`np`)**: Used for efficient numerical operations, including linear algebra and array manipulation.\n- **Pandas (`pd`)**: Provides data structures like DataFrames for handling structured data, essential for data preprocessing.\n- **Polars (`pl`)**: A faster alternative to pandas for DataFrame operations, particularly useful for large datasets.\n- **Matplotlib & Seaborn (`plt`, `sns`)**: Visualization libraries. Matplotlib is used for basic plots, while Seaborn builds on it to create more advanced statistical visualizations.\n- **LightGBM, XGBoost, CatBoost**: Machine learning libraries used for gradient boosting, which is efficient for both regression and classification tasks.\n- **Colorama**: Enhances console output with colored text, making it easier to highlight important results or warnings.\n- **SciPy (`minimize`)**: Provides optimization routines, such as adjusting thresholds to maximize performance metrics like kappa scores.\n- **OS**: Used for file path manipulations and system-related functions.\n- **Scikit-learn (`sklearn`)**: A powerful machine learning library, providing utilities for cross-validation, metrics, and model cloning.\n- **YDF**: A specialized library for machine learning tasks, likely including decision forests.\n- **ThreadPoolExecutor & TQDM**: Tools for parallelizing tasks and displaying progress bars for long-running loops, improving efficiency and usability.\n- **Warnings**: Filters out unwanted warnings to keep the output clean, useful when dealing with noisy outputs from multiple libraries.\n- **IPython display (`clear_output`)**: A utility for clearing the Jupyter notebook output, often used to avoid clutter in long-running scripts.\n","metadata":{}},{"cell_type":"code","source":"!pip -q install /kaggle/input/pytorchtabnet/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:02:23.778061Z","iopub.execute_input":"2024-12-12T15:02:23.778632Z","iopub.status.idle":"2024-12-12T15:03:08.485649Z","shell.execute_reply.started":"2024-12-12T15:02:23.778591Z","shell.execute_reply":"2024-12-12T15:03:08.483987Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pytorch_tabnet.tab_model import TabNetRegressor\nimport torch","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:08.489260Z","iopub.execute_input":"2024-12-12T15:03:08.489921Z","iopub.status.idle":"2024-12-12T15:03:08.499164Z","shell.execute_reply.started":"2024-12-12T15:03:08.489840Z","shell.execute_reply":"2024-12-12T15:03:08.497601Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import RepeatedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nimport polars as pl\nimport polars.selectors as cs\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\n\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T15:03:08.501633Z","iopub.execute_input":"2024-12-12T15:03:08.502322Z","iopub.status.idle":"2024-12-12T15:03:08.522314Z","shell.execute_reply.started":"2024-12-12T15:03:08.502263Z","shell.execute_reply":"2024-12-12T15:03:08.520859Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\nseed_everything(2024)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T15:03:08.524337Z","iopub.execute_input":"2024-12-12T15:03:08.524815Z","iopub.status.idle":"2024-12-12T15:03:08.546681Z","shell.execute_reply.started":"2024-12-12T15:03:08.524771Z","shell.execute_reply":"2024-12-12T15:03:08.545746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_labels = ['None', 'Mild', 'Moderate', 'Severe']","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:08.550513Z","iopub.execute_input":"2024-12-12T15:03:08.551362Z","iopub.status.idle":"2024-12-12T15:03:08.560521Z","shell.execute_reply.started":"2024-12-12T15:03:08.551306Z","shell.execute_reply":"2024-12-12T15:03:08.559072Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"season_dtype = pl.Enum(['Spring', 'Summer', 'Fall', 'Winter'])\n\ntrain = (\n    pl.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n    .with_columns(pl.col('^.*Season$').cast(season_dtype))\n)\n\ntest = (\n    pl.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n    .with_columns(pl.col('^.*Season$').cast(season_dtype))\n)\n\ntrain\ntest","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:08.562096Z","iopub.execute_input":"2024-12-12T15:03:08.562605Z","iopub.status.idle":"2024-12-12T15:03:08.604131Z","shell.execute_reply.started":"2024-12-12T15:03:08.562544Z","shell.execute_reply":"2024-12-12T15:03:08.602918Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"For a supervised learning, we need the target value, but some (sii) are missing. So we only use the part with valid target value(sii).","metadata":{}},{"cell_type":"code","source":"supervised_usable = (\n    train\n    .filter(pl.col('sii').is_not_null())\n)\n\nmissing_count = (\n    supervised_usable\n    .null_count()\n    .transpose(include_header=True,\n               header_name='feature',\n               column_names=['null_count'])\n    .sort('null_count', descending=True)\n    .with_columns((pl.col('null_count') / len(supervised_usable)).alias('null_ratio'))\n)\nplt.figure(figsize=(6, 15))\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\nplt.barh(np.arange(len(missing_count)), missing_count.get_column('null_ratio'), color='coral', label='missing')\nplt.barh(np.arange(len(missing_count)), \n         1 - missing_count.get_column('null_ratio'),\n         left=missing_count.get_column('null_ratio'),\n         color='darkseagreen', label='available')\nplt.yticks(np.arange(len(missing_count)), missing_count.get_column('feature'))\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\nplt.xlim(0, 1)\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:08.605762Z","iopub.execute_input":"2024-12-12T15:03:08.606239Z","iopub.status.idle":"2024-12-12T15:03:09.743230Z","shell.execute_reply.started":"2024-12-12T15:03:08.606190Z","shell.execute_reply":"2024-12-12T15:03:09.741975Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train.select(pl.col('PCIAT-PCIAT_Total').is_null() == pl.col('sii').is_null()).to_series().mean())\n\n(train\n .select(pl.col('PCIAT-PCIAT_Total'))\n .group_by(train.get_column('sii'))\n .agg(pl.col('PCIAT-PCIAT_Total').min().alias('PCIAT-PCIAT_Total min'),\n      pl.col('PCIAT-PCIAT_Total').max().alias('PCIAT-PCIAT_Total max'),\n      pl.col('PCIAT-PCIAT_Total').len().alias('count'))\n .sort('sii')\n)","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:09.744534Z","iopub.execute_input":"2024-12-12T15:03:09.744971Z","iopub.status.idle":"2024-12-12T15:03:09.761052Z","shell.execute_reply.started":"2024-12-12T15:03:09.744927Z","shell.execute_reply":"2024-12-12T15:03:09.759652Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Insight:**\n\nThis dataset is imbalanced. Half of the samples are in class 0, while very few in class 3.\n","metadata":{}},{"cell_type":"code","source":"print('Columns missing in test:')\nprint([f for f in train.columns if f not in test.columns])","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:09.763054Z","iopub.execute_input":"2024-12-12T15:03:09.763408Z","iopub.status.idle":"2024-12-12T15:03:09.770645Z","shell.execute_reply.started":"2024-12-12T15:03:09.763375Z","shell.execute_reply":"2024-12-12T15:03:09.769353Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Demographics\n","metadata":{}},{"cell_type":"markdown","source":"Now we look at some basic demographics.","metadata":{}},{"cell_type":"code","source":"vc = train.get_column('Basic_Demos-Enroll_Season').value_counts()\nplt.pie(vc.get_column('count'), labels=vc.get_column('Basic_Demos-Enroll_Season'))\nplt.title('Season of enrollment')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:09.772559Z","iopub.execute_input":"2024-12-12T15:03:09.773010Z","iopub.status.idle":"2024-12-12T15:03:09.910198Z","shell.execute_reply.started":"2024-12-12T15:03:09.772954Z","shell.execute_reply":"2024-12-12T15:03:09.908633Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vc = train.get_column('Basic_Demos-Sex').value_counts()\nplt.pie(vc.get_column('count'), labels=['boys', 'girls'])\nplt.title('Sex of participant')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:09.912274Z","iopub.execute_input":"2024-12-12T15:03:09.912985Z","iopub.status.idle":"2024-12-12T15:03:10.060608Z","shell.execute_reply.started":"2024-12-12T15:03:09.912919Z","shell.execute_reply":"2024-12-12T15:03:10.059302Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, axs = plt.subplots(2, 1, sharex=True)\nfor sex in range(2):\n    ax = axs.ravel()[sex]\n    vc = train.filter(pl.col('Basic_Demos-Sex') == sex).get_column('Basic_Demos-Age').value_counts()\n    ax.bar(vc.get_column('Basic_Demos-Age'),\n           vc.get_column('count'),\n           color=['lightblue', 'coral'][sex],\n           label=['boys', 'girls'][sex])\n    ax.xaxis.set_major_locator(MaxNLocator(integer=True))\n    ax.set_ylabel('count')\n    ax.legend()\nplt.suptitle('Age distribution')\naxs.ravel()[1].set_xlabel('years')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:10.062335Z","iopub.execute_input":"2024-12-12T15:03:10.063103Z","iopub.status.idle":"2024-12-12T15:03:10.552174Z","shell.execute_reply.started":"2024-12-12T15:03:10.062864Z","shell.execute_reply":"2024-12-12T15:03:10.550954Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, axs = plt.subplots(2, 1, sharex=True, sharey=True)\nfor sex in range(2):\n    ax = axs.ravel()[sex]\n    vc = train.filter(pl.col('Basic_Demos-Sex') == sex).get_column('sii').value_counts()\n    ax.bar(vc.get_column('sii'),\n           vc.get_column('count') / vc.get_column('count').sum(),\n           color=['lightblue', 'coral'][sex],\n           label=['boys', 'girls'][sex])\n    ax.set_xticks(np.arange(4), target_labels)\n    ax.yaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\n    ax.set_ylabel('count')\n    ax.legend()\nplt.suptitle('Target distribution')\naxs.ravel()[1].set_xlabel('Severity Impairment Index (sii)')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:10.553641Z","iopub.execute_input":"2024-12-12T15:03:10.554083Z","iopub.status.idle":"2024-12-12T15:03:10.898623Z","shell.execute_reply.started":"2024-12-12T15:03:10.554046Z","shell.execute_reply":"2024-12-12T15:03:10.897554Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Now we look at correlations","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(14, 12))\ncorr_matrix = supervised_usable.select([\n    'PCIAT-PCIAT_Total', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-BMI', \n    'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n    'Physical-Diastolic_BP', 'Physical-Systolic_BP', 'Physical-HeartRate',\n    'PreInt_EduHx-computerinternet_hoursday', 'SDS-SDS_Total_T', 'PAQ_A-PAQ_A_Total',\n    'PAQ_C-PAQ_C_Total', 'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins','Fitness_Endurance-Time_Sec',\n    'FGC-FGC_CU', 'FGC-FGC_GSND','FGC-FGC_GSD','FGC-FGC_PU','FGC-FGC_SRL','FGC-FGC_SRR','FGC-FGC_TL','BIA-BIA_Activity_Level_num', \n    'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n    'BIA-BIA_FFMI','BIA-BIA_FMI', 'BIA-BIA_Fat','BIA-BIA_Frame_num','BIA-BIA_ICW','BIA-BIA_LDM','BIA-BIA_LST',\n    'BIA-BIA_SMM','BIA-BIA_TBW'\n    # Add other relevant columns\n]).to_pandas().corr()\n\nsii_corr = corr_matrix['PCIAT-PCIAT_Total'].drop('PCIAT-PCIAT_Total')\nfiltered_corr = sii_corr[(sii_corr > 0.1) | (sii_corr < -0.1)]\n\nprint(filtered_corr)\n\nplt.figure(figsize=(8, 6))\nfiltered_corr.sort_values().plot(kind='barh', color='coral')\nplt.title('Features with Correlation > 0.1 or < -0.1 with PCIAT-PCIAT_Total')\nplt.xlabel('Correlation coefficient')\nplt.ylabel('Features')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:10.904154Z","iopub.execute_input":"2024-12-12T15:03:10.904647Z","iopub.status.idle":"2024-12-12T15:03:11.298758Z","shell.execute_reply.started":"2024-12-12T15:03:10.904583Z","shell.execute_reply":"2024-12-12T15:03:11.297682Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Actigraphy (time series)","metadata":{}},{"cell_type":"code","source":"actigraphy = pl.read_parquet('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id=0417c91e/part-0.parquet')\nactigraphy","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:11.300683Z","iopub.execute_input":"2024-12-12T15:03:11.301166Z","iopub.status.idle":"2024-12-12T15:03:11.333660Z","shell.execute_reply.started":"2024-12-12T15:03:11.301115Z","shell.execute_reply":"2024-12-12T15:03:11.332530Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_actigraphy(id, only_one_week=False, small=False):\n    actigraphy = pl.read_parquet(f'/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id={id}/part-0.parquet')\n    day = actigraphy.get_column('relative_date_PCIAT') + actigraphy.get_column('time_of_day') / 86400e9\n    sample = train.filter(pl.col('id') == id)\n    age = sample.get_column('Basic_Demos-Age').item()\n    sex = ['boy', 'girl'][sample.get_column('Basic_Demos-Sex').item()]\n    actigraphy = (\n        actigraphy\n        .with_columns(\n            (day.diff() * 86400).alias('diff_seconds'),\n            (np.sqrt(np.square(pl.col('X')) + np.square(pl.col('Y')) + np.square(pl.col('Z'))).alias('norm'))\n        )\n    )\n\n    if only_one_week:\n        start = np.ceil(day.min())\n        mask = (start <= day.to_numpy()) & (day.to_numpy() <= start + 7*3)\n        mask &= ~ actigraphy.get_column('non-wear_flag').cast(bool).to_numpy()\n    else:\n        mask = np.full(len(day), True)\n        \n    if small:\n        timelines = [\n            ('enmo', 'forestgreen'),\n            ('light', 'orange'),\n        ]\n    else:\n        timelines = [\n            ('X', 'm'),\n            ('Y', 'm'),\n            ('Z', 'm'),\n#             ('norm', 'c'),\n            ('enmo', 'forestgreen'),\n            ('anglez', 'lightblue'),\n            ('light', 'orange'),\n            ('non-wear_flag', 'chocolate')\n    #         ('diff_seconds', 'k'),\n        ]\n        \n    _, axs = plt.subplots(len(timelines), 1, sharex=True, figsize=(12, len(timelines) * 1.1 + 0.5))\n    for ax, (feature, color) in zip(axs, timelines):\n        ax.set_facecolor('#eeeeee')\n        ax.scatter(day.to_numpy()[mask],\n                   actigraphy.get_column(feature).to_numpy()[mask],\n                   color=color, label=feature, s=1)\n        ax.legend(loc='upper left', facecolor='#eeeeee')\n        if feature == 'diff_seconds':\n            ax.set_ylim(-0.5, 20.5)\n    axs[-1].set_xlabel('day')\n    axs[-1].xaxis.set_major_locator(MaxNLocator(integer=True))\n    plt.tight_layout()\n    axs[0].set_title(f'id={id}, {sex}, age={age}')\n    plt.show()\n\nanalyze_actigraphy('0417c91e', only_one_week=False)","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:11.335752Z","iopub.execute_input":"2024-12-12T15:03:11.336246Z","iopub.status.idle":"2024-12-12T15:03:13.692682Z","shell.execute_reply.started":"2024-12-12T15:03:11.336197Z","shell.execute_reply":"2024-12-12T15:03:13.691541Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42\nn_splits = 5","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:13.694274Z","iopub.execute_input":"2024-12-12T15:03:13.694728Z","iopub.status.idle":"2024-12-12T15:03:13.700657Z","shell.execute_reply.started":"2024-12-12T15:03:13.694680Z","shell.execute_reply":"2024-12-12T15:03:13.699385Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering\n\n- **Feature Selection**: The dataset contains features related to physical characteristics (e.g., BMI, Height, Weight), behavioral aspects (e.g., internet usage), and fitness data (e.g., endurance time). \n- **Categorical Feature Encoding**: Categorical features are mapped to numerical values using custom mappings for each unique category within the dataset. This ensures compatibility with machine learning algorithms that require numerical input.\n- **Time Series Aggregation**: Time series statistics (e.g., mean, standard deviation) from the actigraphy data are computed and merged into the main dataset to create additional features for model training.\n","metadata":{}},{"cell_type":"code","source":"def process_file(filename, dirname):\n\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n\n    df.drop('step', axis=1, inplace=True)\n\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\n\n\ndef load_time_series(dirname) -> pd.DataFrame:\n\n    ids = os.listdir(dirname)\n\n    \n\n    with ThreadPoolExecutor() as executor:\n\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n\n    \n\n    stats, indexes = zip(*results)\n\n    \n\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n\n    df['id'] = indexes\n\n    return df\n\nfrom torch.utils.data import Dataset,DataLoader\nfrom sklearn.preprocessing import StandardScaler\n\nclass CustomDataset(Dataset):\n    \n    def __init__(self, dataframe):\n        # Apply StandardScaler to the input features\n        self.scaler = StandardScaler()\n        if 'sii' in dataframe.columns:\n            self.train = True\n            features = dataframe.drop(['id', 'sii'], axis=1)  # Drop ID and target column\n            \n\n            self.targets = dataframe['sii'].values  # Keep target values (sii)\n        else:\n            self.train = False\n            features = dataframe.drop(['id'],axis=1)\n            \n        self.scaled_data = self.scaler.fit_transform(features)  # Scale features\n            \n\n    def __len__(self):\n        return len(self.scaled_data)\n    \n    def __getitem__(self, idx):\n        # Return the scaled input features and target value\n        if self.train:\n            return torch.tensor(self.scaled_data[idx], dtype=torch.float32), torch.tensor(self.targets[idx], dtype=torch.long)  # Ensure targets are long for classification\n        else:\n            return torch.tensor(self.scaled_data[idx], dtype=torch.float32)\n            \nclass LSTMEncoder(nn.Module):\n    \n    def __init__(self, input_size, hidden_size, latent_dim, num_classes, num_layers=1):\n        super(LSTMEncoder, self).__init__()\n        # LSTM layer\n        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)\n        # Fully connected layers for latent dimension and classification\n        self.fc_latent = nn.Linear(hidden_size, latent_dim)  # Latent space\n        self.fc_class = nn.Linear(latent_dim, num_classes)   # Classifier layer for output\n        \n    def forward(self, x):\n        _, (hn, _) = self.lstm(x)  # Get hidden state from LSTM (hn)\n        latent = self.fc_latent(hn[-1])  # Compress the last hidden state to latent space\n        out = self.fc_class(latent)  # Classify using the latent space\n        return latent, out\n    \nclass VAE(nn.Module):\n    def __init__(self, input_channels, seq_length, latent_dim, dropout_prob=0.4):\n        super(VAE, self).__init__()\n        \n        # Encoder: deeper Conv1d layers with BatchNorm1d\n        self.encoder = nn.Sequential(\n            nn.Conv1d(input_channels, 16, kernel_size=3, stride=2, padding=1),\n            nn.BatchNorm1d(16),\n            nn.ReLU(0.2),\n            nn.Dropout(dropout_prob),\n            \n            nn.Conv1d(16, 32, kernel_size=3, stride=2, padding=1),\n            nn.BatchNorm1d(32),\n            nn.ReLU(0.2),\n            nn.Dropout(dropout_prob),\n            \n            nn.Conv1d(32, 64, kernel_size=3, stride=2, padding=1),\n            nn.BatchNorm1d(64),\n            nn.ReLU(0.2),\n            nn.Flatten()\n        )\n        \n        # Calculate size after Conv1d layers\n        conv_output_size = (seq_length // 8) * 64\n        \n        self.fc_mu = nn.Linear(conv_output_size, latent_dim)\n        self.fc_logvar = nn.Linear(conv_output_size, latent_dim)\n        \n        # Decoder: Fully connected and ConvTranspose1d layers\n        self.decoder_input = nn.Linear(latent_dim, conv_output_size)\n        self.decoder = nn.Sequential(\n            nn.Unflatten(1, (64, seq_length // 8)),\n            nn.ConvTranspose1d(64, 32, kernel_size=3, stride=2, padding=1, output_padding=1),\n            nn.BatchNorm1d(32),\n            nn.ReLU(0.2),\n            nn.Dropout(dropout_prob),\n            \n            nn.ConvTranspose1d(32, 16, kernel_size=3, stride=2, padding=1, output_padding=1),\n            nn.BatchNorm1d(16),\n            nn.ReLU(0.2),\n            nn.Dropout(dropout_prob),\n            \n            nn.ConvTranspose1d(16, input_channels, kernel_size=3, stride=2, padding=1, output_padding=1),\n            nn.Sigmoid()\n        )\n    \n    def reparameterize(self, mu, logvar):\n        std = torch.exp(0.5 * logvar)\n        eps = torch.randn_like(std)\n        return mu + eps * std\n    \n    def forward(self, x):\n        encoded = self.encoder(x)\n        mu = self.fc_mu(encoded)\n        logvar = self.fc_logvar(encoded)\n        z = self.reparameterize(mu, logvar)\n        decoded = self.decoder(self.decoder_input(z))\n        return decoded, mu, logvar\n\ndef reconstruction_loss(original, reconstructed, alpha=0.5, beta=0.5):\n    \"\"\"\n    定义重建损失，MSE + 余弦相似度的组合\n    \"\"\"\n    # Mean Squared Error\n    mse_loss = F.mse_loss(reconstructed, original)\n    \n    # Cosine Similarity -> Cosine Distance\n    cosine_sim = F.cosine_similarity(reconstructed, original, dim=-1)  # 按最后一维计算相似度\n    cosine_dist = 1 - cosine_sim.mean()  # 余弦距离（越小越相似）\n    \n    # 总损失：MSE + Cosine Distance\n    total_loss = alpha * mse_loss + beta * cosine_dist\n    return total_loss\n    \ndef perform_autoencoder(df, latent_dim=64, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled).unsqueeze(1)\n\n    input_channels = data_tensor.shape[1]\n    seq_length = data_tensor.shape[2]\n\n    autoencoder = VAE(input_channels, seq_length, latent_dim)\n    optimizer = optim.AdamW(autoencoder.parameters(), lr=5e-3, weight_decay=1e-4)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5)\n    criterion = nn.MSELoss()  # Reconstruction loss\n    \n    # 权重初始化函数\n    def init_weights(m):\n        if isinstance(m, (nn.Conv1d, nn.ConvTranspose1d, nn.Linear)):\n            nn.init.xavier_uniform_(m.weight)  # 使用Xavier初始化\n            if m.bias is not None:\n                nn.init.constant_(m.bias, 0.01)  # 偏置设置为0.01\n    \n    # Apply weight initialization\n    autoencoder.apply(init_weights)\n\n    for epoch in range(epochs):\n        epoch_loss = 0\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed, mu, logvar = autoencoder(batch)\n            \n            # 重建损失：MSE + Cosine Distance\n            loss = reconstruction_loss(batch, reconstructed)\n            \n            # 加入KL散度和稀疏性正则化\n            kl_divergence = -0.5 * torch.mean(1 + logvar - mu.pow(2) - logvar.exp())\n            l1_regularization = sum(param.abs().sum() for param in autoencoder.encoder.parameters())\n            \n            total_loss = loss + kl_divergence + 1e-4 * l1_regularization\n            total_loss.backward()\n            optimizer.step()\n            epoch_loss += total_loss.item()\n        \n        scheduler.step(epoch_loss)\n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {epoch_loss / len(data_tensor):.4f}')    \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n\n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    return df_encoded\n\ndef feature_engineering(df):\n    season_cols = [col for col in df.columns if 'Season' in col]\n    df = df.drop(season_cols, axis=1) \n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    \n    return df\n\n\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-12-12T15:03:13.702679Z","iopub.execute_input":"2024-12-12T15:03:13.703187Z","iopub.status.idle":"2024-12-12T15:04:41.169605Z","shell.execute_reply.started":"2024-12-12T15:03:13.703139Z","shell.execute_reply":"2024-12-12T15:04:41.168363Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.shape\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T15:04:41.170963Z","iopub.execute_input":"2024-12-12T15:04:41.171330Z","iopub.status.idle":"2024-12-12T15:04:41.178372Z","shell.execute_reply.started":"2024-12-12T15:04:41.171298Z","shell.execute_reply":"2024-12-12T15:04:41.177341Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"###############################################################################\ntemp_df = train[['id','sii']]\n\ntime_series_df_with_target = pd.merge(train_ts,temp_df,how=\"left\",on='id')\n\ntemp_df = test[['id']]\n\ntime_series_df_without_target = pd.merge(test_ts,temp_df,how=\"left\",on='id')\n\ntrain_dataset = CustomDataset(time_series_df_with_target)\ntrain_dataloader = DataLoader(train_dataset,batch_size=32,shuffle=True)\n\ntest_dataset = CustomDataset(time_series_df_without_target)\ntest_dataloader = DataLoader(test_dataset,batch_size=1,shuffle=True)\n\n# Example parameters\ninput_size = df_train.shape[1]  # Number of features\nhidden_size = 64  # Number of hidden units in LSTM\nlatent_dim = 64  # Latent space dimension\nnum_classes = len(train['sii'].unique())  # Number of target classes\nnum_layers = 2  # Number of LSTM layers\n\n# Model initialization\nmodel = LSTMEncoder(input_size, hidden_size, latent_dim, num_classes, num_layers)\n\n# Define loss function and optimizer\ncriterion = nn.CrossEntropyLoss()  # For multi-class classification\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n\n# Training loop\nepochs = 30  # Number of epochs to train\n\nfor epoch in range(epochs):\n    total_loss = 0\n    for batch_data, batch_targets in train_dataloader:\n        optimizer.zero_grad()\n        # Forward pass: latent representation and predictions\n        latent, predictions = model(batch_data.unsqueeze(1))\n        # Compute the loss\n        loss = criterion(predictions, batch_targets)\n        total_loss+=loss.item()\n        loss.backward()  # Backpropagate the error\n        optimizer.step()  # Update the weights\n\n    print(f\"Epoch [{epoch+1}], Loss: {total_loss/32}\")\n\n###############################################################################\nfinal_list = []\n\n# After training, you can use the encoder to extract both the latent dimensions and class predictions\nwith torch.no_grad():\n    for batch_data, batch_targets in train_dataloader:\n        batch_data = batch_data.unsqueeze(1)  # Add sequence length dim if needed\n        latent, predictions = model(batch_data)\n        final_list.append(latent)\n\n# Step 1: Concatenate all the latent representations into a single tensor\n# Assuming each 'latent' is of shape (batch_size, latent_dim)\nall_latents = torch.cat(final_list, dim=0)  # Concatenate along the first dimension (batch dimension)\n\n# Step 2: Convert the concatenated tensor to a NumPy array\nlatent_array = all_latents.numpy()  # Convert to NumPy array\n\n# Step 3: Create a DataFrame from the NumPy array\nnum_latent_dims = latent_array.shape[1]  # Get the number of latent dimensions\ntrain_latent = pd.DataFrame(latent_array, columns=[f'enc_{i + 1}' for i in range(num_latent_dims)])\n\ntrain_latent['id'] = time_series_df_with_target['id']\n\ntrain = pd.merge(train, train_latent, how=\"left\", on='id')\n###############################################################################\nfinal_list = []\n\n# After training, you can use the encoder to extract both the latent dimensions and class predictions\nwith torch.no_grad():\n    for batch_data in test_dataloader:\n        batch_data = batch_data.unsqueeze(1)  # Add sequence length dim if needed\n        latent, predictions = model(batch_data)\n        final_list.append(latent)\n\n# Step 1: Concatenate all the latent representations into a single tensor\n# Assuming each 'latent' is of shape (batch_size, latent_dim)\nall_latents = torch.cat(final_list, dim=0)  # Concatenate along the first dimension (batch dimension)\n\n# Step 2: Convert the concatenated tensor to a NumPy array\nlatent_array = all_latents.numpy()  # Convert to NumPy array\n\n# Step 3: Create a DataFrame from the NumPy array\nnum_latent_dims = latent_array.shape[1]  # Get the number of latent dimensions\ntest_latent = pd.DataFrame(latent_array, columns=[f'enc_{i + 1}' for i in range(num_latent_dims)])\n\ntest_latent['id'] = time_series_df_without_target['id']\ntest = pd.merge(test, test_latent, how=\"left\", on='id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T15:04:41.180099Z","iopub.execute_input":"2024-12-12T15:04:41.180424Z","iopub.status.idle":"2024-12-12T15:04:45.909607Z","shell.execute_reply.started":"2024-12-12T15:04:41.180393Z","shell.execute_reply":"2024-12-12T15:04:45.908680Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Training and Evaluation\n\n- **Model Types**: Various models are used, including:\n  - **LightGBM**: A gradient-boosting framework known for its speed and efficiency with large datasets.\n  - **XGBoost**: Another powerful gradient-boosting model used for structured data.\n  - **CatBoost**: Optimized for categorical features without the need for extensive preprocessing.\n  - **Voting Regressor**: An ensemble model that combines the predictions of LightGBM, XGBoost, and CatBoost for better accuracy.\n- **Cross-Validation**: Stratified K-Folds cross-validation is employed to split the data into training and validation sets, ensuring balanced class distribution in each fold.\n- **Quadratic Weighted Kappa (QWK)**: The performance of the models is evaluated using QWK, which measures the agreement between predicted and actual values, taking into account the ordinal nature of the target variable.\n- **Threshold Optimization**: The `minimize` function from `scipy.optimize` is used to fine-tune decision thresholds that map continuous predictions to discrete categories (None, Mild, Moderate, Severe).\n","metadata":{}},{"cell_type":"code","source":"feature_col = train.drop(['sii'], axis=1).columns\n# 各モデルの平均重要度を結合（Noneでなければ）\nall_importances = pd.DataFrame({'feature': feature_col})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T15:04:45.911291Z","iopub.execute_input":"2024-12-12T15:04:45.911590Z","iopub.status.idle":"2024-12-12T15:04:45.919739Z","shell.execute_reply.started":"2024-12-12T15:04:45.911561Z","shell.execute_reply":"2024-12-12T15:04:45.918537Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n# Hyperparameter Tuning\n\n- **LightGBM Parameters**: Hyperparameters such as `learning_rate`, `max_depth`, `num_leaves`, and `feature_fraction` are tuned to improve the performance of the LightGBM model. These parameters control the complexity of the model and its ability to generalize to new data.\n- **XGBoost and CatBoost Parameters**: Similar tuning is applied for XGBoost and CatBoost, adjusting parameters such as `n_estimators`, `max_depth`, `learning_rate`, `subsample`, and `regularization` terms (`reg_alpha`, `reg_lambda`). These help in controlling overfitting and ensuring the model's robustness.","metadata":{}},{"cell_type":"code","source":"# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01,  # Increased from 2.68e-06\n    'device': 'cpu'\n\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED,\n    'tree_method': 'gpu_hist',\n\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU'\n\n}\n# New: TabNet\n\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom pytorch_tabnet.callbacks import Callback\nimport os\nimport torch\nfrom pytorch_tabnet.callbacks import Callback\n\nclass TabNetWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, **kwargs):\n        self.model = TabNetRegressor(**kwargs)\n        self.kwargs = kwargs\n        self.imputer = SimpleImputer(strategy='median')\n        self.best_model_path = 'best_tabnet_model.pt'\n        \n    def fit(self, X, y):\n        # Handle missing values\n        X_imputed = self.imputer.fit_transform(X)\n        \n        if hasattr(y, 'values'):\n            y = y.values\n            \n        # Create internal validation set\n        X_train, X_valid, y_train, y_valid = train_test_split(\n            X_imputed, \n            y, \n            test_size=0.2,\n            random_state=42\n        )\n        \n        # Train TabNet model\n        history = self.model.fit(\n            X_train=X_train,\n            y_train=y_train.reshape(-1, 1),\n            eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n            eval_name=['valid'],\n            eval_metric=['mse'],\n            max_epochs=500,\n            patience=50,\n            batch_size=1024,\n            virtual_batch_size=128,\n            num_workers=0,\n            drop_last=False,\n            callbacks=[\n                TabNetPretrainedModelCheckpoint(\n                    filepath=self.best_model_path,\n                    monitor='valid_mse',\n                    mode='min',\n                    save_best_only=True,\n                    verbose=True\n                )\n            ]\n        )\n        \n        # Load the best model\n        if os.path.exists(self.best_model_path):\n            self.model.load_model(self.best_model_path)\n            os.remove(self.best_model_path)  # Remove temporary file\n        \n        return self\n    \n    def predict(self, X):\n        X_imputed = self.imputer.transform(X)\n        return self.model.predict(X_imputed).flatten()\n    \n    def __deepcopy__(self, memo):\n        # Add deepcopy support for scikit-learn\n        cls = self.__class__\n        result = cls.__new__(cls)\n        memo[id(self)] = result\n        for k, v in self.__dict__.items():\n            setattr(result, k, deepcopy(v, memo))\n        return result\n\n# TabNet hyperparameters\nTabNet_Params = {\n    'n_d': 64,              # Width of the decision prediction layer\n    'n_a': 64,              # Width of the attention embedding for each step\n    'n_steps': 5,           # Number of steps in the architecture\n    'gamma': 1.5,           # Coefficient for feature selection regularization\n    'n_independent': 2,     # Number of independent GLU layer in each GLU block\n    'n_shared': 2,          # Number of shared GLU layer in each GLU block\n    'lambda_sparse': 1e-4,  # Sparsity regularization\n    'optimizer_fn': torch.optim.Adam,\n    'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n    'mask_type': 'entmax',\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'verbose': 1,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n    def __init__(self, filepath, monitor='val_loss', mode='min', \n                 save_best_only=True, verbose=1):\n        super().__init__()  # Initialize parent class\n        self.filepath = filepath\n        self.monitor = monitor\n        self.mode = mode\n        self.save_best_only = save_best_only\n        self.verbose = verbose\n        self.best = float('inf') if mode == 'min' else -float('inf')\n        \n    def on_train_begin(self, logs=None):\n        self.model = self.trainer  # Use trainer itself as model\n        \n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is None:\n            return\n        \n        # Check if current metric is better than best\n        if (self.mode == 'min' and current < self.best) or \\\n           (self.mode == 'max' and current > self.best):\n            if self.verbose:\n                print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n            self.best = current\n            if self.save_best_only:\n                self.model.save_model(self.filepath)  # Save the entire model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T15:04:45.921621Z","iopub.execute_input":"2024-12-12T15:04:45.922026Z","iopub.status.idle":"2024-12-12T15:04:45.946964Z","shell.execute_reply.started":"2024-12-12T15:04:45.921987Z","shell.execute_reply":"2024-12-12T15:04:45.945840Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Final Results and Performance Metrics\n\n- **Train and Validation Scores**: After training across multiple folds, the mean Quadratic Weighted Kappa (QWK) score is calculated for both the training and validation datasets, providing an indicator of model performance. \n- **Optimized QWK Score**: The final optimized QWK score after threshold tuning is displayed, showcasing the model's ability to predict the severity levels effectively.\n- **Test Predictions**: The test set predictions are evaluated, and a breakdown of the predicted severity levels (None, Mild, Moderate, Severe) is shown, along with their respective counts.","metadata":{}},{"cell_type":"markdown","source":"# Ensemble Learning and Submission Preparation\n\n- **Ensemble Learning**: The model uses a **Voting Regressor**, which combines the predictions from LightGBM, XGBoost, and CatBoost. This approach is beneficial as it leverages the strengths of multiple models, reducing overfitting and improving overall model performance.\n- **Out-of-Fold (OOF) Predictions**: During cross-validation, out-of-fold predictions are generated for the training set, which helps in model evaluation without data leakage.\n- **Kappa Optimizer**: The Kappa Optimizer ensures that the predicted values are as close to the actual values as possible by adjusting the thresholds used to convert raw model outputs into class labels.\n- **Test Set Predictions**: After the model is trained and thresholds are optimized, the test dataset is processed, and predictions are generated using the ensemble model. These predictions are converted into the appropriate format for submission.\n- **Submission File Creation**: The predictions are saved in a CSV file following the required format for submission (e.g., for a Kaggle competition), which includes columns like `id` and `sii` (Severity Impairment Index).","metadata":{}},{"cell_type":"code","source":"# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\nTabNet_Model = TabNetWrapper(**TabNet_Params) # New","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T15:22:44.135083Z","iopub.execute_input":"2024-12-12T15:22:44.135529Z","iopub.status.idle":"2024-12-12T15:22:44.143419Z","shell.execute_reply.started":"2024-12-12T15:22:44.135474Z","shell.execute_reply":"2024-12-12T15:22:44.142216Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n# **》》》Model2**\n---","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.preprocessing import StandardScaler\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\n# 讀取資料\ntrain_ori = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n# 檢查 'sii' 欄位\nprint(\"資料集的欄位：\", train_ori.columns.tolist())\nif 'sii' in train_ori.columns:\n    print(\"sii 欄位的唯一值：\", train_ori['sii'].unique())\nelse:\n    raise ValueError(\"錯誤：資料集中沒有 'sii' 欄位。\")\n\n# 將資料依 sii 分組\ndata_sii_0 = train_ori[train_ori['sii'] == 0]\ndata_sii_1 = train_ori[train_ori['sii'] == 1]\ndata_sii_2 = train_ori[train_ori['sii'] == 2]\ndata_sii_3 = train_ori[train_ori['sii'] == 3]\n\n# 計算各類別樣本數\ncount_0 = len(data_sii_0)\ncount_1 = len(data_sii_1)\ncount_2 = len(data_sii_2)\ncount_3 = len(data_sii_3)\n\nprint(\"各類別的樣本數：\")\nprint(f\"sii=0: {count_0}, sii=1: {count_1}, sii=2: {count_2}, sii=3: {count_3}\")\n\n# 找出最大樣本數\nmax_count = max(count_0, count_1, count_2, count_3)\nprint(f\"最大樣本數為：{max_count}\")\n\n# 假設 X 是除 sii 外的特徵\nX = train_ori.drop('sii', axis=1)\ny = train_ori['sii']\n\n# 檢查特徵類型並進行編碼（如果有必要）\ncategorical_features = X.select_dtypes(include=['object', 'category']).columns.tolist()\nif categorical_features:\n    X = pd.get_dummies(X, columns=categorical_features)\n    print(\"進行 One-Hot Encoding 後的特徵數量：\", X.shape[1])\n\n# 檢查是否有缺失值\nif X.isnull().sum().sum() > 0:\n    print(\"存在缺失值，進行填補。\")\n    X = X.fillna(X.mean())\n\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X)\n\n# 重新組合scaled資料\ntrain_scaled = pd.DataFrame(X_scaled, columns=X.columns)\ntrain_scaled['sii'] = y.values\n\n# 定義模型\nlatent_dim =1024   # 雜訊維度\nnum_features = X.shape[1]\nnum_classes = 4   # sii有四類：0,1,2,3\n\nimport torch\nimport torch.nn as nn\n\nclass Generator(nn.Module):\n    def __init__(self, latent_dim, num_features, num_classes):\n        super(Generator, self).__init__()\n        self.label_emb = nn.Embedding(num_classes, num_classes)\n        self.model = nn.Sequential(\n            nn.Linear(latent_dim + num_classes, 32),\n            nn.BatchNorm1d(32),\n            nn.ReLU(True),\n            \n            nn.Linear(32, 64),\n            nn.BatchNorm1d(64),\n            nn.ReLU(True),\n            \n            nn.Linear(64, num_features),\n            nn.Tanh()  # 通常生成器的输出使用 Tanh 激活\n        )\n    \n    def forward(self, z, labels):\n        c = self.label_emb(labels)\n        x = torch.cat([z, c], dim=1)\n        out = self.model(x)\n        return out\n\nclass Discriminator(nn.Module):\n    def __init__(self, num_features, num_classes):\n        super(Discriminator, self).__init__()\n        self.label_emb = nn.Embedding(num_classes, num_classes)\n        self.model = nn.Sequential(\n            nn.Linear(num_features + num_classes, 64),\n            nn.LeakyReLU(0.2, inplace=True),\n            nn.BatchNorm1d(64),\n            \n            nn.Linear(64, 32),\n            nn.LeakyReLU(0.2, inplace=True),\n            nn.BatchNorm1d(32),\n            \n            nn.Linear(32, 1),\n            nn.Sigmoid()\n        )\n    \n    def forward(self, x, labels):\n        c = self.label_emb(labels)\n        d_in = torch.cat([x, c], dim=1)\n        out = self.model(d_in)\n        return out\n\n# 準備裝置、模型、優化器\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"使用的裝置：{device}\")\n\ngenerator = Generator(latent_dim, num_features, num_classes).to(device)\ndiscriminator = Discriminator(num_features, num_classes).to(device)\n\nadversarial_loss = nn.BCELoss()\noptimizer_G = optim.Adam(generator.parameters(), lr=0.0002, betas=(0.5, 0.999))\noptimizer_D = optim.Adam(discriminator.parameters(), lr=0.0002, betas=(0.5, 0.999))\n\n# 將特定類別的資料轉為Tensor\ndef prepare_data_for_class(data_frame, target_class):\n    class_data = data_frame[data_frame['sii'] == target_class]\n    X_class = class_data.drop('sii', axis=1).values\n    y_class = class_data['sii'].values\n    \n    X_tensor = torch.tensor(X_class, dtype=torch.float32)\n    y_tensor = torch.tensor(y_class, dtype=torch.long)  # 用 long 為 embedding\n    return X_tensor, y_tensor\nX_0, y_0 = prepare_data_for_class(train_scaled, 0)\nX_1, y_1 = prepare_data_for_class(train_scaled, 1)\nX_2, y_2 = prepare_data_for_class(train_scaled, 2)\nX_3, y_3 = prepare_data_for_class(train_scaled, 3)\nprint(f\"sii=0 的樣本數：{len(X_0)}\")\nprint(f\"sii=1 的樣本數：{len(X_1)}\")\nprint(f\"sii=2 的樣本數：{len(X_2)}\")\nprint(f\"sii=3 的樣本數：{len(X_3)}\")\n\n# 設定批次大小與訓練週期\nbatch_size = 128\nepochs = 1  # 初步測試時可減少為100\n\ndef wasserstein_loss(y_real, y_fake):\n    return y_fake.mean() - y_real.mean()\n\n# 定義一個通用函式來訓練特定類別的GAN並生成樣本\ndef train_gan_for_class(X_class, y_class, target_class):\n    # 檢查需要補多少\n    needed = max_count - len(X_class)\n    print(f\"Class {target_class}: Current count = {len(X_class)}, Needed = {needed}\")\n    if needed <= 0:\n        # 不需要生成，直接回傳空DataFrame\n        print(f\"Class {target_class} 已經達到或超過最大樣本數，不需要生成合成樣本。\")\n        return pd.DataFrame(columns=X.columns.tolist() + ['sii'])\n    \n    # 訓練GAN\n    for epoch in range(epochs):\n        # 訓練Discriminator\n        discriminator.zero_grad()\n        \n        # 抽樣真實樣本索引，確保不越界\n        if len(X_class) < batch_size:\n            # 若資料量小於batch_size，可調整batch_size或重複取樣\n            real_idx = torch.randint(0, len(X_class), (len(X_class),))\n        else:\n            real_idx = torch.randint(0, len(X_class), (batch_size,))\n        \n        real_samples = X_class[real_idx].to(device)\n        real_labels = y_class[real_idx].to(device)\n        real_validity = discriminator(real_samples, real_labels)\n\n        valid = torch.ones((real_validity.shape[0], 1), device=device)\n        fake = torch.zeros((real_validity.shape[0], 1), device=device)\n        \n        real_loss = wasserstein_loss(real_validity, valid)\n\n        # 生成假樣本\n        z = torch.randn(real_validity.shape[0], latent_dim, device=device)\n        gen_labels = torch.full((real_validity.shape[0],), target_class, dtype=torch.long, device=device)  \n        gen_samples = generator(z, gen_labels)\n        fake_validity = discriminator(gen_samples.detach(), gen_labels)\n        fake_loss = wasserstein_loss(fake_validity, fake)\n        \n        d_loss = real_loss + fake_loss\n        d_loss.backward()\n        optimizer_D.step()\n\n        # 訓練Generator\n        generator.zero_grad()\n        fake_validity = discriminator(gen_samples, gen_labels)\n        g_loss = wasserstein_loss(fake_validity, valid)  # 生成器希望被視為真\n        g_loss.backward()\n        optimizer_G.step()\n\n        if (epoch + 1) % 1 == 0:\n            print(f\"Class {target_class} | Epoch [{epoch + 1}/{epochs}]  D_loss: {d_loss.item():.4f}  G_loss: {g_loss.item():.4f}\")\n\n# 定義一個通用函式來訓練特定類別的GAN並生成樣本\ndef train_gan_for_class_2(X_class, y_class, target_class):\n    # 檢查需要補多少\n    needed = 2*max_count - len(X_class)\n    print(f\"Class {target_class}: Current count = {len(X_class)}, Needed = {needed}\")\n    if needed <= 0:\n        # 不需要生成，直接回傳空DataFrame\n        print(f\"Class {target_class} 已經達到或超過最大樣本數，不需要生成合成樣本。\")\n        return pd.DataFrame(columns=X.columns.tolist() + ['sii'])\n    \n    # 訓練GAN\n    for epoch in range(epochs):\n        # 訓練Discriminator\n        discriminator.zero_grad()\n        \n        # 抽樣真實樣本索引，確保不越界\n        if len(X_class) < batch_size:\n            # 若資料量小於batch_size，可調整batch_size或重複取樣\n            real_idx = torch.randint(0, len(X_class), (len(X_class),))\n        else:\n            real_idx = torch.randint(0, len(X_class), (batch_size,))\n        \n        real_samples = X_class[real_idx].to(device)\n        real_labels = y_class[real_idx].to(device)\n        real_validity = discriminator(real_samples, real_labels)\n\n        valid = torch.ones((real_validity.shape[0], 1), device=device)\n        fake = torch.zeros((real_validity.shape[0], 1), device=device)\n        \n        real_loss = wasserstein_loss(real_validity, valid)\n\n        # 生成假樣本\n        z = torch.randn(real_validity.shape[0], latent_dim, device=device)\n        gen_labels = torch.full((real_validity.shape[0],), target_class, dtype=torch.long, device=device)  \n        gen_samples = generator(z, gen_labels)\n        fake_validity = discriminator(gen_samples.detach(), gen_labels)\n        fake_loss = wasserstein_loss(fake_validity, fake)\n        \n        d_loss = real_loss + fake_loss\n        d_loss.backward()\n        optimizer_D.step()\n\n        # 訓練Generator\n        generator.zero_grad()\n        fake_validity = discriminator(gen_samples, gen_labels)\n        g_loss = wasserstein_loss(fake_validity, valid)  # 生成器希望被視為真\n        g_loss.backward()\n        optimizer_G.step()\n\n        if (epoch + 1) % 1 == 0:\n            print(f\"Class {target_class} | Epoch [{epoch + 1}/{epochs}]  D_loss: {d_loss.item():.4f}  G_loss: {g_loss.item():.4f}\")\n\n\n    # 訓練完成後生成所需數量的合成樣本\n    z = torch.randn(needed, latent_dim, device=device)\n    gen_labels = torch.full((needed,), target_class, dtype=torch.long, device=device)\n    gen_samples_final = generator(z, gen_labels).detach().cpu().numpy()\n\n    # 反標準化\n    try:\n        gen_samples_final = scaler.inverse_transform(gen_samples_final)\n    except Exception as e:\n        print(f\"反標準化時發生錯誤：{e}\")\n        return pd.DataFrame(columns=X.columns.tolist() + ['sii'])\n\n    # 組成DataFrame\n    gen_df = pd.DataFrame(gen_samples_final, columns=X.columns)\n    gen_df['sii'] = target_class\n    print(f\"Class {target_class} | 生成了 {needed} 個合成樣本。\")\n    return gen_df\n\n# 訓練 GAN 並產生 sii=2 和 sii=3 的合成資料\nprint(\"\\n開始訓練 GAN 並生成合成樣本...\\n\")\ngen_df_0 = train_gan_for_class(X_0, y_0, 0)\ngen_df_1 = train_gan_for_class(X_1, y_1, 1)\ngen_df_2 = train_gan_for_class_2(X_2, y_2, 2)\ngen_df_3 = train_gan_for_class(X_3, y_3, 3)\n\n# 最終合併資料\ntrain = pd.concat([data_sii_0, data_sii_1, data_sii_2, data_sii_3, gen_df_0, gen_df_1, gen_df_2, gen_df_3], ignore_index=True)\nprint(\"\\n資料平衡後各類別數量：\")\nprint(train['sii'].value_counts())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T19:17:22.461612Z","iopub.execute_input":"2024-12-12T19:17:22.462061Z","iopub.status.idle":"2024-12-12T19:17:26.569221Z","shell.execute_reply.started":"2024-12-12T19:17:22.462025Z","shell.execute_reply":"2024-12-12T19:17:26.567988Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T19:18:31.999689Z","iopub.execute_input":"2024-12-12T19:18:32.000878Z","iopub.status.idle":"2024-12-12T19:18:32.087396Z","shell.execute_reply.started":"2024-12-12T19:18:32.000827Z","shell.execute_reply":"2024-12-12T19:18:32.086224Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featuresCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\n#train_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\n#test_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n\ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tp_rounded = threshold_Rounder(tpm, KappaOPtimizer.x)\n\n    return tp_rounded\n\nimputer = SimpleImputer(strategy='median')\n\nensemble = VotingRegressor(estimators=[\n    ('lgb', Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(random_state=SEED))])),\n    ('xgb', Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(random_state=SEED))])),\n    ('cat', Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(random_state=SEED, silent=True))])),\n    ('rf', Pipeline(steps=[('imputer', imputer), ('regressor', RandomForestRegressor(random_state=SEED))])),\n    ('gb', Pipeline(steps=[('imputer', imputer), ('regressor', GradientBoostingRegressor(random_state=SEED))]))\n])\n\nSubmission3 = TrainML(ensemble, test)\nSubmission3 = pd.DataFrame({\n    'id': sample['id'],\n    'sii': Submission3\n})\nSubmission3.to_csv('submission.csv', index=False)\nprint(Submission3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T19:18:35.824877Z","iopub.execute_input":"2024-12-12T19:18:35.825290Z","iopub.status.idle":"2024-12-12T19:21:59.569718Z","shell.execute_reply.started":"2024-12-12T19:18:35.825252Z","shell.execute_reply":"2024-12-12T19:21:59.568313Z"}},"outputs":[],"execution_count":null}]}