{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":7453542,"sourceType":"datasetVersion","datasetId":921302}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Based\n- https://www.kaggle.com/code/honganzhu/cmi-piu-competition?scriptVersionId=201912528 Version44 LB0.492","metadata":{}},{"cell_type":"markdown","source":"》# Description of Imported Libraries\n\n- **NumPy (`np`)**: Used for efficient numerical operations, including linear algebra and array manipulation.\n- **Pandas (`pd`)**: Provides data structures like DataFrames for handling structured data, essential for data preprocessing.\n- **Polars (`pl`)**: A faster alternative to pandas for DataFrame operations, particularly useful for large datasets.\n- **Matplotlib & Seaborn (`plt`, `sns`)**: Visualization libraries. Matplotlib is used for basic plots, while Seaborn builds on it to create more advanced statistical visualizations.\n- **LightGBM, XGBoost, CatBoost**: Machine learning libraries used for gradient boosting, which is efficient for both regression and classification tasks.\n- **Colorama**: Enhances console output with colored text, making it easier to highlight important results or warnings.\n- **SciPy (`minimize`)**: Provides optimization routines, such as adjusting thresholds to maximize performance metrics like kappa scores.\n- **OS**: Used for file path manipulations and system-related functions.\n- **Scikit-learn (`sklearn`)**: A powerful machine learning library, providing utilities for cross-validation, metrics, and model cloning.\n- **YDF**: A specialized library for machine learning tasks, likely including decision forests.\n- **ThreadPoolExecutor & TQDM**: Tools for parallelizing tasks and displaying progress bars for long-running loops, improving efficiency and usability.\n- **Warnings**: Filters out unwanted warnings to keep the output clean, useful when dealing with noisy outputs from multiple libraries.\n- **IPython display (`clear_output`)**: A utility for clearing the Jupyter notebook output, often used to avoid clutter in long-running scripts.\n","metadata":{}},{"cell_type":"code","source":"!pip -q install /kaggle/input/pytorchtabnet/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pytorch_tabnet.tab_model import TabNetRegressor\nimport torch","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nimport polars as pl\nimport polars.selectors as cs\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator, FormatStrFormatter, PercentFormatter\nimport seaborn as sns\n\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\nseed_everything(2024)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42\nn_splits = 5","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering\n\n- **Feature Selection**: The dataset contains features related to physical characteristics (e.g., BMI, Height, Weight), behavioral aspects (e.g., internet usage), and fitness data (e.g., endurance time). \n- **Categorical Feature Encoding**: Categorical features are mapped to numerical values using custom mappings for each unique category within the dataset. This ensures compatibility with machine learning algorithms that require numerical input.\n- **Time Series Aggregation**: Time series statistics (e.g., mean, standard deviation) from the actigraphy data are computed and merged into the main dataset to create additional features for model training.\n","metadata":{}},{"cell_type":"code","source":"# def process_file(filename, dirname):\n#     df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n#     df.drop('step', axis=1, inplace=True)\n#     return df.describe().values.reshape(-1), filename.split('=')[1]\n\n# def load_time_series(dirname) -> pd.DataFrame:\n#     ids = os.listdir(dirname)\n    \n#     with ThreadPoolExecutor() as executor:\n#         results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n#     stats, indexes = zip(*results)\n    \n#     df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n#     df['id'] = indexes\n#     return df\n\n\n# class AutoEncoder(nn.Module):\n#     def __init__(self, input_dim, encoding_dim):\n#         super(AutoEncoder, self).__init__()\n#         self.encoder = nn.Sequential(\n#             nn.Linear(input_dim, encoding_dim*3),\n#             nn.ReLU(),\n#             nn.Linear(encoding_dim*3, encoding_dim*2),\n#             nn.ReLU(),\n#             nn.Linear(encoding_dim*2, encoding_dim),\n#             nn.ReLU()\n#         )\n#         self.decoder = nn.Sequential(\n#             nn.Linear(encoding_dim, input_dim*2),\n#             nn.ReLU(),\n#             nn.Linear(input_dim*2, input_dim*3),\n#             nn.ReLU(),\n#             nn.Linear(input_dim*3, input_dim),\n#             nn.Sigmoid()\n#         )\n        \n#     def forward(self, x):\n#         encoded = self.encoder(x)\n#         decoded = self.decoder(encoded)\n#         return decoded\n\n\n# def perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n#     # 标准化\n#     scaler = StandardScaler()\n#     df_scaled = scaler.fit_transform(df)\n#     # 转化为tensor\n#     data_tensor = torch.FloatTensor(df_scaled)\n    \n#     input_dim = data_tensor.shape[1]\n#     autoencoder = AutoEncoder(input_dim, encoding_dim)\n#     # MSE\n#     criterion = nn.MSELoss()\n#     optimizer = optim.Adam(autoencoder.parameters())\n    \n#     for epoch in range(epochs):\n#         for i in range(0, len(data_tensor), batch_size):\n#             batch = data_tensor[i : i + batch_size]\n#             optimizer.zero_grad()\n#             reconstructed = autoencoder(batch)\n#             loss = criterion(reconstructed, batch)\n#             loss.backward()\n#             optimizer.step()\n            \n#         if (epoch + 1) % 10 == 0:\n#             print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n                 \n#     with torch.no_grad():\n#         encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n#     df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n#     return df_encoded\n\n# def feature_engineering(df):\n#     season_cols = [col for col in df.columns if 'Season' in col]\n#     df = df.drop(season_cols, axis=1) \n#     df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n#     df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n#     df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n#     df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n#     df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n#     df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n#     df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n#     df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n#     df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n#     df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n#     df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n#     df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n#     df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n#     df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n#     df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    \n#     return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n# test = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n# sample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n# train_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\n# test_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\n# df_train = train_ts.drop('id', axis=1)\n# df_test = test_ts.drop('id', axis=1)\n\n\n# ####\n\n# train_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=100, batch_size=32)\n# test_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=100, batch_size=32)\n\n# time_series_cols = train_ts_encoded.columns.tolist()\n# train_ts_encoded[\"id\"]=train_ts[\"id\"]\n# test_ts_encoded['id']=test_ts[\"id\"]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# ###\n\n\n# # all_ts = pd.concat([train_ts, test_ts])\n# # all_ts = all_ts.drop('id', axis=1)\n# # all_ts_encoded = perform_autoencoder(all_ts, encoding_dim=64, epochs=500, batch_size=32)\n# # train_ts_encoded = all_ts_encoded.head(996)\n# # test_ts_encoded = all_ts_encoded.tail(2)\n# # time_series_cols = train_ts_encoded.columns.tolist()\n# # train_ts_encoded[\"id\"]=train_ts[\"id\"]\n# # test_ts_encoded['id']=test_ts[\"id\"]\n\n\n# train = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\n# test = pd.merge(test, test_ts_encoded, how=\"left\", on='id')\n\n# imputer = KNNImputer(n_neighbors=5)\n\n# numeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\n\n# imputed_data = imputer.fit_transform(train[numeric_cols])\n\n# train_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\n\n# train_imputed['sii'] = train_imputed['sii'].round().astype(int)\n# for col in train.columns:\n#     if col not in numeric_cols:\n#         train_imputed[col] = train[col]\n        \n# train = train_imputed\n\n# train = feature_engineering(train)\n# train = train.dropna(thresh=10, axis=0)\n# test = feature_engineering(test)\n\n# train = train.drop('id', axis=1)\n# test  = test .drop('id', axis=1)   \n\n\n# featuresCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n#                 'CGAS-CGAS_Score', 'Physical-BMI',\n#                 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n#                 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n#                 'Fitness_Endurance-Max_Stage',\n#                 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n#                 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n#                 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n#                 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n#                 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n#                 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n#                 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n#                 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n#                 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n#                 'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n#                 'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n#                 'SDS-SDS_Total_T',\n#                 'PreInt_EduHx-computerinternet_hoursday', 'sii', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n#                 'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n#                 'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW']\n\n# featuresCols += time_series_cols\n\n# train = train[featuresCols]\n# train = train.dropna(subset='sii')\n\n# featuresCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n#                 'CGAS-CGAS_Score', 'Physical-BMI',\n#                 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n#                 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n#                 'Fitness_Endurance-Max_Stage',\n#                 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n#                 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n#                 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n#                 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n#                 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n#                 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n#                 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n#                 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n#                 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n#                 'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n#                 'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n#                 'SDS-SDS_Total_T',\n#                 'PreInt_EduHx-computerinternet_hoursday', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n#                 'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n#                 'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW']\n\n# featuresCols += time_series_cols\n# test = test[featuresCols]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# if np.any(np.isinf(train)):\n#     train = train.replace([np.inf, -np.inf], np.nan)\n\n# def quadratic_weighted_kappa(y_true, y_pred):\n#     return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n# def threshold_Rounder(oof_non_rounded, thresholds):\n#     return np.where(oof_non_rounded < thresholds[0], 0,\n#                     np.where(oof_non_rounded < thresholds[1], 1,\n#                              np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\n# def evaluate_predictions(thresholds, y_true, oof_non_rounded):\n#     rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n#     return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Training and Evaluation\n\n- **Model Types**: Various models are used, including:\n  - **LightGBM**: A gradient-boosting framework known for its speed and efficiency with large datasets.\n  - **XGBoost**: Another powerful gradient-boosting model used for structured data.\n  - **CatBoost**: Optimized for categorical features without the need for extensive preprocessing.\n  - **Voting Regressor**: An ensemble model that combines the predictions of LightGBM, XGBoost, and CatBoost for better accuracy.\n- **Cross-Validation**: Stratified K-Folds cross-validation is employed to split the data into training and validation sets, ensuring balanced class distribution in each fold.\n- **Quadratic Weighted Kappa (QWK)**: The performance of the models is evaluated using QWK, which measures the agreement between predicted and actual values, taking into account the ordinal nature of the target variable.\n- **Threshold Optimization**: The `minimize` function from `scipy.optimize` is used to fine-tune decision thresholds that map continuous predictions to discrete categories (None, Mild, Moderate, Severe).\n","metadata":{}},{"cell_type":"code","source":"# def TrainML(model_class, test_data):\n#     X = train.drop(['sii'], axis=1)\n#     y = train['sii']\n\n#     SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n#     train_S = []\n#     test_S = []\n    \n#     oof_non_rounded = np.zeros(len(y), dtype=float) \n#     oof_rounded = np.zeros(len(y), dtype=int) \n#     test_preds = np.zeros((len(test_data), n_splits))\n\n#     for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n#         X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n#         y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n#         model = clone(model_class)\n#         model.fit(X_train, y_train)\n\n#         y_train_pred = model.predict(X_train)\n#         y_val_pred = model.predict(X_val)\n\n#         oof_non_rounded[test_idx] = y_val_pred\n#         y_val_pred_rounded = y_val_pred.round(0).astype(int)\n#         oof_rounded[test_idx] = y_val_pred_rounded\n\n#         train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n#         val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n#         train_S.append(train_kappa)\n#         test_S.append(val_kappa)\n        \n#         test_preds[:, fold] = model.predict(test_data)\n        \n#         print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n#         clear_output(wait=True)\n\n#     print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n#     print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n#     KappaOPtimizer = minimize(evaluate_predictions,\n#                               x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n#                               method='Nelder-Mead')\n#     assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n#     oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n#     tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n#     print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n#     tpm = test_preds.mean(axis=1)\n#     tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n#     submission = pd.DataFrame({\n#         'id': sample['id'],\n#         'sii': tpTuned\n#     })\n\n#     return submission","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n# Hyperparameter Tuning\n\n- **LightGBM Parameters**: Hyperparameters such as `learning_rate`, `max_depth`, `num_leaves`, and `feature_fraction` are tuned to improve the performance of the LightGBM model. These parameters control the complexity of the model and its ability to generalize to new data.\n- **XGBoost and CatBoost Parameters**: Similar tuning is applied for XGBoost and CatBoost, adjusting parameters such as `n_estimators`, `max_depth`, `learning_rate`, `subsample`, and `regularization` terms (`reg_alpha`, `reg_lambda`). These help in controlling overfitting and ensuring the model's robustness.","metadata":{}},{"cell_type":"code","source":"# # Model parameters for LightGBM\n# Params = {\n#     'learning_rate': 0.046,\n#     'max_depth': 12,\n#     'num_leaves': 478,\n#     'min_data_in_leaf': 13,\n#     'feature_fraction': 0.893,\n#     'bagging_fraction': 0.784,\n#     'bagging_freq': 4,\n#     'lambda_l1': 10,  # Increased from 6.59\n#     'lambda_l2': 0.01,  # Increased from 2.68e-06\n#     'device': 'cpu'\n\n# }\n\n\n# # XGBoost parameters\n# XGB_Params = {\n#     'learning_rate': 0.05,\n#     'max_depth': 6,\n#     'n_estimators': 200,\n#     'subsample': 0.8,\n#     'colsample_bytree': 0.8,\n#     'reg_alpha': 1,  # Increased from 0.1\n#     'reg_lambda': 5,  # Increased from 1\n#     'random_state': SEED,\n#     'tree_method': 'gpu_hist',\n\n# }\n\n\n# CatBoost_Params = {\n#     'learning_rate': 0.05,\n#     'depth': 6,\n#     'iterations': 200,\n#     'random_seed': SEED,\n#     'verbose': 0,\n#     'l2_leaf_reg': 10,  # Increase this value\n#     'task_type': 'GPU'\n\n# }","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # New: TabNet\n\n# from sklearn.base import BaseEstimator, RegressorMixin\n# from sklearn.impute import SimpleImputer\n# from sklearn.model_selection import train_test_split\n# from pytorch_tabnet.callbacks import Callback\n# import os\n# import torch\n# from pytorch_tabnet.callbacks import Callback\n\n# class TabNetWrapper(BaseEstimator, RegressorMixin):\n#     def __init__(self, **kwargs):\n#         self.model = TabNetRegressor(**kwargs)\n#         self.kwargs = kwargs\n#         self.imputer = SimpleImputer(strategy='median')\n#         self.best_model_path = 'best_tabnet_model.pt'\n        \n#     def fit(self, X, y):\n#         # Handle missing values\n#         X_imputed = self.imputer.fit_transform(X)\n        \n#         if hasattr(y, 'values'):\n#             y = y.values\n            \n#         # Create internal validation set\n#         X_train, X_valid, y_train, y_valid = train_test_split(\n#             X_imputed, \n#             y, \n#             test_size=0.2,\n#             random_state=42\n#         )\n        \n#         # Train TabNet model\n#         history = self.model.fit(\n#             X_train=X_train,\n#             y_train=y_train.reshape(-1, 1),\n#             eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n#             eval_name=['valid'],\n#             eval_metric=['mse'],\n#             max_epochs=200,\n#             patience=20,\n#             batch_size=1024,\n#             virtual_batch_size=128,\n#             num_workers=0,\n#             drop_last=False,\n#             callbacks=[\n#                 TabNetPretrainedModelCheckpoint(\n#                     filepath=self.best_model_path,\n#                     monitor='valid_mse',\n#                     mode='min',\n#                     save_best_only=True,\n#                     verbose=True\n#                 )\n#             ]\n#         )\n        \n#         # Load the best model\n#         if os.path.exists(self.best_model_path):\n#             self.model.load_model(self.best_model_path)\n#             os.remove(self.best_model_path)  # Remove temporary file\n        \n#         return self\n    \n#     def predict(self, X):\n#         X_imputed = self.imputer.transform(X)\n#         return self.model.predict(X_imputed).flatten()\n    \n#     def __deepcopy__(self, memo):\n#         # Add deepcopy support for scikit-learn\n#         cls = self.__class__\n#         result = cls.__new__(cls)\n#         memo[id(self)] = result\n#         for k, v in self.__dict__.items():\n#             setattr(result, k, deepcopy(v, memo))\n#         return result\n\n# # TabNet hyperparameters\n# TabNet_Params = {\n#     'n_d': 64,              # Width of the decision prediction layer\n#     'n_a': 64,              # Width of the attention embedding for each step\n#     'n_steps': 5,           # Number of steps in the architecture\n#     'gamma': 1.5,           # Coefficient for feature selection regularization\n#     'n_independent': 2,     # Number of independent GLU layer in each GLU block\n#     'n_shared': 2,          # Number of shared GLU layer in each GLU block\n#     'lambda_sparse': 1e-4,  # Sparsity regularization\n#     'optimizer_fn': torch.optim.Adam,\n#     'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n#     'mask_type': 'entmax',\n#     'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n#     'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n#     'verbose': 1,\n#     'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n# }\n\n# class TabNetPretrainedModelCheckpoint(Callback):\n#     def __init__(self, filepath, monitor='val_loss', mode='min', \n#                  save_best_only=True, verbose=1):\n#         super().__init__()  # Initialize parent class\n#         self.filepath = filepath\n#         self.monitor = monitor\n#         self.mode = mode\n#         self.save_best_only = save_best_only\n#         self.verbose = verbose\n#         self.best = float('inf') if mode == 'min' else -float('inf')\n        \n#     def on_train_begin(self, logs=None):\n#         self.model = self.trainer  # Use trainer itself as model\n        \n#     def on_epoch_end(self, epoch, logs=None):\n#         logs = logs or {}\n#         current = logs.get(self.monitor)\n#         if current is None:\n#             return\n        \n#         # Check if current metric is better than best\n#         if (self.mode == 'min' and current < self.best) or \\\n#            (self.mode == 'max' and current > self.best):\n#             if self.verbose:\n#                 print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n#             self.best = current\n#             if self.save_best_only:\n#                 self.model.save_model(self.filepath)  # Save the entire model","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ensemble Learning and Submission Preparation\n\n- **Ensemble Learning**: The model uses a **Voting Regressor**, which combines the predictions from LightGBM, XGBoost, and CatBoost. This approach is beneficial as it leverages the strengths of multiple models, reducing overfitting and improving overall model performance.\n- **Out-of-Fold (OOF) Predictions**: During cross-validation, out-of-fold predictions are generated for the training set, which helps in model evaluation without data leakage.\n- **Kappa Optimizer**: The Kappa Optimizer ensures that the predicted values are as close to the actual values as possible by adjusting the thresholds used to convert raw model outputs into class labels.\n- **Test Set Predictions**: After the model is trained and thresholds are optimized, the test dataset is processed, and predictions are generated using the ensemble model. These predictions are converted into the appropriate format for submission.\n- **Submission File Creation**: The predictions are saved in a CSV file following the required format for submission (e.g., for a Kaggle competition), which includes columns like `id` and `sii` (Severity Impairment Index).","metadata":{}},{"cell_type":"markdown","source":"# Final Results and Performance Metrics\n\n- **Train and Validation Scores**: After training across multiple folds, the mean Quadratic Weighted Kappa (QWK) score is calculated for both the training and validation datasets, providing an indicator of model performance. \n- **Optimized QWK Score**: The final optimized QWK score after threshold tuning is displayed, showcasing the model's ability to predict the severity levels effectively.\n- **Test Predictions**: The test set predictions are evaluated, and a breakdown of the predicted severity levels (None, Mild, Moderate, Severe) is shown, along with their respective counts.","metadata":{}},{"cell_type":"code","source":"# # Create model instances\n# Light = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\n# XGB_Model = XGBRegressor(**XGB_Params)\n# CatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n# TabNet_Model = TabNetWrapper(**TabNet_Params) # New","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n# **》》》Model1.Train**\n---","metadata":{}},{"cell_type":"code","source":"# voting_model = VotingRegressor(estimators=[\n#     ('lightgbm', Light),\n#     ('xgboost', XGB_Model),\n#     ('catboost', CatBoost_Model),\n#     ('tabnet', TabNet_Model)\n# ],weights=[4.0,4.0,5.0,4.0])\n\n# Submission1 = TrainML(voting_model, test)\n\n# Submission1","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"```\n1.原始\n],weights=[5.0,4.0,4.0,4.0])\nMean Train QWK --> 0.7424\nMean Validation QWK ---> 0.4735\n----> || Optimized QWK SCORE ::  0.533\n\n```\n\n```\n2. 60 1000 epoch\nTraining Folds: 100%|██████████| 5/5 [01:15<00:00, 15.12s/it]\r\nMean Train QWK --> 0.7300\r\nMean Validation QWK ---> 0.4725\r\n----> || Optimized QWK SCORE ::  0.53```\n\n\n```\n2. 64 500 epoch\nTraining Folds: 100%|██████████| 5/5 [01:28<00:00, 17.69s/it]\r\nMean Train QWK --> 0.7312\r\nMean Validation QWK ---> 0.4814\r\n----> || Optimized QWK SCORE ::  0.538\n\n```'''","metadata":{}},{"cell_type":"markdown","source":"# Model2","metadata":{}},{"cell_type":"code","source":"def _max(data):\n    return np.max(data)\n\n\ndef _min(data):\n    return np.min(data)\n\n\ndef _peak(data):\n    \"\"\"\n        求峰值的函数\n    :param data: 信号数据\n    :return: 峰值\n    \"\"\"\n\n    data = np.abs(data)\n    return np.max(data)\n\n\ndef _ppeak(data):\n    \"\"\"\n        求峰峰值的函数\n    :param data: 信号数据\n    :return: 峰峰值\n    \"\"\"\n    return np.max(data) - np.min(data)\n\n\ndef _rms(data):\n    \"\"\"\n        计算有效值\n    :param data: 信号数据\n    :return:有效值\n    \"\"\"\n    data = data ** 2\n    data_mean = data.mean()\n    return data_mean ** 0.5\n\n\ndef _mean(data):\n    \"\"\"\n        求均值\n    :param data:\n    :return:\n    \"\"\"\n\n    return data.mean()\n\n\ndef _median(data):\n    \"\"\"\n        中位数\n    :param data:\n    :return:\n    \"\"\"\n\n    return np.median(data)\n\n\ndef _var(data):\n    \"\"\"\n        方差\n    :param data:\n    :return:\n    \"\"\"\n    return data.var()\n\n\ndef _std(data):\n    \"\"\"\n        标准差\n    :param data:\n    :return:\n    \"\"\"\n\n    return data.std()\n\n\ndef _peak_factor(data):\n    \"\"\"\n        峰值因子\n\n    :param data:\n    :return:\n    \"\"\"\n    rms = _rms(data)\n    ppeak = _ppeak(data)\n\n    return ppeak / (2 * rms)\n\n\ndef _skew(data):\n    \"\"\"\n        偏度\n    :param data:\n    :return:\n    \"\"\"\n\n    std = _std(data)\n    mean = _mean(data)\n\n    return np.mean(((data - mean) / std) ** 3)\n\n\ndef _kurt(data):\n    \"\"\"\n        峭度\n    :param data:\n    :return:\n    \"\"\"\n\n    std = _std(data)\n    mean = _mean(data)\n    return np.mean(((data - mean) / std) ** 4)\n\n\ndef _kurt_factor(data):\n    \"\"\"\n        峭度因子\n    :param data: 峭度\n    :return:\n    \"\"\"\n\n    beta = _kurt(data)\n    rms = _rms(data)\n    return beta / (rms ** 4)\n\n\n    \n\ndef _kurt_factor(data):\n    \"\"\"\n        峭度因子\n    :param data: 峭度\n    :return:\n    \"\"\"\n\n    beta = _kurt(data)\n    rms = _rms(data)\n    return beta / (rms ** 4)\n\n\ndef _margin_factor(data):\n    \"\"\"\n    裕度因子  信号峰值与方根幅值的比值  方根幅值是算术平方根的平均值的平方\n    :param data:\n    :return:\n    \"\"\"\n    peak = _peak(data)\n    root_amp = (np.sum(np.sqrt(np.abs(data))) / len(data)) ** 2\n    return peak / root_amp\n\n\ndef pulse_index(data):\n    \"\"\"\n    脉冲指标\n    :param data:\n    :return:\n    \"\"\"\n    max = _max(data)\n    min = _min(data)\n    p2p = max - min\n    average = np.mean(np.abs(data))  ##### 是否取绝对值\n\n    return p2p / average\n\n\ndef wave_factor(data):\n    \"\"\"\n    轴承波形因子\n    :param data:\n    :return:\n    \"\"\"\n    n = len(data)\n    sum = np.sum(np.abs(data))  #####  是否绝对值\n    rms = _rms(data)\n    return rms / (sum / n)\n\n\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_fft(data, fs, n_fft):\n    \"\"\"\n\n    :param data:\n    :param fs:\n    :param n_fft:\n    :return:  振幅数据和频率分辨率\n    \"\"\"\n    w = np.hanning(n_fft)\n    data_size = len(data)\n    cnt = data_size // n_fft\n    if cnt == 0:\n        data_pad = np.pad(data, (0, n_fft - data_size))\n    else:\n        data_pad = np.pad(data, (0, data_size - cnt * n_fft))\n\n    cnt = len(data_pad) // n_fft\n    tmp = []\n    for i in range(cnt):\n        p = np.fft.fft(w * data_pad[i * n_fft:(i + 1) * n_fft])\n        tmp.append(p)\n    fft_result = np.mean(tmp, axis=0)\n\n    amp = abs(fft_result) * 2 / (n_fft / 2)\n    # 对直流分量额外调整\n    amp[0] = 0.00000001\n    # 根据FFT特性，取一半频谱即可\n    amp_half = amp[:int(len(amp) / 2) + 1]\n    freq = fs / n_fft\n    # freq_array = np.arange(0, len(amp_half)) * freq\n\n    return amp_half, [0,len(amp_half),freq]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def _extract_features(data):\n    features = []\n    features.append(_max(data))\n    features.append(_min(data))\n    features.append(_peak(data))\n    features.append(_ppeak(data))\n    features.append(_rms(data))\n    \n    features.append(_mean(data))\n    features.append(_median(data))\n    features.append(_var(data))\n    features.append(_std(data))\n    features.append(_peak_factor(data))\n    \n    features.append(_skew(data))\n    features.append(_kurt(data))\n    features.append(_kurt_factor(data))\n    features.append(_margin_factor(data))\n    features.append(pulse_index(data))\n    \n    features.append(wave_factor(data))\n    \n    \n    fft_data, _ = get_fft(data, fs=1/5, n_fft=50)\n    features.extend(fft_data[1:])\n    \n    return features","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_s_feat(ser, ):\n\n    return _extract_features(ser)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_file2(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    # print(df.shape)\n\n    num = 120960 if  df.shape[0] > 120960 else  df.shape[0]\n\n    ana_cols = ['X','Y','Z','enmo','anglez','light']\n    # df.drop('step', axis=1, inplace=True)\n    \n    ret = []\n    for col in ana_cols:\n        ret.extend(list(get_s_feat(df[col].values[:num])))\n    \n    \n    return np.array(ret).reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file2(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nclass AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n        \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32,patience=20):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    \n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n    best_loss = float('inf')\n    early_stopping_counter = 0\n    # patience = 20\n    # train\n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n\n         # 检查是否需要早停\n        if loss.item() < best_loss:\n            best_loss = loss.item()\n            early_stopping_counter = 0\n        else:\n            early_stopping_counter += 1\n            if early_stopping_counter >= patience:\n                print(\"Early stopping\")\n                break         \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n    # 编码数据\n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded\n\ndef feature_engineering(df):\n    season_cols = [col for col in df.columns if 'Season' in col]\n    df = df.drop(season_cols, axis=1) \n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    \n    return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\n\n####\n\n# train_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=300, batch_size=32)\n# test_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=300, batch_size=32)\n\ntime_series_cols = df_train.columns.tolist()\n# train_ts_encoded[\"id\"]=train_ts[\"id\"]\n# test_ts_encoded['id']=test_ts[\"id\"]\n\n###\n\n\n# all_ts = pd.concat([train_ts, test_ts])\n# all_ts = all_ts.drop('id', axis=1)\n# all_ts_encoded = perform_autoencoder(all_ts, encoding_dim=64, epochs=500, batch_size=32)\n# train_ts_encoded = all_ts_encoded.head(996)\n# test_ts_encoded = all_ts_encoded.tail(2)\n# time_series_cols = train_ts_encoded.columns.tolist()\n# train_ts_encoded[\"id\"]=train_ts[\"id\"]\n# test_ts_encoded['id']=test_ts[\"id\"]\n\n\n\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n# 删除无效的sii的行\ntrain = train.dropna(subset=['sii'])\n# imputer = KNNImputer(n_neighbors=5)\n\n# numeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\n\n# imputed_data = imputer.fit_transform(train[numeric_cols])\n\n# train_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\n\n# train_imputed['sii'] = train_imputed['sii'].round().astype(int)\n# for col in train.columns:\n#     if col not in numeric_cols:\n#         train_imputed[col] = train[col]\n        \n# train = train_imputed\n\ntrain = feature_engineering(train)\ntrain = train.dropna(thresh=10, axis=0)\ntest = feature_engineering(test)\n\ntrain = train.drop('id', axis=1)\ntest  = test .drop('id', axis=1)   \n\n\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW']\n\nfeaturesCols += time_series_cols\ntest = test[featuresCols]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if np.any(np.isinf(train)):\n    train = train.replace([np.inf, -np.inf], np.nan)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01,  # Increased from 2.68e-06\n    'device': 'cpu'\n\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED,\n    'tree_method': 'gpu_hist',\n\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n    'task_type': 'GPU'\n\n}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# New: TabNet\n\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom pytorch_tabnet.callbacks import Callback\nimport os\nimport torch\nfrom pytorch_tabnet.callbacks import Callback\n\nclass TabNetWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, **kwargs):\n        self.model = TabNetRegressor(**kwargs)\n        self.kwargs = kwargs\n        self.imputer = SimpleImputer(strategy='median')\n        self.best_model_path = 'best_tabnet_model.pt'\n        \n    def fit(self, X, y):\n        # Handle missing values\n        X_imputed = self.imputer.fit_transform(X)\n        \n        if hasattr(y, 'values'):\n            y = y.values\n            \n        # Create internal validation set\n        X_train, X_valid, y_train, y_valid = train_test_split(\n            X_imputed, \n            y, \n            test_size=0.2,\n            random_state=42\n        )\n        \n        # Train TabNet model\n        history = self.model.fit(\n            X_train=X_train,\n            y_train=y_train.reshape(-1, 1),\n            eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n            eval_name=['valid'],\n            eval_metric=['mse'],\n            max_epochs=500,\n            patience=20,\n            batch_size=1024,\n            virtual_batch_size=128,\n            num_workers=0,\n            drop_last=False,\n            callbacks=[\n                TabNetPretrainedModelCheckpoint(\n                    filepath=self.best_model_path,\n                    monitor='valid_mse',\n                    mode='min',\n                    save_best_only=True,\n                    verbose=True\n                )\n            ]\n        )\n        \n        # Load the best model\n        if os.path.exists(self.best_model_path):\n            self.model.load_model(self.best_model_path)\n            os.remove(self.best_model_path)  # Remove temporary file\n        \n        return self\n    \n    def predict(self, X):\n        X_imputed = self.imputer.transform(X)\n        return self.model.predict(X_imputed).flatten()\n    \n    def __deepcopy__(self, memo):\n        # Add deepcopy support for scikit-learn\n        cls = self.__class__\n        result = cls.__new__(cls)\n        memo[id(self)] = result\n        for k, v in self.__dict__.items():\n            setattr(result, k, deepcopy(v, memo))\n        return result\n\n# TabNet hyperparameters\nTabNet_Params = {\n    'n_d': 64,              # Width of the decision prediction layer\n    'n_a': 64,              # Width of the attention embedding for each step\n    'n_steps': 5,           # Number of steps in the architecture\n    'gamma': 1.5,           # Coefficient for feature selection regularization\n    'n_independent': 2,     # Number of independent GLU layer in each GLU block\n    'n_shared': 2,          # Number of shared GLU layer in each GLU block\n    'lambda_sparse': 1e-4,  # Sparsity regularization\n    'optimizer_fn': torch.optim.Adam,\n    'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n    'mask_type': 'entmax',\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'verbose': 1,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n    def __init__(self, filepath, monitor='val_loss', mode='min', \n                 save_best_only=True, verbose=1):\n        super().__init__()  # Initialize parent class\n        self.filepath = filepath\n        self.monitor = monitor\n        self.mode = mode\n        self.save_best_only = save_best_only\n        self.verbose = verbose\n        self.best = float('inf') if mode == 'min' else -float('inf')\n        \n    def on_train_begin(self, logs=None):\n        self.model = self.trainer  # Use trainer itself as model\n        \n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is None:\n            return\n        \n        # Check if current metric is better than best\n        if (self.mode == 'min' and current < self.best) or \\\n           (self.mode == 'max' and current > self.best):\n            if self.verbose:\n                print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n            self.best = current\n            if self.save_best_only:\n                self.model.save_model(self.filepath)  # Save the entire model","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\nTabNet_Model = TabNetWrapper(**TabNet_Params) # New","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from mlxtend.regressor import StackingCVRegressor\nfrom sklearn.ensemble import StackingRegressor\nfrom sklearn.metrics import make_scorer\nfrom sklearn.linear_model import LassoCV,RidgeCV\nqwk_scorer = make_scorer(quadratic_weighted_kappa, greater_is_better=True)\nstacking_model = StackingRegressor(\n    estimators=[\n        ('lightgbm', Light),\n        ('xgboost', XGB_Model),\n        ('catboost', CatBoost_Model),\n    ],\n    final_estimator=RidgeCV(cv=10),\n    cv=10\n)\nstacking_model\n\n# stackingCV_model = StackingCVRegressor","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission2 = TrainML(stacking_model,test)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission2","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission2.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# sub1 = Submission1\n# sub2 = Submission2\n# # sub3 = Submission3\n# # sub4 = Submission4\n# # sub5 = Submission5\n\n# sub1 = sub1.sort_values(by='id').reset_index(drop=True)\n# sub2 = sub2.sort_values(by='id').reset_index(drop=True)\n# # sub3 = sub3.sort_values(by='id').reset_index(drop=True)\n# # sub4 = sub4.sort_values(by='id').reset_index(drop=True)\n# # sub5 = sub5.sort_values(by='id').reset_index(drop=True)\n\n# combined = pd.DataFrame({\n#     'id': sub1['id'],\n#     'sii_1': sub1['sii'],\n#     'sii_2': sub2['sii'],\n#     # 'sii_3': sub3['sii'],\n#     # 'sii_4': sub4['sii'],\n#     # 'sii_5': sub5['sii'],\n# })\n\n# def majority_vote(row):\n#     return row.mode()[0]\n\n# combined['final_sii'] = combined[['sii_1', 'sii_2']].apply(majority_vote, axis=1)\n\n# final_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\n# # final_submission.iloc[0,1] = 2\n# final_submission.to_csv('submission.csv', index=False)\n\n# print(\"Majority voting completed and saved to 'Final_Submission.csv'\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}