{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":9761094,"sourceType":"datasetVersion","datasetId":5977374}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-12-19T05:38:32.413223Z","iopub.execute_input":"2024-12-19T05:38:32.413665Z","iopub.status.idle":"2024-12-19T05:38:32.419672Z","shell.execute_reply.started":"2024-12-19T05:38:32.413630Z","shell.execute_reply":"2024-12-19T05:38:32.418354Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip -q install /kaggle/input/tabnet/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2024-12-19T05:38:32.423194Z","iopub.execute_input":"2024-12-19T05:38:32.424342Z","iopub.status.idle":"2024-12-19T05:39:17.751161Z","shell.execute_reply.started":"2024-12-19T05:38:32.424261Z","shell.execute_reply":"2024-12-19T05:39:17.749280Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# IMPORT","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nimport random\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.ensemble import StackingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_selection import RFE, RFECV\nfrom sklearn.metrics import make_scorer\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\n\nfrom pytorch_tabnet.callbacks import Callback\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nSEED = 42\nn_splits = 5","metadata":{"execution":{"iopub.status.busy":"2024-12-19T05:39:17.753393Z","iopub.execute_input":"2024-12-19T05:39:17.754051Z","iopub.status.idle":"2024-12-19T05:39:40.372013Z","shell.execute_reply.started":"2024-12-19T05:39:17.753986Z","shell.execute_reply":"2024-12-19T05:39:40.370580Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Function define and read data","metadata":{}},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"Stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    \n    return df\n\ndef feature_engineering(df, is_drope_season = True):\n    if is_drope_season:\n        season_cols = [col for col in df.columns if 'Season' in col]\n        df = df.drop(season_cols, axis=1)\n        \n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    df['BMI_PHR'] = df['Physical-BMI'] * df['Physical-HeartRate']\n\n    return df\n\ndef feature_select():\n    pass\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n    \n    \n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n    \n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_yal = y.iloc[train_idx], y.iloc[test_idx]\n        \n        model = clone(model_class)\n        model.fit(X_train, y_train)\n        \n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n        \n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n        \n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_yal, y_val_pred_rounded)\n            \n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n        \n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n    \n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission","metadata":{"execution":{"iopub.status.busy":"2024-12-19T05:39:40.373897Z","iopub.execute_input":"2024-12-19T05:39:40.374873Z","iopub.status.idle":"2024-12-19T05:39:40.414942Z","shell.execute_reply.started":"2024-12-19T05:39:40.374815Z","shell.execute_reply":"2024-12-19T05:39:40.413106Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_origin = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest_origin = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample_origin = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n\ntrain_ts_origin = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts_origin = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n","metadata":{"execution":{"iopub.status.busy":"2024-12-19T05:39:40.419486Z","iopub.execute_input":"2024-12-19T05:39:40.421182Z","iopub.status.idle":"2024-12-19T05:41:28.910426Z","shell.execute_reply.started":"2024-12-19T05:39:40.421135Z","shell.execute_reply":"2024-12-19T05:41:28.908986Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# New: TabNet\n\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom pytorch_tabnet.callbacks import Callback\nimport os\nimport torch\nfrom pytorch_tabnet.callbacks import Callback\n\nfrom pytorch_tabnet.tab_model import TabNetRegressor\nimport torch\n\nclass TabNetWrapper(BaseEstimator, RegressorMixin):\n    def __init__(self, **kwargs):\n        self.model = TabNetRegressor(**kwargs)\n        self.kwargs = kwargs\n        self.imputer = SimpleImputer(strategy='median')\n        self.best_model_path = 'best_tabnet_model.pt'\n        \n    def fit(self, X, y):\n        # Handle missing values\n        X_imputed = self.imputer.fit_transform(X)\n        \n        if hasattr(y, 'values'):\n            y = y.values\n            \n        # Create internal validation set\n        X_train, X_valid, y_train, y_valid = train_test_split(\n            X_imputed, \n            y, \n            test_size=0.2,\n            random_state=42\n        )\n        \n        # Train TabNet model\n        history = self.model.fit(\n            X_train=X_train,\n            y_train=y_train.reshape(-1, 1),\n            eval_set=[(X_valid, y_valid.reshape(-1, 1))],\n            eval_name=['valid'],\n            eval_metric=['mse'],\n            max_epochs=500,\n            patience=50,\n            batch_size=1024,\n            virtual_batch_size=128,\n            num_workers=0,\n            drop_last=False,\n            callbacks=[\n                TabNetPretrainedModelCheckpoint(\n                    filepath=self.best_model_path,\n                    monitor='valid_mse',\n                    mode='min',\n                    save_best_only=True,\n                    verbose=True\n                )\n            ]\n        )\n        \n        # Load the best model\n        if os.path.exists(self.best_model_path):\n            self.model.load_model(self.best_model_path)\n            os.remove(self.best_model_path)  # Remove temporary file\n        \n        return self\n    \n    def predict(self, X):\n        X_imputed = self.imputer.transform(X)\n        return self.model.predict(X_imputed).flatten()\n    \n    def __deepcopy__(self, memo):\n        # Add deepcopy support for scikit-learn\n        cls = self.__class__\n        result = cls.__new__(cls)\n        memo[id(self)] = result\n        for k, v in self.__dict__.items():\n            setattr(result, k, deepcopy(v, memo))\n        return result\n\n# TabNet hyperparameters\nTabNet_Params = {\n    'n_d': 64,              # Width of the decision prediction layer\n    'n_a': 64,              # Width of the attention embedding for each step\n    'n_steps': 5,           # Number of steps in the architecture\n    'gamma': 1.5,           # Coefficient for feature selection regularization\n    'n_independent': 2,     # Number of independent GLU layer in each GLU block\n    'n_shared': 2,          # Number of shared GLU layer in each GLU block\n    'lambda_sparse': 1e-4,  # Sparsity regularization\n    'optimizer_fn': torch.optim.Adam,\n    'optimizer_params': dict(lr=2e-2, weight_decay=1e-5),\n    'mask_type': 'entmax',\n    'scheduler_params': dict(mode=\"min\", patience=10, min_lr=1e-5, factor=0.5),\n    'scheduler_fn': torch.optim.lr_scheduler.ReduceLROnPlateau,\n    'verbose': -1,\n    'device_name': 'cuda' if torch.cuda.is_available() else 'cpu'\n}\n\nclass TabNetPretrainedModelCheckpoint(Callback):\n    def __init__(self, filepath, monitor='val_loss', mode='min', \n                 save_best_only=True, verbose=1):\n        super().__init__()  # Initialize parent class\n        self.filepath = filepath\n        self.monitor = monitor\n        self.mode = mode\n        self.save_best_only = save_best_only\n        self.verbose = verbose\n        self.best = float('inf') if mode == 'min' else -float('inf')\n        \n    def on_train_begin(self, logs=None):\n        self.model = self.trainer  # Use trainer itself as model\n        \n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is None:\n            return\n        \n        # Check if current metric is better than best\n        if (self.mode == 'min' and current < self.best) or \\\n           (self.mode == 'max' and current > self.best):\n            if self.verbose:\n                print(f'\\nEpoch {epoch}: {self.monitor} improved from {self.best:.4f} to {current:.4f}')\n            self.best = current\n            if self.save_best_only:\n                self.model.save_model(self.filepath)  # Save the entire model","metadata":{"execution":{"iopub.status.busy":"2024-12-19T05:41:28.913213Z","iopub.execute_input":"2024-12-19T05:41:28.913879Z","iopub.status.idle":"2024-12-19T05:41:28.950226Z","shell.execute_reply.started":"2024-12-19T05:41:28.913805Z","shell.execute_reply":"2024-12-19T05:41:28.948195Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"train = train_origin.copy()\ntest = test_origin.copy()\nsample = sample_origin.copy()\ntrain_ts = train_ts_origin.copy()\ntest_ts = test_ts_origin.copy()\n\n\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\nclass ResidualBlock(nn.Module):\n    def __init__(self, input_dim, output_dim):\n        super(ResidualBlock, self).__init__()\n        self.layer = nn.Sequential(\n            nn.Linear(input_dim, output_dim),\n            nn.ReLU(),\n            nn.BatchNorm1d(output_dim)\n        )\n\n    def forward(self, x):\n        return self.layer(x) + x  # 残差连接\n\nclass AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n        \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n    \n    \ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32, seed=SEED, patience=50):\n    # 设置随机种子以确保结果可重复\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n\n    # 数据标准化\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    \n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n                 \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded\n\n\n# 948\n# _, train_ts_encoded = perform_autoencoder_founder(df_train, encoding_dim=60, epochs=100, batch_size=32, target_loss = 1.6646)\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=90, batch_size=32, seed=948)\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=90, batch_size=32, seed=948)\n\ntime_series_cols = train_ts_encoded.columns.tolist()\ntrain_ts_encoded[\"id\"]=train_ts[\"id\"]\ntest_ts_encoded['id']=test_ts[\"id\"]\ntrain = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\ntest = pd.merge(test, test_ts_encoded, how=\"left\", on='id')\n\nimputer = KNNImputer(n_neighbors=5)\nnumeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\nimputed_data = imputer.fit_transform(train[numeric_cols])\ntrain_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\ntrain_imputed['sii'] = train_imputed['sii'].round().astype(int)\nfor col in train.columns:\n    if col not in numeric_cols:\n        train_imputed[col] = train[col]\n        \ntrain = train_imputed\n\n\ntrain = feature_engineering(train)\ntrain = train.dropna(thresh=10,axis=0) ## thresh=10\ntest = feature_engineering(test)\n\n\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW','BMI_PHR']\n\nfeaturesCols += time_series_cols\n\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday', 'BMI_Age','Internet_Hours_Age','BMI_Internet_Hours',\n                'BFP_BMI', 'FFMI_BFP', 'FMI_BFP', 'LST_TBW', 'BFP_BMR', 'BFP_DEE', 'BMR_Weight', 'DEE_Weight',\n                'SMM_Height', 'Muscle_to_Fat', 'Hydration_Status', 'ICW_TBW','BMI_PHR']\nfeaturesCols += time_series_cols\n\ntest = test[featuresCols]\n\n\nif np.any(np.isinf(train)):\n    train = train.replace([np.inf, -np.inf], np.nan) # np.nan\n\nimputer = SimpleImputer(strategy='most_frequent')\n\n# TabNet_Model = TabNetWrapper(**TabNet_Params) # New\nTabNet_Model = Pipeline(steps=[\n                               ('regressor',TabNetWrapper(**TabNet_Params))])\n\n# TabNet_Model = TabNetWrapper(**TabNet_Params)\n# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01,  # Increased from 2.68e-06\n#     'device': 'gpu'\n}\n\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED,\n    'tree_method': 'exact',\n#     'tree_method': 'gpu_hist',\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6, # 6\n    'iterations': 200, # 200\n    'random_seed': SEED,\n    'verbose': 0,\n    'l2_leaf_reg': 10,  # Increase this value\n#     'task_type': 'GPU'\n}\n# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\nLight = Pipeline(steps=[\n                               ('regressor',Light)])\n\nXGB_Model = Pipeline(steps=[\n                               ('regressor',XGB_Model)])\n\nCatBoost_Model = Pipeline(steps=[ \n                               ('regressor',CatBoost_Model)])\n\n# voting_model = VotingRegressor(estimators=[\n#     ('lightgbm',  Light),\n#     ('xgboost',  XGB_Model),\n#     ('catboost', CatBoost_Model),\n# #     ('rf', Pipeline(steps=[('imputer', imputer),  ('regressor', RandomForestRegressor(random_state=SEED))])),\n# #     ('gb', Pipeline(steps=[('imputer', imputer),  ('regressor', GradientBoostingRegressor(random_state=SEED))])),\n#     ('tab', Pipeline(steps=[('imputer', imputer), ('regressor',TabNet_Model)]))\n# ])\n\n\n# Combine models using Voting Regressor\nfrom sklearn.model_selection import cross_val_score\n\nweights = [4.0,4.0,5.0,4.0]\nmodels = [Light, XGB_Model, CatBoost_Model, TabNet_Model]\n\nX = train.drop(['sii'], axis=1)\ny = train['sii']\n\n# for model in models:\n#     score = cross_val_score(model, X, y, cv=5, scoring='neg_mean_squared_error').mean()\n#     weights.append(1 / abs(score))\n\nvoting_model = VotingRegressor(\n    estimators=[\n        ('lightgbm', Light),\n        ('xgboost', XGB_Model),\n        ('catboost', CatBoost_Model),\n        ('tabnet', TabNet_Model)\n    ],\n    weights=weights\n)\n\n\nSubmission1 = TrainML(voting_model, test)\nprint(weights)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T06:22:18.197087Z","iopub.execute_input":"2024-12-19T06:22:18.197671Z","iopub.status.idle":"2024-12-19T06:27:56.596322Z","shell.execute_reply.started":"2024-12-19T06:22:18.197629Z","shell.execute_reply":"2024-12-19T06:27:56.595196Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Mean Train QWK --> 0.7853\r\nMean Validation QWK ---> 0.5013\r\n----> || Optimized QWK SCORE ::  0.542","metadata":{}},{"cell_type":"markdown","source":"### 0.494 [80.0,20.0,30.0,20.0]\nMean Train QWK --> 0.7839\r\nMean Validation QWK ---> 0.5065\r\n----> || Optimized QWK SCORE ::  0.543","metadata":{}},{"cell_type":"markdown","source":"<!-- ### 0.494 best\n[6.0,2.0,2.0,2.0]\n\nMean Train QWK --> 0.7389\r\nMean Validation QWK ---> 0.4788\r\n----> || Optimized QWK SCORE ::  0.524\r\n[4.0, 4.0, 5.0, 4 -->.0]44","metadata":{}},{"cell_type":"markdown","source":"### 0.494\n\ndf_train, encoding_dim=60, **epochs=130**, batch_size=16, seed=948\n\nMean Train QWK --> 0.7581\n\nMean Validation QWK ---> 0.4922\n\n----> || Optimized QWK SCORE ::  0.531\n","metadata":{}},{"cell_type":"markdown","source":"### 0.493\n\ndf_train, encoding_dim=60, **epochs=90**, batch_size=16, seed=948\n\nMean Train QWK --> 0.7442\n\nMean Validation QWK ---> 0.4864\n\n0.6535877452297769\n\n----> || Optimized QWK SCORE ::  0.529","metadata":{}},{"cell_type":"markdown","source":"### 0.493\n\ndf_train, encoding_dim=60, **epochs=110**, batch_size=16, seed=948\n\nMean Train QWK --> 0.7539\n\nMean Validation QWK ---> 0.4880\n\n----> || Optimized QWK SCORE ::  0.521\n","metadata":{}},{"cell_type":"markdown","source":"### 0.491\ndf_train, **encoding_dim=70**, **epochs=110**, batch_size=16, seed=948\n\nMean Train QWK --> 0.7397\n\nMean Validation QWK ---> 0.4830\n\n----> || Optimized QWK SCORE ::  0.532","metadata":{}},{"cell_type":"markdown","source":"### 0.491\n\ndf_train, encoding_dim=60, epochs=1000, batch_size=16, seed=948","metadata":{}},{"cell_type":"markdown","source":"### 0.490\n\ndf_train, encoding_dim=70, epochs=1000, batch_size=16, seed=948\n\nMean Train QWK --> 0.7484\n\nMean Validation QWK ---> 0.4919\n\n----> || Optimized QWK SCORE ::  0.542","metadata":{}},{"cell_type":"markdown","source":"### 0.490\n\ndf_train, encoding_dim=60, **epochs=140**, batch_size=32, seed=948\n\nMean Train QWK --> 0.7440\n\nMean Validation QWK ---> 0.4775\n\n0.6418010752688171\n\n----> || Optimized QWK SCORE ::  0.527","metadata":{}},{"cell_type":"markdown","source":"### 0.488\n\ndf_train, encoding_dim=70, **epochs=160**, batch_size=16, seed=948\n\nMean Train QWK --> 0.7386\n\nMean Validation QWK ---> 0.4792\n\n----> || Optimized QWK SCORE ::  0.530","metadata":{}},{"cell_type":"markdown","source":"### 0.487\n\ndf_train, encoding_dim=70, **epochs=300**, batch_size=16, seed=948\n\nMean Train QWK --> 0.7484\n\nMean Validation QWK ---> 0.4898\n\n----> || Optimized QWK SCORE ::  0.532\n","metadata":{}},{"cell_type":"markdown","source":"### 0.484\n\ndf_train, encoding_dim=60, **epochs=80**, batch_size=16, seed=948\n\nMean Train QWK --> 0.7337\n\nMean Validation QWK ---> 0.4777\n\n0.6510835491345236\n\n----> || Optimized QWK SCORE ::  0.537","metadata":{}},{"cell_type":"markdown","source":"### 0.482\n\ndf_train, encoding_dim=70, **epochs=100**, batch_size=16, seed=948\n\nMean Train QWK --> 0.7490\n\nMean Validation QWK ---> 0.4848\n\n----> || Optimized QWK SCORE ::  0.533","metadata":{}},{"cell_type":"markdown","source":"### 0.494\nMean Train QWK --> 0.7354\n\nMean Validation QWK ---> 0.4742\n\n0.6448191460429697\n\n----> || Optimized QWK SCORE ::  0.527","metadata":{}},{"cell_type":"markdown","source":"## 1","metadata":{}},{"cell_type":"markdown","source":"0.538 0.529  0.534 **0.538** 0.531 0.526  0.528（leakrelu）0.522(relu) 0.537（resnet）","metadata":{}},{"cell_type":"code","source":"Submission1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T05:46:51.448578Z","iopub.execute_input":"2024-12-19T05:46:51.449730Z","iopub.status.idle":"2024-12-19T05:46:51.470124Z","shell.execute_reply.started":"2024-12-19T05:46:51.449659Z","shell.execute_reply":"2024-12-19T05:46:51.468885Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 0.494\nMean Train QWK --> 0.7595\n\nMean Validation QWK ---> 0.3926\n\n----> || Optimized QWK SCORE ::  0.457","metadata":{}},{"cell_type":"markdown","source":"## 2","metadata":{}},{"cell_type":"markdown","source":"0.448(with fearture engineering) 0.458(without fearture engineering) 0.449 **0.457**","metadata":{}},{"cell_type":"code","source":"train = train_origin.copy()\ntest = test_origin.copy()\nsample = sample_origin.copy()\ntrain_ts = train_ts_origin.copy()\ntest_ts = test_ts_origin.copy()\n\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)   \n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        \ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n    \ntrain = train.dropna(thresh=10,axis=0) ## thresh=10\n\n\ndef TrainML2(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.49, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    thresholds = KappaOPtimizer.x\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, thresholds)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    fold_weights = [1.25, 1.0, 1.0, 1.0, 1.0]\n    tpm = test_preds.dot(fold_weights) / np.sum(fold_weights)\n    tpTuned = threshold_Rounder(tpm, thresholds)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission\n\n# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01  # Increased from 2.68e-06\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED,\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'cat_features': cat_c,\n    'verbose': 0,\n    'l2_leaf_reg': 10  # Increase this value\n}\n\n# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm',  Light),\n    ('xgboost',  XGB_Model),\n    ('catboost', CatBoost_Model),\n])\n\n# Create stacking model\nstacking_model = StackingRegressor(estimators=[\n    ('lightgbm',  Light),\n    ('xgboost',  XGB_Model),\n    ('catboost', CatBoost_Model),\n#     ('tabnet', TabNet_Model)  # New:TabNet\n])\n\n\n# Train the ensemble model\nSubmission2 = TrainML2(voting_model, test)\n\n\n# Save submission\n#Submission2.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T05:46:51.471766Z","iopub.execute_input":"2024-12-19T05:46:51.472141Z","iopub.status.idle":"2024-12-19T05:48:01.946005Z","shell.execute_reply.started":"2024-12-19T05:46:51.472105Z","shell.execute_reply":"2024-12-19T05:48:01.944787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featuresCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ntrain = train_origin.copy()\ntest = test_origin.copy()\nsample = sample_origin.copy()\ntrain_ts = train_ts_origin.copy()\ntest_ts = test_ts_origin.copy()\n\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n\ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n    \n\n\nimputer = SimpleImputer(strategy='median')\n\nlgb_pip = Pipeline(steps=[('imputer', imputer),  ('regressor', LGBMRegressor(random_state=SEED))])\nxgb_pip = Pipeline(steps=[('imputer', imputer),  ('regressor', XGBRegressor(random_state=SEED))])\ncat_pip = Pipeline(steps=[('imputer', imputer),  ('regressor', CatBoostRegressor(random_state=SEED, silent=True))])\nrf_pip = Pipeline(steps=[('imputer', imputer),  ('regressor', RandomForestRegressor(random_state=SEED))])\ngb_pip = Pipeline(steps=[('imputer', imputer),  ('regressor', GradientBoostingRegressor(random_state=SEED))])\n\n\n\nvoting_model = VotingRegressor(estimators=[\n    ('lgb', lgb_pip),\n    ('xgb', xgb_pip),\n    ('cat', cat_pip),\n    ('rf', rf_pip),\n    ('gb', gb_pip),\n#     ('tab', Pipeline(steps=[('imputer', imputer), ('regressor',TabNet_Model)]))\n])\n\n\nstacking_model = StackingRegressor(estimators=[\n    ('lgb', Pipeline(steps=[('imputer', imputer),  ('regressor', LGBMRegressor(random_state=SEED))])),\n    ('xgb', Pipeline(steps=[('imputer', imputer),  ('regressor', XGBRegressor(random_state=SEED))])),\n    ('cat', Pipeline(steps=[('imputer', imputer),  ('regressor', CatBoostRegressor(random_state=SEED, silent=True))])),\n    ('rf', Pipeline(steps=[('imputer', imputer),  ('regressor', RandomForestRegressor(random_state=SEED))])),\n    ('gb', Pipeline(steps=[('imputer', imputer),  ('regressor', GradientBoostingRegressor(random_state=SEED))])),\n    #('tab', Pipeline(steps=[('imputer', imputer), ('regressor',TabNet_Model)]))\n])\n\nSubmission3 = TrainML(voting_model, test)\n# Submission3 = pd.DataFrame({\n#     'id': sample['id'],\n#     'sii': Submission3\n# })","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T05:48:01.948285Z","iopub.execute_input":"2024-12-19T05:48:01.948803Z","iopub.status.idle":"2024-12-19T05:50:41.983906Z","shell.execute_reply.started":"2024-12-19T05:48:01.948749Z","shell.execute_reply":"2024-12-19T05:50:41.982539Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 0.494\nMean Train QWK --> 0.9175\n\nMean Validation QWK ---> 0.3803\n\n----> || Optimized QWK SCORE ::  0.450","metadata":{}},{"cell_type":"markdown","source":"## 3","metadata":{}},{"cell_type":"markdown","source":"0.469 **0.441** 0.417 0.443","metadata":{}},{"cell_type":"code","source":"Submission3","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T05:50:41.985879Z","iopub.execute_input":"2024-12-19T05:50:41.986384Z","iopub.status.idle":"2024-12-19T05:50:41.999390Z","shell.execute_reply.started":"2024-12-19T05:50:41.986320Z","shell.execute_reply":"2024-12-19T05:50:41.998320Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Merge Submission","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# 假设你有多个 Submission 数据框\nsubmissions = [Submission1 ,Submission2, Submission3]\nsubmission_names = ['sii_1', 'sii_2', 'sii_3']\n\n# 对每个 Submission 进行排序和重置索引\nfor i, sub in enumerate(submissions):\n    submissions[i] = sub.sort_values(by='id').reset_index(drop=True)\n\n# 创建一个新的 DataFrame 用于合并结果\ncombined = pd.DataFrame({'id': submissions[0]['id']})\n\n# 将每个 Submission 的结果添加到 combined DataFrame\nfor name, sub in zip(submission_names, submissions):\n    combined[name] = sub['sii']\n\ndef majority_vote(row):\n    return row.mode()[0]\n\n# 进行投票\ncombined['final_sii'] = combined[submission_names].apply(majority_vote, axis=1)\n\n# 准备最终的提交结果\nfinal_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\n# 保存到 CSV 文件\nfinal_submission.to_csv('submission.csv', index=False)\n\nprint(\"Majority voting completed and saved to 'submission.csv'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T05:50:42.000928Z","iopub.execute_input":"2024-12-19T05:50:42.001287Z","iopub.status.idle":"2024-12-19T05:50:42.033111Z","shell.execute_reply.started":"2024-12-19T05:50:42.001252Z","shell.execute_reply":"2024-12-19T05:50:42.031515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T05:50:42.035162Z","iopub.execute_input":"2024-12-19T05:50:42.035548Z","iopub.status.idle":"2024-12-19T05:50:42.048662Z","shell.execute_reply.started":"2024-12-19T05:50:42.035510Z","shell.execute_reply":"2024-12-19T05:50:42.046798Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- 0\t00008ff9\t1\n- 1\t000fd460\t0\n- 2\t00105258\t0\n- 3\t00115b9f\t0\n- 4\t0016bb22\t0\n- 5\t001f3379\t1\n- 6\t0038ba98\t0\n- 7\t0068a485\t0\n- 8\t0069fbed\t1\n- 9\t0083e397\t0\n- 10\t0087dd65\t0\n- 11\t00abe655\t0\n- 12\t00ae59c9\t1\n- 13\t00af6387\t1\n- 14\t00bd4359\t1\n- 15\t00c0cd71\t1\n- 16\t00d56d4b\t0\n- 17\t00d9913d\t0\n- 18\t00e6167c\t0\n- 19\t00ebc35d\t1\n","metadata":{}}]}