{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"},{"sourceId":7453542,"sourceType":"datasetVersion","datasetId":921302}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Install Required Package\n!pip -q install /kaggle/input/d/ryati131457/pytorchtabnet/pytorch_tabnet-4.1.0-py3-none-any.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T20:22:08.280167Z","iopub.execute_input":"2024-12-08T20:22:08.280523Z","iopub.status.idle":"2024-12-08T20:22:51.971088Z","shell.execute_reply.started":"2024-12-08T20:22:08.280492Z","shell.execute_reply":"2024-12-08T20:22:51.969267Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone, BaseEstimator, RegressorMixin, ClassifierMixin\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score, mean_squared_error\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.decomposition import PCA\nfrom sklearn.datasets import make_classification\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nimport matplotlib.pyplot as plt\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom pytorch_tabnet.tab_model import TabNetRegressor\nfrom pytorch_tabnet.callbacks import Callback\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.pipeline import Pipeline\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nSEED = 42\nn_splits = 5\n\nimport random\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\nseed_everything(2024)\n\n# Add this after the imports and before any function definitions\nfeaturesCols = ['Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-CGAS_Score', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total',\n                'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T',\n                'PreInt_EduHx-computerinternet_hoursday']\n\n# Also add model parameters\nLightGBM_Params = {\n    'learning_rate': 0.02,\n    'max_depth': 10,\n    'num_leaves': 512,\n    'min_data_in_leaf': 15,\n    'feature_fraction': 0.85,\n    'bagging_fraction': 0.85,\n    'bagging_freq': 5,\n    'lambda_l1': 0.3,\n    'lambda_l2': 0.3,\n    'min_gain_to_split': 0.01,\n    'max_bin': 255,\n    'n_estimators': 1000,\n    'verbose': -1,\n    'n_jobs': -1\n}\n\nXGB_Params = {\n    'learning_rate': 0.02,\n    'max_depth': 8,\n    'min_child_weight': 2,\n    'subsample': 0.85,\n    'colsample_bytree': 0.85,\n    'gamma': 0.05,\n    'reg_alpha': 0.1,\n    'reg_lambda': 1,\n    'n_estimators': 1000,\n    'tree_method': 'hist',\n    'n_jobs': -1\n}\n\nCatBoost_Params = {\n    'learning_rate': 0.02,\n    'depth': 8,\n    'l2_leaf_reg': 2,\n    'min_data_in_leaf': 15,\n    'random_strength': 0.1,\n    'one_hot_max_size': 10,\n    'iterations': 1000,\n    'verbose': 0,\n    'thread_count': -1,\n    'bootstrap_type': 'Bernoulli',\n    'subsample': 0.85,\n    'early_stopping_rounds': 50\n}\n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    scores = []\n    predictions = []\n    \n    for seed in range(5):\n        print(f\"\\nSeed {seed + 1}/5\")\n        SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n        \n        fold_scores = []\n        oof_non_rounded = np.zeros(len(y), dtype=float)\n        test_preds = np.zeros((len(test_data), n_splits))\n        \n        for fold, (train_idx, val_idx) in enumerate(SKF.split(X, y)):\n            X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n            y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n            \n            model = clone(model_class)\n            model.fit(X_train, y_train)\n            \n            val_pred = model.predict(X_val)\n            test_fold_pred = model.predict(test_data)\n            \n            oof_non_rounded[val_idx] = val_pred\n            test_preds[:, fold] = test_fold_pred\n            \n            score = quadratic_weighted_kappa(y_val, val_pred.round())\n            fold_scores.append(score)\n            print(f\"Fold {fold + 1}: QWK = {score:.4f}\")\n        \n        print(f\"Seed {seed + 1} mean QWK: {np.mean(fold_scores):.4f}\")\n        predictions.append(test_preds.mean(axis=1))\n    \n    final_predictions = np.mean(predictions, axis=0)\n    \n    # Optimize thresholds\n    KappaOPtimizer = minimize(evaluate_predictions,\n                          x0=[0.5, 1.5, 2.5], \n                          args=(y, oof_non_rounded),\n                          method='Nelder-Mead')\n    \n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    final_predictions = threshold_Rounder(final_predictions, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': final_predictions\n    })\n    \n    return submission\n\ndef feature_engineering(df):\n    season_cols = [col for col in df.columns if 'Season' in col]\n    df = df.drop(season_cols, axis=1) \n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    df['BMI_PHR'] = df['Physical-BMI'] * df['Physical-HeartRate']\n    \n    return df\n\ndef preprocess_data(train_df, test_df):\n    # Handle outliers in vital signs\n    vital_cols = ['Physical-HeartRate', 'Physical-Systolic_BP', 'Physical-Diastolic_BP']\n    for col in vital_cols:\n        if col in train_df.columns:\n            q1 = train_df[col].quantile(0.25)\n            q3 = train_df[col].quantile(0.75)\n            iqr = q3 - q1\n            lower = q1 - 1.5 * iqr\n            upper = q3 + 1.5 * iqr\n            train_df[col] = train_df[col].clip(lower, upper)\n            test_df[col] = test_df[col].clip(lower, upper)\n    \n    # Better imputation strategy\n    numeric_imputer = KNNImputer(n_neighbors=5)\n    categorical_imputer = SimpleImputer(strategy='most_frequent')\n    \n    # Handle train data\n    train_numeric_cols = train_df.select_dtypes(include=['float64', 'int64']).columns\n    train_categorical_cols = train_df.select_dtypes(include=['object']).columns\n    \n    train_numeric_cols = [col for col in train_numeric_cols if col != 'sii']  # Exclude sii from numeric columns\n    \n    if len(train_numeric_cols) > 0:\n        train_df[train_numeric_cols] = numeric_imputer.fit_transform(train_df[train_numeric_cols])\n    \n    if len(train_categorical_cols) > 0:\n        train_df[train_categorical_cols] = categorical_imputer.fit_transform(train_df[train_categorical_cols])\n    \n    # Handle test data\n    test_numeric_cols = test_df.select_dtypes(include=['float64', 'int64']).columns\n    test_categorical_cols = test_df.select_dtypes(include=['object']).columns\n    \n    if len(test_numeric_cols) > 0:\n        test_df[test_numeric_cols] = numeric_imputer.transform(test_df[test_numeric_cols])\n    \n    if len(test_categorical_cols) > 0:\n        test_df[test_categorical_cols] = categorical_imputer.transform(test_df[test_categorical_cols])\n    \n    # Scale numeric features\n    scaler = StandardScaler()\n    if len(train_numeric_cols) > 0:\n        train_df[train_numeric_cols] = scaler.fit_transform(train_df[train_numeric_cols])\n        test_df[test_numeric_cols] = scaler.transform(test_df[test_numeric_cols])\n    \n    # Handle any remaining infinities\n    train_df = train_df.replace([np.inf, -np.inf], np.nan)\n    test_df = test_df.replace([np.inf, -np.inf], np.nan)\n    \n    # Final imputation for any remaining NaNs\n    train_df = train_df.fillna(train_df.mean())\n    test_df = test_df.fillna(test_df.mean())\n    \n    return train_df, test_df\n\nclass AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim*2),\n            nn.ReLU(),\n            nn.Linear(input_dim*2, input_dim*3),\n            nn.ReLU(),\n            nn.Linear(input_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n        \n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return decoded\n    \ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    \n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n                 \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded\n\nclass StackingWithMetaFeatures(BaseEstimator, RegressorMixin):\n    def __init__(self, base_models, meta_model):\n        self.base_models = base_models\n        self.meta_model = meta_model\n        \n    def fit(self, X, y):\n        # Convert to numpy if pandas\n        if isinstance(X, pd.DataFrame):\n            X = X.values\n        if isinstance(y, pd.Series):\n            y = y.values\n            \n        # Generate meta-features\n        meta_features = np.zeros((X.shape[0], len(self.base_models)))\n        for i, model in enumerate(self.base_models):\n            kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n            for train_idx, val_idx in kf.split(X, y):\n                X_train, X_val = X[train_idx], X[val_idx]\n                y_train, y_val = y[train_idx], y[val_idx]\n                \n                model.fit(X_train, y_train)\n                meta_features[val_idx, i] = model.predict(X_val)\n                \n        # Add original features to meta-features\n        final_features = np.hstack([meta_features, X])\n        \n        # Train meta-model\n        self.meta_model.fit(final_features, y)\n        return self\n\n    def predict(self, X):\n        # Convert to numpy if pandas\n        if isinstance(X, pd.DataFrame):\n            X = X.values\n            \n        # Generate meta-features\n        meta_features = np.zeros((X.shape[0], len(self.base_models)))\n        for i, model in enumerate(self.base_models):\n            meta_features[:, i] = model.predict(X)\n            \n        # Add original features\n        final_features = np.hstack([meta_features, X])\n        \n        return self.meta_model.predict(final_features)\n\nclass OrdinalRegressor(BaseEstimator, RegressorMixin):\n    def __init__(self, clf):\n        self.clf = clf\n        self.clfs = {}\n        \n    def fit(self, X, y):\n        self.unique_classes = np.sort(np.unique(y))\n        for i in range(len(self.unique_classes)-1):\n            binary_y = (y > self.unique_classes[i]).astype(int)\n            self.clfs[i] = clone(self.clf).fit(X, binary_y)\n        return self\n    \n    def predict(self, X):\n        # Get probabilities for each threshold\n        probs = np.zeros((X.shape[0], len(self.unique_classes)-1))\n        for i, clf in self.clfs.items():\n            probs[:, i] = clf.predict(X)\n        \n        # Convert probabilities to regression values\n        predictions = np.sum(probs > 0.5, axis=1)\n        return predictions\n\nclass QuantileEnsemble(BaseEstimator, RegressorMixin):\n    def __init__(self, base_model=GradientBoostingRegressor):\n        self.base_model = base_model\n        self.quantiles = [0.1, 0.5, 0.9]\n        self.models = {q: base_model(loss='quantile', alpha=q) for q in self.quantiles}\n        \n    def fit(self, X, y):\n        for q, model in self.models.items():\n            model.fit(X, y)\n        return self\n        \n    def predict(self, X):\n        predictions = np.zeros((X.shape[0], len(self.quantiles)))\n        for i, (q, model) in enumerate(self.models.items()):\n            predictions[:, i] = model.predict(X)\n        return np.median(predictions, axis=1)\n    \n    def get_params(self, deep=True):\n        return {\"base_model\": self.base_model}\n    \n    def set_params(self, **parameters):\n        for parameter, value in parameters.items():\n            setattr(self, parameter, value)\n        return self\n\ndef create_advanced_ensemble():\n    # Base models\n    base_models = [\n        LGBMRegressor(**LightGBM_Params),\n        XGBRegressor(**XGB_Params),\n        CatBoostRegressor(**CatBoost_Params)\n    ]\n    \n    # Create different types of models\n    stacking_model = StackingWithMetaFeatures(\n        base_models=base_models,\n        meta_model=LGBMRegressor(**LightGBM_Params)\n    )\n    \n    ordinal_model = OrdinalRegressor(\n        clf=LGBMRegressor(**LightGBM_Params)\n    )\n    \n    quantile_model = QuantileEnsemble(\n        base_model=GradientBoostingRegressor\n    )\n    \n    # Combine all models in final ensemble\n    final_ensemble = VotingRegressor([\n        ('stacking', stacking_model),\n        ('ordinal', ordinal_model),\n        ('quantile', quantile_model),\n        ('lgb', LGBMRegressor(**LightGBM_Params)),\n        ('xgb', XGBRegressor(**XGB_Params)),\n        ('cat', CatBoostRegressor(**CatBoost_Params))\n    ], weights=[1.5, 1.2, 1.0, 1.0, 1.0, 1.0])\n    \n    return final_ensemble\n\n# Main execution code\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n# Remove rows with NaN in target variable\ntrain = train.dropna(subset=['sii'])\n\n# Load time series data\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\n# Process time series data\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=100, batch_size=32)\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=100, batch_size=32)\n\ntime_series_cols = train_ts_encoded.columns.tolist()\ntrain_ts_encoded[\"id\"] = train_ts[\"id\"]\ntest_ts_encoded['id'] = test_ts[\"id\"]\n\n# Merge time series features\ntrain = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\ntest = pd.merge(test, test_ts_encoded, how=\"left\", on='id')\n\n# Select features\ntrain = train[featuresCols + ['sii']]\ntest = test[featuresCols]\n\n# Make sure sii is integer type\ntrain['sii'] = train['sii'].astype(int)\n\n# Feature engineering and preprocessing\ntrain = feature_engineering(train)\ntest = feature_engineering(test)\n\n# Preprocess data\ntrain, test = preprocess_data(train, test)\n\n# Create and train advanced ensemble\nadvanced_model = create_advanced_ensemble()\n\n# Generate predictions\nfinal_submission = TrainML(advanced_model, test)\n\n# Save submission\nfinal_submission.to_csv('submission.csv', index=False)\n\nprint(\"Advanced ensemble predictions saved to 'submission.csv'\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T21:08:27.058049Z","iopub.execute_input":"2024-12-08T21:08:27.059093Z","iopub.status.idle":"2024-12-08T22:23:09.347322Z","shell.execute_reply.started":"2024-12-08T21:08:27.059026Z","shell.execute_reply":"2024-12-08T22:23:09.346096Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}