{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\nimport os\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom tensorflow.keras import layers, Model\nfrom tensorflow.keras.optimizers import Adam\nimport matplotlib.pyplot as plt\n\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\n\nfrom sklearn.experimental import enable_iterative_imputer  \nfrom sklearn.impute import IterativeImputer","metadata":{"execution":{"iopub.status.busy":"2024-11-05T13:13:31.363346Z","iopub.execute_input":"2024-11-05T13:13:31.363849Z","iopub.status.idle":"2024-11-05T13:13:31.372195Z","shell.execute_reply.started":"2024-11-05T13:13:31.363803Z","shell.execute_reply":"2024-11-05T13:13:31.370745Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-11-05T13:10:10.569147Z","iopub.execute_input":"2024-11-05T13:10:10.570771Z","iopub.status.idle":"2024-11-05T13:10:10.673359Z","shell.execute_reply.started":"2024-11-05T13:10:10.570694Z","shell.execute_reply":"2024-11-05T13:10:10.671357Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_engineering(df):\n\n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n    df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n    df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n    df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n    df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n    df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n    df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n    df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n    df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n    df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:10:11.548398Z","iopub.execute_input":"2024-11-05T13:10:11.549021Z","iopub.status.idle":"2024-11-05T13:10:11.562214Z","shell.execute_reply.started":"2024-11-05T13:10:11.548967Z","shell.execute_reply":"2024-11-05T13:10:11.560554Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    \n    df['hour_of_day'] = df['time_of_day'] // (6 * 60 * 60 * 10**9)\n    df['ZY'] = (df['Z'])**2 + (df['Y'])**2\n    df['cos_angle'] = np.cos(df['anglez'] * np.pi / 180)\n    df['abs_x'] = abs(df['X'])\n    df['abs_z'] = abs(df['Z'])\n\n    df['pos_x'] = np.where(df['X'] <= 0, 0, df['X'])\n    df['neg_x'] = np.where(df['X'] >= 0, 0, df['X'])\n\n\n    \n    average_by_hour = df.groupby('hour_of_day')[['enmo','Z','ZY','abs_x', \n                                                           'abs_z', 'cos_angle', 'pos_x',\n                                                           'neg_x']].mean().reset_index().T\n    \n    average_by_hour = average_by_hour.drop('hour_of_day', axis=0)[[1,2]]\n    hour_feature = average_by_hour.values.reshape(-1)\n\n\n    return np.append(df.describe().values.reshape(-1), hour_feature), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n\n\nclass AutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(AutoEncoder, self).__init__()\n        \n        # Encoder: Using Sequential blocks\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim),\n            nn.ReLU()\n        )\n        \n        # Decoder: Using Sequential blocks\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, encoding_dim*2),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*2, encoding_dim*3),\n            nn.ReLU(),\n            nn.Linear(encoding_dim*3, input_dim),\n            nn.Sigmoid()\n        )\n        \n    def forward(self, x):\n        # Encoder\n        enc1 = self.encoder[0:2](x)  # First block of encoder\n        enc2 = self.encoder[2:4](enc1)  # Second block of encoder\n        encoded = self.encoder[4:](enc2)  # Final encoding layer\n        \n        # Decoder with skip connections\n        dec1 = self.decoder[0:2](encoded) + enc2  # Skip connection from enc2 to dec1\n        dec2 = self.decoder[2:4](dec1) + enc1  # Skip connection from enc1 to dec2\n        decoded = self.decoder[4:](dec2)  # Final output layer\n        \n        return decoded\n\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    data_tensor = torch.FloatTensor(df_scaled)\n    \n    input_dim = data_tensor.shape[1]\n    autoencoder = AutoEncoder(input_dim, encoding_dim)\n    \n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(autoencoder.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            reconstructed = autoencoder(batch)\n            loss = criterion(reconstructed, batch)\n            loss.backward()\n            optimizer.step()\n            \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n                 \n    with torch.no_grad():\n        encoded_data = autoencoder.encoder(data_tensor).numpy()\n        \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:14:21.633418Z","iopub.execute_input":"2024-11-05T13:14:21.633898Z","iopub.status.idle":"2024-11-05T13:14:21.654866Z","shell.execute_reply.started":"2024-11-05T13:14:21.633857Z","shell.execute_reply":"2024-11-05T13:14:21.653449Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)\n\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=90, epochs=100, batch_size=32)\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=90, epochs=100, batch_size=32)\n\ntime_series_cols = train_ts_encoded.columns.tolist()\ntrain_ts_encoded[\"id\"]=train_ts[\"id\"]\ntest_ts_encoded['id']=test_ts[\"id\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:14:24.271644Z","iopub.execute_input":"2024-11-05T13:14:24.272110Z","iopub.status.idle":"2024-11-05T13:17:30.821596Z","shell.execute_reply.started":"2024-11-05T13:14:24.272069Z","shell.execute_reply":"2024-11-05T13:17:30.819957Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = feature_engineering(test)\ntrain = feature_engineering(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:17:41.708357Z","iopub.execute_input":"2024-11-05T13:17:41.709204Z","iopub.status.idle":"2024-11-05T13:17:41.744373Z","shell.execute_reply.started":"2024-11-05T13:17:41.709149Z","shell.execute_reply":"2024-11-05T13:17:41.742831Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_id = test['id']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:18:02.560274Z","iopub.execute_input":"2024-11-05T13:18:02.560777Z","iopub.status.idle":"2024-11-05T13:18:02.566764Z","shell.execute_reply.started":"2024-11-05T13:18:02.560736Z","shell.execute_reply":"2024-11-05T13:18:02.565291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numeric_cols = train.select_dtypes(include=['float32', 'float64', 'int64']).columns\ntarget_cols = [col for col in train.columns if col == 'sii' or 'PCIAT' in col]\nnon_target_cols = [col for col in train.columns if col != 'sii' and 'PCIAT' not in col]\nnumeric_cols_non_target = [i for i in non_target_cols if i in numeric_cols]\ncatagorial_cols_non_target = [i for i in non_target_cols if i not in numeric_cols]","metadata":{"execution":{"iopub.status.busy":"2024-11-05T13:28:47.301383Z","iopub.execute_input":"2024-11-05T13:28:47.303141Z","iopub.status.idle":"2024-11-05T13:28:47.314640Z","shell.execute_reply.started":"2024-11-05T13:28:47.303073Z","shell.execute_reply":"2024-11-05T13:28:47.313315Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_column = 'PCIAT-PCIAT_Total'\ncategorical_features = catagorial_cols_non_target.copy()\ncategorical_features.remove('id')\nnumerical_features = numeric_cols_non_target.copy()","metadata":{"execution":{"iopub.status.busy":"2024-11-05T13:28:48.852638Z","iopub.execute_input":"2024-11-05T13:28:48.853130Z","iopub.status.idle":"2024-11-05T13:28:48.858832Z","shell.execute_reply.started":"2024-11-05T13:28:48.853083Z","shell.execute_reply":"2024-11-05T13:28:48.857500Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def separate_missing_target_rows(data, target_column):\n    rows_with_target = data.dropna(subset=target_column)\n    rows_missing_target = data[data[target_column].isna()]\n    return rows_with_target, rows_missing_target\n\nrows_with_target, rows_missing_target = separate_missing_target_rows(train, target_column)","metadata":{"execution":{"iopub.status.busy":"2024-11-05T13:28:50.224132Z","iopub.execute_input":"2024-11-05T13:28:50.224586Z","iopub.status.idle":"2024-11-05T13:28:50.238331Z","shell.execute_reply.started":"2024-11-05T13:28:50.224543Z","shell.execute_reply":"2024-11-05T13:28:50.237081Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = rows_with_target.drop(columns=[target_column] + ['sii', 'id'])\ny = rows_with_target['sii']\nX_missing_target = rows_missing_target.drop(columns=[target_column]+ ['sii', 'id'])","metadata":{"execution":{"iopub.status.busy":"2024-11-05T13:28:51.516441Z","iopub.execute_input":"2024-11-05T13:28:51.516930Z","iopub.status.idle":"2024-11-05T13:28:51.529253Z","shell.execute_reply.started":"2024-11-05T13:28:51.516884Z","shell.execute_reply":"2024-11-05T13:28:51.527856Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = X[numerical_features]\nX_missing_target = X_missing_target[numerical_features]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:28:52.858348Z","iopub.execute_input":"2024-11-05T13:28:52.858803Z","iopub.status.idle":"2024-11-05T13:28:52.868487Z","shell.execute_reply.started":"2024-11-05T13:28:52.858758Z","shell.execute_reply":"2024-11-05T13:28:52.867195Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = test[numerical_features]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:29:00.367001Z","iopub.execute_input":"2024-11-05T13:29:00.367433Z","iopub.status.idle":"2024-11-05T13:29:01.410283Z","shell.execute_reply.started":"2024-11-05T13:29:00.367395Z","shell.execute_reply":"2024-11-05T13:29:01.408465Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if np.any(np.isinf(X)):\n    X = X.replace([np.inf, -np.inf], np.nan)\nif np.any(np.isinf(test)):\n    train = test.replace([np.inf, -np.inf], np.nan)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:29:14.680739Z","iopub.execute_input":"2024-11-05T13:29:14.681271Z","iopub.status.idle":"2024-11-05T13:29:14.694049Z","shell.execute_reply.started":"2024-11-05T13:29:14.681227Z","shell.execute_reply":"2024-11-05T13:29:14.692264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numerical_features = X.select_dtypes(include=['float32','float64', 'int64']).columns\n\nnumerical_transformer = Pipeline(steps=[\n    ('scaler', StandardScaler())\n])\n\n# categorical_transformer = Pipeline(steps=[\n#     ('onehot', OneHotEncoder(handle_unknown='ignore'))\n# ])\n\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('num', numerical_transformer, numerical_features)\n    ]\n)\n\n\nX_train = preprocessor.fit_transform(X)\n# X_missing_target_processed = preprocessor.transform(X_missing_target)","metadata":{"execution":{"iopub.status.busy":"2024-11-05T13:30:00.894874Z","iopub.execute_input":"2024-11-05T13:30:00.896022Z","iopub.status.idle":"2024-11-05T13:30:00.934410Z","shell.execute_reply.started":"2024-11-05T13:30:00.895974Z","shell.execute_reply":"2024-11-05T13:30:00.932873Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_scale = preprocessor.fit_transform(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:30:02.876064Z","iopub.execute_input":"2024-11-05T13:30:02.876520Z","iopub.status.idle":"2024-11-05T13:30:03.294791Z","shell.execute_reply.started":"2024-11-05T13:30:02.876480Z","shell.execute_reply":"2024-11-05T13:30:03.293174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.DataFrame(test_scale, columns = test.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:29:20.467747Z","iopub.execute_input":"2024-11-05T13:29:20.468215Z","iopub.status.idle":"2024-11-05T13:29:20.475524Z","shell.execute_reply.started":"2024-11-05T13:29:20.468172Z","shell.execute_reply":"2024-11-05T13:29:20.473887Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = pd.DataFrame(X_train, columns = X.columns)\n# X_missing_target_processed = pd.DataFrame(X_missing_target_processed, columns = X.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:30:08.923982Z","iopub.execute_input":"2024-11-05T13:30:08.924459Z","iopub.status.idle":"2024-11-05T13:30:08.930287Z","shell.execute_reply.started":"2024-11-05T13:30:08.924417Z","shell.execute_reply":"2024-11-05T13:30:08.928997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def impute_missing_values_complex(X_train, max_iter=10, random_state=42):\n    imputer = IterativeImputer(max_iter=max_iter, random_state=random_state)\n    \n    X_train_imputed = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns)\n    \n    return X_train_imputed \n\nX_train = impute_missing_values_complex(X_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:30:11.791518Z","iopub.execute_input":"2024-11-05T13:30:11.791962Z","iopub.status.idle":"2024-11-05T13:33:48.670453Z","shell.execute_reply.started":"2024-11-05T13:30:11.791923Z","shell.execute_reply":"2024-11-05T13:33:48.669018Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = impute_missing_values_complex(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:20:50.592491Z","iopub.execute_input":"2024-11-05T13:20:50.593010Z","iopub.status.idle":"2024-11-05T13:20:51.899356Z","shell.execute_reply.started":"2024-11-05T13:20:50.592969Z","shell.execute_reply":"2024-11-05T13:20:51.898307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = y.reset_index(drop=True)\ny","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:21:17.214444Z","iopub.execute_input":"2024-11-05T13:21:17.215035Z","iopub.status.idle":"2024-11-05T13:21:17.227175Z","shell.execute_reply.started":"2024-11-05T13:21:17.214969Z","shell.execute_reply":"2024-11-05T13:21:17.225743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, VotingClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, cohen_kappa_score, classification_report, confusion_matrix\nfrom sklearn.cluster import DBSCAN\nfrom sklearn.model_selection import StratifiedKFold\nfrom imblearn.over_sampling import SMOTE\nimport numpy as np\nimport pandas as pd\n\ndef hierarchical_cv_model(X, y, n_splits=5, smote_strategy='minority'):\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    \n    # Metrics storage\n    accuracy_scores, f1_scores, precision_scores, recall_scores, qwk_scores = [], [], [], [], []\n    per_class_results = {\"precision\": [], \"recall\": [], \"f1-score\": [], \"support\": []}\n    \n    for fold, (train_index, test_index) in enumerate(skf.split(X, y)):\n        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n        \n        # Apply SMOTE for class balancing in rare classes\n        smote = SMOTE(sampling_strategy=smote_strategy, random_state=42)\n        X_train_balanced, y_train_balanced = smote.fit_resample(X_train, y_train)\n        \n        # Stage 1: Voting Classifier for SII 0 vs. Non-SII 0\n        y_train_stage1 = (y_train_balanced == 0).astype(int)\n        model_stage1 = VotingClassifier(estimators=[\n            ('rf', RandomForestClassifier(random_state=42)),\n            ('gb', GradientBoostingClassifier(random_state=42)),\n            ('lr', LogisticRegression(random_state=42))\n        ], voting='soft')\n        model_stage1.fit(X_train_balanced, y_train_stage1)\n        \n        # Predict Stage 1\n        stage1_pred = model_stage1.predict(X_test)\n        \n        # Stage 2: Voting Classifier for SII 1 vs. Greater Than 1\n        non_sii_0_mask = (stage1_pred == 0)\n        X_train_stage2 = X_train_balanced[y_train_balanced != 0]\n        y_train_stage2 = (y_train_balanced[y_train_balanced != 0] == 1).astype(int)\n        \n        model_stage2 = VotingClassifier(estimators=[\n            ('rf', RandomForestClassifier(random_state=42)),\n            ('gb', GradientBoostingClassifier(random_state=42)),\n            ('lr', LogisticRegression(random_state=42))\n        ], voting='soft')\n        model_stage2.fit(X_train_stage2, y_train_stage2)\n        \n        # Predict Stage 2\n        X_test_stage2 = X_test[non_sii_0_mask]\n        stage2_pred = np.full(len(y_test), 0)  # Initialize with SII 0 predictions\n        stage2_pred[non_sii_0_mask] = model_stage2.predict(X_test_stage2)\n        \n        # Stage 3: Unsupervised Clustering for SII 2 vs. SII 3 on remaining samples\n        greater_than_1_mask = (stage2_pred == 0) & non_sii_0_mask\n        X_train_stage3 = X_train_balanced[(y_train_balanced != 0) & (y_train_balanced != 1)]\n        \n        # Use DBSCAN for clustering\n        dbscan_model = DBSCAN(eps=0.5, min_samples=5)\n        dbscan_model.fit(X_train_stage3)\n        \n        # Predict Stage 3 using DBSCAN clusters and handle -1 outliers\n        X_test_stage3 = X_test[greater_than_1_mask]\n        cluster_pred = dbscan_model.fit_predict(X_test_stage3)\n        \n        # Map cluster predictions to SII 2 and SII 3, handling -1 as a default class\n        cluster_to_sii_map = {0: 2, 1: 3}  # Adjust mapping if necessary\n        stage3_pred = np.copy(stage2_pred)\n        filtered_cluster_pred = [\n            cluster_to_sii_map.get(cluster, 2)  # Assign a default SII value of 2 for -1 outliers\n            for cluster in cluster_pred\n        ]\n        stage3_pred[greater_than_1_mask] = filtered_cluster_pred\n        \n        # Final combined predictions\n        final_predictions = np.where(stage1_pred == 1, 0, np.where(stage2_pred == 1, 1, stage3_pred))\n        \n        # Evaluate metrics for this fold\n        accuracy = accuracy_score(y_test, final_predictions)\n        f1 = f1_score(y_test, final_predictions, average='weighted')\n        precision = precision_score(y_test, final_predictions, average='weighted')\n        recall = recall_score(y_test, final_predictions, average='weighted')\n        qwk = cohen_kappa_score(y_test, final_predictions, weights='quadratic')\n        \n        accuracy_scores.append(accuracy)\n        f1_scores.append(f1)\n        precision_scores.append(precision)\n        recall_scores.append(recall)\n        qwk_scores.append(qwk)\n        \n        # Per-class metrics\n        class_report = classification_report(y_test, final_predictions, output_dict=True)\n        for metric in [\"precision\", \"recall\", \"f1-score\", \"support\"]:\n            per_class_results[metric].append({label: score for label, score in class_report.items() if label.isdigit()})\n        \n        # Print fold metrics and confusion matrix\n        print(f\"Fold {fold + 1} Metrics - Accuracy: {accuracy:.3f}, F1 Score: {f1:.3f}, Precision: {precision:.3f}, Recall: {recall:.3f}, QWK: {qwk:.3f}\")\n        print(\"Confusion Matrix:\\n\", confusion_matrix(y_test, final_predictions))\n    \n    # Aggregate average metrics across folds\n    print(\"\\nAverage Cross-Validation Metrics:\")\n    print(f\"Accuracy: {np.mean(accuracy_scores):.3f} ± {np.std(accuracy_scores):.3f}\")\n    print(f\"F1 Score: {np.mean(f1_scores):.3f} ± {np.std(f1_scores):.3f}\")\n    print(f\"Precision: {np.mean(precision_scores):.3f} ± {np.std(precision_scores):.3f}\")\n    print(f\"Recall: {np.mean(recall_scores):.3f} ± {np.std(recall_scores):.3f}\")\n    print(f\"QWK (Main Metric): {np.mean(qwk_scores):.3f} ± {np.std(qwk_scores):.3f}\")\n    \n# Example usage\nhierarchical_cv_model(X_train, y, n_splits=5, smote_strategy='minority')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:35:38.458456Z","iopub.execute_input":"2024-11-05T13:35:38.459017Z","iopub.status.idle":"2024-11-05T13:38:02.921668Z","shell.execute_reply.started":"2024-11-05T13:35:38.458974Z","shell.execute_reply":"2024-11-05T13:38:02.920415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, ExtraTreesClassifier, VotingClassifier\nfrom xgboost import XGBClassifier\nfrom sklearn.ensemble import IsolationForest\nfrom catboost import CatBoostClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.decomposition import PCA\nfrom sklearn.cluster import DBSCAN\nfrom sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, cohen_kappa_score, classification_report, confusion_matrix\nfrom sklearn.model_selection import StratifiedKFold, RandomizedSearchCV\nfrom sklearn.neighbors import LocalOutlierFactor\nfrom sklearn.preprocessing import StandardScaler\nfrom imblearn.over_sampling import SMOTE\nimport numpy as np\nimport pandas as pd\n\ndef hierarchical_cv_model(X, y, n_splits=5, smote_strategy='minority'):\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    \n    # Metrics storage\n    accuracy_scores, f1_scores, precision_scores, recall_scores, qwk_scores = [], [], [], [], []\n    per_class_results = {\"precision\": [], \"recall\": [], \"f1-score\": [], \"support\": []}\n    \n    for fold, (train_index, test_index) in enumerate(skf.split(X, y)):\n        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n        \n        # Step 1: Outlier Detection using Isolation Forest and Local Outlier Factor\n        iso_forest = IsolationForest(contamination=0.05, random_state=42)\n        lof = LocalOutlierFactor(n_neighbors=20)\n        outlier_preds = (iso_forest.fit_predict(X_train) == 1) & (lof.fit_predict(X_train) == 1)\n        X_train, y_train = X_train[outlier_preds], y_train[outlier_preds]\n        \n        # Step 2: SMOTE for Class Balancing\n        smote = SMOTE(sampling_strategy=smote_strategy, random_state=42)\n        X_train_balanced, y_train_balanced = smote.fit_resample(X_train, y_train)\n        \n        # Stage 1: Ensemble Voting Classifier for SII 0 vs. Non-SII 0\n        y_train_stage1 = (y_train_balanced == 0).astype(int)\n        model_stage1 = VotingClassifier(estimators=[\n            ('rf', RandomForestClassifier(random_state=42)),\n            ('gb', GradientBoostingClassifier(random_state=42)),\n            ('et', ExtraTreesClassifier(random_state=42)),\n            ('xgb', XGBClassifier(random_state=42)),\n            ('cat', CatBoostClassifier(silent=True, random_state=42)),\n            ('lr', LogisticRegression(random_state=42))\n        ], voting='soft')\n        model_stage1.fit(X_train_balanced, y_train_stage1)\n        \n        # Predict Stage 1\n        stage1_pred = model_stage1.predict(X_test)\n        \n        # Stage 2: Voting Classifier for SII 1 vs. Greater Than 1\n        non_sii_0_mask = (stage1_pred == 0)\n        X_train_stage2 = X_train_balanced[y_train_balanced != 0]\n        y_train_stage2 = (y_train_balanced[y_train_balanced != 0] == 1).astype(int)\n        \n        model_stage2 = VotingClassifier(estimators=[\n            ('rf', RandomForestClassifier(random_state=42)),\n            ('gb', GradientBoostingClassifier(random_state=42)),\n            ('et', ExtraTreesClassifier(random_state=42)),\n            ('xgb', XGBClassifier(random_state=42)),\n            ('cat', CatBoostClassifier(silent=True, random_state=42)),\n            ('lr', LogisticRegression(random_state=42))\n        ], voting='soft')\n        model_stage2.fit(X_train_stage2, y_train_stage2)\n        \n        # Predict Stage 2\n        X_test_stage2 = X_test[non_sii_0_mask]\n        stage2_pred = np.full(len(y_test), 0)  # Initialize with SII 0 predictions\n        stage2_pred[non_sii_0_mask] = model_stage2.predict(X_test_stage2)\n        \n        # Stage 3: Unsupervised Clustering for SII 2 vs. SII 3 with Label Propagation\n        greater_than_1_mask = (stage2_pred == 0) & non_sii_0_mask\n        X_train_stage3 = X_train_balanced[(y_train_balanced != 0) & (y_train_balanced != 1)]\n        \n        # Use DBSCAN for clustering and Label Propagation\n        dbscan_model = DBSCAN(eps=0.5, min_samples=10)\n        dbscan_model.fit(X_train_stage3)\n        \n        # Predict Stage 3 using DBSCAN clusters and handle -1 outliers\n        X_test_stage3 = X_test[greater_than_1_mask]\n        cluster_pred = dbscan_model.fit_predict(X_test_stage3)\n        \n        # Map cluster predictions to SII 2 and SII 3, handling -1 as a default class\n        cluster_to_sii_map = {0: 2, 1: 3}  # Adjust mapping if necessary\n        stage3_pred = np.copy(stage2_pred)\n        filtered_cluster_pred = [\n            cluster_to_sii_map.get(cluster, 2)  # Assign a default SII value of 2 for -1 outliers\n            for cluster in cluster_pred\n        ]\n        stage3_pred[greater_than_1_mask] = filtered_cluster_pred\n        \n        # Final combined predictions\n        final_predictions = np.where(stage1_pred == 1, 0, np.where(stage2_pred == 1, 1, stage3_pred))\n        \n        # Evaluate metrics for this fold\n        accuracy = accuracy_score(y_test, final_predictions)\n        f1 = f1_score(y_test, final_predictions, average='weighted')\n        precision = precision_score(y_test, final_predictions, average='weighted')\n        recall = recall_score(y_test, final_predictions, average='weighted')\n        qwk = cohen_kappa_score(y_test, final_predictions, weights='quadratic')\n        \n        accuracy_scores.append(accuracy)\n        f1_scores.append(f1)\n        precision_scores.append(precision)\n        recall_scores.append(recall)\n        qwk_scores.append(qwk)\n        \n        # Per-class metrics\n        class_report = classification_report(y_test, final_predictions, output_dict=True)\n        for metric in [\"precision\", \"recall\", \"f1-score\", \"support\"]:\n            per_class_results[metric].append({label: score for label, score in class_report.items() if label.isdigit()})\n        \n        # Print fold metrics and confusion matrix\n        print(f\"Fold {fold + 1} Metrics - Accuracy: {accuracy:.3f}, F1 Score: {f1:.3f}, Precision: {precision:.3f}, Recall: {recall:.3f}, QWK: {qwk:.3f}\")\n        print(\"Confusion Matrix:\\n\", confusion_matrix(y_test, final_predictions))\n    \n    # Aggregate average metrics across folds\n    print(\"\\nAverage Cross-Validation Metrics:\")\n    print(f\"Accuracy: {np.mean(accuracy_scores):.3f} ± {np.std(accuracy_scores):.3f}\")\n    print(f\"F1 Score: {np.mean(f1_scores):.3f} ± {np.std(f1_scores):.3f}\")\n    print(f\"Precision: {np.mean(precision_scores):.3f} ± {np.std(precision_scores):.3f}\")\n    print(f\"Recall: {np.mean(recall_scores):.3f} ± {np.std(recall_scores):.3f}\")\n    print(f\"QWK (Main Metric): {np.mean(qwk_scores):.3f} ± {np.std(qwk_scores):.3f}\")\n    \n# Example usage\nhierarchical_cv_model(X_train, y, n_splits=5, smote_strategy='minority')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-05T13:39:18.491943Z","iopub.execute_input":"2024-11-05T13:39:18.492412Z","iopub.status.idle":"2024-11-05T13:44:08.331344Z","shell.execute_reply.started":"2024-11-05T13:39:18.492370Z","shell.execute_reply":"2024-11-05T13:44:08.329753Z"}},"outputs":[],"execution_count":null}]}