{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 1. Uninstall existing packages to ensure a clean slate\n!pip uninstall scikit-learn imbalanced-learn -y\n\n# 2. Install the known stable versions\n!pip install scikit-learn==1.1.3\n!pip install imbalanced-learn==0.10.1","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:12:56.775719Z","iopub.execute_input":"2025-12-18T04:12:56.776427Z","iopub.status.idle":"2025-12-18T04:13:11.182980Z","shell.execute_reply.started":"2025-12-18T04:12:56.776391Z","shell.execute_reply":"2025-12-18T04:13:11.181766Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Image Preprocessing**","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image, ImageDraw\n\n\nimport matplotlib.pyplot as plt\nfrom scipy.ndimage import gaussian_filter\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\n# Paths\ndata_dir = '/kaggle/input/aptos2019-blindness-detection'\ntrain_csv_path = os.path.join(data_dir, 'train.csv')\nimg_dir = os.path.join(data_dir, 'train_images')\nprocessed_dir = '/kaggle/working/processed_images'\nos.makedirs(processed_dir, exist_ok=True)\n\n# Step 1: Visualize Initial Data\ndef visualize_initial_data(num_samples_per_class=2):\n    train_csv = pd.read_csv(train_csv_path)\n    fig, axs = plt.subplots(5, num_samples_per_class, figsize=(10, 20))\n    for class_label in range(5):\n        class_samples = train_csv[train_csv['diagnosis'] == class_label].sample(num_samples_per_class, random_state=42)\n        for i, (_, row) in enumerate(class_samples.iterrows()):\n            img_path = os.path.join(img_dir, f\"{row['id_code']}.png\")\n            img = Image.open(img_path)\n            axs[class_label, i].imshow(img)\n            axs[class_label, i].set_title(f\"Class {class_label}: {row['id_code']}\")\n            axs[class_label, i].axis('off')\n    plt.tight_layout()\n    plt.show()\n\n# Call\nvisualize_initial_data()\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:11.184871Z","iopub.execute_input":"2025-12-18T04:13:11.185231Z","iopub.status.idle":"2025-12-18T04:13:24.498542Z","shell.execute_reply.started":"2025-12-18T04:13:11.185186Z","shell.execute_reply":"2025-12-18T04:13:24.497371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_csv = pd.read_csv(train_csv_path)\nprint(train_csv.shape[0])\nprint(train_csv['diagnosis'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:24.499661Z","iopub.execute_input":"2025-12-18T04:13:24.500315Z","iopub.status.idle":"2025-12-18T04:13:24.519394Z","shell.execute_reply.started":"2025-12-18T04:13:24.500273Z","shell.execute_reply":"2025-12-18T04:13:24.518338Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sizes = train_csv['diagnosis'].value_counts().values\nlabels = ['No Dr','Moderate DR','Mild Dr','Proliferative DR', 'Sever DR']\n\nfig, ax = plt.subplots()\nax.pie(sizes, labels=labels, autopct='%1.1f%%');","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:24.521723Z","iopub.execute_input":"2025-12-18T04:13:24.522009Z","iopub.status.idle":"2025-12-18T04:13:24.655278Z","shell.execute_reply.started":"2025-12-18T04:13:24.521988Z","shell.execute_reply":"2025-12-18T04:13:24.653660Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import cv2\n\ndef smart_resize(img, size=(1024,1024)):\n    h, w = img.shape[:2]\n    if h > size[0] and w > size[1]:\n        interp = cv2.INTER_AREA\n    elif h < size[0] and w < size[1]:\n        interp = cv2.INTER_CUBIC\n    else:\n        interp = cv2.INTER_LINEAR\n    return cv2.resize(img, size, interpolation = interp)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:24.656501Z","iopub.execute_input":"2025-12-18T04:13:24.657302Z","iopub.status.idle":"2025-12-18T04:13:24.916708Z","shell.execute_reply.started":"2025-12-18T04:13:24.657245Z","shell.execute_reply":"2025-12-18T04:13:24.915652Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def crop_circular(img):\n    gray = cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)\n    mask = gray > 30\n    coords = np.argwhere(mask)\n    y0, x0 = coords.min(axis = 0)\n    y1, x1 = coords.max(axis = 0)\n    cropped = img[y0:y1, x0:x1]\n    return cropped","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:24.917827Z","iopub.execute_input":"2025-12-18T04:13:24.918185Z","iopub.status.idle":"2025-12-18T04:13:24.924565Z","shell.execute_reply.started":"2025-12-18T04:13:24.918155Z","shell.execute_reply":"2025-12-18T04:13:24.923390Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def ben_graham_preprocess(img, sigmaX = 10):\n    blur = cv2.GaussianBlur(img, (0,0), sigmaX)\n    img = cv2.addWeighted(img, 4, blur, -4, 128)\n    return img","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:24.925833Z","iopub.execute_input":"2025-12-18T04:13:24.926446Z","iopub.status.idle":"2025-12-18T04:13:24.954761Z","shell.execute_reply.started":"2025-12-18T04:13:24.926412Z","shell.execute_reply":"2025-12-18T04:13:24.953211Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_path = os.path.join(img_dir, f\"{train_csv.iloc[0,0]}.png\")\nimg = cv2.imread(img_path)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nimg.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:24.955831Z","iopub.execute_input":"2025-12-18T04:13:24.956117Z","iopub.status.idle":"2025-12-18T04:13:25.275436Z","shell.execute_reply.started":"2025-12-18T04:13:24.956096Z","shell.execute_reply":"2025-12-18T04:13:25.273802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_resized = smart_resize(img)\nimg_resized.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:25.276782Z","iopub.execute_input":"2025-12-18T04:13:25.277182Z","iopub.status.idle":"2025-12-18T04:13:25.319504Z","shell.execute_reply.started":"2025-12-18T04:13:25.277154Z","shell.execute_reply":"2025-12-18T04:13:25.318460Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_cropped = crop_circular(img_resized)\nimg_cropped.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:25.322752Z","iopub.execute_input":"2025-12-18T04:13:25.323137Z","iopub.status.idle":"2025-12-18T04:13:25.347774Z","shell.execute_reply.started":"2025-12-18T04:13:25.323116Z","shell.execute_reply":"2025-12-18T04:13:25.346515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_braham = ben_graham_preprocess(img_cropped)\nimg_braham.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:25.348813Z","iopub.execute_input":"2025-12-18T04:13:25.349212Z","iopub.status.idle":"2025-12-18T04:13:25.409168Z","shell.execute_reply.started":"2025-12-18T04:13:25.349188Z","shell.execute_reply":"2025-12-18T04:13:25.408137Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(img_braham);","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:25.410154Z","iopub.execute_input":"2025-12-18T04:13:25.410437Z","iopub.status.idle":"2025-12-18T04:13:25.813397Z","shell.execute_reply.started":"2025-12-18T04:13:25.410417Z","shell.execute_reply":"2025-12-18T04:13:25.811865Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(img);","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:25.814761Z","iopub.execute_input":"2025-12-18T04:13:25.815276Z","iopub.status.idle":"2025-12-18T04:13:27.318795Z","shell.execute_reply.started":"2025-12-18T04:13:25.815221Z","shell.execute_reply":"2025-12-18T04:13:27.317074Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_image(img_path, target_size=512):\n    img = cv2.imread(img_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    # Order: Crop -> Resize -> Ben Graham (per Kaggle load_ben_color)\n    img = smart_resize(img,(1024, 1024))\n    img = crop_circular(img)\n    img = ben_graham_preprocess(img, sigmaX=10)\n    img = smart_resize(img,(target_size, target_size))\n    return img","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:27.320552Z","iopub.execute_input":"2025-12-18T04:13:27.320950Z","iopub.status.idle":"2025-12-18T04:13:27.327859Z","shell.execute_reply.started":"2025-12-18T04:13:27.320917Z","shell.execute_reply":"2025-12-18T04:13:27.326380Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n# Step 3: Visualize Processed Data\ndef visualize_processed_data(num_samples_per_class=2):\n    train_csv = pd.read_csv(train_csv_path)\n    fig, axs = plt.subplots(5, 2 * num_samples_per_class, figsize=(20, 20))\n    for class_label in range(5):\n        class_samples = train_csv[train_csv['diagnosis'] == class_label].sample(num_samples_per_class, random_state=42)\n        for i, (_, row) in enumerate(class_samples.iterrows()):\n            img_path = os.path.join(img_dir, f\"{row['id_code']}.png\")\n            original = cv2.imread(img_path)\n            original = cv2.cvtColor(original, cv2.COLOR_BGR2RGB)\n            processed = preprocess_image(img_path)\n            \n            axs[class_label, 2*i].imshow(original)\n            axs[class_label, 2*i].set_title(f\"Original Class {class_label}: {row['id_code']}\")\n            axs[class_label, 2*i].axis('off')\n            \n            axs[class_label, 2*i + 1].imshow(processed)\n            axs[class_label, 2*i + 1].set_title(f\"Processed Class {class_label}: {row['id_code']}\")\n            axs[class_label, 2*i + 1].axis('off')\n    plt.tight_layout()\n    plt.show()\n\n# Call\nvisualize_processed_data()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:27.329195Z","iopub.execute_input":"2025-12-18T04:13:27.329918Z","iopub.status.idle":"2025-12-18T04:13:40.520035Z","shell.execute_reply.started":"2025-12-18T04:13:27.329891Z","shell.execute_reply":"2025-12-18T04:13:40.518596Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 4: Save Processed Images\ndef save_processed_images():\n    train_csv = pd.read_csv(train_csv_path)\n    for _, row in train_csv.iterrows():\n        img_path = os.path.join(img_dir, f\"{row['id_code']}.png\")\n        processed_img = preprocess_image(img_path)\n        save_path = os.path.join(processed_dir, f\"{row['id_code']}.png\")\n        cv2.imwrite(save_path, cv2.cvtColor(processed_img, cv2.COLOR_RGB2BGR))\n    print(f\"Processed and saved {len(train_csv)} images to {processed_dir}\")\n\n# Call\nsave_processed_images()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:13:40.521218Z","iopub.execute_input":"2025-12-18T04:13:40.521531Z","iopub.status.idle":"2025-12-18T04:29:05.419582Z","shell.execute_reply.started":"2025-12-18T04:13:40.521512Z","shell.execute_reply":"2025-12-18T04:29:05.417148Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Data Augmentation**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom PIL import Image\nimport torch\nfrom torch.utils.data import Dataset\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:29:05.424464Z","iopub.execute_input":"2025-12-18T04:29:05.425074Z","iopub.status.idle":"2025-12-18T04:29:08.054112Z","shell.execute_reply.started":"2025-12-18T04:29:05.425025Z","shell.execute_reply":"2025-12-18T04:29:08.052850Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport os\n\n# Paths\ndata_dir = '/kaggle/input/aptos2019-blindness-detection'\ntrain_csv_path = os.path.join(data_dir, 'train.csv')\nprocessed_dir = '/kaggle/working/processed_images'\nos.makedirs(processed_dir, exist_ok=True)\n\n# Load CSV\ndf = pd.read_csv(train_csv_path)\n\n# --- Step 1: Train+Val vs Test (80:20) ---\ntrain_val_df, test_df = train_test_split(\n    df,\n    test_size=0.2,\n    stratify=df['diagnosis'],\n    random_state=42\n)\n\n# --- Step 2: Train vs Val (80:20 of train_val -> 64:16 overall) ---\ntrain_df, val_df = train_test_split(\n    train_val_df,\n    test_size=0.2,  # 20% of 80% = 16% of total\n    stratify=train_val_df['diagnosis'],\n    random_state=42\n)\n\n# --- Optional: Print class distribution to verify ---\nprint(\"Train class distribution:\\n\", train_df['diagnosis'].value_counts(normalize=True))\nprint(\"Val class distribution:\\n\", val_df['diagnosis'].value_counts(normalize=True))\nprint(\"Test class distribution:\\n\", test_df['diagnosis'].value_counts(normalize=True))\n\n# --- Save CSVs ---\ntrain_df.to_csv(os.path.join(processed_dir, 'train_split.csv'), index=False)\nval_df.to_csv(os.path.join(processed_dir, 'val_split.csv'), index=False)\ntest_df.to_csv(os.path.join(processed_dir, 'test_split.csv'), index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:29:08.055174Z","iopub.execute_input":"2025-12-18T04:29:08.055944Z","iopub.status.idle":"2025-12-18T04:29:08.701691Z","shell.execute_reply.started":"2025-12-18T04:29:08.055913Z","shell.execute_reply":"2025-12-18T04:29:08.700439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Set up subplots\nfig, axes = plt.subplots(1, 2, figsize=(12, 5))\n\n# Train distribution\nsns.countplot(x='diagnosis', data=train_df, ax=axes[0], palette='viridis')\naxes[0].set_title('Train Class Distribution')\naxes[0].set_xlabel('Diagnosis')\naxes[0].set_ylabel('Count')\n\n# Validation distribution\nsns.countplot(x='diagnosis', data=val_df, ax=axes[1], palette='magma')\naxes[1].set_title('Validation Class Distribution')\naxes[1].set_xlabel('Diagnosis')\naxes[1].set_ylabel('Count')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:29:08.703434Z","iopub.execute_input":"2025-12-18T04:29:08.704455Z","iopub.status.idle":"2025-12-18T04:29:09.416624Z","shell.execute_reply.started":"2025-12-18T04:29:08.704417Z","shell.execute_reply":"2025-12-18T04:29:09.415508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport torch\nfrom torch.utils.data import Dataset\nfrom PIL import Image\nimport numpy as np\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n\n\nclass APTOSDataset(Dataset):\n    def __init__(self, df, processed_dir, transform=None):\n        self.data = df\n        self.processed_dir = processed_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        row = self.data.iloc[idx]\n        img_name = f\"{row['id_code']}.png\"\n        img_path = os.path.join(self.processed_dir, img_name)\n        \n        # --- Read image using PIL ---\n        image = Image.open(img_path).convert('RGB')  # ensure 3 channels\n        image = np.array(image)  # convert to numpy for Albumentations\n        \n        label = torch.tensor(row['diagnosis'], dtype=torch.long)\n        \n        if self.transform:\n            image = self.transform(image=image)['image']\n        \n        return image, label\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:29:09.417770Z","iopub.execute_input":"2025-12-18T04:29:09.418537Z","iopub.status.idle":"2025-12-18T04:29:09.427201Z","shell.execute_reply.started":"2025-12-18T04:29:09.418501Z","shell.execute_reply":"2025-12-18T04:29:09.426038Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\ntrain_transform = A.Compose([\n    A.Resize(256, 256),                  # intermediate resize\n    A.CenterCrop(224, 224),              # clean center patch\n    A.HorizontalFlip(p=0.5),             # light augmentation\n    A.RandomBrightnessContrast(0.1,0.1, p=0.3),\n    A.Rotate(limit=5, border_mode=0, p=0.3),\n    A.Normalize(mean=(0.485,0.456,0.406), std=(0.229,0.224,0.225)),\n    ToTensorV2()\n])\n\nval_test_transform = A.Compose([\n    A.Resize(256, 256),\n    A.CenterCrop(224, 224),\n    A.Normalize(mean=(0.485,0.456,0.406), std=(0.229,0.224,0.225)),\n    ToTensorV2()\n])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:29:09.428487Z","iopub.execute_input":"2025-12-18T04:29:09.428781Z","iopub.status.idle":"2025-12-18T04:29:09.465697Z","shell.execute_reply.started":"2025-12-18T04:29:09.428759Z","shell.execute_reply":"2025-12-18T04:29:09.464535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training dataset\ntrain_dataset = APTOSDataset(df=train_df,\n                             processed_dir=processed_dir,\n                             transform=train_transform)\n\n# Validation dataset\nval_dataset = APTOSDataset(df=val_df,\n                           processed_dir=processed_dir,\n                           transform=val_test_transform)\n\n# Test dataset\ntest_dataset = APTOSDataset(df=test_df,\n                            processed_dir=processed_dir,\n                            transform=val_test_transform)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:29:09.466946Z","iopub.execute_input":"2025-12-18T04:29:09.467290Z","iopub.status.idle":"2025-12-18T04:29:09.476992Z","shell.execute_reply.started":"2025-12-18T04:29:09.467239Z","shell.execute_reply":"2025-12-18T04:29:09.475686Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\ndef visualize_augmentation(dataset, num_samples=5):\n    \"\"\"\n    Visualize original and augmented images from a dataset.\n\n    Args:\n        dataset (Dataset): Instance of APTOSDataset.\n        num_samples (int): Number of images to display.\n    \"\"\"\n    plt.figure(figsize=(12, num_samples * 3))\n    \n    for i in range(num_samples):\n        # Get raw image path\n        row = dataset.data.iloc[i]\n        img_name = f\"{row['id_code']}.png\"\n        img_path = os.path.join(dataset.processed_dir, img_name)\n        \n        # Load original image with PIL\n        orig_image = np.array(Image.open(img_path).convert('RGB'))\n        \n        # Apply augmentation / transform\n        if dataset.transform:\n            aug_image = dataset.transform(image=orig_image)['image']\n            # Convert tensor to numpy for visualization\n            if isinstance(aug_image, torch.Tensor):\n                aug_image = aug_image.permute(1, 2, 0).numpy()\n                # Undo normalization for visualization\n                mean = np.array([0.485, 0.456, 0.406])\n                std = np.array([0.229, 0.224, 0.225])\n                aug_image = np.clip((aug_image * std + mean), 0, 1)\n        else:\n            aug_image = orig_image / 255.0  # normalize for display\n        \n        # Plot original\n        plt.subplot(num_samples, 2, i*2 + 1)\n        plt.imshow(orig_image)\n        plt.title(f\"Original - Label {row['diagnosis']}\")\n        plt.axis('off')\n        \n        # Plot augmented\n        plt.subplot(num_samples, 2, i*2 + 2)\n        plt.imshow(aug_image)\n        plt.title(f\"Augmented - Label {row['diagnosis']}\")\n        plt.axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:29:09.478433Z","iopub.execute_input":"2025-12-18T04:29:09.479384Z","iopub.status.idle":"2025-12-18T04:29:09.504740Z","shell.execute_reply.started":"2025-12-18T04:29:09.479345Z","shell.execute_reply":"2025-12-18T04:29:09.503472Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize training dataset\nvisualize_augmentation(train_dataset, num_samples=5)\n\n# Visualize validation dataset\nvisualize_augmentation(val_dataset, num_samples=5)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:29:09.506142Z","iopub.execute_input":"2025-12-18T04:29:09.506583Z","iopub.status.idle":"2025-12-18T04:29:12.848659Z","shell.execute_reply.started":"2025-12-18T04:29:09.506558Z","shell.execute_reply":"2025-12-18T04:29:12.847067Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_workers = min(6, os.cpu_count() or multiprocessing.cpu_count())\nprint(f\"🧠 Using {num_workers} workers\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:29:12.850545Z","iopub.execute_input":"2025-12-18T04:29:12.851639Z","iopub.status.idle":"2025-12-18T04:29:12.857570Z","shell.execute_reply.started":"2025-12-18T04:29:12.851608Z","shell.execute_reply":"2025-12-18T04:29:12.856356Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_loader = DataLoader(\n    train_dataset,\n    batch_size=32,\n    shuffle=True,\n    num_workers=num_workers,\n    pin_memory=True,\n)\n\nval_loader = DataLoader(val_dataset, batch_size=64, shuffle=False, pin_memory=True)\ntest_loader = DataLoader(test_dataset, batch_size=64, shuffle=False, pin_memory=True,)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:29:12.858677Z","iopub.execute_input":"2025-12-18T04:29:12.858947Z","iopub.status.idle":"2025-12-18T04:29:12.880648Z","shell.execute_reply.started":"2025-12-18T04:29:12.858926Z","shell.execute_reply":"2025-12-18T04:29:12.879385Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Model**","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torchvision import models\nimport os\nimport numpy as np\nfrom tqdm.notebook import tqdm\n\ndef extract_efficientnet_b3_features(\n    train_loader,\n    val_loader,\n    test_loader,\n    save_dir,\n    device=\"cuda\"\n):\n    \"\"\"\n    Extract features from EfficientNet-B3 (pretrained) using Global Average Pooling\n    and save as .npy files.\n    \"\"\"\n\n    os.makedirs(save_dir, exist_ok=True)\n\n    # ---- Load pretrained EfficientNet-B3 ----\n    model = models.efficientnet_b3(\n        weights=models.EfficientNet_B3_Weights.IMAGENET1K_V1\n    )\n    model.eval()\n    model.to(device)\n\n    # ---- Feature extractor (no classifier) ----\n    feature_extractor = nn.Sequential(\n        model.features,                    # convolutional backbone\n        nn.AdaptiveAvgPool2d((1, 1))        # global average pooling\n    ).to(device)\n\n    feature_extractor.eval()\n\n    def extract_features(loader, split_name):\n        features_list = []\n        labels_list = []\n\n        with torch.no_grad():\n            for imgs, labels in tqdm(loader, desc=f\"Extracting {split_name} features\"):\n                imgs = imgs.to(device)\n\n                feats = feature_extractor(imgs)      # [B, 1536, 1, 1]\n                feats = feats.flatten(1)             # [B, 1536]\n\n                features_list.append(feats.cpu().numpy())\n                labels_list.append(labels.numpy())\n\n        features_array = np.vstack(features_list)\n        labels_array = np.hstack(labels_list)\n\n        np.save(os.path.join(save_dir, f\"{split_name}_features.npy\"), features_array)\n        np.save(os.path.join(save_dir, f\"{split_name}_labels.npy\"), labels_array)\n\n        print(\n            f\"{split_name} features saved: {features_array.shape}, \"\n            f\"labels saved: {labels_array.shape}\"\n        )\n\n        return features_array, labels_array\n\n    train_feats, train_labels = extract_features(train_loader, \"train\")\n    val_feats, val_labels = extract_features(val_loader, \"val\")\n    test_feats, test_labels = extract_features(test_loader, \"test\")\n\n    return (\n        (train_feats, train_labels),\n        (val_feats, val_labels),\n        (test_feats, test_labels),\n    )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T04:29:12.882003Z","iopub.execute_input":"2025-12-18T04:29:12.882550Z","iopub.status.idle":"2025-12-18T04:29:12.910600Z","shell.execute_reply.started":"2025-12-18T04:29:12.882517Z","shell.execute_reply":"2025-12-18T04:29:12.908875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"save_folder = '/kaggle/working/resnet50_features'\n\n(train_feats, train_labels), (val_feats, val_labels), (test_feats, test_labels) = extract_efficientnet_b3_features(\n    train_loader=train_loader,\n    val_loader=val_loader,\n    test_loader=test_loader,\n    save_dir=save_folder,\n    device='cpu'\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T05:02:24.979780Z","iopub.execute_input":"2025-12-18T05:02:24.984180Z","iopub.status.idle":"2025-12-18T05:10:21.100660Z","shell.execute_reply.started":"2025-12-18T05:02:24.984093Z","shell.execute_reply":"2025-12-18T05:10:21.098395Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **ML pipline**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\nfrom imblearn.over_sampling import SMOTE\n\ndef preprocess_features_with_smote_verbose(train_features, train_labels,\n                                           val_features, val_labels,\n                                           test_features, test_labels,\n                                           pca_components=0.95,\n                                           random_state=42):\n    \"\"\"\n    Apply scaling -> PCA -> SMOTE on training features and print shapes after each step.\n    Apply same scaling & PCA to val/test (no SMOTE).\n\n    Args:\n        train_features, val_features, test_features: numpy arrays [N_samples, N_features]\n        train_labels, val_labels, test_labels: numpy arrays [N_samples]\n        pca_components: number of components or float for variance ratio\n        random_state: for reproducibility\n\n    Returns:\n        X_train_res, y_train_res, X_val_pca_scaled, y_val, X_test_pca_scaled, y_test\n        scaler1, pca, scaler2: fitted objects for reference or reuse\n    \"\"\"\n    print(\"Original shapes:\")\n    print(f\"  Train: {train_features.shape}, Val: {val_features.shape}, Test: {test_features.shape}\")\n\n    # 1. Standard scale on training features\n    scaler1 = StandardScaler()\n    X_train_scaled = scaler1.fit_transform(train_features)\n    X_val_scaled = scaler1.transform(val_features)\n    X_test_scaled = scaler1.transform(test_features)\n    print(\"After first StandardScaler:\")\n    print(f\"  Train: {X_train_scaled.shape}, Val: {X_val_scaled.shape}, Test: {X_test_scaled.shape}\")\n\n    # 2. PCA on scaled features\n    pca = PCA(n_components=pca_components, random_state=random_state)\n    X_train_pca = pca.fit_transform(X_train_scaled)\n    X_val_pca = pca.transform(X_val_scaled)\n    X_test_pca = pca.transform(X_test_scaled)\n    print(f\"After PCA (n_components={pca.n_components_}):\")\n    print(f\"  Train: {X_train_pca.shape}, Val: {X_val_pca.shape}, Test: {X_test_pca.shape}\")\n\n    # 3. Optional: second scaling after PCA\n    scaler2 = StandardScaler()\n    X_train_pca_scaled = scaler2.fit_transform(X_train_pca)\n    X_val_pca_scaled = scaler2.transform(X_val_pca)\n    X_test_pca_scaled = scaler2.transform(X_test_pca)\n    print(\"After second StandardScaler (post-PCA):\")\n    print(f\"  Train: {X_train_pca_scaled.shape}, Val: {X_val_pca_scaled.shape}, Test: {X_test_pca_scaled.shape}\")\n\n    # 4. Apply SMOTE only on training set\n    smote = SMOTE(random_state=random_state)\n    X_train_res, y_train_res = smote.fit_resample(X_train_pca_scaled, train_labels)\n    print(\"After SMOTE on training set:\")\n    print(f\"  Train: {X_train_res.shape}, Labels: {y_train_res.shape}\")\n\n    return X_train_res, y_train_res, X_val_pca_scaled, val_labels, X_test_pca_scaled, test_labels, scaler1, pca, scaler2\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T05:10:21.103774Z","iopub.execute_input":"2025-12-18T05:10:21.104206Z","iopub.status.idle":"2025-12-18T05:10:21.734851Z","shell.execute_reply.started":"2025-12-18T05:10:21.104170Z","shell.execute_reply":"2025-12-18T05:10:21.733567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load saved features\ntrain_features = np.load('resnet50_features/train_features.npy')\ntrain_labels   = np.load('resnet50_features/train_labels.npy')\nval_features   = np.load('resnet50_features/val_features.npy')\nval_labels     = np.load('resnet50_features/val_labels.npy')\ntest_features  = np.load('resnet50_features/test_features.npy')\ntest_labels    = np.load('resnet50_features/test_labels.npy')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T05:10:21.736058Z","iopub.execute_input":"2025-12-18T05:10:21.736390Z","iopub.status.idle":"2025-12-18T05:10:21.757754Z","shell.execute_reply.started":"2025-12-18T05:10:21.736366Z","shell.execute_reply":"2025-12-18T05:10:21.756239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_res, y_train_res, X_val_proc, y_val, X_test_proc, y_test, scaler1, pca, scaler2 = \\\n    preprocess_features_with_smote_verbose(train_features, train_labels,\n                                           val_features, val_labels,\n                                           test_features, test_labels,\n                                           pca_components=0.95)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T05:10:21.760630Z","iopub.execute_input":"2025-12-18T05:10:21.762188Z","iopub.status.idle":"2025-12-18T05:10:23.922905Z","shell.execute_reply.started":"2025-12-18T05:10:21.762141Z","shell.execute_reply":"2025-12-18T05:10:23.921359Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import accuracy_score, f1_score, roc_curve, auc, confusion_matrix, ConfusionMatrixDisplay\nfrom sklearn.preprocessing import label_binarize\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.base import clone\n\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\nfrom sklearn.linear_model import LogisticRegression, RidgeClassifier, SGDClassifier, PassiveAggressiveClassifier\nfrom sklearn.neighbors import KNeighborsClassifier, NearestCentroid\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neural_network import MLPClassifier\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T05:10:23.924110Z","iopub.execute_input":"2025-12-18T05:10:23.924461Z","iopub.status.idle":"2025-12-18T05:10:23.943953Z","shell.execute_reply.started":"2025-12-18T05:10:23.924437Z","shell.execute_reply":"2025-12-18T05:10:23.942146Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -----------------------------\n# 1️⃣ Base Classifier Training\n# -----------------------------\nfrom sklearn.model_selection import GridSearchCV\n# Define all 13 classifiers\nbase_classifiers = {\n    'SVM': SVC(probability=True, random_state=42),\n    'LogisticRegression': LogisticRegression(max_iter=1000, random_state=42),\n    'RandomForest': RandomForestClassifier(random_state=42),\n    'KNN': KNeighborsClassifier(),\n    'GaussianNB': GaussianNB(),\n    'ExtraTrees': ExtraTreesClassifier(random_state=42),\n    'MLP': MLPClassifier(max_iter=1000, random_state=42),\n    'PassiveAggressive': PassiveAggressiveClassifier(max_iter=1000, random_state=42),\n    'RidgeClassifier': RidgeClassifier(),\n    'SGDClassifier': SGDClassifier(max_iter=1000, random_state=42),\n    'NearestCentroid': NearestCentroid(),\n    'DecisionTree': DecisionTreeClassifier(random_state=42)\n}\n\nclass_names = [0, 1, 2, 3, 4]\n\n# -----------------------------\n# Define hyperparameter grids for each classifier\n# -----------------------------\nparam_grids = {\n    'SVM': {'C':[0.001,0.01,0.1,1,10,100], 'kernel':['linear','rbf','poly'], 'gamma':['scale','auto']},\n    'LogisticRegression': {'C':[0.01,0.1,1,10,100], 'penalty':['l2'], 'solver':['lbfgs']},\n    'RandomForest': {'n_estimators':[100,200,300,400], 'max_depth':[None,10,20,30,40], 'min_samples_split':[2,5,10]},\n    'KNN': {'n_neighbors':[1,3,5,7,9,11], 'weights':['uniform','distance'], 'p':[1,2], 'algorithm':['auto','ball_tree','kd_tree']},\n    'GaussianNB': {},  # no tuning\n    'ExtraTrees': {'n_estimators':[100,200,300,400], 'max_depth':[None,10,20,30,40], 'min_samples_split':[2,5,10], 'max_features':['auto','sqrt','log2']},\n    'MLP': {'hidden_layer_sizes':[(100,),(100,50),(50,50,25),(64,32)], 'alpha':[0.0001,0.001,0.01], 'activation':['relu','tanh'], 'learning_rate':['constant','adaptive']},\n    'PassiveAggressive': {'C':[0.01,0.1,1,10,100]},\n    'RidgeClassifier': {'alpha':[0.01,0.1,1,10,100]},\n    'SGDClassifier': {'alpha':[0.0001,0.001,0.01,0.1], 'loss':['hinge','log','perceptron'], 'penalty':['l2','l1','elasticnet']},\n    'NearestCentroid': {},  # no tuning\n    'DecisionTree': {'max_depth':[None,10,20,30,40], 'min_samples_split':[2,5,10,20], 'criterion':['gini','entropy']}\n}\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T05:10:23.945602Z","iopub.execute_input":"2025-12-18T05:10:23.945949Z","iopub.status.idle":"2025-12-18T05:10:23.962673Z","shell.execute_reply.started":"2025-12-18T05:10:23.945919Z","shell.execute_reply":"2025-12-18T05:10:23.961486Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_tune_select_classifiers(X_train, y_train, X_val, y_val, classifiers_dict, param_grids,\n                                  class_names, scoring='f1_weighted', qwk_thresh=0.5, roc_folder='prediction/ML_base_ROC'):\n    \"\"\"\n    Train all classifiers with expanded hyperparameter search.\n    Only retain classifiers that exceed qwk_thresh on validation for stacking.\n    \n    Returns:\n        tuned_models_selected: dict of tuned classifiers that pass QWK threshold\n        tuning_results: dict of best parameters\n        qwk_scores: dict of QWK scores for each classifier\n    \"\"\"\n    import os\n    from sklearn.metrics import cohen_kappa_score, f1_score, roc_curve, auc\n    from sklearn.preprocessing import label_binarize\n    from sklearn.model_selection import GridSearchCV\n    from sklearn.base import clone\n    import matplotlib.pyplot as plt\n    \n    os.makedirs(roc_folder, exist_ok=True)\n    n_classes = len(class_names)\n    y_val_bin = label_binarize(y_val, classes=list(range(n_classes)))\n    \n    tuned_models_selected = {}\n    tuning_results = {}\n    qwk_scores = {}\n    \n    for name, model in classifiers_dict.items():\n        print(f\"\\n=== {name} ===\")\n        \n        # Hyperparameter tuning if grid exists\n        grid = param_grids.get(name, {})\n        if grid:\n            gscv = GridSearchCV(clone(model), grid, scoring=scoring, cv=3, n_jobs=-1)\n            gscv.fit(X_train, y_train)\n            best_model = gscv.best_estimator_\n            tuning_results[name] = gscv.best_params_\n        else:\n            best_model = clone(model)\n            best_model.fit(X_train, y_train)\n            tuning_results[name] = None\n        \n        # Evaluate on validation\n        y_pred = best_model.predict(X_val)\n        qk = cohen_kappa_score(y_val, y_pred, weights='quadratic')\n        f1 = f1_score(y_val, y_pred, average='weighted')\n        qwk_scores[name] = qk\n        print(f\"[Tuned] QWK: {qk:.4f}, Weighted F1: {f1:.4f}\")\n        \n        # Save ROC\n        if hasattr(best_model, \"predict_proba\"):\n            y_score = best_model.predict_proba(X_val)\n        elif hasattr(best_model, \"decision_function\"):\n            y_score = best_model.decision_function(X_val)\n            if y_score.ndim == 1:\n                y_score = np.vstack([1-y_score, y_score]).T\n        else:\n            y_score = None\n        \n        if y_score is not None:\n            plt.figure(figsize=(7,6))\n            for i in range(n_classes):\n                fpr, tpr, _ = roc_curve(y_val_bin[:,i], y_score[:,i])\n                roc_auc = auc(fpr, tpr)\n                plt.plot(fpr, tpr, label=f\"Class {class_names[i]} (AUC={roc_auc:.2f})\")\n            plt.plot([0,1],[0,1],'k--')\n            plt.xlabel(\"False Positive Rate\")\n            plt.ylabel(\"True Positive Rate\")\n            plt.title(f\"{name} ROC Curve (Tuned)\")\n            plt.legend(loc='lower right')\n            plt.tight_layout()\n            plt.savefig(os.path.join(roc_folder, f\"{name}_tuned_ROC.png\"))\n            plt.close()\n        \n        # Only keep classifiers above QWK threshold\n        if qk >= qwk_thresh:\n            tuned_models_selected[name] = best_model\n        else:\n            print(f\"Classifier {name} below QWK threshold ({qwk_thresh}), excluded from stacking.\")\n    \n    return tuned_models_selected, tuning_results, qwk_scores\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T05:10:23.963550Z","iopub.execute_input":"2025-12-18T05:10:23.963823Z","iopub.status.idle":"2025-12-18T05:10:23.988078Z","shell.execute_reply.started":"2025-12-18T05:10:23.963803Z","shell.execute_reply":"2025-12-18T05:10:23.986383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tuned_models_selected,tuning_results, qwk_scores = train_tune_select_classifiers(\n    X_train_res, y_train_res,\n    X_val=X_val_proc, y_val=y_val,\n    classifiers_dict=base_classifiers,\n    param_grids=param_grids,\n    class_names=class_names,\n    roc_folder='prediction/ML_base_ROC'\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T05:20:35.369528Z","iopub.execute_input":"2025-12-18T05:20:35.371525Z","iopub.status.idle":"2025-12-18T06:08:00.976903Z","shell.execute_reply.started":"2025-12-18T05:20:35.371475Z","shell.execute_reply":"2025-12-18T06:08:00.974274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom sklearn.base import clone\nfrom sklearn.preprocessing import StandardScaler, label_binarize\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.metrics import cohen_kappa_score, f1_score, confusion_matrix, ConfusionMatrixDisplay, roc_curve, auc\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\n\n# -----------------------------\n# 1️⃣ Generate OOF Probabilities for Stacking\n# -----------------------------\ndef generate_oof_predictions(fitted_models, X_train, y_train, n_splits=5):\n    n_samples = X_train.shape[0]\n    n_classifiers = len(fitted_models)\n    n_classes = len(np.unique(y_train))\n    oof_probs = np.zeros((n_samples, n_classifiers * n_classes))\n    oof_labels = np.zeros(n_samples, dtype=y_train.dtype)\n\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n\n    for i, (name, model) in enumerate(fitted_models.items()):\n        fold_probs = np.zeros((n_samples, n_classes))\n        for train_idx, val_idx in skf.split(X_train, y_train):\n            clf = clone(model)\n            clf.fit(X_train[train_idx], y_train[train_idx])\n            if hasattr(clf, \"predict_proba\"):\n                probs = clf.predict_proba(X_train[val_idx])\n            elif hasattr(clf, \"decision_function\"):\n                probs = clf.decision_function(X_train[val_idx])\n                if probs.ndim == 1:\n                    probs = np.vstack([1-probs, probs]).T\n            else:\n                probs = np.zeros((len(val_idx), n_classes))\n            fold_probs[val_idx] = probs\n            oof_labels[val_idx] = y_train[val_idx]\n        oof_probs[:, i*n_classes:(i+1)*n_classes] = fold_probs\n\n    return oof_probs, oof_labels\n\n# -----------------------------\n# 2️⃣ Generate Validation/Test Probabilities\n# -----------------------------\ndef generate_base_predictions(fitted_models, X_data):\n    n_samples = X_data.shape[0]\n    n_classifiers = len(fitted_models)\n    n_classes = len(fitted_models[list(fitted_models.keys())[0]].classes_)\n    probs = np.zeros((n_samples, n_classifiers * n_classes))\n\n    for i, (name, model) in enumerate(fitted_models.items()):\n        if hasattr(model, \"predict_proba\"):\n            p = model.predict_proba(X_data)\n        elif hasattr(model, \"decision_function\"):\n            p = model.decision_function(X_data)\n            if p.ndim==1:\n                p = np.vstack([1-p, p]).T\n        else:\n            p = np.zeros((n_samples, n_classes))\n        probs[:, i*n_classes:(i+1)*n_classes] = p\n    return probs\n\n# -----------------------------\n# 3️⃣ Stacked Model Training and Evaluation\n# -----------------------------\ndef train_stacked_mlp(X_stack_train, y_stack_train, X_stack_val, y_val, class_names, save_folder='prediction/stacked'):\n    os.makedirs(save_folder, exist_ok=True)\n\n    # Scale stacked features\n    scaler_stack = StandardScaler()\n    X_stack_train_scaled = scaler_stack.fit_transform(X_stack_train)\n    X_stack_val_scaled = scaler_stack.transform(X_stack_val)\n\n    # Train MLP meta-learner\n    stack_model = MLPClassifier(hidden_layer_sizes=(64,32),\n                                activation='relu', solver='adam',\n                                max_iter=1000, random_state=42,\n                                early_stopping=True, alpha=0.0001)\n    stack_model.fit(X_stack_train_scaled, y_stack_train)\n\n    # Predictions\n    y_stack_pred = stack_model.predict(X_stack_val_scaled)\n    y_stack_probs = stack_model.predict_proba(X_stack_val_scaled)\n\n    # QWK and weighted F1\n    qk_stack = cohen_kappa_score(y_val, y_stack_pred, weights='quadratic')\n    f1_stack = f1_score(y_val, y_stack_pred, average='weighted')\n    print(f\"\\nStacked Model QWK: {qk_stack:.4f}, Weighted F1: {f1_stack:.4f}\")\n\n    # Confusion Matrix\n    cm = confusion_matrix(y_val, y_stack_pred)\n    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_names)\n    disp.plot(cmap=plt.cm.Blues)\n    plt.title(\"Stacked Model Confusion Matrix\")\n    plt.savefig(os.path.join(save_folder, \"stacked_confusion.png\"))\n    plt.show()\n\n    # ROC Curves\n    n_classes = len(class_names)\n    y_bin = label_binarize(y_val, classes=list(range(n_classes)))\n\n    plt.figure(figsize=(7,6))\n    for i in range(n_classes):\n        fpr, tpr, _ = roc_curve(y_bin[:,i], y_stack_probs[:,i])\n        roc_auc = auc(fpr, tpr)\n        plt.plot(fpr, tpr, label=f\"{class_names[i]} (AUC={roc_auc:.2f})\")\n    plt.plot([0,1],[0,1],'k--')\n    plt.xlabel(\"False Positive Rate\")\n    plt.ylabel(\"True Positive Rate\")\n    plt.title(\"Stacked Model ROC Curve\")\n    plt.legend(loc='lower right')\n    plt.tight_layout()\n    plt.savefig(os.path.join(save_folder, \"stacked_ROC.png\"))\n    plt.show()\n\n    return stack_model, scaler_stack\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T05:20:13.511373Z","iopub.status.idle":"2025-12-18T05:20:13.511799Z","shell.execute_reply.started":"2025-12-18T05:20:13.511613Z","shell.execute_reply":"2025-12-18T05:20:13.511634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tuned_models_selected","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T05:20:13.513921Z","iopub.status.idle":"2025-12-18T05:20:13.514466Z","shell.execute_reply.started":"2025-12-18T05:20:13.514194Z","shell.execute_reply":"2025-12-18T05:20:13.514213Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1️⃣ Generate OOF predictions for stacking\nX_stack_train, y_stack_train = generate_oof_predictions(tuned_models_selected, X_train_res, y_train_res)\n\n# 2️⃣ Generate validation probabilities\nX_stack_val = generate_base_predictions(tuned_models_selected, X_val_proc)\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T05:20:13.515790Z","iopub.status.idle":"2025-12-18T05:20:13.516157Z","shell.execute_reply.started":"2025-12-18T05:20:13.515996Z","shell.execute_reply":"2025-12-18T05:20:13.516010Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 3️⃣ Train stacked MLP and evaluate\nstack_model, scaler_stack = train_stacked_mlp(X_stack_train, y_stack_train,\n                                               X_stack_val, y_val,\n                                               class_names,\n                                               save_folder='prediction/stacked')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T05:20:13.518654Z","iopub.status.idle":"2025-12-18T05:20:13.519216Z","shell.execute_reply.started":"2025-12-18T05:20:13.518958Z","shell.execute_reply":"2025-12-18T05:20:13.518978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}