{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --upgrade scikit-learn\n!pip install --upgrade imbalanced-learn","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Image Preprocessing**","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image, ImageDraw\n\n\nimport matplotlib.pyplot as plt\nfrom scipy.ndimage import gaussian_filter\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\n# Paths\ndata_dir = '/kaggle/input/aptos2019-blindness-detection'\ntrain_csv_path = os.path.join(data_dir, 'train.csv')\nimg_dir = os.path.join(data_dir, 'train_images')\nprocessed_dir = '/kaggle/working/processed_images'\nos.makedirs(processed_dir, exist_ok=True)\n\n# Step 1: Visualize Initial Data\ndef visualize_initial_data(num_samples_per_class=2):\n    train_csv = pd.read_csv(train_csv_path)\n    fig, axs = plt.subplots(5, num_samples_per_class, figsize=(10, 20))\n    for class_label in range(5):\n        class_samples = train_csv[train_csv['diagnosis'] == class_label].sample(num_samples_per_class, random_state=42)\n        for i, (_, row) in enumerate(class_samples.iterrows()):\n            img_path = os.path.join(img_dir, f\"{row['id_code']}.png\")\n            img = Image.open(img_path)\n            axs[class_label, i].imshow(img)\n            axs[class_label, i].set_title(f\"Class {class_label}: {row['id_code']}\")\n            axs[class_label, i].axis('off')\n    plt.tight_layout()\n    plt.show()\n\n# Call\nvisualize_initial_data()\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:45:20.423448Z","iopub.execute_input":"2025-10-28T18:45:20.424232Z","iopub.status.idle":"2025-10-28T18:45:30.314010Z","shell.execute_reply.started":"2025-10-28T18:45:20.424203Z","shell.execute_reply":"2025-10-28T18:45:30.312639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import cv2\n\ndef smart_resize(img, size=(1024,1024)):\n    h, w = img.shape[:2]\n    if h > size[0] and w > size[1]:\n        interp = cv2.INTER_AREA\n    elif h < size[0] and w < size[1]:\n        interp = cv2.INTER_CUBIC\n    else:\n        interp = cv2.INTER_LINEAR\n    return cv2.resize(img, size, interpolation = interp)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:45:47.329215Z","iopub.execute_input":"2025-10-28T18:45:47.329788Z","iopub.status.idle":"2025-10-28T18:45:47.535989Z","shell.execute_reply.started":"2025-10-28T18:45:47.329766Z","shell.execute_reply":"2025-10-28T18:45:47.535389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def crop_circular(img):\n    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)\n    mask = gray > 30\n    coords = np.argwhere(mask)\n    y0, x0 = coords.min(axis = 0)\n    y1, x1 = coords.max(axis = 0)\n    cropped = img[y0:y1, x0:x1]\n    return cropped","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:45:56.398136Z","iopub.execute_input":"2025-10-28T18:45:56.398704Z","iopub.status.idle":"2025-10-28T18:45:56.403056Z","shell.execute_reply.started":"2025-10-28T18:45:56.398680Z","shell.execute_reply":"2025-10-28T18:45:56.402245Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def ben_graham_preprocess(img, sigmaX = 10):\n    blur = cv2.GaussianBlur(img, (0,0), sigmaX)\n    img = cv2.addWeighted(img, 4, blur, -4, 128)\n    return img","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:45:57.027527Z","iopub.execute_input":"2025-10-28T18:45:57.028064Z","iopub.status.idle":"2025-10-28T18:45:57.032106Z","shell.execute_reply.started":"2025-10-28T18:45:57.028039Z","shell.execute_reply":"2025-10-28T18:45:57.031309Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_csv = pd.read_csv(train_csv_path)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:45:58.681544Z","iopub.execute_input":"2025-10-28T18:45:58.682320Z","iopub.status.idle":"2025-10-28T18:45:58.689040Z","shell.execute_reply.started":"2025-10-28T18:45:58.682296Z","shell.execute_reply":"2025-10-28T18:45:58.688397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_path = os.path.join(img_dir, f\"{train_csv.iloc[0,0]}.png\")\nimg = cv2.imread(img_path)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nimg.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:45:58.879138Z","iopub.execute_input":"2025-10-28T18:45:58.879377Z","iopub.status.idle":"2025-10-28T18:45:59.067578Z","shell.execute_reply.started":"2025-10-28T18:45:58.879360Z","shell.execute_reply":"2025-10-28T18:45:59.066858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_resized = smart_resize(img)\nimg_resized.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:45:59.102863Z","iopub.execute_input":"2025-10-28T18:45:59.103203Z","iopub.status.idle":"2025-10-28T18:45:59.135560Z","shell.execute_reply.started":"2025-10-28T18:45:59.103177Z","shell.execute_reply":"2025-10-28T18:45:59.135007Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_cropped = crop_circular(img_resized)\nimg_cropped.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:45:59.361910Z","iopub.execute_input":"2025-10-28T18:45:59.362509Z","iopub.status.idle":"2025-10-28T18:45:59.382350Z","shell.execute_reply.started":"2025-10-28T18:45:59.362483Z","shell.execute_reply":"2025-10-28T18:45:59.381649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_braham = ben_graham_preprocess(img_cropped)\nimg_braham.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:46:01.347478Z","iopub.execute_input":"2025-10-28T18:46:01.348195Z","iopub.status.idle":"2025-10-28T18:46:01.383836Z","shell.execute_reply.started":"2025-10-28T18:46:01.348171Z","shell.execute_reply":"2025-10-28T18:46:01.383225Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(img_braham);","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:46:01.521762Z","iopub.execute_input":"2025-10-28T18:46:01.522041Z","iopub.status.idle":"2025-10-28T18:46:01.824634Z","shell.execute_reply.started":"2025-10-28T18:46:01.522021Z","shell.execute_reply":"2025-10-28T18:46:01.823911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(img);","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:46:01.825623Z","iopub.execute_input":"2025-10-28T18:46:01.825866Z","iopub.status.idle":"2025-10-28T18:46:03.032620Z","shell.execute_reply.started":"2025-10-28T18:46:01.825849Z","shell.execute_reply":"2025-10-28T18:46:03.031930Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_image(img_path, target_size=512):\n    img = cv2.imread(img_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    # Order: Crop -> Resize -> Ben Graham (per Kaggle load_ben_color)\n    img = smart_resize(img,(1024, 1024))\n    img = crop_circular(img)\n    img = ben_graham_preprocess(img, sigmaX=10)\n    img = smart_resize(img,(target_size, target_size))\n    return img","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:46:03.033747Z","iopub.execute_input":"2025-10-28T18:46:03.033966Z","iopub.status.idle":"2025-10-28T18:46:03.038170Z","shell.execute_reply.started":"2025-10-28T18:46:03.033949Z","shell.execute_reply":"2025-10-28T18:46:03.037381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n# Step 3: Visualize Processed Data\ndef visualize_processed_data(num_samples_per_class=2):\n    train_csv = pd.read_csv(train_csv_path)\n    fig, axs = plt.subplots(5, 2 * num_samples_per_class, figsize=(20, 20))\n    for class_label in range(5):\n        class_samples = train_csv[train_csv['diagnosis'] == class_label].sample(num_samples_per_class, random_state=42)\n        for i, (_, row) in enumerate(class_samples.iterrows()):\n            img_path = os.path.join(img_dir, f\"{row['id_code']}.png\")\n            original = cv2.imread(img_path)\n            original = cv2.cvtColor(original, cv2.COLOR_BGR2RGB)\n            processed = preprocess_image(img_path)\n            \n            axs[class_label, 2*i].imshow(original)\n            axs[class_label, 2*i].set_title(f\"Original Class {class_label}: {row['id_code']}\")\n            axs[class_label, 2*i].axis('off')\n            \n            axs[class_label, 2*i + 1].imshow(processed)\n            axs[class_label, 2*i + 1].set_title(f\"Processed Class {class_label}: {row['id_code']}\")\n            axs[class_label, 2*i + 1].axis('off')\n    plt.tight_layout()\n    plt.show()\n\n# Call\nvisualize_processed_data()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:46:03.038982Z","iopub.execute_input":"2025-10-28T18:46:03.039219Z","iopub.status.idle":"2025-10-28T18:46:13.042696Z","shell.execute_reply.started":"2025-10-28T18:46:03.039194Z","shell.execute_reply":"2025-10-28T18:46:13.041373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 4: Save Processed Images\ndef save_processed_images():\n    train_csv = pd.read_csv(train_csv_path)\n    for _, row in train_csv.iterrows():\n        img_path = os.path.join(img_dir, f\"{row['id_code']}.png\")\n        processed_img = preprocess_image(img_path)\n        save_path = os.path.join(processed_dir, f\"{row['id_code']}.png\")\n        cv2.imwrite(save_path, cv2.cvtColor(processed_img, cv2.COLOR_RGB2BGR))\n    print(f\"Processed and saved {len(train_csv)} images to {processed_dir}\")\n\n# Call\nsave_processed_images()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:46:13.045377Z","iopub.execute_input":"2025-10-28T18:46:13.045615Z","iopub.status.idle":"2025-10-28T18:57:23.385460Z","shell.execute_reply.started":"2025-10-28T18:46:13.045597Z","shell.execute_reply":"2025-10-28T18:57:23.384705Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Data Augmentation**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom PIL import Image\nimport torch\nfrom torch.utils.data import Dataset\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:57:23.386321Z","iopub.execute_input":"2025-10-28T18:57:23.386601Z","iopub.status.idle":"2025-10-28T18:57:25.002446Z","shell.execute_reply.started":"2025-10-28T18:57:23.386580Z","shell.execute_reply":"2025-10-28T18:57:25.001835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Paths\ndata_dir = '/kaggle/input/aptos2019-blindness-detection'\ntrain_csv_path = os.path.join(data_dir, 'train.csv')\nprocessed_dir = '/kaggle/working/processed_images'\n\n# Step 1: Stratified Train/Val/Test Split\ndef get_stratified_split(csv_path, train_size=0.8, val_size=0.1):\n    df = pd.read_csv(csv_path)\n    # Train + Val vs Test\n    train_val_df, test_df = train_test_split(\n        df, test_size=0.1, stratify=df['diagnosis'], random_state=42\n    )\n    # Train vs Val\n    relative_val_size = val_size / (train_size + val_size)\n    train_df, val_df = train_test_split(\n        train_val_df, test_size=relative_val_size, stratify=train_val_df['diagnosis'], random_state=42\n    )\n    return train_df, val_df, test_df\n\n# Save splits (optional, for reference)\ntrain_df, val_df, test_df = get_stratified_split(train_csv_path)\ntrain_df.to_csv(os.path.join(processed_dir, 'train_split.csv'), index=False)\nval_df.to_csv(os.path.join(processed_dir, 'val_split.csv'), index=False)\ntest_df.to_csv(os.path.join(processed_dir, 'test_split.csv'), index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:57:25.003151Z","iopub.execute_input":"2025-10-28T18:57:25.003331Z","iopub.status.idle":"2025-10-28T18:57:25.390853Z","shell.execute_reply.started":"2025-10-28T18:57:25.003317Z","shell.execute_reply":"2025-10-28T18:57:25.390106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Set up subplots\nfig, axes = plt.subplots(1, 2, figsize=(12, 5))\n\n# Train distribution\nsns.countplot(x='diagnosis', data=train_df, ax=axes[0], palette='viridis')\naxes[0].set_title('Train Class Distribution')\naxes[0].set_xlabel('Diagnosis')\naxes[0].set_ylabel('Count')\n\n# Validation distribution\nsns.countplot(x='diagnosis', data=val_df, ax=axes[1], palette='magma')\naxes[1].set_title('Validation Class Distribution')\naxes[1].set_xlabel('Diagnosis')\naxes[1].set_ylabel('Count')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:57:25.391718Z","iopub.execute_input":"2025-10-28T18:57:25.391938Z","iopub.status.idle":"2025-10-28T18:57:25.904128Z","shell.execute_reply.started":"2025-10-28T18:57:25.391922Z","shell.execute_reply":"2025-10-28T18:57:25.903251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class APTOSDataset(Dataset):\n    def __init__(self, df, processed_dir, transform=None):\n        self.data = df\n        self.processed_dir = processed_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n\n        row = self.data.iloc[idx]\n        img_name = f\"{row['id_code']}.png\"\n        img_path = os.path.join(self.processed_dir, img_name)\n        image = cv2.imread(img_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        label = torch.tensor(row['diagnosis'], dtype=torch.long)\n        \n        if self.transform:\n            if type(self.transform) == list:\n                if label.item() in [1,3, 4]:\n                    image = self.transform[1](image = image)['image']\n                else:\n                    image = self.transform[0](image = image)['image']\n            else:\n                image = self.transform(image = image)['image']\n        \n        return image, label\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:57:25.905011Z","iopub.execute_input":"2025-10-28T18:57:25.905384Z","iopub.status.idle":"2025-10-28T18:57:25.912139Z","shell.execute_reply.started":"2025-10-28T18:57:25.905358Z","shell.execute_reply":"2025-10-28T18:57:25.911397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n\n# Normal transformation (for all)\nbase_transform = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.RandomBrightnessContrast(brightness_limit=0.15, contrast_limit=0.15, p=0.3),\n    A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=15, val_shift_limit=10, p=0.2),\n    A.GaussianBlur(blur_limit=(3, 5), p=0.3),\n    A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n    ToTensorV2()\n])\n\n# Strong transformation (for minority classes)\nstrong_transform = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.RandomBrightnessContrast(brightness_limit=0.25, contrast_limit=0.25, p=0.7),\n    A.HueSaturationValue(hue_shift_limit=20, sat_shift_limit=25, val_shift_limit=20, p=0.6),\n    A.CLAHE(clip_limit=2.0, tile_grid_size=(8, 8), p=0.4),\n    A.GaussNoise(var_limit=(5.0, 25.0), p=0.4),\n    A.GaussianBlur(blur_limit=(3, 7), p=0.4),\n    A.RandomGamma(gamma_limit=(80, 120), p=0.3),\n    A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n    ToTensorV2()\n])\n\ntrain_transforms = [base_transform, strong_transform]\nval_test_transforms = A.Compose([\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ToTensorV2(),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:57:25.913102Z","iopub.execute_input":"2025-10-28T18:57:25.913311Z","iopub.status.idle":"2025-10-28T18:57:25.944311Z","shell.execute_reply.started":"2025-10-28T18:57:25.913294Z","shell.execute_reply":"2025-10-28T18:57:25.943595Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 4: Visualize Augmented Images\ndef visualize_augmented_data(df, processed_dir, transform, num_samples_per_class=2):\n    dataset = APTOSDataset(df, processed_dir, transform=transform)\n    fig, axs = plt.subplots(5, num_samples_per_class, figsize=(15, 25))\n    for class_label in range(5):\n        class_samples = df[df['diagnosis'] == class_label].sample(num_samples_per_class, random_state=42)\n        for i, (_, row) in enumerate(class_samples.iterrows()):\n            img_name = f\"{row['id_code']}.png\"\n            img_path = os.path.join(processed_dir, img_name)\n            print(img_path)\n            img = cv2.imread(img_path)\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            # Apply transforms\n            augmented = transform[0](image = img)\n            img_tensor = augmented[\"image\"]\n            # Convert back to PIL for display (denormalize)\n            img_array = img_tensor.permute(1, 2, 0).numpy()\n            img_array = img_array * np.array([0.229, 0.224, 0.225]) + np.array([0.485, 0.456, 0.406])\n            img_array = np.clip(img_array, 0, 1)\n            axs[class_label, i].imshow(img_array)\n            axs[class_label, i].set_title(f\"Augmented Class {class_label}: {row['id_code']}\")\n            axs[class_label, i].axis('off')\n            axs[class_label, i].imshow(img)\n            axs[class_label, i].set_title(f\"Not Augmented Class {class_label}: {row['id_code']}\")\n            axs[class_label, i].axis('off')\n    plt.tight_layout()\n    plt.show()\n\n# Call\nvisualize_augmented_data(train_df, processed_dir, train_transforms)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T18:57:25.947004Z","iopub.execute_input":"2025-10-28T18:57:25.947227Z","execution_failed":"2025-10-28T20:47:16.555Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset = APTOSDataset(train_df, processed_dir, transform=train_transforms)\nval_dataset = APTOSDataset(val_df, processed_dir, transform=val_test_transforms)\ntest_dataset = APTOSDataset(test_df, processed_dir, transform=val_test_transforms)\n\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-28T20:47:16.555Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True, num_workers=4)\nval_loader = DataLoader(val_dataset, batch_size=64, shuffle=False, num_workers=1)\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=1)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-28T20:47:16.556Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Model**","metadata":{}},{"cell_type":"code","source":"# Compute class weights for sampling and loss\ndef get_class_weights(df):\n    class_counts = df['diagnosis'].value_counts().sort_index()\n    total_samples = len(df)\n    num_classes = 5\n    weights = [total_samples / (num_classes * class_counts[i]) for i in range(num_classes)]\n    weights = np.array(weights)\n    weights = weights / weights.sum()\n    return weights\n\nclass_weights = get_class_weights(train_df)\nloss_weights = torch.tensor(class_weights, dtype=torch.float).to(torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"))","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-28T20:47:16.556Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch.nn.functional as F\n\nclass FocalLoss(torch.nn.Module):\n    def __init__(self, alpha = None, gamma = 2.0, reduction ='mean'):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduction = reduction\n    def forward(self, inputs, targets):\n        logpt = -F.cross_entropy(inputs, targets, reduction = 'none')\n        pt = logpt.exp()\n        if self.alpha is not None:\n            at = self.alpha[targets].to(inputs.device)\n            logpt = at * logpt\n        loss = -((1 - pt) ** self.gamma) * logpt\n        if self.reduction == 'mean':\n            return loss.mean()\n        elif self.reducton == 'sum':\n            return loss.sum()\n        return loss\nalpha = torch.tensor(loss_weights/ loss_weights.sum(), dtype=torch.float32)\ncriterion = FocalLoss(alpha=alpha, gamma = 1.0)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-28T20:47:16.556Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# === Dropout Model ===\nclass DenseNetWithDropout(nn.Module):\n    def __init__(self, num_classes=5, dropout_rate=0.4):\n        super(DenseNetWithDropout, self).__init__()\n        base = models.densenet121(weights=models.DenseNet121_Weights.IMAGENET1K_V1)\n        self.features = base.features\n        num_features = base.classifier.in_features\n        self.dropout = nn.Dropout(dropout_rate)\n        self.classifier = nn.Linear(num_features, num_classes)\n\n    def forward(self, x):\n        x = self.features(x)\n        x = nn.functional.relu(x, inplace=True)\n        x = nn.functional.adaptive_avg_pool2d(x, (1, 1)).view(x.size(0), -1)\n        x = self.dropout(x)\n        x = self.classifier(x)\n        return x\n\n# === MixUp Function ===\ndef mixup_data(x, y, alpha=0.4):\n    \"\"\"Apply MixUp with probability proportional to minority class.\"\"\"\n    if alpha <= 0:\n        return x, y, 1.0\n\n    lam = np.random.beta(alpha, alpha)\n    batch_size = x.size(0)\n    index = torch.randperm(batch_size).to(x.device)\n    mixed_x = lam * x + (1 - lam) * x[index, :]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam\n\n\ndef mixup_criterion(criterion, preds, y_a, y_b, lam):\n    return lam * criterion(preds, y_a) + (1 - lam) * criterion(preds, y_b)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T19:33:22.249831Z","iopub.execute_input":"2025-10-28T19:33:22.250319Z","iopub.status.idle":"2025-10-28T19:33:22.257359Z","shell.execute_reply.started":"2025-10-28T19:33:22.250296Z","shell.execute_reply":"2025-10-28T19:33:22.256544Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.optim.lr_scheduler as lr_scheduler\nfrom torchvision import models\nfrom tqdm.notebook import tqdm\nimport numpy as np\nfrom sklearn.metrics import f1_score\nimport random\n\n# === Device ===\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nBEST_MODEL_PATH = \"best_f1_model.pth\"\n\n\n\n\n\n\n\n\n\n# === Initialize ===\nmodel = DenseNetWithDropout(num_classes=5, dropout_rate=0.4).to(device)\ncriterion = FocalLoss(alpha=alpha, gamma=2.0)\n\n\n# === Helper Training Function ===\ndef train_phase(model, train_loader, val_loader, lr, unfrozen_layers, num_epochs, best_f1):\n    print(f\"\\n🚀 Starting Phase with LR={lr} | Unfrozen layers: {unfrozen_layers}\")\n\n    # Freeze all first\n    for name, param in model.named_parameters():\n        param.requires_grad = False\n\n    # Unfreeze specified parts\n    for layer_name in unfrozen_layers:\n        for name, param in model.named_parameters():\n            if layer_name in name:\n                param.requires_grad = True\n\n    optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=lr)\n    scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=6, verbose=True)\n\n    for epoch in tqdm(range(num_epochs), desc=f\"Phase ({unfrozen_layers})\"):\n        # --- TRAIN ---\n        model.train()\n        running_loss, preds, labels = 0.0, [], []\n\n        for imgs, lbls in tqdm(train_loader, desc=f\"Epoch {epoch+1} Train\", leave=False):\n            imgs, lbls = imgs.to(device), lbls.to(device)\n            optimizer.zero_grad()\n\n            # === Apply MixUp augmentation ===\n            # More probability for minority classes\n            mix_prob = 0.6 if torch.rand(1).item() < 0.6 else 0.2\n            if random.random() < mix_prob:\n                imgs, y_a, y_b, lam = mixup_data(imgs, lbls, alpha=0.4)\n                outputs = model(imgs)\n                loss = mixup_criterion(criterion, outputs, y_a, y_b, lam)\n            else:\n                outputs = model(imgs)\n                loss = criterion(outputs, lbls)\n\n            loss.backward()\n            nn.utils.clip_grad_norm_(model.parameters(), max_norm=2.0)\n            optimizer.step()\n\n            running_loss += loss.item()\n            preds.extend(outputs.argmax(1).detach().cpu().numpy())\n            labels.extend(lbls.cpu().numpy())\n\n        train_loss = running_loss / len(train_loader)\n        train_f1 = f1_score(labels, preds, average='macro')\n\n        # --- VALIDATE ---\n        model.eval()\n        val_loss, val_preds, val_labels = 0.0, [], []\n        with torch.no_grad():\n            for imgs, lbls in val_loader:\n                imgs, lbls = imgs.to(device), lbls.to(device)\n                outputs = model(imgs)\n                loss = criterion(outputs, lbls)\n                val_loss += loss.item()\n                val_preds.extend(outputs.argmax(1).cpu().numpy())\n                val_labels.extend(lbls.cpu().numpy())\n\n        val_loss /= len(val_loader)\n        val_f1 = f1_score(val_labels, val_preds, average='macro')\n\n        print(f\"Epoch {epoch+1}: Train F1={train_f1:.4f}, Val F1={val_f1:.4f}, Loss={val_loss:.4f}\")\n\n        if val_f1 > best_f1:\n            best_f1 = val_f1\n            torch.save(model.state_dict(), BEST_MODEL_PATH)\n            print(f\"✅ Model saved! New best F1: {best_f1:.4f}\")\n\n        scheduler.step(val_f1)\n\n    return best_f1\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T19:33:35.511722Z","iopub.execute_input":"2025-10-28T19:33:35.512311Z","iopub.status.idle":"2025-10-28T19:33:35.744280Z","shell.execute_reply.started":"2025-10-28T19:33:35.512287Z","shell.execute_reply":"2025-10-28T19:33:35.743711Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# === PHASES ===\nbest_f1 = 0.0\n\n# Phase 1: Train classifier only\nbest_f1 = train_phase(model, train_loader, val_loader, lr=1e-4, unfrozen_layers=[\"classifier\"], num_epochs=10, best_f1=best_f1)\n\n# Phase 2: Unfreeze denseblock4 + classifier\nbest_f1 = train_phase(model, train_loader, val_loader, lr=5e-5, unfrozen_layers=[\"denseblock4\", \"classifier\"], num_epochs=15, best_f1=best_f1)\n\n# Phase 3: Unfreeze denseblock3–4 + classifier\nbest_f1 = train_phase(model, train_loader, val_loader, lr=2e-5, unfrozen_layers=[\"denseblock3\", \"denseblock4\", \"classifier\"], num_epochs=20, best_f1=best_f1)\n\n# Phase 4: Fine-tune entire model\nbest_f1 = train_phase(model, train_loader, val_loader, lr=1e-5, unfrozen_layers=[\"features\", \"classifier\"], num_epochs=20, best_f1=best_f1)\n\nprint(f\"\\n🏁 Training Complete! Best Macro F1: {best_f1:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-28T19:33:38.219845Z","iopub.execute_input":"2025-10-28T19:33:38.220310Z","iopub.status.idle":"2025-10-28T20:07:19.891779Z","shell.execute_reply.started":"2025-10-28T19:33:38.220286Z","shell.execute_reply":"2025-10-28T20:07:19.890272Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_dir = '/kaggle/working/features'","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport numpy as np\n\n# Extract features\nmodel.eval()\ndef extract_features(loader, model, save_path):\n    features, labels = [], []\n    with torch.no_grad():\n        for images, lbls in loader:\n            images = images.to(device)\n            feats = model(images).cpu().numpy().reshape(len(images), -1)\n            features.append(feats)\n            labels.append(lbls.numpy())\n    features = np.concatenate(features)\n    labels = np.concatenate(labels)\n    np.savez(save_path, features=features, labels=labels)\n    return features, labels\n\n# Extract and save\ntrain_features, train_labels = extract_features(train_loader, model, os.path.join(feature_dir, 'train_features.npz'))\nval_features, val_labels = extract_features(val_loader, model, os.path.join(feature_dir, 'val_features.npz'))\ntest_features, test_labels = extract_features(test_loader, model, os.path.join(feature_dir, 'test_features.npz'))\nprint(f\"Train features shape: {train_features.shape}, Val features shape: {val_features.shape}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import time\n\n# Sleep for 1 hour (3600 seconds)\ntime.sleep(3600)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import StandardScaler, label_binarize\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.metrics import f1_score, roc_curve, auc\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, StackingClassifier # New Import\nfrom sklearn.svm import SVC\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.linear_model import (\n    LogisticRegression, PassiveAggressiveClassifier, Perceptron,\n    RidgeClassifier, SGDClassifier\n)\nfrom sklearn.neural_network import MLPClassifier\nfrom scipy.stats import randint, loguniform\nfrom imblearn.combine import SMOTEENN\nfrom itertools import cycle\nimport joblib\n\n# === Plot style ===\nplt.style.use('seaborn-v0_8')\nsns.set(font_scale=1.2)\n\n# === Paths ===\n# NOTE: Using a relative path for feature_dir, assuming execution context\nfeature_dir = 'features' # Assuming features were saved in './features'\noutput_dir = \"tuned_classifiers\"\nroc_dir = os.path.join(\"images\", \"tunedModel\")\nos.makedirs(output_dir, exist_ok=True)\nos.makedirs(roc_dir, exist_ok=True)\nprint(f\"Output directory created: {output_dir}\")\n\n# === Load data ===\ntry:\n    train_data = np.load(os.path.join(feature_dir, 'train_features.npz'))\n    val_data = np.load(os.path.join(feature_dir, 'val_features.npz'))\n    test_data = np.load(os.path.join(feature_dir, 'test_features.npz'))\nexcept FileNotFoundError:\n    print(f\"Error: Feature files not found in {feature_dir}. Please ensure the previous step ran correctly.\")\n    exit()\n\ntrain_features, train_labels = train_data['features'], train_data['labels']\nval_features, val_labels = val_data['features'], val_data['labels']\ntest_features, test_labels = test_data['features'], test_data['labels']\nprint(f\"Initial training set size: {len(train_features)} samples\")\n\n# === Apply SMOTEENN for hybrid resampling (CRITICAL for imbalance) ===\nprint(\"⚖️ Balancing training data using SMOTEENN...\")\n# SMOTEENN combines oversampling (SMOTE) and undersampling (Edited Nearest Neighbors)\nsmote_enn = SMOTEENN(random_state=42, n_jobs=-1)\ntrain_features_res, train_labels_res = smote_enn.fit_resample(train_features, train_labels)\nprint(f\"Resampled training set size: {train_features_res.shape[0]} samples\")\n\n# === Standardize features (CRITICAL for distance-based models like SVM/MLP) ===\nscaler = StandardScaler()\n# Fit only on the RESAMPLED training data\ntrain_features_scaled = scaler.fit_transform(train_features_res)\n# Transform validation and test data using the fitted scaler\nval_features_scaled = scaler.transform(val_features)\ntest_features_scaled = scaler.transform(test_features)\njoblib.dump(scaler, os.path.join(output_dir, 'scaler.pkl'))\nprint(f\"Scaled training features shape: {train_features_scaled.shape}\")\n\n# === Define classifiers (Level 0 Base Estimators) ===\nclassifiers = {\n    'dt': DecisionTreeClassifier(random_state=42),\n    'rf': RandomForestClassifier(random_state=42, n_jobs=-1),\n    # SVC requires probability=True for predict_proba (used for ROC plotting)\n    'svm': SVC(probability=True, random_state=42), \n    'knn': KNeighborsClassifier(n_jobs=-1),\n    'lr': LogisticRegression(max_iter=1000, random_state=42, n_jobs=-1),\n    'et': ExtraTreesClassifier(random_state=42, n_jobs=-1),\n    'mlp': MLPClassifier(max_iter=1000, random_state=42),\n    'ridge': RidgeClassifier(random_state=42),\n    # Removed NC, PA, Perceptron, SGD for faster runtime and focus on key models\n}\n\n# === Parameter distributions for RandomizedSearch ===\nparam_distributions = {\n    'dt': {'max_depth': [5, 10, 20, None],\n           'min_samples_split': randint(2, 10),\n           'min_samples_leaf': randint(1, 5)},\n    'rf': {'n_estimators': randint(100, 300),\n           'max_depth': [10, 20, None],\n           'min_samples_split': randint(2, 10),\n           'min_samples_leaf': randint(1, 5)},\n    'svm': {'C': loguniform(0.1, 10),\n            'kernel': ['rbf', 'linear'],\n            'gamma': ['scale', 'auto']},\n    'knn': {'n_neighbors': randint(3, 10),\n            'weights': ['uniform', 'distance'],\n            'p': [1, 2]},\n    'lr': {'C': loguniform(0.01, 10),\n           'solver': ['lbfgs', 'liblinear'],\n           'penalty': ['l2']},\n    'et': {'n_estimators': randint(100, 300),\n           'max_depth': [10, 20, None],\n           'min_samples_split': randint(2, 10),\n           'min_samples_leaf': randint(1, 5)},\n    'mlp': {'hidden_layer_sizes': [(64,), (128,), (128, 64), (256, 128)],\n            'alpha': loguniform(1e-5, 1e-2),\n            'learning_rate': ['constant', 'adaptive']},\n    'ridge': {'alpha': loguniform(0.1, 100)},\n}\n\n# === ROC plotting ===\ndef plot_roc(model_name, model, X, y, save_dir):\n    \"\"\"Plot one-vs-rest ROC for multiclass classification.\"\"\"\n    if not hasattr(model, \"predict_proba\"):\n        # RidgeClassifier does not have predict_proba\n        print(f\"⚠️ {model_name} does not support predict_proba, skipping ROC.\")\n        return\n\n    classes = np.unique(y)\n    y_bin = label_binarize(y, classes=classes)\n    y_score = model.predict_proba(X)\n\n    plt.figure(figsize=(7, 6))\n    colors = cycle(plt.cm.tab10.colors)\n\n    for i, color in zip(range(len(classes)), colors):\n        fpr, tpr, _ = roc_curve(y_bin[:, i], y_score[:, i])\n        roc_auc = auc(fpr, tpr)\n        plt.plot(fpr, tpr, color=color, lw=2,\n                 label=f'Class {classes[i]} (AUC={roc_auc:.2f})')\n\n    plt.plot([0, 1], [0, 1], 'k--', lw=2)\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title(f'ROC Curve - {model_name} [Image of ROC curve plot]')\n    plt.legend(loc=\"lower right\")\n    plt.tight_layout()\n    plt.savefig(os.path.join(save_dir, f\"{model_name}_roc.png\"))\n    plt.close()\n\n# Store best estimators for stacking\nbest_estimators = {}\nindividual_results = []\n\n# === Training & Evaluation for Base Models ===\nfor name, clf in classifiers.items():\n    print(f\"\\n🔹 Tuning {name}...\")\n\n    params = param_distributions[name]\n    if params:\n        search = RandomizedSearchCV(\n            clf,\n            param_distributions=params,\n            n_iter=20, # Reduced iterations for faster demo\n            cv=5,\n            scoring='f1_weighted', # Use weighted F1 for tuning\n            random_state=42,\n            n_jobs=-1,\n            verbose=0\n        )\n        search.fit(train_features_scaled, train_labels_res)\n        best_model = search.best_estimator_\n        print(f\"✅ Best Params: {search.best_params_}\")\n    else:\n        # For classifiers without hyperparameters to tune\n        best_model = clf.fit(train_features_scaled, train_labels_res)\n\n    # Save and store the best model\n    joblib.dump(best_model, os.path.join(output_dir, f\"{name}_tuned_model.pkl\"))\n    best_estimators[name] = best_model # Store for stacking\n\n    # Evaluate on Validation and Test sets\n    val_pred = best_model.predict(val_features_scaled)\n    test_pred = best_model.predict(test_features_scaled)\n    val_f1 = f1_score(val_labels, val_pred, average='weighted', zero_division=0)\n    test_f1 = f1_score(test_labels, test_pred, average='weighted', zero_division=0)\n\n    print(f\"Validation F1: {val_f1:.4f} | Test F1: {test_f1:.4f}\")\n    individual_results.append((name, val_f1, test_f1))\n\n    # ROC Plotting\n    plot_roc(name, best_model, test_features_scaled, test_labels, roc_dir)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nimport joblib\n\n# === Paths ===\nfeature_dir = '/kaggle/working/features'  # or your feature directory\noutput_dir = \"tuned_classifiers\"\nmeta_dir = \"meta_features\"\nos.makedirs(meta_dir, exist_ok=True)\n\n# === Load training features & labels ===\ntrain_data = np.load(os.path.join(feature_dir, 'train_features.npz'))\ntrain_features = train_data['features']\ntrain_labels = train_data['labels']\n\n# === Load scaler (used in base models) and scale train features ===\nscaler = joblib.load(os.path.join(output_dir, 'scaler.pkl'))\ntrain_features_scaled = scaler.transform(train_features)\n\n# === Load tuned base models ===\nclassifiers = {\n    'dt': joblib.load(os.path.join(output_dir, 'dt_tuned_model.pkl')),\n    'rf': joblib.load(os.path.join(output_dir, 'rf_tuned_model.pkl')),\n    'svm': joblib.load(os.path.join(output_dir, 'svm_tuned_model.pkl')),\n    'knn': joblib.load(os.path.join(output_dir, 'knn_tuned_model.pkl')),\n    'lr': joblib.load(os.path.join(output_dir, 'lr_tuned_model.pkl')),\n    'et': joblib.load(os.path.join(output_dir, 'et_tuned_model.pkl')),\n    'mlp': joblib.load(os.path.join(output_dir, 'mlp_tuned_model.pkl')),\n    'nc': joblib.load(os.path.join(output_dir, 'nc_tuned_model.pkl')),\n    'pa': joblib.load(os.path.join(output_dir, 'pa_tuned_model.pkl')),\n    'perceptron': joblib.load(os.path.join(output_dir, 'perceptron_tuned_model.pkl')),\n    'ridge': joblib.load(os.path.join(output_dir, 'ridge_tuned_model.pkl')),\n    'sgd': joblib.load(os.path.join(output_dir, 'sgd_tuned_model.pkl'))\n}\n\n# === Base models that support predict_proba ===\nproba_classifiers = ['dt', 'rf', 'svm', 'knn', 'lr', 'et', 'mlp', 'sgd']\n\n# === Prepare one-hot encoder for models without predict_proba ===\nunique_labels = np.unique(train_labels)\nonehot_encoder = OneHotEncoder(sparse_output=False, categories=[unique_labels])\n\n# === Generate predictions (meta-features) ===\nmeta_features = []\nfor name, clf in classifiers.items():\n    print(f\"Generating training meta-features for {name}...\")\n    \n    if name in proba_classifiers:\n        preds = clf.predict_proba(train_features_scaled)\n    else:\n        pred_labels = clf.predict(train_features_scaled)\n        preds = onehot_encoder.fit_transform(pred_labels.reshape(-1, 1))\n    \n    np.save(os.path.join(meta_dir, f\"{name}_train_predictions.npy\"), preds)\n    meta_features.append(preds)\n\n# === Stack meta-features from all base models ===\nmeta_features = np.hstack(meta_features)\n\n# === Scale meta-features ===\nmeta_scaler = StandardScaler()\nmeta_features_scaled = meta_scaler.fit_transform(meta_features)\njoblib.dump(meta_scaler, os.path.join(meta_dir, 'meta_scaler.pkl'))\n\n# === Save meta-features and labels ===\nnp.save(os.path.join(meta_dir, 'meta_features_train.npy'), meta_features_scaled)\nnp.save(os.path.join(meta_dir, 'train_labels.npy'), train_labels)\n\nprint(\"✅ Meta-features for TRAIN set generated and saved successfully.\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}