{"metadata":{"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"accelerator":"GPU","colab":{"gpuType":"T4","machine_shape":"hm","provenance":[]},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing Libraries","metadata":{"id":"XtoHvXTQCPGf"}},{"cell_type":"code","source":"import torch\nimport pathlib\nimport random\nimport cv2\nimport gc\nimport sys\nimport os\nimport glob\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport sklearn.model_selection as ms\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.nn as nn\nimport albumentations as A\nimport shutil\n\n\nfrom sklearn.metrics import balanced_accuracy_score, cohen_kappa_score, accuracy_score, classification_report\nfrom tqdm import tqdm\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import Compose, Normalize, ToTensor\nfrom pathlib import Path\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.utils.data import WeightedRandomSampler\n\n\n\n\nSEED = 42\n\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed(SEED)\nnp.random.seed(SEED)\nrandom.seed(SEED)\n\n# Fixing one of the most annoying \"features\" of opencv\ncv2.ocl.setUseOpenCL(False)\ncv2.setNumThreads(0)","metadata":{"id":"OeJwXOKjtgbV","outputId":"abed1a88-9f84-4299-9622-476bcb818c8f"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Mounting Google Drive and Extracting Data\n\nThis step assumes that you have the dataset in your Google Drive.","metadata":{"id":"wqtAguN-CS6Y"}},{"cell_type":"code","source":"# This will mount your google drive folder\nfrom google.colab import drive","metadata":{"id":"ifGj2LjjnzXG"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"drive.mount('/content/drive')","metadata":{"id":"j1elAsDwnIlY","outputId":"6917e6a7-6dfc-4964-cfd5-a0da2b9f6b0c"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\n# Define paths\nDATA_DIR = '/content/data'\nos.makedirs(DATA_DIR, exist_ok=True)\n\n# Extract the dataset\n!unzip -q /content/drive/MyDrive/aptos2019.zip -d {DATA_DIR}","metadata":{"id":"YMox8MvarLPF","outputId":"98ec8241-f823-4e0b-d25f-eba01839234b"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls /content/data/","metadata":{"id":"oxNK22eorf_d","outputId":"c75bb8d2-8590-4561-a395-01b8958c13ae"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Loading and Exploring Data\n\nIn this section, I have:\n\n- Uploaded the training, validation, and test datasets, examined their structure, and analyzed class distributions.\n- Observed a significant class imbalance, with class `0` making up nearly half of the dataset.\n- Combined the training and validation datasets to create stratified folds for cross-validation, ensuring no data leakage.","metadata":{"id":"4YROk4Y_StUV"}},{"cell_type":"code","source":"# Define file paths\nTRAIN_CSV = f'{DATA_DIR}/train_1.csv'\nVALID_CSV = f'{DATA_DIR}/valid.csv'\nTEST_CSV = f'{DATA_DIR}/test.csv'\n\n# Load data with error handling\ndef load_csv(file_path):\n    \"\"\"Load a CSV file and handle errors gracefully.\"\"\"\n    try:\n        df = pd.read_csv(file_path)\n        print(f\"Loaded {file_path} successfully. Shape: {df.shape}\")\n        return df\n    except FileNotFoundError:\n        print(f\"Error: {file_path} not found!\")\n        sys.exit(1)\n\n# Load datasets\ntrain_df = load_csv(TRAIN_CSV)\nvalid_df = load_csv(VALID_CSV)\ntest_df = load_csv(TEST_CSV)\n\n# Display dataset information\nprint(\"Train Data Preview:\")\nprint(train_df.head(), \"\\n\")\nprint(f\"Training Set Size: {train_df.shape[0]} samples\")\nprint(f\"Validation Set Size: {valid_df.shape[0]} samples\")","metadata":{"id":"tu3LIBffsh2i","outputId":"43a3e36d-bb24-45d1-ae42-6950499231a4"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define Image Paths\nTRAIN_IMAGES_DIR = '/content/data/train_images/train_images/'\nVAL_IMAGES_DIR = '/content/data/val_images/val_images/'\nTEST_IMAGES_DIR = '/content/data/test_images/test_images/'","metadata":{"id":"2quXdimnts4r"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to calculate and display class distributions\ndef print_class_distributions(df, dataset_name):\n    \"\"\"Print and plot class distributions in the dataset.\"\"\"\n    class_counts = df['diagnosis'].value_counts(normalize=True) * 100\n    class_counts.sort_index().plot(\n        kind='bar', title=f'Class Distribution in {dataset_name} (in %)', figsize=(8, 5)\n    )\n    plt.xlabel(\"Class\")\n    plt.ylabel(\"Percentage\")\n    plt.show()\n    print(f\"Class Distribution in {dataset_name} (in %):\\n{class_counts.round(2)}\\n\")\n\n# Plot class distributions for train and validation datasets\nprint_class_distributions(train_df, \"Training Set\")\nprint_class_distributions(valid_df, \"Validation Set\")","metadata":{"id":"d14VyiDPt1b3","outputId":"5da1011a-6876-4603-9503-e5c5ab6beb97"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Data Leakage Check\n\n# Ensure there are no overlaps between training and validation IDs\ntrain_ids = set(train_df['id_code'])\nvalid_ids = set(valid_df['id_code'])\n\n# Check for overlaps\nassert len(train_ids.intersection(valid_ids)) == 0, \"Overlap detected between training and validation sets!\"\nprint(\"No overlap between training and validation sets. Split integrity confirmed!\")","metadata":{"id":"TDLz1jS2t4Q5","outputId":"52eb25ad-f13f-4068-a08c-574607c6be10"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Unified directory for all images\nALL_IMAGES_DIR = '/content/data/all_images'\n\n# Create directory if it doesn't exist and copy images\n!mkdir -p {ALL_IMAGES_DIR}\n!cp {TRAIN_IMAGES_DIR}/*.png {ALL_IMAGES_DIR}\n!cp {VAL_IMAGES_DIR}/*.png {ALL_IMAGES_DIR}","metadata":{"id":"GZH2LcWHjmw3"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\n# Stratified K-Folds\n\n# Combine training and validation datasets\ntrain_full_df = pd.concat([train_df, valid_df], ignore_index=True)\nprint(f\"Combined train+valid shape: {train_full_df.shape}\")\n\n# Define number of folds\nN_FOLDS = 5\n\n# Perform stratified K-Fold split\nskf = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\ny = train_full_df['diagnosis'].values\n\nfolds = []\nfor train_index, val_index in skf.split(train_full_df, y):\n    fold_train_df = train_full_df.iloc[train_index].reset_index(drop=True)\n    fold_val_df = train_full_df.iloc[val_index].reset_index(drop=True)\n    folds.append((fold_train_df, fold_val_df))\n\nprint(f\"Created {N_FOLDS} folds.\")","metadata":{"id":"r1fJ4Hmafz7f","outputId":"ba22eed1-0373-4478-a465-25a3843df424"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Validate Fold Integrity\n\n# Ensure no overlaps and consistency in folds\ntotal_samples = 0\nfor fold_train_df, fold_val_df in folds:\n    # Check for overlaps\n    assert not fold_train_df['id_code'].isin(fold_val_df['id_code']).any(), \"Overlap found between train and val sets\"\n    # Count validation samples\n    total_samples += fold_val_df.shape[0]\n\n# Verify total sample count\nassert total_samples == train_full_df.shape[0], \"Mismatch in the total number of samples\"\nprint(\"Fold integrity validated. All folds are consistent and non-overlapping.\")","metadata":{"id":"kn3nXCQmq1yb","outputId":"0dd0e050-66ca-4929-f9cb-69dd7a907f91"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Image Visualization and Preprocessing\n\n- Visualized sample retinal images from the training set.\n- Applied preprocessing techniques, including resizing, cropping uninformative areas, and enhancing image clarity using Gaussian blurring (based on Ben Graham's method).\n- Compared original images with preprocessed versions to highlight improvements and better prepare the data for model training.\n","metadata":{"id":"JG7spgl4fKNi"}},{"cell_type":"code","source":"# Example preprocessing function\ndef preprocess_image(image_path, size=(256, 256)):\n    \"\"\"\n    Preprocesses an image by resizing it to the specified size.\n    \"\"\"\n    image = cv2.imread(image_path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = cv2.resize(image, size)  # Resize to specified size\n    return image\n\n\n# Preprocess and visualize sample images in one row\ndef visualize_images(image_paths, image_titles):\n    plt.figure(figsize=(20, 5))\n    for idx, (img_path, title) in enumerate(zip(image_paths, image_titles)):\n        img = preprocess_image(img_path)\n        plt.subplot(1, len(image_paths), idx + 1)\n        plt.imshow(img)\n        plt.title(title)\n        plt.axis(\"off\")\n    plt.show()\n\n# Get random sample of 5 images\nsample_images = train_df['id_code'].sample(5).values\nsample_paths = [f\"{TRAIN_IMAGES_DIR}{img_name}.png\" for img_name in sample_images]\nsample_titles = [f\"Image: {img_name}\" for img_name in sample_images]\n\n# Visualize images\nvisualize_images(sample_paths, sample_titles)","metadata":{"id":"TxwPBlq0vXUL","outputId":"d83e782a-6020-4c32-fea5-26c21867662d"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from (https://www.kaggle.com/code/ratthachat/aptos-eye-preprocessing-in-diabetic-retinopathy)\n\n# Function to crop uninformative borders from the image\ndef crop_image_from_gray(img, tol=7):\n    \"\"\"\n    Crop out black borders or uninformative parts from an image.\n    \"\"\"\n    if img.ndim == 2:  # For grayscale images\n        mask = img > tol\n        return img[np.ix_(mask.any(1), mask.any(0))]\n    elif img.ndim == 3:  # For color images\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img > tol\n        if mask.any():\n            return img[np.ix_(mask.any(1), mask.any(0))]\n    return img  # Return original image if too dark\n\n# Function to apply Ben Graham's preprocessing technique\ndef load_ben_color(path, img_size=256, sigmaX=10):\n    \"\"\"\n    Apply Ben Graham's preprocessing to an input image.\n    \"\"\"\n    image = cv2.imread(path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = crop_image_from_gray(image)\n    image = cv2.resize(image, (img_size, img_size))\n    image = cv2.addWeighted(image, 4, cv2.GaussianBlur(image, (0, 0), sigmaX), -4, 128)\n    return image","metadata":{"id":"kOllIFSIT2Pf"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize images before and after preprocessing\ndef visualize_preprocessed_images(df, image_dir, sample_size=5, img_size=256):\n    \"\"\"\n    Visualize original and preprocessed images side by side for a random sample.\n    \"\"\"\n    fig, axs = plt.subplots(2, sample_size, figsize=(20, 10))\n    sample = df.sample(sample_size, random_state=42)\n\n    for count, (_, row) in enumerate(sample.iterrows()):\n        img_path = os.path.join(image_dir, f\"{row['id_code']}.png\")\n\n        # Load original image\n        img_original = cv2.imread(img_path)\n        img_original = cv2.cvtColor(img_original, cv2.COLOR_BGR2RGB)\n\n        # Apply preprocessing\n        img_preprocessed = load_ben_color(img_path, img_size=img_size)\n\n        # Plot original image\n        axs[0, count].imshow(img_original)\n        axs[0, count].set_title(f\"Original (Label: {row['diagnosis']})\")\n        axs[0, count].axis('off')\n\n        # Plot preprocessed image\n        axs[1, count].imshow(img_preprocessed)\n        axs[1, count].set_title(f\"Preprocessed (Label: {row['diagnosis']})\")\n        axs[1, count].axis('off')\n\n    plt.tight_layout()\n    plt.show()\n\n\n# Visualize a random sample of images\nvisualize_preprocessed_images(train_df, TRAIN_IMAGES_DIR)","metadata":{"id":"3dtnHtuqV6hy","outputId":"03f3d4de-dcb5-4d2e-a561-9247baa5df75"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Dataset Preparation and Model Definition\n\nIn this section, I:\n\n- Defined minimal data augmentation techniques using Albumentations for training and validation, including only rotations, flips, and normalization, to avoid conflicting with the Ben Graham preprocessing defined earlier.\n- Created a custom dataset class (`APTOSDataset`) to efficiently load and preprocess retinal images.\n- Built a custom EfficientNet-based model with the flexibility to:\n  - Choose between variants (`b0` or `b4`).\n  - Freeze or partially unfreeze the backbone for transfer learning.\n  - Customize the classifier head to suit the dataset's number of classes.","metadata":{"id":"lX0NeF2DfSnM"}},{"cell_type":"code","source":"def get_train_transforms(img_size=256):\n    \"\"\"\n    Data augmentation for training images.\n    Includes random rotations, flips, scaling, and normalization.\n    \"\"\"\n    return A.Compose([\n        A.Rotate(limit=15, p=0.5),\n        A.HorizontalFlip(p=0.5),\n        A.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.1, rotate_limit=0, p=0.5),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        ToTensorV2()\n    ])\n\ndef get_val_transforms(img_size=256):\n    \"\"\"\n    Transformations for validation images.\n    Only normalization is applied.\n    \"\"\"\n    return A.Compose([\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        ToTensorV2()\n    ])","metadata":{"id":"AC6jXFkiT7HD"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class APTOSDataset(Dataset):\n    \"\"\"\n    Custom Dataset for loading and preprocessing APTOS 2019 retinal images.\n    \"\"\"\n    def __init__(self, images_dir: str, dataset: pd.DataFrame, transforms=None):\n        self.images_dir = images_dir\n        self.dataset = dataset\n        self.transforms = transforms\n\n    def __getitem__(self, idx: int) -> dict:\n        # Get image and label\n        entry = self.dataset.iloc[idx]\n        img_name = entry['id_code']\n        img_path = os.path.join(self.images_dir, f\"{img_name}.png\")\n        label = torch.tensor(int(entry['diagnosis'])).long()\n\n        # Load the image\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        # Apply augmentations if provided\n        if self.transforms:\n            augmented = self.transforms(image=img)\n            img = augmented['image']\n\n        return {\n            \"image\": img,\n            \"label\": label\n        }\n\n    def __len__(self) -> int:\n        return len(self.dataset)","metadata":{"id":"RMR2t0CnyXf5"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define EfficientNet-based model\nfrom torchvision.models import efficientnet_b0, efficientnet_b4, EfficientNet_B0_Weights, EfficientNet_B4_Weights\n\nclass EfficientNet(nn.Module):\n    \"\"\"\n    Custom EfficientNet model with options for transfer learning and classifier customization.\n    \"\"\"\n    def __init__(self, num_classes=5, dropout_rate=0.2, variant=\"b0\", freeze_backbone=True, unfreeze_blocks=0):\n        super(EfficientNet, self).__init__()\n\n        # Select EfficientNet variant\n        if variant == \"b0\":\n            self.backbone = efficientnet_b0(weights=EfficientNet_B0_Weights.IMAGENET1K_V1)\n        elif variant == \"b4\":\n            self.backbone = efficientnet_b4(weights=EfficientNet_B4_Weights.IMAGENET1K_V1)\n        else:\n            raise ValueError(\"Unsupported EfficientNet variant. Choose 'b0' or 'b4'.\")\n\n        # Freeze the backbone if specified\n        if freeze_backbone:\n            for param in self.backbone.features.parameters():\n                param.requires_grad = False\n\n        # Optionally unfreeze the last N blocks\n        if unfreeze_blocks > 0:\n            print(f\"Unfreezing the last {unfreeze_blocks} blocks of the EfficientNet backbone...\")\n            num_blocks = len(self.backbone.features)\n            for idx in range(num_blocks - unfreeze_blocks, num_blocks):\n                for param in self.backbone.features[idx].parameters():\n                    param.requires_grad = True\n\n        # Replace the classifier to match the dataset\n        in_features = self.backbone.classifier[1].in_features\n        self.backbone.classifier = nn.Sequential(\n            nn.Dropout(dropout_rate),\n            nn.Linear(in_features, num_classes)\n        )\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        return self.backbone(x)","metadata":{"id":"YDNfjH2Ox5Ov"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Training Framework: Trainer and Configuration Classes\n\nIn this section, I:\n\n- Built a modular `Trainer` class to manage the training process, including:\n  - Initializing data loaders, optimizer, scheduler, and loss functions (`CrossEntropy` or `FocalLoss`).\n  - Implementing training and validation loops with metrics such as loss, balanced accuracy, quadratic kappa, and accuracy.\n  - Saving the best model for each fold to `.pth` files based on validation performance.\n  - Storing metrics (e.g., loss, accuracy) for each fold to enable analysis and reproducibility.\n- Designed a `Config` class to centralize hyperparameters and settings, such as:\n  - Dataset and model configurations (batch size, learning rate, image size, etc.).\n  - Scheduler options (`CosineAnnealing` or `ReduceLROnPlateau`) for dynamic learning rate adjustment.\n  - Fine-tuning controls for EfficientNet, such as freezing or unfreezing backbone blocks.\n\nThis framework simplifies training, ensures reproducibility, and supports scalability for various deep learning experiments.\n\n\n","metadata":{"id":"rSRZnYIvD3gE"}},{"cell_type":"code","source":"class Trainer:\n    \"\"\"\n    Trainer class to manage the training and validation of a model.\n    Includes data loading, optimizer and scheduler initialization, and metric computation.\n    \"\"\"\n    def __init__(self, train_df, val_df, train_path, val_path, cfg, model, class_weights=None):\n        self.train_df = train_df\n        self.val_df = val_df\n        self.train_path = train_path\n        self.val_path = val_path\n        self.cfg = cfg\n        self.model = model.to(cfg.device)\n        self.class_weights = class_weights\n\n        # Initialize attributes for tracking metrics\n        self.best_score = -float('inf')  # Best Balanced Accuracy (BACC)\n        self.best_kappa = -float('inf')  # Best Kappa\n        self.best_accuracy = -float('inf')  # Best Accuracy\n\n        self.train_losses, self.val_losses, self.val_baccs = [], [], []\n        self.val_kappas, self.val_accuracies = [], []  # Lists for Kappa and Accuracy\n        self.val_preds, self.val_labels = [], []\n\n        # Initialize loaders, optimizer, scheduler, and loss function\n        self.init_loaders()\n        self.init_optimizer_loss()\n\n    def init_loaders(self):\n        \"\"\"Initialize data loaders for training and validation.\"\"\"\n        self.train_loader = DataLoader(\n            APTOSDataset(self.train_path, self.train_df, get_train_transforms(self.cfg.image_size)),\n            batch_size=self.cfg.batch_size, shuffle=True,\n            num_workers=self.cfg.num_workers, drop_last=True\n        )\n\n        self.val_loader = DataLoader(\n            APTOSDataset(self.val_path, self.val_df, get_val_transforms(self.cfg.image_size)),\n            batch_size=self.cfg.batch_size, shuffle=False,\n            num_workers=self.cfg.num_workers\n        )\n\n    def init_optimizer_loss(self):\n        \"\"\"Initialize optimizer, scheduler, and loss function.\"\"\"\n        backbone_params = [p for p in self.model.backbone.features.parameters() if p.requires_grad]\n        classifier_params = [p for p in self.model.backbone.classifier.parameters()]\n\n        self.optimizer = optim.Adam([\n            {'params': backbone_params, 'lr': self.cfg.backbone_learning_rate},\n            {'params': classifier_params, 'lr': self.cfg.learning_rate}\n        ], weight_decay=self.cfg.weight_decay)\n\n        self.scheduler_obj = None\n        if self.cfg.scheduler == 'cosine':\n            self.scheduler_obj = optim.lr_scheduler.CosineAnnealingLR(self.optimizer, T_max=self.cfg.num_epochs)\n        elif self.cfg.scheduler == 'plateau':\n            self.scheduler_obj = optim.lr_scheduler.ReduceLROnPlateau(self.optimizer, patience=self.cfg.lr_patience, factor=self.cfg.lr_factor)\n\n        # Loss function selection: FocalLoss or CrossEntropyLoss\n        if self.cfg.use_focal_loss:\n            # Check explicitly if class_weights is None\n            if self.class_weights is not None:\n                self.loss_fn = FocalLoss(alpha=self.class_weights, gamma=2)\n            else:\n                self.loss_fn = FocalLoss(gamma=2)\n        else:\n            self.loss_fn = nn.CrossEntropyLoss(weight=self.class_weights)\n\n    def train_epoch(self):\n        \"\"\"Train for one epoch.\"\"\"\n        self.model.train()\n        running_loss = 0.0\n        for batch in tqdm(self.train_loader, desc=f\"Training Epoch {self.epoch}\"):\n            images, labels = batch['image'].to(self.cfg.device), batch['label'].to(self.cfg.device)\n            self.optimizer.zero_grad()\n            outputs = self.model(images)\n            loss = self.loss_fn(outputs, labels)\n            loss.backward()\n            self.optimizer.step()\n            running_loss += loss.item()\n        return running_loss / len(self.train_loader)\n\n    def validate_epoch(self):\n        \"\"\"Validate the model and compute metrics.\"\"\"\n        self.model.eval()\n        running_loss = 0.0\n        all_preds, all_labels = [], []\n\n        # Clear previous predictions and labels\n        self.val_preds, self.val_labels = [], []\n\n        with torch.no_grad():\n            for batch in tqdm(self.val_loader, desc=\"Validating\"):\n                images, labels = batch['image'].to(self.cfg.device), batch['label'].to(self.cfg.device)\n                outputs = self.model(images)\n                loss = self.loss_fn(outputs, labels)\n                running_loss += loss.item()\n\n                # Store predictions and labels\n                all_preds.append(outputs.cpu().numpy())  # Convert to numpy\n                all_labels.append(labels.cpu().numpy())\n\n        avg_loss = running_loss / len(self.val_loader)\n        all_preds = np.concatenate(all_preds, axis=0)\n        all_labels = np.concatenate(all_labels, axis=0)\n\n        # Update val_preds and val_labels\n        self.val_preds.extend(all_preds)\n        self.val_labels.extend(all_labels)\n\n        # Compute metrics\n        predicted_classes = all_preds.argmax(axis=1)\n        bacc = balanced_accuracy_score(all_labels, predicted_classes)\n        kappa = cohen_kappa_score(all_labels, predicted_classes, weights='quadratic')\n        accuracy = accuracy_score(all_labels, predicted_classes)\n\n        # Append metrics\n        self.val_baccs.append(bacc)\n        self.val_kappas.append(kappa)\n        self.val_accuracies.append(accuracy)\n\n        return avg_loss, bacc, kappa, accuracy\n\n\n\n    def run(self):\n        \"\"\"Run training and validation for all epochs.\"\"\"\n\n        # Cleanup old files for the current fold only\n        results_path = f\"{self.cfg.save_dir}/fold_{self.cfg.fold}_results.pth\"\n        best_model_path = f\"{self.cfg.save_dir}/best_model_fold_{self.cfg.fold}.pth\"\n\n        if os.path.exists(results_path):\n            print(f\"Deleting previous results file: {results_path}\")\n            os.remove(results_path)\n\n        if os.path.exists(best_model_path):\n            print(f\"Deleting previous best model file: {best_model_path}\")\n            os.remove(best_model_path)\n\n        # Training loop\n        for self.epoch in range(1, self.cfg.num_epochs + 1):\n            train_loss = self.train_epoch()\n            val_loss, val_bacc, val_kappa, val_accuracy = self.validate_epoch()\n\n            # Append metrics\n            self.train_losses.append(train_loss)\n            self.val_losses.append(val_loss)\n\n            # Save the best model\n            if val_bacc > self.best_score:\n                self.best_score = val_bacc\n                torch.save(self.model.state_dict(), best_model_path)\n                print(f\"Saved best model for fold {self.cfg.fold} with Balanced Accuracy: {val_bacc:.4f}\")\n\n            # Print metrics for the epoch\n            print(f\"Epoch {self.epoch}: Train Loss={train_loss:.4f}, Val Loss={val_loss:.4f}, \"\n                  f\"Val BAcc={val_bacc:.4f}, Kappa={val_kappa:.4f}, Accuracy={val_accuracy:.4f}\")\n\n            # Scheduler step\n            if self.scheduler_obj:\n                if self.cfg.scheduler == 'plateau':\n                    self.scheduler_obj.step(val_loss)\n                else:\n                    self.scheduler_obj.step()\n\n        # Save results at the end of training for the fold\n        results = {\n            \"train_losses\": self.train_losses,\n            \"val_losses\": self.val_losses,\n            \"val_baccs\": self.val_baccs,\n            \"val_kappas\": self.val_kappas,\n            \"val_accuracies\": self.val_accuracies,\n        }\n        torch.save(results, results_path)\n        print(f\"Results for fold {self.cfg.fold} saved to {results_path}.\")\n\n\n","metadata":{"id":"JJAyFrgsErxg"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Config:\n    \"\"\"\n    Unified configuration class to handle all hyperparameters and paths for different experiments.\n    \"\"\"\n    def __init__(self):\n        # General Settings\n        self.batch_size = 32  # Default batch size, can be adjusted per experiment\n        self.num_workers = 2  # Number of data loading workers\n        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        self.seed = 42  # Seed for reproducibility\n        self.num_classes = 5  # Number of output classes\n        self.image_size = 256  # Default resolution (e.g., EfficientNet-B0), can be adjusted for other variants\n\n        # Model Configuration\n        self.model_name = \"efficientnet-b0\"  # Default model name\n        self.variant = \"b0\"  # Model variant (e.g., \"b0\", \"b4\")\n        self.freeze_backbone = True  # Whether to freeze the backbone initially\n        self.unfreeze_blocks = 0  # Number of blocks to unfreeze during fine-tuning\n\n        # Training Control\n        self.num_epochs = 20  # Number of training epochs\n        self.learning_rate = 1e-3  # Learning rate for the classifier head\n        self.backbone_learning_rate = 1e-5  # Learning rate for the backbone\n        self.weight_decay = 1e-4  # Weight decay for regularization\n\n        # Scheduler Configuration\n        self.scheduler = 'plateau'  # Options: 'plateau' or 'cosine'\n        self.lr_step_size = 5  # For step-based schedulers\n        self.lr_gamma = 0.1  # LR reduction factor\n        self.lr_patience = 3  # Patience for ReduceLROnPlateau\n        self.lr_factor = 0.1  # Factor for ReduceLROnPlateau\n\n        # Loss Function\n        self.use_focal_loss = False  # Use FocalLoss if True, otherwise CrossEntropyLoss\n\n        # Fold Information\n        self.fold = 0  # Current fold index, updated dynamically during training\n\n        # Save Directory\n        self.save_dir = Path(\"/content/data/snapshots\")  # Default save directory\n        self.save_dir.mkdir(parents=True, exist_ok=True)\n\n","metadata":{"id":"Q9DL9UCg07Rq"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Efficient Training: Preprocessing and Caching Images (from Lecture 9)\n\nTo speed up the training process, I preprocessed and resized all images in advance. Instead of preprocessing images during every epoch, the images are cached in a resized and preprocessed format. This optimization reduces the training epoch time significantly, from approximately 4 minutes to just 5 seconds.\n\nKey steps performed:\n\n- Applied Ben Graham's preprocessing method to all images.\n- Resized images to the target size (256x256) and saved them to a new directory (ALL_IMAGES_RESIZED_DIR) for fast retrieval during training.\n","metadata":{"id":"xNxZK00ODbLj"}},{"cell_type":"code","source":"# Paths for original and resized images\nALL_IMAGES_DIR = '/content/data/all_images'\nALL_IMAGES_RESIZED_DIR = '/content/data/all_images_resized'\n\n# Ensure the resized directory exists\nos.makedirs(ALL_IMAGES_RESIZED_DIR, exist_ok=True)\n\n# Get all image file paths\nimg_paths = glob.glob(os.path.join(ALL_IMAGES_DIR, '*.png'))\nimg_size = 256  # Target image size\n\n# Resize and preprocess all images\nfor img_path in tqdm(img_paths, desc=\"Resizing and Caching Images\"):\n    # Read and preprocess the image\n    img = cv2.imread(img_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n    # Apply Ben Graham's preprocessing\n    img = crop_image_from_gray(img)\n    img = cv2.resize(img, (img_size, img_size))\n    img = cv2.addWeighted(img, 4, cv2.GaussianBlur(img, (0, 0), 10), -4, 128)\n\n    # Save the preprocessed image to the resized directory\n    img_save = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n    base_name = os.path.basename(img_path)\n    cv2.imwrite(os.path.join(ALL_IMAGES_RESIZED_DIR, base_name), img_save)\n","metadata":{"id":"dZIH1iyHonbf","outputId":"73cad67a-5766-4036-e761-c89b6b9e35d4"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training Loop (Baseline Model)\n\nIn this section, I:\n\n- Implemented the `run_cross_validation` function to train and evaluate models across multiple folds:\n  - For each fold:\n    - Initialized the `Trainer` with fold-specific data splits, dynamically scaled class weights (if provided), and model configurations.\n    - Tracked the best balanced accuracy (BACC), quadratic kappa, and accuracy during training.\n    - Saved the best-performing model for each fold in `.pth` files.\n  - At the end:\n    - Computed and displayed the mean and standard deviation of BACC, kappa, and accuracy across folds to summarize performance.\n\n- Used `run_cross_validation` with default configurations to train the baseline model and evaluate its performance.\n\nThis process establishes a robust evaluation framework, providing insights into the model's generalization across folds.\n\n","metadata":{"id":"o5tIbwx6SivJ"}},{"cell_type":"code","source":"def run_cross_validation(cfg, image_dir, custom_scaling_factors=None):\n    \"\"\"\n    Runs cross-validation training with specified configuration, image directory, and optional class weights.\n\n    Args:\n        cfg (Config): Configuration object containing hyperparameters.\n        image_dir (str): Path to the directory containing all images.\n        custom_scaling_factors (list, optional): Scaling factors for class weights. If None, no scaling is applied.\n\n    Returns:\n        dict: Dictionary containing mean and std for balanced accuracy, kappa, and accuracy across folds.\n    \"\"\"\n    # Store results for BACC, Kappa, and Accuracy across folds\n    fold_results_bacc = []\n    fold_results_kappa = []\n    fold_results_acc = []\n\n    for fold_idx, (fold_train_df, fold_val_df) in enumerate(folds):\n        cfg.fold = fold_idx  # Set the current fold index\n        print(f\"Starting Fold {fold_idx + 1}/{len(folds)}...\")\n\n        # Calculate class weights if scaling factors are provided\n        class_weights = None\n        if custom_scaling_factors is not None:\n            # Calculate class counts for the training set of this fold\n            class_counts = fold_train_df['diagnosis'].value_counts().sort_index().values\n            total_samples = sum(class_counts)\n\n            # Compute raw class weights (inverse frequency)\n            raw_class_weights = [total_samples / count for count in class_counts]\n\n            # Apply custom scaling factors\n            scaled_class_weights = [\n                custom_scaling_factors[i] * raw_class_weights[i]\n                for i in range(len(raw_class_weights))\n            ]\n\n            # Convert to PyTorch tensor and move to device\n            class_weights = torch.tensor(scaled_class_weights, dtype=torch.float).to(cfg.device)\n\n        # Initialize model with unfreeze logic from cfg\n        model = EfficientNet(\n            num_classes=cfg.num_classes,\n            variant=cfg.variant,\n            freeze_backbone=cfg.freeze_backbone,\n            unfreeze_blocks=cfg.unfreeze_blocks\n        )\n\n        # Initialize the Trainer\n        trainer = Trainer(\n            train_df=fold_train_df,\n            val_df=fold_val_df,\n            train_path=image_dir,  # Single image directory\n            val_path=image_dir,  # Use the same directory\n            cfg=cfg,\n            model=model,\n            class_weights=class_weights  # Pass fold-specific scaled weights if available\n        )\n\n        # Run the training loop\n        trainer.run()\n\n        # Track the best metrics for the current fold\n        fold_best_bacc = max(trainer.val_baccs)\n        fold_best_kappa = max(trainer.val_kappas)\n        fold_best_acc = max(trainer.val_accuracies)\n\n        fold_results_bacc.append(fold_best_bacc)\n        fold_results_kappa.append(fold_best_kappa)\n        fold_results_acc.append(fold_best_acc)\n\n    # Compute mean and std for each metric\n    mean_bacc = np.mean(fold_results_bacc)\n    std_bacc = np.std(fold_results_bacc)\n    mean_kappa = np.mean(fold_results_kappa)\n    std_kappa = np.std(fold_results_kappa)\n    mean_acc = np.mean(fold_results_acc)\n    std_acc = np.std(fold_results_acc)\n\n    # Print results\n    print(f\"Mean Balanced Accuracy across {len(folds)} folds: {mean_bacc:.4f} ± {std_bacc:.4f}\")\n    print(f\"Mean Kappa across {len(folds)} folds: {mean_kappa:.4f} ± {std_kappa:.4f}\")\n    print(f\"Mean Accuracy across {len(folds)} folds: {mean_acc:.4f} ± {std_acc:.4f}\")\n\n    return {\n        \"mean_bacc\": mean_bacc,\n        \"std_bacc\": std_bacc,\n        \"mean_kappa\": mean_kappa,\n        \"std_kappa\": std_kappa,\n        \"mean_acc\": mean_acc,\n        \"std_acc\": std_acc,\n        \"trainer\": trainer\n    }\n","metadata":{"id":"9J4Xpf1ug_aZ"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cfg = Config()  # Initialize default config\nresults_baseline = run_cross_validation(cfg, ALL_IMAGES_RESIZED_DIR)","metadata":{"id":"eLFnlSKTmIW7","outputId":"b3380383-767d-4cd9-d574-16527a75524c"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Model Evaluation and Visualization\n\nIn this section, I evaluated the baseline model's performance using various metrics and visualizations:\n\n- **Out-of-Fold (OOF) Predictions**:\n  - Retrieved predictions and true labels across all folds using the `get_oof_predictions` function.\n  - Prepared OOF predictions as a baseline for later comparison with an improved model using a bootstrap hypothesis test.\n- **Expected Calibration Error (ECE)**:\n  - Computed ECE to measure how well the model's predicted probabilities align with actual outcomes.\n- **Performance Analysis**:\n  - Evaluated metrics such as balanced accuracy and misclassification percentages per class.\n  - Generated a detailed classification report for the validation set.\n- **Visualization**:\n  - Plotted training and validation trends (loss, balanced accuracy, and kappa) for selected folds.\n  - Visualized examples of correct and incorrect predictions.\n  - Analyzed misclassification percentages per class using bar plots.\n\nThese metrics and visualizations provide a comprehensive understanding of the model’s behavior, calibration, and error distribution. The OOF predictions establish a strong baseline for evaluating improvements in subsequent models.\n","metadata":{"id":"D1ZZOJz2SdDn"}},{"cell_type":"code","source":"# Bootstrapping\n\ndef get_oof_predictions(cfg, image_dir, folds):\n    all_oof_preds = []\n    all_oof_labels = []\n\n    for fold_idx, (_, fold_val_df) in enumerate(folds):\n        # Set the fold index in cfg so paths align if needed\n        cfg.fold = fold_idx\n        best_model_path = f\"{cfg.save_dir}/best_model_fold_{cfg.fold}.pth\"\n\n        # Load the best model for this fold\n        model = EfficientNet(\n            num_classes=cfg.num_classes,\n            variant=cfg.variant,\n            freeze_backbone=cfg.freeze_backbone,\n            unfreeze_blocks=cfg.unfreeze_blocks\n        )\n        model.load_state_dict(torch.load(best_model_path, map_location=cfg.device))\n        model.to(cfg.device)\n        model.eval()\n\n        # Prepare validation loader\n        val_loader = DataLoader(\n            APTOSDataset(image_dir, fold_val_df, get_val_transforms(cfg.image_size)),\n            batch_size=cfg.batch_size, shuffle=False, num_workers=cfg.num_workers\n        )\n\n        # Get predictions for this fold\n        fold_preds = []\n        fold_labels = []\n        with torch.no_grad():\n            for batch in val_loader:\n                images, labels = batch['image'].to(cfg.device), batch['label'].to(cfg.device)\n                outputs = model(images)\n                fold_preds.append(outputs.cpu().numpy())\n                fold_labels.append(labels.cpu().numpy())\n\n        fold_preds = np.concatenate(fold_preds, axis=0)\n        fold_labels = np.concatenate(fold_labels, axis=0)\n\n        all_oof_preds.append(fold_preds)\n        all_oof_labels.append(fold_labels)\n\n    # Concatenate across all folds\n    all_oof_preds = np.concatenate(all_oof_preds, axis=0)\n    all_oof_labels = np.concatenate(all_oof_labels, axis=0)\n\n    return all_oof_preds, all_oof_labels","metadata":{"id":"VP3I0KXRh0Hm"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def bootstrap_test(y_true, preds_model1, preds_model2, n_bootstraps=10000, random_state=42):\n    \"\"\"\n    Perform a bootstrap hypothesis test comparing the balanced accuracy of two models.\n\n    Args:\n        y_true (np.ndarray): Ground truth labels for all out-of-fold samples.\n        preds_model1 (np.ndarray): OOF predictions for model 1 (e.g., baseline).\n                                   Shape: (N, num_classes).\n                                   Predictions are raw logits or probabilities.\n        preds_model2 (np.ndarray): OOF predictions for model 2 (e.g., improved model).\n                                   Shape: (N, num_classes).\n        n_bootstraps (int): Number of bootstrap resamples to perform.\n        random_state (int): Random seed for reproducibility.\n\n    Returns:\n        p_value (float): The p-value from the bootstrap test.\n        differences (np.ndarray): Array of all bootstrap differences (improved - baseline).\n    \"\"\"\n    rng = np.random.default_rng(random_state)\n\n    # Convert predictions to classes\n    model1_classes = preds_model1.argmax(axis=1)\n    model2_classes = preds_model2.argmax(axis=1)\n\n    N = len(y_true)\n    differences = []\n\n    # Perform bootstrap sampling\n    for _ in range(n_bootstraps):\n        indices = rng.integers(0, N, size=N)\n        bacc1 = balanced_accuracy_score(y_true[indices], model1_classes[indices])\n        bacc2 = balanced_accuracy_score(y_true[indices], model2_classes[indices])\n        differences.append(bacc2 - bacc1)\n\n    differences = np.array(differences)\n    # p-value: proportion of bootstrap differences that are <= 0\n    # (i.e., how often improved model is not better than baseline)\n    p_value = np.mean(differences <= 0)\n\n    return p_value, differences\n","metadata":{"id":"hXFT1DUYikRR"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"baseline_oof_preds, baseline_oof_labels = get_oof_predictions(cfg, ALL_IMAGES_RESIZED_DIR, folds)","metadata":{"id":"LCcz_-YMiFDg","outputId":"37914fb9-bf49-4404-e295-c3aa697ead43"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_ece(logits, labels, n_bins=10):\n    from scipy.special import softmax\n    probs = softmax(logits, axis=1)\n    predicted_classes = probs.argmax(axis=1)\n    correct = (predicted_classes == labels)\n    confidences = probs[np.arange(len(probs)), predicted_classes]\n\n    bins = np.linspace(0.0, 1.0, n_bins + 1)\n    bin_indices = np.digitize(confidences, bins) - 1\n\n    ece = 0.0\n    N = len(probs)\n    for b in range(n_bins):\n        in_bin = (bin_indices == b)\n        bin_count = np.sum(in_bin)\n        if bin_count > 0:\n            bin_acc = np.mean(correct[in_bin])\n            bin_conf = np.mean(confidences[in_bin])\n            ece += (bin_count / N) * np.abs(bin_conf - bin_acc)\n    return ece","metadata":{"id":"j1VWsSWgfqF9"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# After training is done for a fold\nfinal_preds = np.array(results_baseline['trainer'].val_preds)\nfinal_labels = np.array(results_baseline['trainer'].val_labels)","metadata":{"id":"gcoqxF9bfphJ"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def visualize_fold_results(fold_index):\n    \"\"\"\n    Visualize training and validation metrics for a given fold.\n    \"\"\"\n    # Fixed path to the results directory\n    results_dir = \"/content/data/snapshots\"\n    results_file = os.path.join(results_dir, f\"fold_{fold_index}_results.pth\")\n\n    # Check if the results file exists\n    if not os.path.exists(results_file):\n        print(f\"No results file found for fold {fold_index}.\")\n        return\n\n    print(f\"Loading results file for fold {fold_index} from {results_file}.\")\n    results = torch.load(results_file, weights_only=False)\n\n    # Extract metrics\n    train_losses = results[\"train_losses\"]\n    val_losses = results[\"val_losses\"]\n    val_baccs = results[\"val_baccs\"]\n    val_kappas = results[\"val_kappas\"]\n    val_accuracies = results[\"val_accuracies\"]\n\n    # Plot training and validation loss\n    plt.figure(figsize=(10, 5))\n    plt.plot(train_losses, label='Train Loss')\n    plt.plot(val_losses, label='Validation Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.title(f'Training and Validation Loss (Fold {fold_index})')\n    plt.legend()\n    plt.show()\n\n    # Plot validation metrics (Balanced Accuracy, Kappa, Accuracy)\n    plt.figure(figsize=(10, 5))\n    plt.plot(val_baccs, label='Balanced Accuracy')\n    plt.plot(val_kappas, label='Kappa')\n    plt.plot(val_accuracies, label='Accuracy')\n    plt.axhline(0.5, color='red', linestyle='--', label='Random Baseline (0.5)')\n    plt.ylim(0, 1)  # Set Y-axis range from 0 to 1\n    plt.xlabel('Epoch')\n    plt.ylabel('Metrics')\n    plt.title(f'Validation Metrics (Fold {fold_index})')\n    plt.legend()\n    plt.show()","metadata":{"id":"jzU3WYbXQBYB"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize metrics for a specific fold\nvisualize_fold_results(fold_index=2)","metadata":{"id":"M5dQrNDyQXd1","outputId":"d9d871f1-c956-4909-88f9-bb33f8bda57e"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def visualize_predictions(val_df, val_preds, val_labels, images_dir, num_samples=5):\n    \"\"\"\n    Visualize some examples of correct and incorrect predictions.\n\n    Args:\n        val_df (pd.DataFrame): Validation dataframe with 'id_code' and 'diagnosis' columns.\n        val_preds (np.ndarray): Model predictions (after argmax) or logits for the final epoch.\n        val_labels (np.ndarray): Ground truth labels for the final epoch.\n        images_dir (str): Directory containing the validation images.\n        num_samples (int): Number of samples to visualize for correct and incorrect predictions.\n    \"\"\"\n    # If val_preds are logits/probs, get predicted classes\n    predicted_classes = val_preds.argmax(axis=1) if val_preds.ndim > 1 else val_preds\n\n    # Identify correct and incorrect samples\n    correct_indices = np.where(predicted_classes == val_labels)[0]\n    incorrect_indices = np.where(predicted_classes != val_labels)[0]\n\n    # Randomly select samples\n    correct_samples = np.random.choice(correct_indices, size=min(num_samples, len(correct_indices)), replace=False) \\\n                      if len(correct_indices) > 0 else []\n    incorrect_samples = np.random.choice(incorrect_indices, size=min(num_samples, len(incorrect_indices)), replace=False) \\\n                        if len(incorrect_indices) > 0 else []\n\n    def plot_samples(indices, title):\n        if len(indices) == 0:\n            print(f\"No {title.lower()} samples to show.\")\n            return\n        plt.figure(figsize=(20, 5))\n        for i, idx in enumerate(indices):\n            entry = val_df.iloc[idx]\n            img_name = entry['id_code']\n            img_path = os.path.join(images_dir, f\"{img_name}.png\")\n\n            img = cv2.imread(img_path)\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n            true_label = val_labels[idx]\n            pred_label = predicted_classes[idx]\n\n            plt.subplot(1, len(indices), i+1)\n            plt.imshow(img)\n            plt.title(f\"Pred: {pred_label}, True: {true_label}\", fontsize=12)\n            plt.axis('off')\n        plt.suptitle(title, fontsize=16)\n        plt.show()\n\n    # Plot correct predictions\n    plot_samples(correct_samples, \"Correct Predictions\")\n\n    # Plot incorrect predictions\n    plot_samples(incorrect_samples, \"Incorrect Predictions\")\n","metadata":{"id":"SKTp1LKVd_4H"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize correct and incorrect predictions\nvisualize_predictions(results_baseline['trainer'].val_df, final_preds, final_labels, ALL_IMAGES_RESIZED_DIR, num_samples=5)","metadata":{"id":"Af7Q7nhDfkFX","outputId":"0fe5354d-9376-4cb7-d407-4cb729e8818e"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_misclassification_percentages(final_preds, final_labels):\n    # Convert predictions to class indices if necessary\n    predicted_classes = final_preds.argmax(axis=1) if final_preds.ndim > 1 else final_preds\n\n    # Identify indices of incorrect predictions\n    incorrect_indices = np.where(predicted_classes != final_labels)[0]\n\n    # Extract the true labels of the incorrect samples\n    incorrect_true_labels = final_labels[incorrect_indices]\n\n    # Count occurrences of incorrect predictions per class\n    unique_classes, incorrect_counts = np.unique(incorrect_true_labels, return_counts=True)\n\n    # Count total samples per class in the validation set\n    total_unique, total_counts = np.unique(final_labels, return_counts=True)\n    class_counts_dict = dict(zip(total_unique, total_counts))\n\n    # Convert misclassification counts to percentages\n    percentages = {\n        unique_classes[i]: (incorrect_counts[i] / class_counts_dict[unique_classes[i]]) * 100\n        for i in range(len(unique_classes))\n    }\n\n    # Plot the percentage of misclassifications per class\n    plt.figure(figsize=(8, 5))\n    plt.bar(percentages.keys(), percentages.values())\n    plt.ylim(0, 100)\n    plt.xlabel('Class')\n    plt.ylabel('Percentage of Class Misclassified (%)')\n    plt.title('Percentage of Each Class Misclassified')\n    plt.xticks(list(class_counts_dict.keys()))  # Ensure all class labels are displayed\n    plt.show()\n\n    return percentages\n","metadata":{"id":"xLKkpWGjL9-N"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate_predictions(final_preds, final_labels, n_bins=10):\n    \"\"\"\n    Evaluate model predictions using Expected Calibration Error (ECE),\n    misclassification percentages, and a classification report.\n\n    Args:\n        final_preds (np.ndarray): Predicted probabilities or logits from the model.\n        final_labels (np.ndarray): Ground truth labels.\n        n_bins (int, optional): Number of bins for ECE computation. Default is 10.\n\n    Returns:\n        dict: A dictionary containing ECE, misclassification percentages, and the classification report.\n    \"\"\"\n    # Compute Expected Calibration Error (ECE)\n    ece = compute_ece(final_preds, final_labels, n_bins=n_bins)\n    print(f\"Expected Calibration Error (ECE): {ece:.4f}\")\n\n    # Analyze misclassification percentages\n    percentages = plot_misclassification_percentages(final_preds, final_labels)\n    print(\"Misclassification percentages per class:\")\n    for cls, pct in percentages.items():\n        print(f\"Class {cls}: {pct:.2f}%\")\n\n    # Print classification report\n    classification_report_text = classification_report(final_labels, final_preds.argmax(axis=1), digits=2)\n    print(\"\\nClassification Report:\\n\", classification_report_text)\n\n    # Return results as a dictionary\n    return {\n        \"ece\": ece,\n        \"misclassification_percentages\": percentages,\n        \"classification_report\": classification_report_text\n    }\n","metadata":{"id":"vki-lv_Dp3hO"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Evaluate predictions\nevaluation_results = evaluate_predictions(final_preds, final_labels)","metadata":{"id":"q_SlBWDckp4Z","outputId":"a5160b3e-bad5-482f-dc85-c2c32b8261b8"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Model Evaluation and Visualization\n\nIn this section, I evaluated the baseline model's performance and identified the following key points:\n\n- **High Misclassification for Class 1**: The model struggles to distinguish Class 1, with a misclassification rate of approximately **47%**.\n- **Severe Misclassification for Classes 3 and 4**: These are the least-represented classes, with very high misclassification rates exceeding **70%**, highlighting the model's difficulty in handling minority classes.\n- **Exceptional Performance on Class 0**: The model achieves outstanding precision and F1-score for Class 0, indicating its dominance in predictions.\n- **Dominance of Class 0**: Metrics are heavily influenced by the prevalence of Class 0, masking poor performance on minority classes.\n\n- **Class Imbalance**: The model's struggles with minority classes highlight the need for techniques to address class imbalance, such as oversampling, data augmentation, or class weighting.\n- **Calibration**: The Expected Calibration Error (ECE) is low (around **0.03**), indicating the model's probability outputs are reliable.\n\nThis evaluation underscores the need to address class imbalance while maintaining the model's strong calibration. Such improvements are essential for achieving better performance across all classes.\n\n","metadata":{"id":"6OHdGtNl24wR"}},{"cell_type":"markdown","source":"# Improve Baseline Model","metadata":{"id":"zhjDS55VSXn9"}},{"cell_type":"markdown","source":"### Baseline + Class Weight\n\nIn this section, I improved the baseline model by incorporating class weights to address class imbalance.\n\n#### Key Observations:\n- **Balanced Accuracy**: Improved significantly compared to the baseline, indicating better performance across all classes.\n- **Class 1**: Misclassification rate reduced substantially, demonstrating improved differentiation for this class.\n- **Class 3**: Saw a notable decrease in misclassification rate, showing the model's better ability to handle minority classes.\n- **Class 4**: Although there was some improvement, it remains a challenging class for the model.\n- **Calibration**: The Expected Calibration Error (ECE) remained low, reflecting reliable probability outputs.\n\n#### Trade-offs:\n- **Overall Accuracy**: Decreased slightly compared to the baseline, suggesting the model is prioritizing performance on minority classes at the cost of dominant classes.\n- **Class 0**: Misclassification increased slightly but remained low, with the model maintaining exceptional performance for this majority class.\n\n#### Statistical Significance:\n- **Bootstrap Test**: A bootstrap hypothesis test confirmed the improvement in balanced accuracy is statistically significant, with a p-value of **0.0000** and a 95% confidence interval indicating a meaningful positive difference.\n\nThis analysis highlights the trade-offs involved in improving minority class performance, with class weights proving to be an effective strategy for addressing imbalance while maintaining overall reliability.\n","metadata":{"id":"BK6kkOEK1uox"}},{"cell_type":"code","source":"# Initialize Config\ncfg = Config()\n\n# Define custom scaling factors\ncustom_scaling_factors = [1.0, 1.2, 1.2, 1.5, 1.5]  # Emphasize minority classes\n\n# Run cross-validation with class weights\nresults_class_weight = run_cross_validation(cfg, ALL_IMAGES_RESIZED_DIR, custom_scaling_factors)","metadata":{"id":"5Xop2bX0hCFQ","outputId":"42758a1b-ebe5-4e33-d5d5-a12d495e98a1"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class_weight_oof_preds, class_weight_oof_labels = get_oof_predictions(cfg, ALL_IMAGES_RESIZED_DIR, folds)","metadata":{"id":"922_7yqWofQV","outputId":"6bf4b9ab-2e12-4782-cc31-7f581104cb89"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ensure that baseline_oof_labels and class_weight_oof_labels match\nassert np.array_equal(baseline_oof_labels, class_weight_oof_labels), \"Labels must match for a fair comparison!\"\ny_true = baseline_oof_labels\n\np_value, diff_distribution = bootstrap_test(\n    y_true=y_true,\n    preds_model1=baseline_oof_preds,  # baseline model preds\n    preds_model2=class_weight_oof_preds,  # baseline + class weight preds\n    n_bootstraps=10000,\n    random_state=42\n)\n\nprint(f\"Bootstrap p-value comparing baseline vs baseline+class_weight: {p_value:.8f}\")\n","metadata":{"id":"wLG7lPJ-oaqv","outputId":"90ef33b1-dc18-47e0-b171-9d342f03ea68"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.hist(diff_distribution, bins=50, alpha=0.7)\nplt.axvline(0, color='red', linestyle='--')\nplt.title(\"Bootstrap Distribution of Balanced Accuracy Differences\")\nplt.xlabel(\"Difference in Balanced Accuracy (Model2 - Model1)\")\nplt.ylabel(\"Frequency\")\nplt.show()\n\nlower_bound = np.percentile(diff_distribution, 2.5)\nupper_bound = np.percentile(diff_distribution, 97.5)\nprint(f\"95% Confidence Interval: [{lower_bound:.4f}, {upper_bound:.4f}]\")","metadata":{"id":"9dmScaXkrxgT","outputId":"fa68a40e-1ab1-4ac1-ae71-9e6571de91cb"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# After training is done for a fold\nfinal_preds = np.array(results_class_weight['trainer'].val_preds)\nfinal_labels = np.array(results_class_weight['trainer'].val_labels)\n\n# Evaluate predictions\nevaluation_results = evaluate_predictions(final_preds, final_labels)","metadata":{"id":"fqlIANkAg7To","outputId":"640bc047-cd37-4cf6-83a1-87c305d7afe6"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Best Model: Baseline + Class Weight + Unfreeze 2 Blocks\n\nIn this section, I extended the previous model by unfreezing the last two blocks of the EfficientNet backbone while retaining class weights. This allowed the model to fine-tune deeper features and adapt better to the dataset.\n\n#### Key Observations:\n- **Balanced Accuracy**: Achieved a slight improvement over the previous model, indicating better performance across all classes.\n- **Class Performance**:\n  - **Class 1**: Significant improvement, with a noticeable drop in misclassification rates and a higher F1-score.\n  - **Class 4**: Marked reduction in misclassification rates and an improved F1-score, though it remains challenging.\n  - **Class 3**: Performance deteriorated slightly, with an increase in misclassification rates, highlighting the need for further intervention.\n  - **Class 0**: Maintained exceptional performance with minor increases in misclassification, still the best-performing class.\n- **Calibration**: The Expected Calibration Error (ECE) showed further improvement, indicating more reliable predicted probabilities.\n\n#### Statistical Significance:\n- **Bootstrap Test**: The improvement in balanced accuracy was statistically significant, with a p-value of **0.0000** and a 95% confidence interval confirming the meaningful positive difference.\n\n#### Trade-offs:\n- **Overall Accuracy**: Improved slightly compared to the previous model, balancing overall performance and minority class focus.\n- **Macro-Averaged Metrics**: Both the F1-score and recall improved, reflecting better generalization to minority classes, but challenges persist for Class 3.\n\n#### Conclusion:\nUnfreezing two backbone blocks helped the model achieve better differentiation for minority classes like Class 4 while maintaining strong calibration and overall performance. However, challenges with Class 3 suggest that additional targeted interventions, such as data augmentation or advanced loss functions, may further enhance performance.\n","metadata":{"id":"aYoPRvbo8BGi"}},{"cell_type":"code","source":"cfg = Config()\ncfg.unfreeze_blocks = 2  # Unfreeze the last 2 blocks of the backbone\ncustom_scaling_factors = [1.0, 1.2, 1.2, 1.5, 1.5]\nresults_unfreeze = run_cross_validation(cfg, ALL_IMAGES_RESIZED_DIR, custom_scaling_factors)","metadata":{"id":"TKY9bmCDPhhQ","outputId":"e46ce8cc-b4e3-4d06-f493-b3ca63d22ba5"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"improved_oof_preds, improved_oof_labels = get_oof_predictions(cfg, ALL_IMAGES_RESIZED_DIR, folds)","metadata":{"id":"qnxMZiMNiJyi","outputId":"53890f10-0c8d-409a-beaa-18ef909a7e4a"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ensure that baseline_oof_labels == improved_oof_labels for a fair comparison.\nassert np.array_equal(baseline_oof_labels, improved_oof_labels), \"Labels must match!\"\ny_true = baseline_oof_labels  # They are the same for both models\n\np_value, diff_distribution = bootstrap_test(\n    y_true=y_true,\n    preds_model1=baseline_oof_preds,\n    preds_model2=improved_oof_preds,\n    n_bootstraps=10000,\n    random_state=42\n)\n\nprint(f\"Bootstrap p-value: {p_value:.4f}\")","metadata":{"id":"7O0N6bqAiS2v","outputId":"54242eb7-d029-4a92-9fda-d4bb86592ba8"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.hist(diff_distribution, bins=50, alpha=0.7)\nplt.axvline(0, color='red', linestyle='--')\nplt.title(\"Bootstrap Distribution of Balanced Accuracy Differences\")\nplt.xlabel(\"Difference in Balanced Accuracy (Model2 - Model1)\")\nplt.ylabel(\"Frequency\")\nplt.show()\n\nlower_bound = np.percentile(diff_distribution, 2.5)\nupper_bound = np.percentile(diff_distribution, 97.5)\nprint(f\"95% Confidence Interval: [{lower_bound:.4f}, {upper_bound:.4f}]\")","metadata":{"id":"v_Gf43Znzmsa","outputId":"c89ae221-e77b-4c70-b08a-cb74afc99c05"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# After training is done for a fold\nfinal_preds = np.array(results_unfreeze['trainer'].val_preds)\nfinal_labels = np.array(results_unfreeze['trainer'].val_labels)\n\n# Evaluate predictions\nevaluation_results = evaluate_predictions(final_preds, final_labels)","metadata":{"id":"CpUd668C8SlV","outputId":"976f27a5-568b-46e4-c890-36556845eb5d"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Pre-caching Resize Images to 380x380","metadata":{"id":"hGwq65JNIblT"}},{"cell_type":"code","source":"ALL_IMAGES_DIR = '/content/data/all_images'\nALL_IMAGES_RESIZED_DIR_380 = '/content/data/all_images_resized_380'\nIMG_SIZE = 380\n\nos.makedirs(ALL_IMAGES_RESIZED_DIR_380, exist_ok=True)\n\nimg_paths = glob.glob(os.path.join(ALL_IMAGES_DIR, '*.png'))\n\nfor img_path in tqdm(img_paths, desc=\"Resizing\"):\n    img = cv2.imread(img_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n    #Ben Graham's Pre-processing\n    img = crop_image_from_gray(img)\n    img = cv2.resize(img, (IMG_SIZE, IMG_SIZE))\n    img = cv2.addWeighted(img, 4, cv2.GaussianBlur(img, (0,0), 10), -4, 128)\n\n\n    img_save = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n    base_name = os.path.basename(img_path)\n    cv2.imwrite(os.path.join(ALL_IMAGES_RESIZED_DIR_380, base_name), img_save)","metadata":{"id":"tPv2CACjIaXR","outputId":"283ea4b1-6326-4ec2-8c89-596952b8a3b4"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### EfficientNet-B4\n\nIn this section, I experimented with EfficientNet-B4, a larger and more complex architecture, to evaluate its impact on performance. Images were resized to **380x380**, and training followed the same strategies as the EfficientNet-B0 experiment, including class weights.\n\n#### Key Observations:\n- **Balanced Accuracy**: Improved slightly compared to EfficientNet-B0, achieving a marginal gain across classes.\n- **Overall Accuracy**: Increased slightly, suggesting some improvement in overall predictions.\n- **Calibration**: The Expected Calibration Error (ECE) improved compared to previous experiments, indicating more reliable probability estimates.\n\n- **Class Performance**:\n  - **Class 2**: Continued to show challenges, with a high misclassification rate but steady performance compared to EfficientNet-B0.\n  - **Class 3 and Class 4**: These classes remain difficult to classify, with persistently high misclassification rates, though some improvement was observed for Class 4.\n\n#### Trade-offs:\n- **Training Time**: Increased significantly, from **~10 minutes (EfficientNet-B0)** to **~50 minutes (EfficientNet-B4)**, making this approach computationally expensive.\n- **Complexity vs. Performance**: The increased complexity provided limited performance gains, suggesting that EfficientNet-B4 might not be the most practical choice for this task.\n\n#### Conclusion:\nEfficientNet-B4 offered slight improvements in balanced accuracy, overall accuracy, and calibration compared to EfficientNet-B0. However, the computational cost and persistent challenges with minority classes, especially Class 3, make EfficientNet-B0 a more efficient and balanced choice for this dataset.\n\n","metadata":{"id":"QhIy497IL_Sc"}},{"cell_type":"code","source":"cfg = Config()\ncfg.model_name = \"efficientnet-b4\"  # Switch to EfficientNet-B4\ncfg.variant = \"b4\"\ncfg.image_size = 380  # Update resolution for B4\ncfg.batch_size = 16  # Reduce batch size for memory constraints\ncfg.save_dir = Path(\"/content/data/snapshots-B4\")  # Change save directory for B4\ncfg.save_dir.mkdir(parents=True, exist_ok=True)\ncustom_scaling_factors = [1.0, 1.2, 1.2, 1.5, 1.5]  # Define custom weights for B4\n\n\n\nresults_b4 = run_cross_validation(cfg, ALL_IMAGES_RESIZED_DIR_380, custom_scaling_factors)","metadata":{"id":"yAtdhCbNQhUo","outputId":"35ec1d03-19c1-4c4a-e27e-e4c9665a5ebb"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# After training is done for a fold\nfinal_preds = np.array(results_b4['trainer'].val_preds)\nfinal_labels = np.array(results_b4['trainer'].val_labels)\n\n# Evaluate predictions\nevaluation_results = evaluate_predictions(final_preds, final_labels)","metadata":{"id":"5vndeg-NQ9Rf","outputId":"eb9c4b80-f10c-4644-f96f-680662250056"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Advanced Data Augmentation & FocalLoss\n\nIn this final section, I explored advanced data augmentation techniques combined with the FocalLoss function to address class imbalance and improve model generalization. The objective was to focus the model's learning on harder-to-classify samples while enhancing data diversity.\n\n#### Key Observations:\n- **Minimal Improvement**:\n  - **Balanced Accuracy**: Remained comparable to the setup with class weights and unfreezing 2 blocks, showing no meaningful improvement.\n  - **Kappa and Accuracy**: Showed marginal or no gains, with accuracy slightly lower than earlier setups.\n- **Calibration**: The Expected Calibration Error (ECE) worsened, indicating less reliable probability estimates.\n\n#### Classification Metrics:\n- **Class 3**: Misclassification reduced slightly, but recall and F1-score remained suboptimal, showing the class is still challenging for the model.\n- **Class 4**: Some improvement in misclassification rates, but overall performance remains limited.\n- **Class 0**: Maintained strong performance with minimal misclassification, consistent across setups.\n\n#### Trade-offs:\n- **No Significant Performance Gains**: Despite the added complexity, the model's overall performance, especially for minority classes, showed only marginal improvements.\n- **Calibration vs. Class Focus**: While FocalLoss aimed to improve minority class handling, it resulted in worse calibration.\n- **Increased Computational Overhead**: The inclusion of advanced augmentations added training complexity without yielding meaningful benefits.\n\n#### Conclusion:\nThe combination of advanced augmentations and FocalLoss did not lead to significant improvements over simpler approaches like class weights and unfreezing 2 blocks. Calibration worsened, and performance gains for minority classes were marginal. This suggests that further exploration of strategies for handling imbalanced data is needed to achieve meaningful improvements.\n","metadata":{"id":"i2IewV4LEEH4"}},{"cell_type":"code","source":"def get_train_transforms(img_size=256):\n    return A.Compose([\n        A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.5),\n        A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10, p=0.5),\n        A.ElasticTransform(alpha=1, sigma=50, p=0.3),\n        A.Rotate(limit=15, p=0.5),\n        A.HorizontalFlip(p=0.5),\n        A.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.1, rotate_limit=0, p=0.5),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        ToTensorV2()\n    ])","metadata":{"id":"oVOUMKn6BWhk"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, alpha=None, gamma=2):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha  # Class weights\n        self.gamma = gamma\n\n    def forward(self, logits, labels):\n        ce_loss = nn.CrossEntropyLoss(weight=self.alpha, reduction='none')(logits, labels)\n        pt = torch.exp(-ce_loss)  # Probability of true class\n        focal_loss = ((1 - pt) ** self.gamma) * ce_loss\n        return focal_loss.mean()\n","metadata":{"id":"vfBTYD0rI2LR"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Update Config for Advanced Data Augmentation & FocalLoss\ncfg = Config()\ncfg.model_name = \"efficientnet-b0\"  # Continue using EfficientNet-B0\ncfg.variant = \"b0\"  # EfficientNet-B0 variant\n\ncfg.image_size = 256  # Default resolution\ncfg.batch_size = 32  # Batch size suitable for EfficientNet-B0\ncfg.use_focal_loss = True  # Switch to FocalLoss for this experiment\n\ncfg.unfreeze_blocks = 2  # Unfreeze the last 2 blocks of the backbone\n\n# Advanced Data Augmentation\ncustom_scaling_factors = [1.0, 1.2, 1.2, 1.5, 1.5]  # Emphasize minority classes\nresults_advanced = run_cross_validation(cfg, ALL_IMAGES_RESIZED_DIR, custom_scaling_factors)","metadata":{"id":"4VxafnG-M50L","outputId":"e053928b-223f-45c0-a7ce-11841fa28ab0"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# After training is done for a fold\nfinal_preds = np.array(results_advanced['trainer'].val_preds)\nfinal_labels = np.array(results_advanced['trainer'].val_labels)\n\n# Evaluate predictions\nevaluation_results = evaluate_predictions(final_preds, final_labels)","metadata":{"id":"eXWP6Ep_M3I0","outputId":"92605b89-a5ad-447f-c6d6-c8d3368a2d4a"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize metrics for a specific fold\nvisualize_fold_results(fold_index=2)","metadata":{"id":"i85BuvcHQbLm","outputId":"43af02ab-b3d2-46d9-8a39-883f23959ba6"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"FoBN7mbkRdS0"},"outputs":[],"execution_count":null}]}