{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"},{"sourceId":63056,"databundleVersionId":9094797,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install torch torchvision albumentations opencv-python scikit-learn seaborn","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# ==========================================\n# 1. IMPORTS AND SETUP\n# ==========================================\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import classification_report, confusion_matrix, roc_auc_score, roc_curve\nfrom sklearn.metrics import precision_recall_curve, average_precision_score\nfrom sklearn.model_selection import train_test_split\nimport cv2\nfrom PIL import Image\nimport zipfile\nimport shutil\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Deep Learning Libraries\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\nimport torchvision.transforms as transforms\nimport torchvision.models as models\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, OneCycleLR\n\n# Augmentation - First Prize Winner Strategy\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# Set device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\n\n# Set random seeds for reproducibility\nimport random\ndef set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(42)\n\n# ==========================================\n# 2. ENHANCED CONFIGURATION\n# ==========================================\n\n# Model configuration\nIMG_SIZE = 224  # Using winner's optimal size\nBATCH_SIZE = 32  # Increased for better gradient estimates\nNUM_EPOCHS = 50  # More epochs for better convergence\nLEARNING_RATE = 3e-4  # Optimized learning rate\nWEIGHT_DECAY = 1e-4\nNUM_CLASSES = 2\nACCUMULATION_STEPS = 2  # Gradient accumulation for larger effective batch size\n\n# Dataset paths\nISIC_2024_TRAIN_DIR = '/kaggle/input/isic-2024-challenge/train-image/image'\nISIC_2024_METADATA = '/kaggle/input/isic-2024-challenge/train-metadata.csv'\nSIIM_TRAIN_DIR = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train'\nSIIM_METADATA = '/kaggle/input/siim-isic-melanoma-classification/train.csv'\nSKIN_CANCER_FOLDER = '/kaggle/input/isic-melanoma-v4/skin cancer.v4i.folder/train'  # Updated path\n\n# Output directories\nOUTPUT_DIR = 'enhanced_skin_cancer_dataset'\nTRAIN_DIR = f'{OUTPUT_DIR}/train'\nVALID_DIR = f'{OUTPUT_DIR}/valid'\nTEST_DIR = f'{OUTPUT_DIR}/test'\n\n# Class names\nCLASS_NAMES = ['Nevus', 'Melanoma']\n\n# ==========================================\n# 3. FIRST PRIZE WINNER AUGMENTATION STRATEGY\n# ==========================================\n\n# Training augmentations - First Prize Winner Strategy\ntrain_transforms = A.Compose([\n    A.Transpose(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.HorizontalFlip(p=0.5),\n    A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.75),\n    A.OneOf([\n        A.MotionBlur(blur_limit=5),\n        A.MedianBlur(blur_limit=5),\n        A.GaussianBlur(blur_limit=5),\n        A.GaussNoise(var_limit=(5.0, 30.0)),\n    ], p=0.7),\n    A.OneOf([\n        A.OpticalDistortion(distort_limit=1.0),\n        A.GridDistortion(num_steps=5, distort_limit=1.0),\n        A.ElasticTransform(alpha=3),\n    ], p=0.7),\n    A.CLAHE(clip_limit=4.0, p=0.5),\n    A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10, p=0.5),\n    A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, \n                       border_mode=cv2.BORDER_REFLECT_101, p=0.85),\n    A.Resize(IMG_SIZE, IMG_SIZE),\n    A.CoarseDropout(\n        max_holes=1,\n        max_height=int(IMG_SIZE * 0.3),\n        max_width=int(IMG_SIZE * 0.3),\n        num_holes_range=(1, 1),\n        p=0.5\n    ),\n    A.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],\n        max_pixel_value=255.0,\n        p=1.0\n    ),\n    ToTensorV2()\n], p=1.0)\n\n# Validation transforms\nvalid_transforms = A.Compose([\n    A.Resize(IMG_SIZE, IMG_SIZE),\n    A.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],\n        max_pixel_value=255.0,\n        p=1.0\n    ),\n    ToTensorV2()\n], p=1.0)\n\n# Test Time Augmentation transforms\ntta_transforms = [\n    A.Compose([A.Resize(IMG_SIZE, IMG_SIZE), A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ToTensorV2()]),\n    A.Compose([A.Resize(IMG_SIZE, IMG_SIZE), A.HorizontalFlip(p=1.0), A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ToTensorV2()]),\n    A.Compose([A.Resize(IMG_SIZE, IMG_SIZE), A.VerticalFlip(p=1.0), A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ToTensorV2()]),\n    A.Compose([A.Resize(IMG_SIZE, IMG_SIZE), A.Transpose(p=1.0), A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ToTensorV2()]),\n    A.Compose([A.Resize(IMG_SIZE, IMG_SIZE), A.RandomBrightnessContrast(brightness_limit=0.1, contrast_limit=0.1, p=1.0), A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ToTensorV2()]),\n]\n\n# ==========================================\n# 4. ENHANCED DATA AGGREGATION\n# ==========================================\n\ndef extract_and_process_datasets():\n    \"\"\"Extract and process all three datasets into a unified structure\"\"\"\n    \n    print(\"Creating output directories...\")\n    os.makedirs(OUTPUT_DIR, exist_ok=True)\n    for split in ['train', 'valid', 'test']:\n        for class_id in ['0', '1']:  # 0: Nevus, 1: Melanoma\n            os.makedirs(f'{OUTPUT_DIR}/{split}/{class_id}', exist_ok=True)\n    \n    all_samples = []\n    \n    # 1. Process ISIC 2024 Dataset\n    print(\"Processing ISIC 2024 dataset...\")\n    try:\n        metadata_2024 = pd.read_csv(ISIC_2024_METADATA)\n        filtered_2024 = metadata_2024[metadata_2024['target'].isin([0, 1])].copy()\n        \n        # Get more balanced samples - prioritize melanoma\n        melanoma_2024 = filtered_2024[filtered_2024['target'] == 1].head(800)\n        nevus_2024 = filtered_2024[filtered_2024['target'] == 0].head(800)\n        selected_2024 = pd.concat([melanoma_2024, nevus_2024], ignore_index=True)\n        \n        for idx, row in selected_2024.iterrows():\n            img_name = row['isic_id'] + '.jpg'\n            img_path = os.path.join(ISIC_2024_TRAIN_DIR, img_name)\n            if os.path.exists(img_path):\n                all_samples.append({\n                    'image_path': img_path,\n                    'label': row['target'],\n                    'source': 'isic_2024',\n                    'image_id': row['isic_id']\n                })\n        \n        print(f\"ISIC 2024: {len(selected_2024)} samples processed\")\n    except Exception as e:\n        print(f\"Error processing ISIC 2024: {e}\")\n    \n    # 2. Process SIIM-ISIC Dataset\n    print(\"Processing SIIM-ISIC dataset...\")\n    try:\n        metadata_siim = pd.read_csv(SIIM_METADATA)\n        filtered_siim = metadata_siim[metadata_siim['target'].isin([0, 1])].copy()\n        \n        # Get more melanoma samples from SIIM\n        melanoma_siim = filtered_siim[filtered_siim['target'] == 1].head(1200)\n        nevus_siim = filtered_siim[filtered_siim['target'] == 0].head(400)\n        selected_siim = pd.concat([melanoma_siim, nevus_siim], ignore_index=True)\n        \n        for idx, row in selected_siim.iterrows():\n            img_name = row['image_name'] + '.jpg'\n            img_path = os.path.join(SIIM_TRAIN_DIR, img_name)\n            if os.path.exists(img_path):\n                all_samples.append({\n                    'image_path': img_path,\n                    'label': row['target'],\n                    'source': 'siim_isic',\n                    'image_id': row['image_name']\n                })\n        \n        print(f\"SIIM-ISIC: {len(selected_siim)} samples processed\")\n    except Exception as e:\n        print(f\"Error processing SIIM-ISIC: {e}\")\n    \n    # 3. Process Third Dataset (from folder structure)\n    print(\"Processing third dataset from folder...\")\n    try:\n        # Check if the folder exists\n        if not os.path.exists(SKIN_CANCER_FOLDER):\n            print(f\"Warning: Folder {SKIN_CANCER_FOLDER} does not exist\")\n        else:\n            print(f\"Found folder: {SKIN_CANCER_FOLDER}\")\n            \n            # Process class 1 (nevus) and class 2 (melanoma)\n            class_mapping = {'1': 0, '2': 1}  # 1->nevus(0), 2->melanoma(1)\n            \n            for original_class, new_class in class_mapping.items():\n                class_dir = os.path.join(SKIN_CANCER_FOLDER, original_class)\n                print(f\"Looking for class folder: {class_dir}\")\n                \n                if os.path.exists(class_dir):\n                    image_files = [f for f in os.listdir(class_dir) if f.lower().endswith(('.jpg', '.jpeg', '.png'))]\n                    print(f\"Found {len(image_files)} images in class {original_class}\")\n                    \n                    # Get more samples, especially melanoma\n                    if new_class == 1:  # melanoma\n                        selected_files = image_files[:1500]  # More melanoma\n                    else:  # nevus\n                        selected_files = image_files[:800]   # Moderate nevus\n                    \n                    for img_file in selected_files:\n                        img_path = os.path.join(class_dir, img_file)\n                        all_samples.append({\n                            'image_path': img_path,\n                            'label': new_class,\n                            'source': 'skin_cancer_v4',\n                            'image_id': img_file.split('.')[0]\n                        })\n                    \n                    print(f\"Processed {len(selected_files)} images from class {original_class}\")\n                else:\n                    print(f\"Class folder {class_dir} does not exist\")\n                    # Let's explore the actual folder structure\n                    if os.path.exists(SKIN_CANCER_FOLDER):\n                        print(f\"Contents of {SKIN_CANCER_FOLDER}:\")\n                        for item in os.listdir(SKIN_CANCER_FOLDER):\n                            item_path = os.path.join(SKIN_CANCER_FOLDER, item)\n                            if os.path.isdir(item_path):\n                                print(f\"  Directory: {item}\")\n                            else:\n                                print(f\"  File: {item}\")\n        \n        print(f\"Third dataset: processed successfully\")\n    except Exception as e:\n        print(f\"Error processing third dataset: {e}\")\n        import traceback\n        traceback.print_exc()\n    \n    # Convert to DataFrame and analyze\n    df_all = pd.DataFrame(all_samples)\n    print(f\"\\nTotal samples collected: {len(df_all)}\")\n    if len(df_all) > 0:\n        print(\"Class distribution:\")\n        print(df_all['label'].value_counts())\n        print(\"Source distribution:\")\n        print(df_all['source'].value_counts())\n    else:\n        print(\"No samples were collected. Please check the dataset paths.\")\n    \n    return df_all\n\n\ndef copy_and_split_data(df_all):\n    \"\"\"Copy images and split into train/val/test with stratification\"\"\"\n    \n    # Stratified split - 70% train, 15% val, 15% test\n    train_df, temp_df = train_test_split(df_all, test_size=0.3, random_state=42, stratify=df_all['label'])\n    val_df, test_df = train_test_split(temp_df, test_size=0.5, random_state=42, stratify=temp_df['label'])\n    \n    print(f\"\\nDataset splits:\")\n    print(f\"Train: {len(train_df)} samples\")\n    print(f\"Validation: {len(val_df)} samples\")\n    print(f\"Test: {len(test_df)} samples\")\n    \n    # Copy images to respective directories\n    def copy_images_to_split(df, split_name):\n        split_dir = os.path.join(OUTPUT_DIR, split_name)\n        copied = 0\n        \n        for idx, row in df.iterrows():\n            src_path = row['image_path']\n            label = str(row['label'])\n            img_id = row['image_id']\n            \n            # Create unique filename to avoid conflicts\n            ext = os.path.splitext(src_path)[1]\n            dest_filename = f\"{row['source']}_{img_id}{ext}\"\n            dest_path = os.path.join(split_dir, label, dest_filename)\n            \n            try:\n                shutil.copy2(src_path, dest_path)\n                copied += 1\n            except Exception as e:\n                print(f\"Error copying {src_path}: {e}\")\n        \n        print(f\"{split_name}: {copied} images copied\")\n    \n    copy_images_to_split(train_df, 'train')\n    copy_images_to_split(val_df, 'valid')\n    copy_images_to_split(test_df, 'test')\n    \n    return train_df, val_df, test_df\n\n# ==========================================\n# 5. ENHANCED DATASET CLASS\n# ==========================================\n\nclass EnhancedSkinCancerDataset(Dataset):\n    def __init__(self, data_dir, transform=None, use_mixup=False, mixup_alpha=0.2):\n        self.data_dir = data_dir\n        self.transform = transform\n        self.use_mixup = use_mixup\n        self.mixup_alpha = mixup_alpha\n        self.samples = []\n        \n        # Load all image paths and labels\n        for class_idx, class_name in enumerate(['0', '1']):\n            class_dir = os.path.join(data_dir, class_name)\n            if os.path.exists(class_dir):\n                for img_name in os.listdir(class_dir):\n                    if img_name.lower().endswith(('.jpg', '.jpeg', '.png')):\n                        img_path = os.path.join(class_dir, img_name)\n                        self.samples.append((img_path, class_idx))\n        \n        print(f\"Dataset {data_dir}: {len(self.samples)} samples\")\n        \n    def __len__(self):\n        return len(self.samples)\n    \n    def __getitem__(self, idx):\n        img_path, label = self.samples[idx]\n        \n        # Load image\n        try:\n            image = cv2.imread(img_path)\n            if image is None:\n                # Fallback to PIL if cv2 fails\n                from PIL import Image\n                image = Image.open(img_path).convert('RGB')\n                image = np.array(image)\n            else:\n                image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        except Exception as e:\n            print(f\"Error loading image {img_path}: {e}\")\n            # Return a black image as fallback\n            image = np.zeros((224, 224, 3), dtype=np.uint8)\n        \n        # Apply transformations\n        if self.transform:\n            try:\n                augmented = self.transform(image=image)\n                image = augmented['image']\n            except Exception as e:\n                print(f\"Error in transform for {img_path}: {e}\")\n                # Create a simple transform as fallback\n                image = cv2.resize(image, (224, 224))\n                image = image.astype(np.float32) / 255.0\n                image = (image - np.array([0.485, 0.456, 0.406])) / np.array([0.229, 0.224, 0.225])\n                image = torch.from_numpy(image.transpose(2, 0, 1)).float()\n        \n        # Always return consistent format - no mixup in __getitem__\n        # Mixup will be handled in a custom collate function\n        return image, label\n\n\n# ==========================================\n# 6. ADVANCED MODEL ARCHITECTURE\n# ==========================================\n\nclass AdvancedSkinCancerClassifier(nn.Module):\n    def __init__(self, model_name='efficientnet_v2_l', num_classes=2, pretrained=True, dropout_rate=0.3):\n        super(AdvancedSkinCancerClassifier, self).__init__()\n        \n        # Load backbone\n        if model_name == 'efficientnet_v2_l':\n            self.backbone = models.efficientnet_v2_l(pretrained=pretrained)\n            in_features = self.backbone.classifier[1].in_features\n            self.backbone.classifier = nn.Identity()\n        elif model_name == 'convnext_large':  \n            self.backbone = models.convnext_large(pretrained=pretrained)\n            in_features = self.backbone.classifier[2].in_features\n            self.backbone.classifier = nn.Identity()\n        elif model_name == 'swin_v2_b':\n            self.backbone = models.swin_v2_b(pretrained=pretrained)\n            in_features = self.backbone.head.in_features\n            self.backbone.head = nn.Identity()\n        \n        # Advanced multi-scale feature extraction\n        self.global_pool = nn.AdaptiveAvgPool2d(1)\n        self.global_max_pool = nn.AdaptiveMaxPool2d(1)\n        \n        # Advanced classifier with attention\n        self.attention = nn.Sequential(\n            nn.Linear(in_features, in_features // 4),\n            nn.ReLU(inplace=True),\n            nn.Linear(in_features // 4, in_features),\n            nn.Sigmoid()\n        )\n        \n        self.classifier = nn.Sequential(\n            nn.Dropout(dropout_rate),\n            nn.Linear(in_features * 2, 1024),  # *2 for avg+max pooling\n            nn.ReLU(inplace=True),\n            nn.BatchNorm1d(1024),\n            nn.Dropout(dropout_rate),\n            nn.Linear(1024, 512),\n            nn.ReLU(inplace=True),\n            nn.BatchNorm1d(512),\n            nn.Dropout(dropout_rate / 2),\n            nn.Linear(512, 256),\n            nn.ReLU(inplace=True),\n            nn.BatchNorm1d(256),\n            nn.Dropout(dropout_rate / 2),\n            nn.Linear(256, num_classes)\n        )\n        \n        self._initialize_weights()\n    \n    def _initialize_weights(self):\n        for m in self.classifier.modules():\n            if isinstance(m, nn.Linear):\n                nn.init.xavier_uniform_(m.weight)\n                nn.init.constant_(m.bias, 0)\n    \n    def forward(self, x):\n        # Extract features\n        features = self.backbone(x)\n        \n        if len(features.shape) == 4:  # If backbone returns feature maps\n            # Apply attention mechanism\n            attention_weights = self.attention(features.mean(dim=[2, 3]))\n            features = features * attention_weights.unsqueeze(-1).unsqueeze(-1)\n            \n            # Global pooling\n            avg_pool = self.global_pool(features).flatten(1)\n            max_pool = self.global_max_pool(features).flatten(1)\n            features = torch.cat([avg_pool, max_pool], dim=1)\n        else:  # If backbone returns flattened features\n            features = features\n            features = torch.cat([features, features], dim=1)  # Duplicate for consistency\n        \n        # Classification\n        output = self.classifier(features)\n        return output\n\n\ndef mixup_collate_fn(batch, mixup_alpha=0.2, use_mixup=True):\n    \"\"\"Custom collate function that applies mixup at batch level\"\"\"\n    images, labels = zip(*batch)\n    images = torch.stack(images)\n    labels = torch.tensor(labels, dtype=torch.long)\n    \n    if use_mixup and np.random.random() < 0.5:\n        # Apply mixup\n        batch_size = images.size(0)\n        indices = torch.randperm(batch_size)\n        \n        lam = np.random.beta(mixup_alpha, mixup_alpha)\n        mixed_images = lam * images + (1 - lam) * images[indices]\n        labels_a = labels\n        labels_b = labels[indices]\n        \n        return mixed_images, labels_a, labels_b, torch.tensor(lam)\n    else:\n        return images, labels\n# ==========================================\n# 7. ADVANCED LOSS FUNCTIONS\n# ==========================================\n\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, reduce=True):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduce = reduce\n\n    def forward(self, inputs, targets):\n        ce_loss = F.cross_entropy(inputs, targets, reduction='none')\n        pt = torch.exp(-ce_loss)\n        focal_loss = self.alpha * (1-pt)**self.gamma * ce_loss\n        return focal_loss.mean() if self.reduce else focal_loss\n\nclass LabelSmoothingLoss(nn.Module):\n    def __init__(self, num_classes, smoothing=0.1):\n        super(LabelSmoothingLoss, self).__init__()\n        self.num_classes = num_classes\n        self.smoothing = smoothing\n        \n    def forward(self, inputs, targets):\n        log_probs = F.log_softmax(inputs, dim=1)\n        targets_smooth = torch.zeros_like(log_probs).scatter_(1, targets.unsqueeze(1), 1)\n        targets_smooth = targets_smooth * (1 - self.smoothing) + self.smoothing / self.num_classes\n        loss = (-targets_smooth * log_probs).sum(dim=1).mean()\n        return loss\n\ndef mixup_loss(criterion, pred, y_a, y_b, lam):\n    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)\n\n# ==========================================\n# 8. ENHANCED TRAINING FUNCTION\n# ==========================================\n\ndef train_advanced_model(model, train_loader, val_loader, criterion, optimizer, scheduler, num_epochs):\n    scaler = GradScaler()\n    best_val_acc = 0.0\n    train_losses, val_losses = [], []\n    train_accs, val_accs = [], []\n    \n    for epoch in range(num_epochs):\n        print(f'Epoch {epoch+1}/{num_epochs}')\n        print('-' * 50)\n        \n        # Training phase\n        model.train()\n        running_loss = 0.0\n        correct_predictions = 0\n        total_predictions = 0\n        \n        optimizer.zero_grad()\n        \n        for batch_idx, batch_data in enumerate(train_loader):\n            try:\n                if len(batch_data) == 4:  # Mixup batch\n                    images, labels_a, labels_b, lam = batch_data\n                    images = images.to(device, non_blocking=True)\n                    labels_a, labels_b = labels_a.to(device, non_blocking=True), labels_b.to(device, non_blocking=True)\n                    lam = lam.item() if isinstance(lam, torch.Tensor) else lam\n                    \n                    with autocast():\n                        outputs = model(images)\n                        loss = mixup_loss(criterion, outputs, labels_a, labels_b, lam)\n                    \n                    # For accuracy calculation with mixup\n                    _, predicted = torch.max(outputs.data, 1)\n                    total_predictions += labels_a.size(0)\n                    correct_predictions += (lam * (predicted == labels_a).float() + \n                                          (1 - lam) * (predicted == labels_b).float()).sum().item()\n                else:  # Regular batch\n                    images, labels = batch_data\n                    images, labels = images.to(device, non_blocking=True), labels.to(device, non_blocking=True)\n                    \n                    with autocast():\n                        outputs = model(images)\n                        loss = criterion(outputs, labels)\n                    \n                    _, predicted = torch.max(outputs.data, 1)\n                    total_predictions += labels.size(0)\n                    correct_predictions += (predicted == labels).sum().item()\n                \n                # Gradient accumulation\n                loss = loss / ACCUMULATION_STEPS\n                scaler.scale(loss).backward()\n                \n                if (batch_idx + 1) % ACCUMULATION_STEPS == 0:\n                    scaler.unscale_(optimizer)\n                    torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n                    scaler.step(optimizer)\n                    scaler.update()\n                    optimizer.zero_grad()\n                \n                running_loss += loss.item() * ACCUMULATION_STEPS\n                \n                if batch_idx % 20 == 0:\n                    print(f'Batch {batch_idx}/{len(train_loader)}, Loss: {loss.item() * ACCUMULATION_STEPS:.4f}')\n                    \n            except Exception as e:\n                print(f\"Error in batch {batch_idx}: {e}\")\n                continue\n        \n        train_loss = running_loss / len(train_loader)\n        train_acc = correct_predictions / total_predictions if total_predictions > 0 else 0\n        \n        # Validation phase\n        model.eval()\n        val_running_loss = 0.0\n        val_correct = 0\n        val_total = 0\n        \n        with torch.no_grad():\n            for images, labels in val_loader:\n                try:\n                    images, labels = images.to(device, non_blocking=True), labels.to(device, non_blocking=True)\n                    \n                    with autocast():\n                        outputs = model(images)\n                        loss = criterion(outputs, labels)\n                    \n                    val_running_loss += loss.item()\n                    _, predicted = torch.max(outputs, 1)\n                    val_total += labels.size(0)\n                    val_correct += (predicted == labels).sum().item()\n                except Exception as e:\n                    print(f\"Error in validation batch: {e}\")\n                    continue\n        \n        val_loss = val_running_loss / len(val_loader) if len(val_loader) > 0 else float('inf')\n        val_acc = val_correct / val_total if val_total > 0 else 0\n        \n        # Store metrics\n        train_losses.append(train_loss)\n        val_losses.append(val_loss)\n        train_accs.append(train_acc)\n        val_accs.append(val_acc)\n        \n        print(f'Train Loss: {train_loss:.4f}, Train Acc: {train_acc:.4f}')\n        print(f'Val Loss: {val_loss:.4f}, Val Acc: {val_acc:.4f}')\n        \n        # Save best model\n        if val_acc > best_val_acc:\n            best_val_acc = val_acc\n            torch.save({\n                'epoch': epoch,\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'val_acc': val_acc,\n                'val_loss': val_loss\n            }, 'best_enhanced_model.pth')\n            print(f'New best model saved! Val Acc: {best_val_acc:.4f}')\n        \n        # Learning rate scheduling\n        scheduler.step()\n        \n        # Early stopping check\n        if epoch > 20 and val_acc < 0.7:  # If not improving after 20 epochs\n            print(\"Performance not improving, adjusting learning rate...\")\n            for param_group in optimizer.param_groups:\n                param_group['lr'] *= 0.5\n        \n        print()\n    \n    return train_losses, val_losses, train_accs, val_accs\n\n\n# ==========================================\n# 9. ENHANCED EVALUATION WITH TTA\n# ==========================================\n\ndef evaluate_with_advanced_tta(model, test_loader, tta_transforms):\n    model.eval()\n    all_predictions = []\n    all_labels = []\n    all_probabilities = []\n    \n    with torch.no_grad():\n        for images, labels in test_loader:\n            labels = labels.to(device)\n            batch_probs = []\n            \n            # Original prediction\n            images_device = images.to(device)\n            with autocast():\n                outputs = model(images_device)\n                probs = F.softmax(outputs, dim=1)\n                batch_probs.append(probs.cpu())\n            \n            # TTA predictions\n            for tta_transform in tta_transforms:\n                tta_images = []\n                for img in images:\n                    # Convert tensor back to numpy for albumentations\n                    img_np = img.permute(1, 2, 0).numpy()\n                    img_np = (img_np * np.array([0.229, 0.224, 0.225]) + np.array([0.485, 0.456, 0.406])) * 255\n                    img_np = np.clip(img_np, 0, 255).astype(np.uint8)\n                    \n                    augmented = tta_transform(image=img_np)\n                    tta_images.append(augmented['image'])\n                \n                tta_batch = torch.stack(tta_images).to(device)\n                with autocast():\n                    outputs = model(tta_batch)\n                    probs = F.softmax(outputs, dim=1)\n                    batch_probs.append(probs.cpu())\n            \n            # Average TTA predictions\n            avg_probs = torch.stack(batch_probs).mean(dim=0)\n            predictions = torch.argmax(avg_probs, dim=1)\n            \n            all_predictions.extend(predictions.numpy())\n            all_labels.extend(labels.cpu().numpy())\n            all_probabilities.extend(avg_probs[:, 1].numpy())\n    \n    return np.array(all_predictions), np.array(all_labels), np.array(all_probabilities)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# ==========================================\n# 10. MAIN EXECUTION\n# ==========================================\n\ndef main():\n    print(\"=\"*80)\n    print(\"ENHANCED SKIN CANCER CLASSIFICATION PIPELINE - TARGET: 92%+ ACCURACY\")\n    print(\"=\"*80)\n    \n    # Step 1: Extract and process all datasets\n    print(\"Step 1: Processing all three datasets...\")\n    df_all = extract_and_process_datasets()\n    \n    # Step 2: Copy and split data\n    print(\"\\nStep 2: Copying and splitting data...\")\n    train_df, val_df, test_df = copy_and_split_data(df_all)\n    \n    # Step 3: Create enhanced datasets\n    print(\"\\nStep 3: Creating enhanced datasets...\")\n    train_dataset = EnhancedSkinCancerDataset(TRAIN_DIR, transform=train_transforms, use_mixup=False)  # Mixup handled in collate_fn\n    val_dataset = EnhancedSkinCancerDataset(VALID_DIR, transform=valid_transforms, use_mixup=False)\n    test_dataset = EnhancedSkinCancerDataset(TEST_DIR, transform=valid_transforms, use_mixup=False)\n    \n    # Calculate class weights for balanced training\n    train_labels = [sample[1] for sample in train_dataset.samples]\n    if len(train_labels) == 0:\n        print(\"No training samples found! Please check dataset paths.\")\n        return 0, 0\n        \n    class_counts = np.bincount(train_labels)\n    class_weights = len(train_labels) / (len(class_counts) * class_counts)\n    \n    # Create weighted sampler for balanced training\n    sample_weights = [class_weights[label] for label in train_labels]\n    sampler = WeightedRandomSampler(sample_weights, len(sample_weights))\n    \n    # Create custom collate function for training with mixup\n    def train_collate_fn(batch):\n        return mixup_collate_fn(batch, mixup_alpha=0.2, use_mixup=True)\n    \n    def val_collate_fn(batch):\n        return mixup_collate_fn(batch, use_mixup=False)\n    \n    # Create data loaders with custom collate functions\n    train_loader = DataLoader(\n        train_dataset, \n        batch_size=BATCH_SIZE, \n        sampler=sampler, \n        num_workers=2,  # Reduced num_workers to avoid issues\n        pin_memory=True, \n        collate_fn=train_collate_fn,\n        drop_last=True  # Drop last incomplete batch\n    )\n    val_loader = DataLoader(\n        val_dataset, \n        batch_size=BATCH_SIZE, \n        shuffle=False, \n        num_workers=2, \n        pin_memory=True, \n        collate_fn=val_collate_fn,\n        drop_last=False\n    )\n    test_loader = DataLoader(\n        test_dataset, \n        batch_size=BATCH_SIZE, \n        shuffle=False, \n        num_workers=2, \n        pin_memory=True, \n        collate_fn=val_collate_fn,\n        drop_last=False\n    )\n    print(f\"Train loader: {len(train_loader)} batches\")\n    print(f\"Val loader: {len(val_loader)} batches\") \n    print(f\"Test loader: {len(test_loader)} batches\")\n    \n    # Step 4: Initialize advanced model\n    print(\"\\nStep 4: Initializing advanced model...\")\n    model = AdvancedSkinCancerClassifier(\n        model_name='efficientnet_v2_l', \n        num_classes=NUM_CLASSES, \n        pretrained=True,\n        dropout_rate=0.3\n    ).to(device)\n    \n    print(f\"Model parameters: {sum(p.numel() for p in model.parameters()):,}\")\n    print(f\"Trainable parameters: {sum(p.numel() for p in model.parameters() if p.requires_grad):,}\")\n    \n    # Step 5: Setup advanced loss and optimization\n    print(\"\\nStep 5: Setting up loss functions and optimizers...\")\n    \n    # Combine multiple loss functions\n    focal_loss = FocalLoss(alpha=1, gamma=2)\n    label_smooth_loss = LabelSmoothingLoss(num_classes=NUM_CLASSES, smoothing=0.1)\n    ce_loss = nn.CrossEntropyLoss(weight=torch.tensor(class_weights, dtype=torch.float32).to(device))\n    \n    def combined_criterion(outputs, targets):\n        loss1 = focal_loss(outputs, targets)\n        loss2 = label_smooth_loss(outputs, targets)\n        loss3 = ce_loss(outputs, targets)\n        return 0.4 * loss1 + 0.3 * loss2 + 0.3 * loss3\n    \n    # Advanced optimizer setup\n    optimizer = optim.AdamW(\n        model.parameters(),\n        lr=LEARNING_RATE,\n        weight_decay=WEIGHT_DECAY,\n        betas=(0.9, 0.999)\n    )\n    \n    # Advanced scheduler\n    scheduler = CosineAnnealingWarmRestarts(\n        optimizer,\n        T_0=10,\n        T_mult=2,\n        eta_min=1e-6\n    )\n    \n    # Step 6: Train the model\n    print(\"\\nStep 6: Training the advanced model...\")\n    train_losses, val_losses, train_accs, val_accs = train_advanced_model(\n        model, train_loader, val_loader, combined_criterion, optimizer, scheduler, NUM_EPOCHS\n    )\n    \n    # Step 7: Load best model and evaluate\n    print(\"\\nStep 7: Loading best model and evaluating...\")\n    checkpoint = torch.load('best_enhanced_model.pth')\n    model.load_state_dict(checkpoint['model_state_dict'])\n    print(f\"Best validation accuracy: {checkpoint['val_acc']:.4f}\")\n    \n    # Step 8: Advanced evaluation with TTA\n    print(\"\\nStep 8: Performing advanced evaluation with TTA...\")\n    test_predictions, test_labels, test_probabilities = evaluate_with_advanced_tta(\n        model, test_loader, tta_transforms\n    )\n    \n    # Calculate comprehensive metrics\n    test_accuracy = np.mean(test_predictions == test_labels)\n    test_auc = roc_auc_score(test_labels, test_probabilities)\n    \n    print(f\"\\nFINAL RESULTS:\")\n    print(f\"Test Accuracy: {test_accuracy:.4f} ({test_accuracy*100:.2f}%)\")\n    print(f\"Test AUC: {test_auc:.4f}\")\n    \n    # Detailed classification report\n    print(\"\\nDetailed Classification Report:\")\n    print(classification_report(test_labels, test_predictions, target_names=CLASS_NAMES))\n    \n    # Step 9: Visualization and analysis\n    print(\"\\nStep 9: Creating visualizations...\")\n    \n    # Plot training curves\n    plt.figure(figsize=(15, 5))\n    \n    plt.subplot(1, 3, 1)\n    plt.plot(train_losses, label='Train Loss', color='blue')\n    plt.plot(val_losses, label='Val Loss', color='red')\n    plt.title('Training and Validation Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend()\n    plt.grid(True)\n    \n    plt.subplot(1, 3, 2)\n    plt.plot(train_accs, label='Train Accuracy', color='blue')\n    plt.plot(val_accs, label='Val Accuracy', color='red')\n    plt.title('Training and Validation Accuracy')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    plt.grid(True)\n    \n    # ROC Curve\n    plt.subplot(1, 3, 3)\n    fpr, tpr, _ = roc_curve(test_labels, test_probabilities)\n    plt.plot(fpr, tpr, color='darkorange', lw=2, label=f'ROC curve (AUC = {test_auc:.3f})')\n    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('ROC Curve')\n    plt.legend(loc=\"lower right\")\n    plt.grid(True)\n    \n    plt.tight_layout()\n    plt.savefig('training_results.png', dpi=300, bbox_inches='tight')\n    plt.show()\n    \n    # Confusion Matrix\n    plt.figure(figsize=(8, 6))\n    cm = confusion_matrix(test_labels, test_predictions)\n    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', \n                xticklabels=CLASS_NAMES, yticklabels=CLASS_NAMES)\n    plt.title('Confusion Matrix')\n    plt.ylabel('True Label')\n    plt.xlabel('Predicted Label')\n    plt.savefig('confusion_matrix.png', dpi=300, bbox_inches='tight')\n    plt.show()\n    \n    # Step 10: Model ensemble for maximum performance\n    print(\"\\nStep 10: Creating model ensemble for maximum performance...\")\n    \n    # Train additional models with different architectures\n    ensemble_models = []\n    ensemble_names = ['efficientnet_v2_l', 'convnext_large']\n    \n    for i, model_name in enumerate(ensemble_names):\n        if i == 0:  # First model already trained\n            ensemble_models.append(model)\n            continue\n            \n        print(f\"Training ensemble model {i+1}: {model_name}\")\n        ensemble_model = AdvancedSkinCancerClassifier(\n            model_name=model_name,\n            num_classes=NUM_CLASSES,\n            pretrained=True,\n            dropout_rate=0.3\n        ).to(device)\n        \n        ensemble_optimizer = optim.AdamW(\n            ensemble_model.parameters(),\n            lr=LEARNING_RATE * 0.8,  # Slightly lower LR\n            weight_decay=WEIGHT_DECAY\n        )\n        \n        ensemble_scheduler = CosineAnnealingWarmRestarts(\n            ensemble_optimizer, T_0=8, T_mult=2, eta_min=1e-6\n        )\n        \n        # Train for fewer epochs\n        _, _, _, _ = train_advanced_model(\n            ensemble_model, train_loader, val_loader, \n            combined_criterion, ensemble_optimizer, ensemble_scheduler, \n            NUM_EPOCHS // 2\n        )\n        \n        ensemble_models.append(ensemble_model)\n    \n    # Ensemble prediction\n    print(\"\\nPerforming ensemble prediction...\")\n    ensemble_predictions = []\n    ensemble_probabilities = []\n    \n    for test_images, test_labels_batch in test_loader:\n        batch_ensemble_probs = []\n        \n        for model_idx, ens_model in enumerate(ensemble_models):\n            ens_model.eval()\n            with torch.no_grad():\n                test_images_device = test_images.to(device)\n                with autocast():\n                    outputs = ens_model(test_images_device)\n                    probs = F.softmax(outputs, dim=1)\n                    batch_ensemble_probs.append(probs.cpu())\n        \n        # Average ensemble predictions\n        avg_ensemble_probs = torch.stack(batch_ensemble_probs).mean(dim=0)\n        ensemble_predictions.extend(torch.argmax(avg_ensemble_probs, dim=1).numpy())\n        ensemble_probabilities.extend(avg_ensemble_probs[:, 1].numpy())\n    \n    # Final ensemble metrics\n    ensemble_accuracy = np.mean(np.array(ensemble_predictions) == test_labels)\n    ensemble_auc = roc_auc_score(test_labels, ensemble_probabilities)\n    \n    print(f\"\\nENSEMBLE RESULTS:\")\n    print(f\"Ensemble Accuracy: {ensemble_accuracy:.4f} ({ensemble_accuracy*100:.2f}%)\")\n    print(f\"Ensemble AUC: {ensemble_auc:.4f}\")\n    \n    # Step 11: Save final results and model\n    print(\"\\nStep 11: Saving final results...\")\n    \n    # Save ensemble models\n    torch.save({\n        'models': [model.state_dict() for model in ensemble_models],\n        'model_names': ensemble_names,\n        'test_accuracy': ensemble_accuracy,\n        'test_auc': ensemble_auc,\n        'class_names': CLASS_NAMES\n    }, 'final_ensemble_model.pth')\n    \n    # Save results summary\n    results_summary = {\n        'single_model_accuracy': test_accuracy,\n        'single_model_auc': test_auc,\n        'ensemble_accuracy': ensemble_accuracy,\n        'ensemble_auc': ensemble_auc,\n        'target_achieved': ensemble_accuracy >= 0.92,\n        'training_epochs': NUM_EPOCHS,\n        'total_samples': len(df_all),\n        'class_distribution': df_all['label'].value_counts().to_dict()\n    }\n    \n    # Save to JSON\n    import json\n    with open('results_summary.json', 'w') as f:\n        json.dump(results_summary, f, indent=2)\n    \n    print(\"\\n\" + \"=\"*80)\n    print(\"FINAL PERFORMANCE SUMMARY\")\n    print(\"=\"*80)\n    print(f\"Single Model Accuracy: {test_accuracy:.4f} ({test_accuracy*100:.2f}%)\")\n    print(f\"Ensemble Accuracy: {ensemble_accuracy:.4f} ({ensemble_accuracy*100:.2f}%)\")\n    print(f\"Target 92% Achieved: {'✓ YES' if ensemble_accuracy >= 0.92 else '✗ NO'}\")\n    print(f\"AUC Score: {ensemble_auc:.4f}\")\n    print(\"=\"*80)\n    \n    \n    return ensemble_accuracy, ensemble_auc\n\n# ==========================================\n# 11. ADDITIONAL UTILITY FUNCTIONS\n# ==========================================\n\ndef predict_single_image(image_path, ensemble_models, transforms):\n    \"\"\"Predict a single image using the ensemble\"\"\"\n    image = cv2.imread(image_path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    \n    ensemble_probs = []\n    \n    for model in ensemble_models:\n        model.eval()\n        with torch.no_grad():\n            # Apply transforms\n            augmented = transforms(image=image)\n            image_tensor = augmented['image'].unsqueeze(0).to(device)\n            \n            with autocast():\n                output = model(image_tensor)\n                prob = F.softmax(output, dim=1)\n                ensemble_probs.append(prob.cpu())\n    \n    # Average predictions\n    avg_prob = torch.stack(ensemble_probs).mean(dim=0)\n    prediction = torch.argmax(avg_prob, dim=1).item()\n    confidence = avg_prob[0][prediction].item()\n    \n    return CLASS_NAMES[prediction], confidence\n\ndef generate_submission(test_loader, ensemble_models, output_file='submission.csv'):\n    \"\"\"Generate submission file for Kaggle\"\"\"\n    predictions = []\n    \n    for images, _ in test_loader:\n        batch_ensemble_probs = []\n        \n        for model in ensemble_models:\n            model.eval()\n            with torch.no_grad():\n                images_device = images.to(device)\n                with autocast():\n                    outputs = model(images_device)\n                    probs = F.softmax(outputs, dim=1)\n                    batch_ensemble_probs.append(probs.cpu())\n        \n        avg_probs = torch.stack(batch_ensemble_probs).mean(dim=0)\n        batch_predictions = avg_probs[:, 1].numpy()  # Probability of melanoma\n        predictions.extend(batch_predictions)\n    \n    # Create submission DataFrame\n    submission_df = pd.DataFrame({\n        'id': range(len(predictions)),\n        'target': predictions\n    })\n    \n    submission_df.to_csv(output_file, index=False)\n    print(f\"Submission file saved as {output_file}\")\n\n# ==========================================\n# 12. EXECUTION\n# ==========================================\n\nif __name__ == \"__main__\":\n    try:\n        accuracy, auc = main()\n        print(f\"\\nTraining completed successfully!\")\n        print(f\"Final Accuracy: {accuracy:.4f}\")\n        print(f\"Final AUC: {auc:.4f}\")\n        \n    except Exception as e:\n        print(f\"Error during training: {e}\")\n        import traceback\n        traceback.print_exc()\n    \n    finally:\n        # Cleanup GPU memory\n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()\n        print(\"GPU memory cleared.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}