{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"}],"dockerImageVersionId":31042,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\nComplete APTOS Diabetic Retinopathy Detection Training Script\nOptimized for Kaggle Dual T4 GPUs (16GB each)\n\"\"\"\n\nimport os\nimport gc\nimport cv2\nimport time\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom typing import Tuple, Dict, List\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\nfrom torch.optim import AdamW\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, CosineAnnealingWarmRestarts\nfrom torch.cuda.amp import GradScaler, autocast\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport timm\n\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score, classification_report, confusion_matrix\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nwarnings.filterwarnings('ignore')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# Configuration Class\n# =============================================================================\n\nclass Config:\n    # Dataset paths (Kaggle paths)\n    TRAIN_CSV = '/kaggle/input/aptos2019-blindness-detection/train.csv'\n    TRAIN_IMAGES_DIR = '/kaggle/input/aptos2019-blindness-detection/train_images'\n    TEST_CSV = '/kaggle/input/aptos2019-blindness-detection/test.csv'\n    TEST_IMAGES_DIR = '/kaggle/input/aptos2019-blindness-detection/test_images'\n    \n    # Model configuration\n    MODEL_NAME = 'tf_efficientnet_b4_ns'  # Best for medical imaging\n    NUM_CLASSES = 5\n    IMAGE_SIZE = 512  # Optimal for retinal images\n    \n    # Training configuration\n    BATCH_SIZE = 8  # Optimized for dual T4 16GB\n    EPOCHS = 80\n    MIN_EPOCHS = 30\n    EARLY_STOPPING_PATIENCE = 15\n    \n    # Optimization\n    LEARNING_RATE = 1e-3\n    MIN_LR = 1e-7\n    WEIGHT_DECAY = 1e-4\n    GRADIENT_CLIP_NORM = 1.0\n    \n    # Hardware\n    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    NUM_WORKERS = 4\n    PIN_MEMORY = True\n    USE_MIXED_PRECISION = True\n    \n    # Reproducibility\n    SEED = 42\n    \n    # Output paths\n    OUTPUT_DIR = '/kaggle/working'\n    MODEL_SAVE_PATH = '/kaggle/working/best_model.pth'\n    CHECKPOINT_DIR = '/kaggle/working/checkpoints'\n    \n    # Cross-validation\n    N_FOLDS = 5\n    FOLD_TO_TRAIN = 0  # Set to -1 to train all folds\n    \n    # Advanced techniques\n    USE_FOCAL_LOSS = True\n    USE_LABEL_SMOOTHING = True\n    LABEL_SMOOTHING_ALPHA = 0.1\n    USE_MIXUP = True\n    MIXUP_ALPHA = 0.2\n    USE_CUTMIX = True\n    CUTMIX_ALPHA = 1.0\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# Utility Functions\n# =============================================================================\n\ndef set_seed(seed: int = 42):\n    \"\"\"Set random seeds for reproducibility\"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\ndef get_device_info():\n    \"\"\"Print GPU information\"\"\"\n    if torch.cuda.is_available():\n        print(f\"🚀 Using GPU: {torch.cuda.get_device_name()}\")\n        print(f\"🔢 Number of GPUs: {torch.cuda.device_count()}\")\n        for i in range(torch.cuda.device_count()):\n            print(f\"   GPU {i}: {torch.cuda.get_device_name(i)}\")\n            print(f\"   Memory: {torch.cuda.get_device_properties(i).total_memory / 1e9:.1f} GB\")\n    else:\n        print(\"❌ No GPU available\")\n\ndef create_output_dirs():\n    \"\"\"Create necessary output directories\"\"\"\n    os.makedirs(Config.OUTPUT_DIR, exist_ok=True)\n    os.makedirs(Config.CHECKPOINT_DIR, exist_ok=True)\n\n# =============================================================================\n# Image Preprocessing Functions\n# =============================================================================\n\ndef crop_image_from_gray(img, tol=7):\n    \"\"\"Crop image to remove black borders\"\"\"\n    if img.ndim == 2:\n        mask = img > tol\n        return img[np.ix_(mask.any(1), mask.any(0))]\n    elif img.ndim == 3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img > tol\n        check_shape = img[:,:,0][np.ix_(mask.any(1), mask.any(0))].shape[0]\n        if check_shape == 0:\n            return img\n        else:\n            img1 = img[:,:,0][np.ix_(mask.any(1), mask.any(0))]\n            img2 = img[:,:,1][np.ix_(mask.any(1), mask.any(0))]\n            img3 = img[:,:,2][np.ix_(mask.any(1), mask.any(0))]\n            img = np.stack([img1, img2, img3], axis=-1)\n    return img\n\ndef preprocess_image(image_path: str, image_size: int = 512) -> np.ndarray:\n    \"\"\"Enhanced preprocessing for retinal images\"\"\"\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    # Crop black borders\n    img = crop_image_from_gray(img)\n    \n    # Resize\n    img = cv2.resize(img, (image_size, image_size))\n    \n    # Apply CLAHE for better contrast\n    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\n    lab = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)\n    lab[:,:,0] = clahe.apply(lab[:,:,0])\n    img = cv2.cvtColor(lab, cv2.COLOR_LAB2RGB)\n    \n    return img\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# Data Augmentation\n# =============================================================================\n\ndef get_transforms(image_size: int = 512):\n    \"\"\"Advanced augmentation pipeline for retinal images\"\"\"\n    \n    train_transform = A.Compose([\n        A.Resize(image_size, image_size),\n        \n        # Geometric transformations\n        A.RandomRotate90(p=0.5),\n        A.Flip(p=0.5),\n        A.ShiftScaleRotate(\n            shift_limit=0.1, scale_limit=0.1, rotate_limit=15, \n            border_mode=cv2.BORDER_CONSTANT, value=0, p=0.5\n        ),\n        \n        # Optical distortions (common in retinal images)\n        A.OneOf([\n            A.OpticalDistortion(distort_limit=0.1, shift_limit=0.1, p=0.3),\n            A.GridDistortion(num_steps=5, distort_limit=0.1, p=0.3),\n            A.ElasticTransform(alpha=1, sigma=50, alpha_affine=50, p=0.3),\n        ], p=0.3),\n        \n        # Color and lighting augmentations\n        A.OneOf([\n            A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.3),\n            A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10, p=0.3),\n            A.CLAHE(clip_limit=2.0, tile_grid_size=(8, 8), p=0.3),\n        ], p=0.5),\n        \n        # Noise and blur\n        A.OneOf([\n            A.GaussNoise(var_limit=(10, 50), p=0.2),\n            A.GaussianBlur(blur_limit=3, p=0.2),\n            A.MotionBlur(blur_limit=3, p=0.2),\n        ], p=0.2),\n        \n        # Cutout variations\n        A.OneOf([\n            A.Cutout(num_holes=8, max_h_size=32, max_w_size=32, p=0.3),\n            A.CoarseDropout(max_holes=8, max_height=32, max_width=32, p=0.3),\n        ], p=0.3),\n        \n        # Normalization\n        A.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225],\n        ),\n        ToTensorV2(),\n    ])\n    \n    val_transform = A.Compose([\n        A.Resize(image_size, image_size),\n        A.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225],\n        ),\n        ToTensorV2(),\n    ])\n    \n    return train_transform, val_transform\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# Dataset Class\n# =============================================================================\n\nclass APTOSDataset(Dataset):\n    \"\"\"Optimized APTOS dataset with advanced preprocessing\"\"\"\n    \n    def __init__(self, df: pd.DataFrame, image_dir: str, transforms=None, is_training: bool = True):\n        self.df = df.reset_index(drop=True)\n        self.image_dir = image_dir\n        self.transforms = transforms\n        self.is_training = is_training\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        image_path = os.path.join(self.image_dir, f\"{row['id_code']}.png\")\n        \n        # Load and preprocess image\n        image = preprocess_image(image_path, Config.IMAGE_SIZE)\n        \n        if self.transforms:\n            transformed = self.transforms(image=image)\n            image = transformed['image']\n        \n        if self.is_training:\n            label = torch.tensor(row['diagnosis'], dtype=torch.long)\n            return image, label\n        else:\n            return image\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# Advanced Loss Functions\n# =============================================================================\n\nclass FocalLoss(nn.Module):\n    \"\"\"Focal Loss for handling class imbalance\"\"\"\n    \n    def __init__(self, alpha=1.0, gamma=2.0, reduction='mean'):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduction = reduction\n    \n    def forward(self, inputs, targets):\n        ce_loss = F.cross_entropy(inputs, targets, reduction='none')\n        pt = torch.exp(-ce_loss)\n        focal_loss = self.alpha * (1-pt)**self.gamma * ce_loss\n        \n        if self.reduction == 'mean':\n            return focal_loss.mean()\n        elif self.reduction == 'sum':\n            return focal_loss.sum()\n        else:\n            return focal_loss\n\nclass LabelSmoothingCrossEntropy(nn.Module):\n    \"\"\"Label smoothing cross entropy loss\"\"\"\n    \n    def __init__(self, smoothing=0.1):\n        super().__init__()\n        self.smoothing = smoothing\n    \n    def forward(self, inputs, targets):\n        log_prob = F.log_softmax(inputs, dim=-1)\n        weight = inputs.new_ones(inputs.size()) * self.smoothing / (inputs.size(-1) - 1.)\n        weight.scatter_(-1, targets.unsqueeze(-1), (1. - self.smoothing))\n        loss = (-weight * log_prob).sum(dim=-1).mean()\n        return loss","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# Mixup and CutMix\n# =============================================================================\n\ndef mixup_data(x, y, alpha=1.0, use_cuda=True):\n    \"\"\"Mixup augmentation\"\"\"\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n    \n    batch_size = x.size(0)\n    if use_cuda:\n        index = torch.randperm(batch_size).cuda()\n    else:\n        index = torch.randperm(batch_size)\n    \n    mixed_x = lam * x + (1 - lam) * x[index, :]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam\n\ndef cutmix_data(x, y, alpha=1.0):\n    \"\"\"CutMix augmentation\"\"\"\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n    \n    batch_size = x.size(0)\n    index = torch.randperm(batch_size)\n    \n    bbx1, bby1, bbx2, bby2 = rand_bbox(x.size(), lam)\n    x[:, :, bbx1:bbx2, bby1:bby2] = x[index, :, bbx1:bbx2, bby1:bby2]\n    \n    # Adjust lambda to exactly match pixel ratio\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.size()[-1] * x.size()[-2]))\n    \n    y_a, y_b = y, y[index]\n    return x, y_a, y_b, lam\n\ndef rand_bbox(size, lam):\n    \"\"\"Generate random bounding box for CutMix\"\"\"\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n    \n    # Uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n    \n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    \n    return bbx1, bby1, bbx2, bby2\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# Model Architecture\n# =============================================================================\n\nclass APTOSModel(nn.Module):\n    \"\"\"Advanced model with attention mechanisms\"\"\"\n    \n    def __init__(self, model_name: str = 'tf_efficientnet_b4_ns', num_classes: int = 5, pretrained: bool = True):\n        super().__init__()\n        \n        # Backbone\n        self.backbone = timm.create_model(model_name, pretrained=pretrained, num_classes=0)\n        \n        # Get feature dimension\n        self.feature_dim = self.backbone.num_features\n        \n        # Attention mechanism\n        self.attention = nn.Sequential(\n            nn.Linear(self.feature_dim, self.feature_dim // 4),\n            nn.ReLU(),\n            nn.Linear(self.feature_dim // 4, self.feature_dim),\n            nn.Sigmoid()\n        )\n        \n        # Classifier head\n        self.classifier = nn.Sequential(\n            nn.Dropout(0.5),\n            nn.Linear(self.feature_dim, self.feature_dim // 2),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(self.feature_dim // 2, num_classes)\n        )\n        \n        # Initialize weights\n        self._init_weights()\n    \n    def _init_weights(self):\n        \"\"\"Initialize classifier weights\"\"\"\n        for m in self.classifier.modules():\n            if isinstance(m, nn.Linear):\n                nn.init.xavier_uniform_(m.weight)\n                nn.init.constant_(m.bias, 0)\n    \n    def forward(self, x):\n        # Extract features\n        features = self.backbone(x)\n        \n        # Apply attention\n        attention_weights = self.attention(features)\n        attended_features = features * attention_weights\n        \n        # Classify\n        output = self.classifier(attended_features)\n        \n        return output\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# Training Functions\n# =============================================================================\n\ndef create_weighted_sampler(df: pd.DataFrame) -> WeightedRandomSampler:\n    \"\"\"Create weighted sampler for handling class imbalance\"\"\"\n    class_counts = df['diagnosis'].value_counts()\n    total_samples = len(df)\n    \n    # Calculate weights (inverse frequency)\n    class_weights = {}\n    for cls in range(Config.NUM_CLASSES):\n        if cls in class_counts:\n            class_weights[cls] = total_samples / (Config.NUM_CLASSES * class_counts[cls])\n        else:\n            class_weights[cls] = 1.0\n    \n    # Create sample weights\n    sample_weights = [class_weights[label] for label in df['diagnosis']]\n    \n    return WeightedRandomSampler(\n        weights=sample_weights,\n        num_samples=len(sample_weights),\n        replacement=True\n    )\n\nclass EarlyStopping:\n    \"\"\"Enhanced early stopping with model restoration\"\"\"\n    \n    def __init__(self, patience=15, min_delta=0.001, mode='max'):\n        self.patience = patience\n        self.min_delta = min_delta\n        self.mode = mode\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        \n    def __call__(self, val_score):\n        if self.best_score is None:\n            self.best_score = val_score\n        elif self.mode == 'max':\n            if val_score > self.best_score + self.min_delta:\n                self.best_score = val_score\n                self.counter = 0\n            else:\n                self.counter += 1\n        else:  # mode == 'min'\n            if val_score < self.best_score - self.min_delta:\n                self.best_score = val_score\n                self.counter = 0\n            else:\n                self.counter += 1\n                \n        if self.counter >= self.patience:\n            self.early_stop = True\n\ndef train_epoch(model, train_loader, criterion, optimizer, scheduler, scaler, device, epoch):\n    \"\"\"Enhanced training epoch with mixup/cutmix\"\"\"\n    model.train()\n    running_loss = 0.0\n    all_preds, all_targets = [], []\n    \n    progress_bar = range(len(train_loader))\n    \n    for batch_idx, (images, targets) in enumerate(train_loader):\n        images, targets = images.to(device, non_blocking=True), targets.to(device, non_blocking=True)\n        \n        # Apply mixup or cutmix randomly\n        if Config.USE_MIXUP and np.random.rand() < 0.3:\n            images, targets_a, targets_b, lam = mixup_data(images, targets, Config.MIXUP_ALPHA)\n            mixed = True\n        elif Config.USE_CUTMIX and np.random.rand() < 0.3:\n            images, targets_a, targets_b, lam = cutmix_data(images, targets, Config.CUTMIX_ALPHA)\n            mixed = True\n        else:\n            mixed = False\n        \n        optimizer.zero_grad()\n        \n        # Forward pass with mixed precision\n        with autocast(enabled=Config.USE_MIXED_PRECISION):\n            outputs = model(images)\n            \n            if mixed:\n                loss = lam * criterion(outputs, targets_a) + (1 - lam) * criterion(outputs, targets_b)\n            else:\n                loss = criterion(outputs, targets)\n        \n        # Backward pass\n        if Config.USE_MIXED_PRECISION:\n            scaler.scale(loss).backward()\n            scaler.unscale_(optimizer)\n            torch.nn.utils.clip_grad_norm_(model.parameters(), Config.GRADIENT_CLIP_NORM)\n            scaler.step(optimizer)\n            scaler.update()\n        else:\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), Config.GRADIENT_CLIP_NORM)\n            optimizer.step()\n        \n        running_loss += loss.item()\n        \n        if not mixed:\n            all_preds.extend(outputs.argmax(dim=1).cpu().numpy())\n            all_targets.extend(targets.cpu().numpy())\n        \n        # Print progress\n        if batch_idx % 50 == 0:\n            print(f'Epoch {epoch}, Batch {batch_idx}/{len(train_loader)}, Loss: {loss.item():.4f}')\n    \n    epoch_loss = running_loss / len(train_loader)\n    \n    if len(all_preds) > 0:\n        epoch_qwk = cohen_kappa_score(all_targets, all_preds, weights='quadratic')\n    else:\n        epoch_qwk = 0.0\n    \n    return epoch_loss, epoch_qwk\n\ndef validate_epoch(model, val_loader, criterion, device):\n    \"\"\"Validation epoch with detailed metrics\"\"\"\n    model.eval()\n    running_loss = 0.0\n    all_preds, all_targets = [], []\n    \n    with torch.no_grad():\n        for images, targets in val_loader:\n            images, targets = images.to(device, non_blocking=True), targets.to(device, non_blocking=True)\n            \n            with autocast(enabled=Config.USE_MIXED_PRECISION):\n                outputs = model(images)\n                loss = criterion(outputs, targets)\n            \n            running_loss += loss.item()\n            all_preds.extend(outputs.argmax(dim=1).cpu().numpy())\n            all_targets.extend(targets.cpu().numpy())\n    \n    epoch_loss = running_loss / len(val_loader)\n    epoch_qwk = cohen_kappa_score(all_targets, all_preds, weights='quadratic')\n    epoch_acc = accuracy_score(all_targets, all_preds)\n    \n    return epoch_loss, epoch_qwk, epoch_acc, all_preds, all_targets","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# Main Training Function\n# =============================================================================\n\ndef train_fold(fold: int, train_df: pd.DataFrame, val_df: pd.DataFrame):\n    \"\"\"Train a single fold\"\"\"\n    print(f\"\\n{'='*60}\")\n    print(f\"🚀 Training Fold {fold}\")\n    print(f\"{'='*60}\")\n    \n    # Create transforms\n    train_transform, val_transform = get_transforms(Config.IMAGE_SIZE)\n    \n    # Create datasets\n    train_dataset = APTOSDataset(train_df, Config.TRAIN_IMAGES_DIR, train_transform, is_training=True)\n    val_dataset = APTOSDataset(val_df, Config.TRAIN_IMAGES_DIR, val_transform, is_training=True)\n    \n    # Create weighted sampler\n    weighted_sampler = create_weighted_sampler(train_df)\n    \n    # Create data loaders\n    train_loader = DataLoader(\n        train_dataset, \n        batch_size=Config.BATCH_SIZE, \n        sampler=weighted_sampler,\n        num_workers=Config.NUM_WORKERS, \n        pin_memory=Config.PIN_MEMORY,\n        drop_last=True\n    )\n    \n    val_loader = DataLoader(\n        val_dataset, \n        batch_size=Config.BATCH_SIZE, \n        shuffle=False,\n        num_workers=Config.NUM_WORKERS, \n        pin_memory=Config.PIN_MEMORY\n    )\n    \n    # Initialize model\n    model = APTOSModel(Config.MODEL_NAME, Config.NUM_CLASSES, pretrained=True)\n    \n    # Multi-GPU support\n    if torch.cuda.device_count() > 1:\n        print(f\"🚀 Using {torch.cuda.device_count()} GPUs!\")\n        model = nn.DataParallel(model)\n    \n    model = model.to(Config.DEVICE)\n    \n    # Loss function\n    if Config.USE_FOCAL_LOSS:\n        criterion = FocalLoss(alpha=1.0, gamma=2.0)\n    elif Config.USE_LABEL_SMOOTHING:\n        criterion = LabelSmoothingCrossEntropy(smoothing=Config.LABEL_SMOOTHING_ALPHA)\n    else:\n        criterion = nn.CrossEntropyLoss()\n    \n    # Optimizer and scheduler\n    optimizer = AdamW(model.parameters(), lr=Config.LEARNING_RATE, weight_decay=Config.WEIGHT_DECAY)\n    scheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=5, verbose=True, min_lr=Config.MIN_LR)\n    \n    # Mixed precision scaler\n    scaler = GradScaler() if Config.USE_MIXED_PRECISION else None\n    \n    # Early stopping\n    early_stopping = EarlyStopping(patience=Config.EARLY_STOPPING_PATIENCE, min_delta=0.001, mode='max')\n    \n    # Training history\n    history = {\n        'train_loss': [], 'val_loss': [], 'train_qwk': [], 'val_qwk': [], 'val_acc': [], 'lr': []\n    }\n    \n    best_qwk = -1.0\n    best_model_path = f'{Config.CHECKPOINT_DIR}/best_model_fold_{fold}.pth'\n    \n    print(f\"📊 Training samples: {len(train_df)}\")\n    print(f\"📊 Validation samples: {len(val_df)}\")\n    print(f\"🏗️ Model parameters: {sum(p.numel() for p in model.parameters()):,}\")\n    \n    # Training loop\n    for epoch in range(Config.EPOCHS):\n        start_time = time.time()\n        \n        # Train\n        train_loss, train_qwk = train_epoch(model, train_loader, criterion, optimizer, scheduler, scaler, Config.DEVICE, epoch)\n        \n        # Validate\n        val_loss, val_qwk, val_acc, val_preds, val_targets = validate_epoch(model, val_loader, criterion, Config.DEVICE)\n        \n        # Step scheduler\n        scheduler.step(val_qwk)\n        current_lr = optimizer.param_groups[0]['lr']\n        \n        # Update history\n        history['train_loss'].append(train_loss)\n        history['val_loss'].append(val_loss)\n        history['train_qwk'].append(train_qwk)\n        history['val_qwk'].append(val_qwk)\n        history['val_acc'].append(val_acc)\n        history['lr'].append(current_lr)\n        \n        # Save best model\n        if val_qwk > best_qwk:\n            best_qwk = val_qwk\n            torch.save({\n                'epoch': epoch,\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'scheduler_state_dict': scheduler.state_dict(),\n                'best_qwk': best_qwk,\n                'fold': fold\n            }, best_model_path)\n        \n        epoch_time = time.time() - start_time\n        \n        # Print progress\n        print(f\"\\nEpoch {epoch+1}/{Config.EPOCHS} - {epoch_time:.2f}s\")\n        print(f\"Train Loss: {train_loss:.4f} | Train QWK: {train_qwk:.4f}\")\n        print(f\"Val Loss: {val_loss:.4f} | Val QWK: {val_qwk:.4f} | Val Acc: {val_acc:.4f}\")\n        print(f\"LR: {current_lr:.6f} | Best QWK: {best_qwk:.4f}\")\n        print(f\"Early Stopping: {early_stopping.counter}/{early_stopping.patience}\")\n        \n        # Early stopping\n        if epoch >= Config.MIN_EPOCHS:\n            early_stopping(val_qwk)\n            if early_stopping.early_stop:\n                print(f\"⏹️ Early stopping at epoch {epoch+1}\")\n                break\n        \n        # Memory cleanup\n        if epoch % 10 == 0:\n            gc.collect()\n            torch.cuda.empty_cache()\n    \n    # Final evaluation\n    print(f\"\\n🎯 Fold {fold} completed!\")\n    print(f\"🏆 Best QWK: {best_qwk:.4f}\")\n    \n    # Load best model for final evaluation\n    checkpoint = torch.load(best_model_path, map_location=Config.DEVICE)\n    model.load_state_dict(checkpoint['model_state_dict'])\n    \n    # Final validation\n    final_val_loss, final_val_qwk, final_val_acc, final_preds, final_targets = validate_epoch(model, val_loader, criterion, Config.DEVICE)\n    \n    # Print classification report\n    print(\"\\n📊 Final Classification Report:\")\n    print(classification_report(final_targets, final_preds, target_names=[f'Class {i}' for i in range(Config.NUM_CLASSES)]))\n    \n    # Plot confusion matrix\n    plt.figure(figsize=(8, 6))\n    cm = confusion_matrix(final_targets, final_preds)\n    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')\n    plt.title(f'Confusion Matrix - Fold {fold}')\n    plt.ylabel('True Label')\n    plt.xlabel('Predicted Label')\n    plt.savefig(f'{Config.OUTPUT_DIR}/confusion_matrix_fold_{fold}.png', dpi=300, bbox_inches='tight')\n    plt.close()\n    \n    return best_qwk, history\n\ndef plot_training_history(history_list: List[Dict], fold_scores: List[float]):\n    \"\"\"Plot training history for all folds\"\"\"\n    fig, axes = plt.subplots(2, 3, figsize=(18, 10))\n    \n    metrics = ['train_loss', 'val_loss', 'train_qwk', 'val_qwk', 'val_acc', 'lr']\n    titles = ['Training Loss', 'Validation Loss', 'Training QWK', 'Validation QWK', 'Validation Accuracy', 'Learning Rate']\n\n    for i, metric in enumerate(metrics):\n        row, col = divmod(i, 3)\n        ax = axes[row, col]\n\n        for j, history in enumerate(history_list):\n            ax.plot(history[metric], label=f'Fold {j+1}')\n\n        ax.set_title(titles[i])\n        ax.set_xlabel('Epoch')\n        ax.set_ylabel(titles[i])\n        ax.legend()\n    plt.tight_layout()\n    plt.savefig(f'{Config.OUTPUT_DIR}/training_history.png', dpi=300, bbox_inches='tight')\n    plt.close()\n    print(\"\\n📊 Training History Plot Saved!\")\n    ","metadata":{},"outputs":[],"execution_count":null}]}