{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# RSNA 2024 Lumbar Spine — Version 7\n## 2.5D CNN Approach (Fundamentally Different)\n\n### Philosophy: Simplicity Over Complexity\n\nPrevious versions (v4-v6) used: `CNN → RNN/GRU → Attention → Classify`  \n**v7 eliminates the sequence model entirely** and uses a **2.5D CNN** approach.\n\n### How 2.5D Works:\n- Take 7 adjacent slices centered on the diagnostic level\n- **Stack them as input channels** (7 channels instead of 3 RGB)\n- Feed through a single CNN (adapted for 7-channel input)\n- The CNN learns cross-slice spatial features naturally\n\n### Why This Should Work Better:\n1. **Fewer parameters** — no GRU, no attention pool = less overfitting\n2. **Joint spatial-temporal learning** — CNN kernels see across slices natively\n3. **Proven approach** — used by top RSNA competition solutions\n4. **Simple + strong baseline** — easier to debug and tune\n\n### Other Key Features:\n- Multi-head classification (main + ordinal)\n- WeightedRandomSampler for class balance\n- DICOM windowing preserved\n- Gradient clipping + SWA\n- No horizontal flip in TTA\n","metadata":{}},{"cell_type":"code","source":"import os\nimport copy\nimport cv2\nimport glob\nimport pydicom\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\nfrom tqdm import tqdm\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import confusion_matrix, classification_report, balanced_accuracy_score\nfrom collections import Counter\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torchvision.models as models\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\nfrom torch.amp import autocast, GradScaler\nfrom torch.optim.swa_utils import AveragedModel, SWALR\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CONFIG = {\n    'seed': 42,\n    'img_size': 224,               # Slightly smaller for 2.5D (saves memory)\n    'num_slices': 7,               # Adjacent slices to stack\n    'batch_size': 16,              # Larger batch (no RNN = less memory)\n    'epochs': 30,\n    \n    'learning_rate': 2e-4,\n    'backbone_lr': 2e-5,\n    'weight_decay': 0.03,\n    'patience': 12,\n    'num_folds': 5,\n    'train_folds': [0],\n    \n    # Loss\n    'focal_gamma': 2.0,\n    'ordinal_weight': 0.5,        # Weight for ordinal auxiliary loss\n    \n    # Training\n    'clip_grad_norm': 1.0,\n    'use_swa': True,\n    'swa_start_epoch': 20,\n    'swa_lr': 5e-6,\n    'warmup_epochs': 3,\n    'freeze_backbone_epochs': 2,\n    \n    # Architecture\n    'dropout': 0.3,               # Less dropout (simpler model needs less regularization)\n    \n    # Mixup\n    'use_mixup': True,\n    'mixup_alpha': 0.2,\n    \n    'device': torch.device('cuda' if torch.cuda.is_available() else 'cpu'),\n    'target_condition': 'spinal_canal_stenosis',\n    'target_series': 'Sagittal T2/STIR'\n}\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_everything(CONFIG['seed'])\nprint(f\"✅ Device: {CONFIG['device']}\")\nprint(f\"   Version: 7 (2.5D CNN — No RNN)\")\nprint(f\"   Image: {CONFIG['img_size']}px, {CONFIG['num_slices']} slices stacked as channels\")\nprint(f\"   Batch: {CONFIG['batch_size']} (larger — no RNN memory overhead)\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. Data Loading","metadata":{}},{"cell_type":"code","source":"DATA_ROOT = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/\"\nTRAIN_IMAGES = os.path.join(DATA_ROOT, \"train_images\")\n\ndf_train = pd.read_csv(f\"{DATA_ROOT}/train.csv\")\ndf_coords = pd.read_csv(f\"{DATA_ROOT}/train_label_coordinates.csv\")\ndf_desc = pd.read_csv(f\"{DATA_ROOT}/train_series_descriptions.csv\")\n\n# Clean & Merge\ndf_train.columns = [col.lower().replace('/', '_') for col in df_train.columns]\ncondition_cols = [c for c in df_train.columns if c != 'study_id']\ndf_labels = pd.melt(df_train, id_vars=['study_id'], value_vars=condition_cols,\n                    var_name='condition_level', value_name='severity')\ndf_labels = df_labels.dropna(subset=['severity'])\ndf_labels['severity'] = df_labels['severity'].astype(str).str.lower().str.replace('/', '_')\n\ndef extract_meta(val):\n    parts = val.split('_')\n    level = parts[-2] + '_' + parts[-1]\n    condition = '_'.join(parts[:-2])\n    return condition, level\n\ndf_labels[['base_condition', 'level_str']] = df_labels['condition_level'].apply(lambda x: pd.Series(extract_meta(x)))\nseverity_map = {'normal_mild': 0, 'moderate': 1, 'severe': 2}\ndf_labels['label'] = df_labels['severity'].map(severity_map)\ndf_labels = df_labels.dropna(subset=['label'])\ndf_labels['label'] = df_labels['label'].astype(int)\n\ndf_coords = df_coords.merge(df_desc, on=['study_id', 'series_id'], how='left')\ndf_coords['condition'] = df_coords['condition'].str.lower().str.replace(' ', '_')\ndf_coords['level'] = df_coords['level'].str.lower().str.replace('/', '_')\ndf_coords['condition_level'] = df_coords['condition'] + '_' + df_coords['level']\n\ndf_model = df_labels[df_labels['base_condition'] == CONFIG['target_condition']].copy()\ndf_coords_filt = df_coords[(df_coords['condition'] == CONFIG['target_condition']) & \n                           (df_coords['series_description'] == CONFIG['target_series'])]\n\ndf_final = df_model.merge(df_coords_filt[['study_id', 'condition_level', 'series_id', 'instance_number', 'x', 'y']],\n                          on=['study_id', 'condition_level'], how='inner')\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Filter valid files\nvalid_rows = []\nfor index, row in tqdm(df_final.iterrows(), total=len(df_final), desc=\"Checking Files\"):\n    path = f\"{TRAIN_IMAGES}/{row['study_id']}/{row['series_id']}/{int(row['instance_number'])}.dcm\"\n    if os.path.exists(path):\n        valid_rows.append(row)\n\ndf_final = pd.DataFrame(valid_rows).reset_index(drop=True)\nlevel_map = {'l1_l2': 0, 'l2_l3': 1, 'l3_l4': 2, 'l4_l5': 3, 'l5_s1': 4}\ndf_final['level_idx'] = df_final['level_str'].map(level_map)\n\nprint(f\"\\n✅ Data Ready: {len(df_final)} samples\")\nclass_counts = df_final['label'].value_counts().sort_index()\nfor i, count in enumerate(class_counts):\n    pct = count / len(df_final) * 100\n    print(f\"   Class {i}: {count} samples ({pct:.1f}%)\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Weighted Sampler","metadata":{}},{"cell_type":"code","source":"def create_weighted_sampler(df):\n    class_counts = np.bincount(df['label'].values, minlength=3).astype(float)\n    class_weights = 1.0 / class_counts\n    sample_weights = class_weights[df['label'].values]\n    sampler = WeightedRandomSampler(\n        weights=sample_weights, num_samples=len(df), replacement=True\n    )\n    print(f\"📊 WeightedRandomSampler: counts={class_counts.astype(int).tolist()}\")\n    return sampler\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. 2.5D Dataset\n\n**Key difference from v4-v6**: Instead of returning `(7, 3, H, W)` sequence tensors,\nthis dataset returns `(7, H, W)` — 7 grayscale slices stacked as channels.\n\nThe CNN's first conv layer is adapted to accept 7 input channels.\n","metadata":{}},{"cell_type":"code","source":"class RSNA25DDataset(Dataset):\n    \"\"\"\n    2.5D Dataset: stacks adjacent slices as channels.\n    Output shape: (num_slices, H, W) instead of (seq, 3, H, W)\n    \"\"\"\n    def __init__(self, df, num_slices=7, img_size=224, transform=None, is_training=False):\n        self.df = df.reset_index(drop=True)\n        self.num_slices = num_slices\n        self.img_size = img_size\n        self.transform = transform\n        self.is_training = is_training\n        self.clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def load_dicom(self, path):\n        try:\n            dcm = pydicom.dcmread(path)\n            img = dcm.pixel_array.astype(np.float32)\n            if hasattr(dcm, 'WindowCenter') and hasattr(dcm, 'WindowWidth'):\n                wc = dcm.WindowCenter\n                ww = dcm.WindowWidth\n                if isinstance(wc, pydicom.multival.MultiValue): wc = float(wc[0])\n                else: wc = float(wc)\n                if isinstance(ww, pydicom.multival.MultiValue): ww = float(ww[0])\n                else: ww = float(ww)\n                img = np.clip((img - (wc - ww/2)) / max(ww, 1) * 255, 0, 255)\n            else:\n                if img.max() > img.min():\n                    img = (img - img.min()) / (img.max() - img.min()) * 255.0\n            img = img.astype(np.uint8)\n            img = self.clahe.apply(img)\n            return img\n        except:\n            return np.zeros((self.img_size, self.img_size), dtype=np.uint8)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        center_inst = int(row['instance_number'])\n        study_path = f\"{TRAIN_IMAGES}/{row['study_id']}/{row['series_id']}\"\n        cx, cy = int(row['x']), int(row['y'])\n        \n        if self.is_training:\n            jitter = self.img_size // 16\n            cx += random.randint(-jitter, jitter)\n            cy += random.randint(-jitter, jitter)\n        \n        half = self.num_slices // 2\n        indices = [center_inst + i - half for i in range(self.num_slices)]\n        \n        # Load all slices as single-channel images\n        slices = []\n        for inst in indices:\n            path = os.path.join(study_path, f\"{inst}.dcm\")\n            if os.path.exists(path):\n                img = self.load_dicom(path)\n            else:\n                img = np.zeros((self.img_size, self.img_size), dtype=np.uint8)\n            \n            h, w = img.shape\n            crop_size = self.img_size // 2\n            x1 = max(0, cx - crop_size)\n            y1 = max(0, cy - crop_size)\n            x2 = min(w, cx + crop_size)\n            y2 = min(h, cy + crop_size)\n            crop = img[y1:y2, x1:x2]\n            \n            if crop.size == 0:\n                crop = np.zeros((self.img_size, self.img_size), dtype=np.uint8)\n            else:\n                crop = cv2.resize(crop, (self.img_size, self.img_size))\n            \n            slices.append(crop)\n        \n        # Stack as multi-channel image: (H, W, num_slices)\n        multi_channel = np.stack(slices, axis=-1)  # (H, W, 7)\n        \n        # Apply augmentation (treats each channel consistently)\n        if self.transform:\n            res = self.transform(image=multi_channel)\n            tensor = res['image']  # (7, H, W) after ToTensorV2\n        else:\n            tensor = torch.tensor(multi_channel).permute(2, 0, 1).float() / 255.0\n        \n        label = torch.tensor(row['label'], dtype=torch.long)\n        level_idx = torch.tensor(row['level_idx'], dtype=torch.long)\n        \n        return tensor, label, level_idx\n\nprint(\"✅ RSNA25DDataset ready\")\nprint(f\"   Output shape: ({CONFIG['num_slices']}, {CONFIG['img_size']}, {CONFIG['img_size']})\")\nprint(f\"   No sequence dimension — slices are channels\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Augmentation (7-channel compatible)","metadata":{}},{"cell_type":"code","source":"# Albumentations handles arbitrary channel counts natively\ntrain_aug = A.Compose([\n    A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.15, rotate_limit=15,\n                       border_mode=cv2.BORDER_CONSTANT, value=0, p=0.7),\n    A.OneOf([\n        A.RandomBrightnessContrast(brightness_limit=0.3, contrast_limit=0.3, p=1.0),\n        A.RandomGamma(gamma_limit=(70, 130), p=1.0),\n    ], p=0.7),\n    A.OneOf([\n        A.GaussNoise(var_limit=(5.0, 40.0), p=1.0),\n        A.MultiplicativeNoise(multiplier=(0.85, 1.15), p=1.0),\n    ], p=0.3),\n    A.OneOf([\n        A.ElasticTransform(alpha=1, sigma=50, alpha_affine=25, p=1.0),\n        A.GridDistortion(num_steps=5, distort_limit=0.1, p=1.0),\n    ], p=0.25),\n    A.CoarseDropout(max_holes=5, max_height=28, max_width=28,\n                    min_holes=1, min_height=12, min_width=12,\n                    fill_value=0, p=0.3),\n    # No Normalize — we handle this in the model's stem\n    ToTensorV2()\n])\n\nval_aug = A.Compose([\n    ToTensorV2()\n])\n\n# TTA (no horizontal flip)\ntta_augs = [\n    val_aug,\n    A.Compose([\n        A.RandomBrightnessContrast(brightness_limit=0.1, contrast_limit=0.1, p=1.0),\n        ToTensorV2()\n    ]),\n    A.Compose([\n        A.ShiftScaleRotate(shift_limit=0, scale_limit=0.05, rotate_limit=0, p=1.0),\n        ToTensorV2()\n    ]),\n]\n\nprint(f\"✅ Augmentation (7-channel compatible): {len(tta_augs)} TTA variants\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. 2.5D CNN Model\n\n**Architecture:**\n```\nInput: (B, 7, 224, 224)  — 7 slices as channels\n  → Stem conv (7 → 3 channels, initialized from pretrained)\n  → EfficientNet-V2-S backbone (pretrained)\n  → Global Average Pool → 1280 features\n  → Level conditioning (FiLM)\n  → Dual head: main classifier + ordinal head\n```\n\nThe stem conv **adapts 7-channel input to 3 channels** while preserving pretrained weights.\nThe center 3 channels use the original ImageNet weights; edge channels start near zero.\n","metadata":{}},{"cell_type":"code","source":"class Spine25DModel(nn.Module):\n    \"\"\"\n    2.5D CNN: no RNN, no attention — pure CNN.\n    Adjacent slices stacked as channels → CNN → classify.\n    \"\"\"\n    def __init__(self, num_classes=3, num_slices=7, dropout=0.3, num_levels=5):\n        super().__init__()\n        self.num_classes = num_classes\n        \n        # Load pretrained backbone\n        effnet = models.efficientnet_v2_s(weights='IMAGENET1K_V1')\n        \n        # Get original first conv weights (3-channel)\n        orig_conv = effnet.features[0][0]  # First Conv2dNormActivation\n        orig_weight = orig_conv.weight.data  # (out_channels, 3, kH, kW)\n        \n        # Create new stem: 7 channels → same output as original first conv\n        out_channels = orig_weight.shape[0]\n        kH, kW = orig_weight.shape[2], orig_weight.shape[3]\n        \n        self.stem = nn.Conv2d(num_slices, out_channels, kernel_size=(kH, kW),\n                             stride=orig_conv.stride, padding=orig_conv.padding,\n                             bias=False)\n        \n        # Initialize stem weights from pretrained\n        with torch.no_grad():\n            new_weight = torch.zeros(out_channels, num_slices, kH, kW)\n            # Center 3 channels get pretrained weights\n            center = num_slices // 2\n            new_weight[:, center-1:center+2, :, :] = orig_weight\n            # Edge channels get small random init (near zero)\n            for i in range(num_slices):\n                if i < center-1 or i > center+1:\n                    new_weight[:, i, :, :] = orig_weight.mean(dim=1) * 0.1\n            self.stem.weight.data = new_weight\n        \n        # Backbone (skip original first conv, use rest)\n        self.backbone_norm = effnet.features[0][1]  # BatchNorm after first conv\n        self.backbone_act = effnet.features[0][2]   # Activation after first conv\n        self.backbone_rest = nn.Sequential(*list(effnet.features.children())[1:])\n        self.avgpool = effnet.avgpool\n        self.feature_dim = 1280\n        \n        # Level conditioning\n        self.level_gamma = nn.Embedding(num_levels, self.feature_dim)\n        self.level_beta = nn.Embedding(num_levels, self.feature_dim)\n        nn.init.ones_(self.level_gamma.weight)\n        nn.init.zeros_(self.level_beta.weight)\n        \n        # Main classifier (3-class)\n        self.classifier = nn.Sequential(\n            nn.LayerNorm(self.feature_dim),\n            nn.Dropout(dropout),\n            nn.Linear(self.feature_dim, 256),\n            nn.GELU(),\n            nn.Dropout(dropout * 0.5),\n            nn.Linear(256, num_classes)\n        )\n        \n        # Ordinal head (K-1 logits for ordinal consistency)\n        self.ordinal_head = nn.Sequential(\n            nn.Linear(self.feature_dim, 128),\n            nn.GELU(),\n            nn.Dropout(dropout * 0.5),\n            nn.Linear(128, num_classes - 1)\n        )\n        \n    def forward(self, x, level_idx=None):\n        # x: (B, 7, H, W)\n        # Normalize to [0, 1] range\n        x = x.float() / 255.0\n        \n        # Stem: 7 channels → backbone channel count\n        x = self.stem(x)\n        x = self.backbone_norm(x)\n        x = self.backbone_act(x)\n        \n        # EfficientNet backbone (rest of the layers)\n        x = self.backbone_rest(x)\n        \n        # Pool\n        x = self.avgpool(x)\n        features = x.view(x.size(0), -1)\n        \n        # Level conditioning (FiLM)\n        if level_idx is not None:\n            gamma = self.level_gamma(level_idx)\n            beta = self.level_beta(level_idx)\n            features = gamma * features + beta\n        \n        # Dual output\n        ce_logits = self.classifier(features)\n        ordinal_logits = self.ordinal_head(features)\n        \n        return {\n            'ce': ce_logits,\n            'ordinal': ordinal_logits\n        }\n    \n    def predict_proba(self, outputs):\n        \"\"\"Combine CE and ordinal predictions for robust inference.\"\"\"\n        # CE probabilities\n        ce_probs = F.softmax(outputs['ce'], dim=1)\n        \n        # Ordinal probabilities\n        cum = torch.sigmoid(outputs['ordinal'])  # P(Y>0), P(Y>1)\n        ord_probs = torch.zeros_like(ce_probs)\n        ord_probs[:, 0] = 1 - cum[:, 0]\n        ord_probs[:, 1] = cum[:, 0] - cum[:, 1]\n        ord_probs[:, 2] = cum[:, 1]\n        ord_probs = ord_probs.clamp(min=0)\n        ord_probs = ord_probs / ord_probs.sum(dim=1, keepdim=True).clamp(min=1e-8)\n        \n        # Average CE and ordinal probabilities\n        return 0.6 * ce_probs + 0.4 * ord_probs\n\nprint(\"✅ Spine25DModel ready\")\nprint(\"   - Stem: 7ch → pretrained weights (center channels = ImageNet)\")\nprint(\"   - Backbone: EfficientNet-V2-S\")\nprint(\"   - Output: CE logits + ordinal logits (averaged at inference)\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Combined Loss (CE + Ordinal)","metadata":{}},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, gamma=2.0, reduction='mean'):\n        super().__init__()\n        self.gamma = gamma\n        self.reduction = reduction\n    def forward(self, inputs, targets):\n        ce = F.cross_entropy(inputs, targets, reduction='none')\n        pt = torch.exp(-ce)\n        focal = ((1 - pt) ** self.gamma) * ce\n        return focal.mean() if self.reduction == 'mean' else focal\n\nclass OrdinalLoss(nn.Module):\n    def __init__(self, num_classes=3):\n        super().__init__()\n        self.num_classes = num_classes\n    def forward(self, logits, labels):\n        levels = torch.arange(self.num_classes - 1, device=labels.device)\n        targets = (labels.unsqueeze(1) > levels.unsqueeze(0)).float()\n        return F.binary_cross_entropy_with_logits(logits, targets, reduction='mean')\n\nclass CombinedLoss(nn.Module):\n    def __init__(self, gamma=2.0, ordinal_weight=0.5):\n        super().__init__()\n        self.focal = FocalLoss(gamma)\n        self.ordinal = OrdinalLoss()\n        self.ordinal_weight = ordinal_weight\n    def forward(self, outputs, labels):\n        ce_loss = self.focal(outputs['ce'], labels)\n        ord_loss = self.ordinal(outputs['ordinal'], labels)\n        total = ce_loss + self.ordinal_weight * ord_loss\n        return total, {'total': total.item(), 'ce': ce_loss.item(), 'ordinal': ord_loss.item()}\n\ndef mixup_data(x, y, alpha=0.2):\n    if alpha > 0: lam = np.random.beta(alpha, alpha)\n    else: lam = 1.0\n    idx = torch.randperm(x.size(0), device=x.device)\n    return lam * x + (1 - lam) * x[idx], y, y[idx], lam\n\nprint(\"✅ FocalLoss + OrdinalLoss combined\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_per_class_metrics(preds, labels, num_classes=3):\n    metrics = {}\n    for c in range(num_classes):\n        mask = (labels == c)\n        if mask.sum() > 0:\n            metrics[f'class_{c}_recall'] = ((preds == c) & mask).sum() / mask.sum()\n        else:\n            metrics[f'class_{c}_recall'] = 0.0\n    return metrics\n\nclass EarlyStopping:\n    def __init__(self, patience=5, min_delta=0.001, mode='max'):\n        self.patience, self.min_delta, self.mode = patience, min_delta, mode\n        self.counter, self.best_score = 0, None\n    def __call__(self, score):\n        if self.best_score is None:\n            self.best_score = score; return False\n        improved = (score > self.best_score + self.min_delta) if self.mode == 'max' \\\n                   else (score < self.best_score - self.min_delta)\n        if improved:\n            self.best_score, self.counter = score, 0; return False\n        self.counter += 1\n        return self.counter >= self.patience\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Training Function","metadata":{}},{"cell_type":"code","source":"def train_one_fold_v7(model, train_loader, val_loader, fold, config):\n    criterion = CombinedLoss(gamma=config['focal_gamma'], ordinal_weight=config['ordinal_weight'])\n    \n    # Separate stem learning rate (new weights need faster learning)\n    optimizer = optim.AdamW([\n        {'params': model.stem.parameters(), 'lr': config['learning_rate']},      # Stem adapts fast\n        {'params': model.backbone_norm.parameters(), 'lr': config['backbone_lr']},\n        {'params': model.backbone_act.parameters(), 'lr': config['backbone_lr']},\n        {'params': model.backbone_rest.parameters(), 'lr': config['backbone_lr']},\n        {'params': model.level_gamma.parameters(), 'lr': config['learning_rate']},\n        {'params': model.level_beta.parameters(), 'lr': config['learning_rate']},\n        {'params': model.classifier.parameters(), 'lr': config['learning_rate']},\n        {'params': model.ordinal_head.parameters(), 'lr': config['learning_rate']},\n    ], weight_decay=config['weight_decay'])\n    \n    warmup_steps = config['warmup_epochs'] * len(train_loader)\n    total_steps = config['epochs'] * len(train_loader)\n    def lr_lambda(step):\n        if step < warmup_steps: return step / max(warmup_steps, 1)\n        progress = (step - warmup_steps) / max(total_steps - warmup_steps, 1)\n        return max(0.5 * (1 + np.cos(np.pi * progress)), 1e-6 / config['learning_rate'])\n    \n    scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)\n    scaler = GradScaler('cuda')\n    \n    swa_model = AveragedModel(model) if config['use_swa'] else None\n    swa_scheduler = SWALR(optimizer, swa_lr=config['swa_lr']) if config['use_swa'] else None\n    \n    early_stopping = EarlyStopping(patience=config['patience'], min_delta=0.003, mode='max')\n    best_balanced_acc = 0.0\n    \n    history = {\n        'train_loss': [], 'train_acc': [],\n        'val_loss': [], 'val_acc': [], 'balanced_acc': [],\n        'class_0_recall': [], 'class_1_recall': [], 'class_2_recall': []\n    }\n    \n    print(f\"\\n🚀 Training Fold {fold+1} (v7 — 2.5D CNN)\")\n    print(f\"   Train: {len(train_loader.dataset)}, Val: {len(val_loader.dataset)}\")\n    print(f\"   Batch: {config['batch_size']}, FocalGamma: {config['focal_gamma']}\")\n    \n    # Freeze backbone initially\n    for p in model.backbone_rest.parameters(): p.requires_grad = False\n    for p in model.backbone_norm.parameters(): p.requires_grad = False\n    backbone_frozen = True\n    \n    for epoch in range(config['epochs']):\n        if backbone_frozen and epoch >= config['freeze_backbone_epochs']:\n            for p in model.backbone_rest.parameters(): p.requires_grad = True\n            for p in model.backbone_norm.parameters(): p.requires_grad = True\n            backbone_frozen = False\n            print(f\"   🔓 Backbone unfrozen at epoch {epoch+1}\")\n        \n        model.train()\n        train_loss, train_correct, train_total = 0, 0, 0\n        is_swa = config['use_swa'] and epoch >= config['swa_start_epoch']\n        \n        tag = \"[FROZEN]\" if backbone_frozen else (\"[SWA]\" if is_swa else \"\")\n        loop = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{config['epochs']} {tag}\")\n        \n        for images, labels, level_idx in loop:\n            images = images.to(config['device'])\n            labels = labels.to(config['device'])\n            level_idx = level_idx.to(config['device'])\n            \n            use_mix = config['use_mixup'] and not is_swa and random.random() < 0.5\n            if use_mix:\n                images, labels_a, labels_b, lam = mixup_data(images, labels, config['mixup_alpha'])\n            \n            optimizer.zero_grad()\n            with autocast('cuda'):\n                outputs = model(images, level_idx)\n                if use_mix:\n                    loss_a, _ = criterion(outputs, labels_a)\n                    loss_b, _ = criterion(outputs, labels_b)\n                    loss = lam * loss_a + (1 - lam) * loss_b\n                else:\n                    loss, _ = criterion(outputs, labels)\n            \n            scaler.scale(loss).backward()\n            scaler.unscale_(optimizer)\n            torch.nn.utils.clip_grad_norm_(model.parameters(), config['clip_grad_norm'])\n            scaler.step(optimizer)\n            scaler.update()\n            \n            if is_swa: swa_scheduler.step()\n            else: scheduler.step()\n            \n            train_loss += loss.item()\n            with torch.no_grad():\n                preds = model.predict_proba(outputs).argmax(dim=1)\n            if use_mix:\n                train_correct += (lam * (preds == labels_a).float() + \n                                  (1-lam) * (preds == labels_b).float()).sum().item()\n            else:\n                train_correct += (preds == labels).sum().item()\n            train_total += labels.size(0)\n            \n            loop.set_postfix(loss=f\"{train_loss/(loop.n+1):.4f}\", \n                           acc=f\"{100*train_correct/train_total:.1f}%\")\n        \n        train_acc = train_correct / train_total\n        if swa_model and is_swa: swa_model.update_parameters(model)\n        \n        # Validation\n        model.eval()\n        val_loss, val_correct, val_total = 0, 0, 0\n        all_preds, all_labels = [], []\n        \n        with torch.no_grad():\n            for images, labels, level_idx in val_loader:\n                images = images.to(config['device'])\n                labels = labels.to(config['device'])\n                level_idx = level_idx.to(config['device'])\n                with autocast('cuda'):\n                    outputs = model(images, level_idx)\n                    loss, _ = criterion(outputs, labels)\n                val_loss += loss.item()\n                preds = model.predict_proba(outputs).argmax(dim=1)\n                val_correct += (preds == labels).sum().item()\n                val_total += labels.size(0)\n                all_preds.extend(preds.cpu().numpy())\n                all_labels.extend(labels.cpu().numpy())\n        \n        val_acc = val_correct / val_total\n        all_preds, all_labels = np.array(all_preds), np.array(all_labels)\n        pc = compute_per_class_metrics(all_preds, all_labels)\n        ba = (pc['class_0_recall'] + pc['class_1_recall'] + pc['class_2_recall']) / 3\n        \n        history['train_loss'].append(train_loss / len(train_loader))\n        history['train_acc'].append(train_acc)\n        history['val_loss'].append(val_loss / len(val_loader))\n        history['val_acc'].append(val_acc)\n        history['balanced_acc'].append(ba)\n        for c in range(3): history[f'class_{c}_recall'].append(pc[f'class_{c}_recall'])\n        \n        print(f\"📊 Train Acc: {100*train_acc:.1f}% | Val Acc: {100*val_acc:.1f}% | \"\n              f\"N={100*pc['class_0_recall']:.1f}% M={100*pc['class_1_recall']:.1f}% \"\n              f\"S={100*pc['class_2_recall']:.1f}% | BA={100*ba:.1f}%\")\n        \n        min_min = min(pc['class_1_recall'], pc['class_2_recall'])\n        if ba > best_balanced_acc and min_min >= 0.15:\n            best_balanced_acc = ba\n            torch.save(model.state_dict(), f\"best_v7_fold{fold}.pth\")\n            print(f\"   ✅ Saved! BA={100*ba:.1f}%\")\n        \n        if early_stopping(ba):\n            print(f\"   ⏹️ Early stop at epoch {epoch+1}\")\n            break\n    \n    model.load_state_dict(torch.load(f\"best_v7_fold{fold}.pth\"))\n    return model, history, best_balanced_acc\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 8. Training","metadata":{}},{"cell_type":"code","source":"kfold = StratifiedGroupKFold(n_splits=CONFIG['num_folds'], shuffle=True, random_state=CONFIG['seed'])\nfold_results = []\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for fold, (train_idx, val_idx) in enumerate(kfold.split(df_final, df_final['label'], df_final['study_id'])):\n    if fold not in CONFIG['train_folds']:\n        continue\n    \n    print(f\"\\n{'='*60}\")\n    print(f\"FOLD {fold+1} (v7 — 2.5D CNN)\")\n    print(f\"{'='*60}\")\n    \n    train_df = df_final.iloc[train_idx].reset_index(drop=True)\n    val_df = df_final.iloc[val_idx].reset_index(drop=True)\n    \n    for i in range(3):\n        c = (train_df['label']==i).sum()\n        print(f\"   Class {i}: {c} ({100*c/len(train_df):.1f}%)\")\n    \n    sampler = create_weighted_sampler(train_df)\n    \n    train_ds = RSNA25DDataset(train_df, num_slices=CONFIG['num_slices'], \n                              img_size=CONFIG['img_size'], transform=train_aug, is_training=True)\n    val_ds = RSNA25DDataset(val_df, num_slices=CONFIG['num_slices'],\n                            img_size=CONFIG['img_size'], transform=val_aug, is_training=False)\n    \n    train_loader = DataLoader(train_ds, batch_size=CONFIG['batch_size'], sampler=sampler,\n                             num_workers=2, pin_memory=True, drop_last=True)\n    val_loader = DataLoader(val_ds, batch_size=CONFIG['batch_size'], shuffle=False,\n                           num_workers=2, pin_memory=True)\n    \n    model = Spine25DModel(\n        num_classes=3, num_slices=CONFIG['num_slices'], dropout=CONFIG['dropout']\n    ).to(CONFIG['device'])\n    \n    params = sum(p.numel() for p in model.parameters())\n    print(f\"   🏗️ Spine25DModel: {params:,} params (no RNN!)\")\n    \n    model, history, best_ba = train_one_fold_v7(model, train_loader, val_loader, fold, CONFIG)\n    fold_results.append({'fold': fold, 'best_balanced_acc': best_ba, 'history': history})\n    print(f\"\\n✅ Fold {fold+1}: Best BA = {100*best_ba:.1f}%\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*60)\nprint(\"TRAINING SUMMARY\")\nprint(\"=\"*60)\nfor r in fold_results:\n    print(f\"Fold {r['fold']+1}: Best BA = {100*r['best_balanced_acc']:.1f}%\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 9. TTA Evaluation","metadata":{}},{"cell_type":"code","source":"def predict_tta_v7(model, df, config, augs):\n    model.eval()\n    all_probs, all_labels = [], None\n    for ai, aug in enumerate(augs):\n        ds = RSNA25DDataset(df, num_slices=config['num_slices'],\n                            img_size=config['img_size'], transform=aug, is_training=False)\n        loader = DataLoader(ds, batch_size=config['batch_size'], shuffle=False, \n                          num_workers=2, pin_memory=True)\n        probs_list = []\n        if ai == 0: lbl = []\n        with torch.no_grad():\n            for imgs, labels, lidx in loader:\n                imgs = imgs.to(config['device'])\n                lidx = lidx.to(config['device'])\n                with autocast('cuda'):\n                    out = model(imgs, lidx)\n                    p = model.predict_proba(out)\n                probs_list.append(p.cpu().numpy())\n                if ai == 0: lbl.extend(labels.numpy())\n        all_probs.append(np.concatenate(probs_list, 0))\n        if ai == 0: all_labels = np.array(lbl)\n    avg = np.mean(all_probs, 0)\n    return np.argmax(avg, 1), all_labels, avg\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.eval()\ntta_preds, tta_labels, _ = predict_tta_v7(model, val_df, CONFIG, tta_augs)\nno_tta_preds, _, _ = predict_tta_v7(model, val_df, CONFIG, [val_aug])\n\npc1 = compute_per_class_metrics(no_tta_preds, tta_labels)\nba1 = np.mean([pc1[f'class_{c}_recall'] for c in range(3)])\npc2 = compute_per_class_metrics(tta_preds, tta_labels)\nba2 = np.mean([pc2[f'class_{c}_recall'] for c in range(3)])\n\nprint(f\"\\n{'='*60}\")\nprint(f\"Without TTA: BA={100*ba1:.1f}%  N={100*pc1['class_0_recall']:.1f}%  \"\n      f\"M={100*pc1['class_1_recall']:.1f}%  S={100*pc1['class_2_recall']:.1f}%\")\nprint(f\"With TTA:    BA={100*ba2:.1f}%  N={100*pc2['class_0_recall']:.1f}%  \"\n      f\"M={100*pc2['class_1_recall']:.1f}%  S={100*pc2['class_2_recall']:.1f}%\")\nprint(f\"TTA delta:   {100*(ba2-ba1):+.1f}%\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*50)\nprint(\"CLASSIFICATION REPORT\")\nprint(\"=\"*50)\nprint(classification_report(tta_labels, tta_preds,\n                           target_names=['Normal/Mild', 'Moderate', 'Severe']))\n\ncm = confusion_matrix(tta_labels, tta_preds)\ncm_norm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\nplt.figure(figsize=(8, 6))\nsns.heatmap(cm_norm, annot=True, fmt='.2%', cmap='Blues',\n            xticklabels=['Normal/Mild', 'Moderate', 'Severe'],\n            yticklabels=['Normal/Mild', 'Moderate', 'Severe'])\nplt.ylabel('True'); plt.xlabel('Predicted')\nplt.title(f'v7 2.5D CNN (BA: {100*ba2:.1f}%)')\nplt.tight_layout(); plt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if fold_results:\n    h = fold_results[0]['history']\n    fig, axes = plt.subplots(1, 3, figsize=(18, 5))\n    ep = range(1, len(h['train_loss'])+1)\n    axes[0].plot(ep, h['train_loss'], 'b-', label='Train')\n    axes[0].plot(ep, h['val_loss'], 'r-', label='Val')\n    axes[0].set_title('Loss'); axes[0].legend(); axes[0].grid(True, alpha=0.3)\n    axes[1].plot(ep, h['class_0_recall'], 'g-o', label='Normal', ms=3)\n    axes[1].plot(ep, h['class_1_recall'], color='orange', marker='s', label='Moderate', ms=3)\n    axes[1].plot(ep, h['class_2_recall'], 'r-^', label='Severe', ms=3)\n    axes[1].set_title('Per-Class Recall'); axes[1].legend(); axes[1].grid(True, alpha=0.3)\n    axes[2].plot(ep, h['balanced_acc'], 'purple', marker='d', lw=2, ms=3)\n    axes[2].set_title(f'BA (Best: {100*max(h[\"balanced_acc\"]):.1f}%)')\n    axes[2].grid(True, alpha=0.3)\n    plt.tight_layout(); plt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*60)\nprint(\"v7 COMPLETE — 2.5D CNN (No RNN)\")\nprint(\"=\"*60)\nprint(f\"  Architecture: 7-slice → stem conv → EfficientNet-V2-S → classify\")\nprint(f\"  No GRU, no LSTM, no attention pooling\")\nprint(f\"  Simpler model = less overfitting with limited minority data\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}