{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Cell 1: Imports, Data Cleaning & YOLO‐label prep\nimport os\nimport glob\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom tqdm import tqdm\n\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom PIL import Image, UnidentifiedImageError\n\n# ─── 1) Load raw CSVs ──────────────────────────────────────────────────────────\nROOT       = Path('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/')\ntrain      = pd.read_csv(ROOT/'train.csv')\nlabel_df   = pd.read_csv(ROOT/'train_label_coordinates.csv')\nseries_df  = pd.read_csv(ROOT/'train_series_descriptions.csv')\n\n# ─── 2) Melt train.csv → long format ───────────────────────────────────────────\ntrain_long = (\n    train\n    .melt(id_vars=['study_id'], var_name='cond_lvl', value_name='score')\n    .assign(\n        lvl1     = lambda df: df['cond_lvl'].str.extract(r'_(l\\d+)_')[0].str.upper(),\n        lvl2     = lambda df: df['cond_lvl'].str.extract(r'_(s\\d+)$')[0].str.upper(),\n        cond_key = lambda df: df['cond_lvl'].str.rsplit('_', n=2).str[0]\n    )\n    .assign(\n        level     = lambda df: df['lvl1'] + '/' + df['lvl2'],\n        condition = lambda df: df['cond_key'].map({\n            'spinal_canal_stenosis':            'Spinal Canal Stenosis',\n            'left_neural_foraminal_narrowing':  'Neural Foraminal Narrowing',\n            'right_neural_foraminal_narrowing': 'Neural Foraminal Narrowing',\n            'left_subarticular_stenosis':       'Subarticular Stenosis',\n            'right_subarticular_stenosis':      'Subarticular Stenosis',\n        })\n    )[['study_id','condition','level','score']]\n)\n\n# ─── 3) Merge everything ───────────────────────────────────────────────────────\nmerged = (\n    label_df\n    .merge(train_long, on=['study_id','condition','level'], how='left')\n    .merge(series_df[['study_id','series_id','series_description']],\n           on=['study_id','series_id'], how='left')\n)\nprint(\"After merge, sample:\\n\", merged.head())\n\n# ─── 4) Build the set of ROI‐containing stems ─────────────────────────────────\nmerged['stem'] = merged.apply(\n    lambda r: f\"{int(r.study_id)}_{int(r.series_id)}_{int(r.instance_number)}\",\n    axis=1\n)\nneeded_stems = set(merged['stem'])\nprint(f\"🔍 Will convert only {len(needed_stems)} images that have an ROI\")\n\n# ─── 5) Map stems → DICOM paths & convert only those ─────────────────────────\nIMG_DIR = Path('data/images/train/');  IMG_DIR.mkdir(parents=True, exist_ok=True)\n\nstem2path = {}\nfor dcm_path in ROOT.rglob('*.dcm'):\n    p = Path(dcm_path)\n    rel = p.relative_to(ROOT).with_suffix('').parts\n    stem = \"_\".join(rel[-3:])\n    if stem in needed_stems:\n        stem2path[stem] = dcm_path\n\nprint(f\"🔍 Found {len(stem2path)} DICOMs matching ROI stems\")\n\nconverted = 0\nfor stem, dcm_path in tqdm(stem2path.items(),\n                           total=len(stem2path),\n                           desc=\"Converting ROI→PNG\"):\n    png_fp = IMG_DIR/f\"{stem}.png\"\n    if png_fp.exists():\n        continue\n    ds = pydicom.dcmread(str(dcm_path))\n    arr = apply_voi_lut(ds.pixel_array, ds).astype(np.float32)\n    arr = ((arr - arr.min())/(arr.max()-arr.min())*255).astype(np.uint8)\n    Image.fromarray(arr).save(png_fp)\n    converted += 1\n\nprint(f\"✅ Converted {converted} ROI DICOMs to PNG in {IMG_DIR}\")\n\n# ─── 6) Gather image shapes from the PNG folder ───────────────────────────────\npng_paths = list(IMG_DIR.glob(\"*.png\"))\nprint(f\"📂 Found {len(png_paths)} PNG files in {IMG_DIR}\")\n\nimage_shapes = {}\nskipped = []\nfor p in tqdm(png_paths, desc=\"Loading image shapes\"):\n    try:\n        H, W = Image.open(p).size[::-1]\n        image_shapes[p.stem] = (H, W)\n    except (UnidentifiedImageError, OSError):\n        skipped.append(p.name)\n\nprint(f\"⚠️ Skipped {len(skipped)} unreadable PNGs\")\nprint(f\"✅ Collected shapes for {len(image_shapes)} images\")\n\n# ─── 7) Write YOLO-format label files ─────────────────────────────────────────\nLABEL_DIR = Path('data/labels/train/');  LABEL_DIR.mkdir(parents=True, exist_ok=True)\n\nROI_SIZE = 128  # pixels\n\ndef get_cls_id(cond_text: str) -> int:\n    \"\"\"Return 0/1/2 based on the substring in the original condition field.\"\"\"\n    if 'Spinal Canal' in cond_text:\n        return 0\n    if 'Neural Foraminal' in cond_text:\n        return 1\n    if 'Subarticular' in cond_text:\n        return 2\n    raise ValueError(f\"Unknown condition text: {cond_text!r}\")\n\ndef write_yolo(stem, cls_id, x_c, y_c, w_n, h_n):\n    with open(LABEL_DIR/f\"{stem}.txt\", 'a') as f:\n        f.write(f\"{cls_id} {x_c:.6f} {y_c:.6f} {w_n:.6f} {h_n:.6f}\\n\")\n\nfor _, row in tqdm(merged.iterrows(),\n                   total=len(merged),\n                   desc=\"Writing YOLO labels\"):\n    stem = row.stem\n    if stem not in image_shapes:\n        continue\n    H, W = image_shapes[stem]\n    x_c, y_c = row.x / W, row.y / H\n    w_n = h_n = ROI_SIZE / W\n    cls_id = get_cls_id(row.condition)\n    write_yolo(stem, cls_id, x_c, y_c, w_n, h_n)\n\nprint(\"✅ Done writing YOLO labels to\", LABEL_DIR)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T06:48:25.799234Z","iopub.execute_input":"2025-06-01T06:48:25.799416Z","iopub.status.idle":"2025-06-01T07:11:23.898194Z","shell.execute_reply.started":"2025-06-01T06:48:25.799400Z","shell.execute_reply":"2025-06-01T07:11:23.897433Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 2: Build train/val splits & data.yaml for YOLOv8\n\nimport random\nfrom pathlib import Path\nfrom sklearn.model_selection import train_test_split\n\n# 1) Gather all the stems you converted\nIMG_DIR = Path('data/images/train/')\nall_stems = [p.stem for p in IMG_DIR.glob(\"*.png\")]\nprint(f\"🖼️  {len(all_stems)} total images found\")\n\n# 2) Split into train / val\ntrain_stems, val_stems = train_test_split(\n    all_stems,\n    test_size=0.2,\n    random_state=42,\n    shuffle=True\n)\nprint(f\"🚂  {len(train_stems)} train  |  {len(val_stems)} val\")\n\n# 3) Write file lists\nwith open('data/train.txt','w') as f:\n    for s in train_stems:\n        f.write(f\"{IMG_DIR}/{s}.png\\n\")\n\nwith open('data/val.txt','w') as f:\n    for s in val_stems:\n        f.write(f\"{IMG_DIR}/{s}.png\\n\")\n\nprint(\"✅ train.txt & val.txt saved\")\n\n# 4) Create data.yaml\nimport yaml\n\ndata = {\n    'path': '.',            # project root\n    'train': 'data/train.txt',\n    'val':   'data/val.txt',\n    'nc':    3,             # number of classes\n    'names': [\n        'Spinal Canal Stenosis',\n        'Neural Foraminal Narrowing',\n        'Subarticular Stenosis'\n    ]\n}\n\nwith open('data/data.yaml','w') as f:\n    yaml.dump(data, f, sort_keys=False)\n\nprint(\"✅ data/data.yaml created:\")\nprint(yaml.dump(data, sort_keys=False))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T07:14:11.068879Z","iopub.execute_input":"2025-06-01T07:14:11.069418Z","iopub.status.idle":"2025-06-01T07:14:12.465848Z","shell.execute_reply.started":"2025-06-01T07:14:11.069394Z","shell.execute_reply":"2025-06-01T07:14:12.465265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 3: Install YOLOv8 dependency\n!pip install ultralytics --upgrade\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T07:14:59.383526Z","iopub.execute_input":"2025-06-01T07:14:59.383927Z","iopub.status.idle":"2025-06-01T07:16:15.101335Z","shell.execute_reply.started":"2025-06-01T07:14:59.383907Z","shell.execute_reply":"2025-06-01T07:16:15.100625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 3 (updated): 5-Fold Cross-Validation Training per Condition with per‐epoch checkpoints\n\nimport yaml\nfrom pathlib import Path\nfrom ultralytics import YOLO\nfrom sklearn.model_selection import StratifiedKFold\n\n# Ensure `merged` from Cell 1 is in scope\nconditions = {\n    'Spinal Canal Stenosis':      'Spinal_Canal_Stenosis',\n    'Neural Foraminal Narrowing': 'Neural_Foraminal_Narrowing',\n    'Subarticular Stenosis':      'Subarticular_Stenosis'\n}\n\nIMG_DIR   = Path('data/images/train/')\nBASE_DATA = Path('data')\n\nfor cond_text, cond_folder in conditions.items():\n    df_cond = merged[merged['condition'].str.contains(cond_text)].copy()\n\n    class_names = sorted(df_cond['level'].unique())\n    cls_map = {lvl: i for i, lvl in enumerate(class_names)}\n    df_cond['class_id'] = df_cond['level'].map(cls_map)\n\n    stems = df_cond['stem'].tolist()\n    labels = df_cond['class_id'].tolist()\n\n    skf = StratifiedKFold(n_splits=2, shuffle=True, random_state=42)\n    for fold, (train_idx, val_idx) in enumerate(skf.split(stems, labels)):\n        fold_dir = BASE_DATA/cond_folder/f'fold_{fold}'\n        fold_dir.mkdir(parents=True, exist_ok=True)\n\n        # write train/val lists\n        train_txt = fold_dir/'train.txt'\n        val_txt   = fold_dir/'val.txt'\n        with open(train_txt, 'w') as f:\n            for i in train_idx:\n                f.write(f\"{IMG_DIR}/{stems[i]}.png\\n\")\n        with open(val_txt, 'w') as f:\n            for i in val_idx:\n                f.write(f\"{IMG_DIR}/{stems[i]}.png\\n\")\n\n        # write data.yaml\n        data_cfg = {\n            'path': '.', \n            'train': str(train_txt), \n            'val':   str(val_txt), \n            'nc':    len(class_names), \n            'names': class_names\n        }\n        data_yaml = fold_dir/'data.yaml'\n        with open(data_yaml, 'w') as f:\n            yaml.dump(data_cfg, f, sort_keys=False)\n\n        # train and save a checkpoint every epoch\n        print(f\"\\n▶️  Training {cond_folder}, fold {fold}\")\n        model = YOLO('yolov8n.pt')\n        model.train(\n            data=str(data_yaml),\n            epochs=2,\n            imgsz=512,\n            batch=16,\n            project='runs/detect',\n            name=f\"{cond_folder}_fold{fold}\",\n            save_period=1,    # save epoch_{i}.pt each epoch\n            verbose=True,\n            plots=True\n        )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T07:21:32.129469Z","iopub.execute_input":"2025-06-01T07:21:32.130152Z","iopub.status.idle":"2025-06-01T07:50:41.463831Z","shell.execute_reply.started":"2025-06-01T07:21:32.130127Z","shell.execute_reply":"2025-06-01T07:50:41.462935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 4 (Debug): Inspect your runs/detect folder for .pt weights\n\nimport os\nfrom pathlib import Path\n\nRUN_DIR = Path('runs/detect')\nif not RUN_DIR.exists():\n    raise RuntimeError(f\"{RUN_DIR} does not exist! Did you train your models?\")\n\nprint(f\"Directory tree under {RUN_DIR}:\\n\")\nfor root, dirs, files in os.walk(RUN_DIR):\n    level = root.replace(str(RUN_DIR), '').count(os.sep)\n    indent = '    ' * level\n    print(f\"{indent}- {Path(root).name}/\")\n    for fname in files:\n        print(f\"{indent}    - {fname}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T20:11:31.043698Z","iopub.status.idle":"2025-05-30T20:11:31.044057Z","shell.execute_reply.started":"2025-05-30T20:11:31.043877Z","shell.execute_reply":"2025-05-30T20:11:31.043894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 4: Inference & Crop ROIs (clean, per-fold, only split images)\n\nimport os\nimport sys\nimport contextlib\nfrom pathlib import Path\nfrom PIL import Image\nfrom ultralytics import YOLO\nfrom tqdm import tqdm\n\n@contextlib.contextmanager\ndef suppress_output():\n    with open(os.devnull, \"w\") as devnull:\n        old_stdout, old_stderr = sys.stdout, sys.stderr\n        try:\n            sys.stdout = devnull\n            sys.stderr = devnull\n            yield\n        finally:\n            sys.stdout = old_stdout\n            sys.stderr = old_stderr\n\nconditions = {\n    'Spinal_Canal_Stenosis':      'Spinal_Canal_Stenosis',\n    'Neural_Foraminal_Narrowing': 'Neural_Foraminal_Narrowing',\n    'Subarticular_Stenosis':      'Subarticular_Stenosis'\n}\n\nIMG_DIR = Path('data/images/train/')\nCROPS_BASE = Path('data/crops/')\nCROPS_BASE.mkdir(parents=True, exist_ok=True)\n\nfor cond_folder in conditions.values():\n    # use actual data/ splits\n    fold_base = Path(f'data/{cond_folder}')\n    if not fold_base.exists():\n        continue\n\n    fold_dirs = sorted(f for f in fold_base.iterdir() if f.name.startswith('fold_'))\n    for fold_dir in fold_dirs:\n        fold_idx = fold_dir.name.split('_')[-1]\n\n        # Validate paths\n        train_path = fold_dir / 'train.txt'\n        val_path = fold_dir / 'val.txt'\n        if not train_path.exists() or not val_path.exists():\n            print(f\"⚠️ Skipping {fold_dir}: missing train/val split files.\")\n            continue\n\n        # Load split image paths\n        with open(train_path) as f1, open(val_path) as f2:\n            crop_list = [Path(p.strip()) for p in f1.readlines() + f2.readlines()]\n\n        # Match YOLO weights from corresponding runs/detect folder\n        yolo_weights = Path(f'runs/detect/{cond_folder}_fold{fold_idx}/weights')\n        best_ckpt = yolo_weights / 'best.pt'\n        if not best_ckpt.exists():\n            print(f\"⚠️ Skipping {cond_folder}_fold{fold_idx}: no best.pt found.\")\n            continue\n\n        with suppress_output():\n            model = YOLO(str(best_ckpt))\n\n        # Output crop folder\n        out_dir = CROPS_BASE / cond_folder / f'fold_{fold_idx}'\n        out_dir.mkdir(parents=True, exist_ok=True)\n\n        for img_fp in tqdm(crop_list, desc=f\"Cropping {cond_folder}_fold{fold_idx}\"):\n            with suppress_output():\n                results = model(img_fp, imgsz=512, conf=0.3, verbose=False)\n\n            boxes = results[0].boxes.xyxy.cpu().numpy()\n            if boxes.size == 0:\n                continue\n\n            im = Image.open(img_fp)\n            stem = img_fp.stem\n            for j, (x1, y1, x2, y2) in enumerate(boxes):\n                crop = im.crop((int(x1), int(y1), int(x2), int(y2)))\n                crop.save(out_dir / f\"{stem}_roi{j}.png\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T07:50:50.544731Z","iopub.execute_input":"2025-06-01T07:50:50.545102Z","iopub.status.idle":"2025-06-01T08:25:32.338044Z","shell.execute_reply.started":"2025-06-01T07:50:50.545044Z","shell.execute_reply":"2025-06-01T08:25:32.337113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add to end of Cell 1\nmerged.to_pickle('data/merged.pkl')\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 5: Train severity classifiers for all 3 conditions from ROI crops\n\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models, transforms\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\nfrom pathlib import Path\nfrom PIL import Image\n\n# Settings\nCROP_ROOT = Path('data/crops')\nMERGED = merged  # from Cell 1\nBATCH_SIZE = 32\nEPOCHS = 10\nIMG_SIZE = 224\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nLABEL_MAP = {'Normal/Mild': 0, 'Moderate': 1, 'Severe': 2}\nIDX2LABEL = {v: k for k, v in LABEL_MAP.items()}\n\nconditions = {\n    'Spinal_Canal_Stenosis':      'Spinal Canal Stenosis',\n    'Neural_Foraminal_Narrowing': 'Neural Foraminal Narrowing',\n    'Subarticular_Stenosis':      'Subarticular Stenosis'\n}\n\n# Dataset class\nclass CropDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img = Image.open(row['path']).convert('RGB')\n        if self.transform:\n            img = self.transform(img)\n        return img, torch.tensor(row['label_id'])\n\n# Transform\ntransform = transforms.Compose([\n    transforms.Resize((IMG_SIZE, IMG_SIZE)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406],\n                         [0.229, 0.224, 0.225])\n])\n\nfor crop_folder, original_condition in conditions.items():\n    crop_dir = CROP_ROOT / crop_folder\n    if not crop_dir.exists():\n        print(f\"⚠️ Skipping {crop_folder}: no crop directory found.\")\n        continue\n\n    # Step 1: Load (path, label_id) pairs\n    records = []\n    for crop_fp in crop_dir.rglob(\"*.png\"):\n        stem = crop_fp.stem.split('_roi')[0]\n        row = MERGED[(MERGED['stem'] == stem) &\n                     (MERGED['condition'] == original_condition)]\n        if row.empty or pd.isna(row.iloc[0]['score']):\n            continue\n        severity = row.iloc[0]['score']\n        if severity not in LABEL_MAP:\n            continue\n        records.append({'path': str(crop_fp), 'label_id': LABEL_MAP[severity]})\n\n    df = pd.DataFrame(records)\n    if df.empty:\n        print(f\"⚠️ Skipping {crop_folder}: no labeled crops found.\")\n        continue\n\n    # Step 2: Train/val split\n    train_df, val_df = train_test_split(\n        df, stratify=df['label_id'], test_size=0.2, random_state=42\n    )\n\n    train_loader = DataLoader(CropDataset(train_df, transform), batch_size=BATCH_SIZE, shuffle=True)\n    val_loader   = DataLoader(CropDataset(val_df, transform), batch_size=BATCH_SIZE)\n\n    # Step 3: Model\n    model = models.resnet18(pretrained=True)\n    model.fc = nn.Linear(model.fc.in_features, len(LABEL_MAP))\n    model = model.to(DEVICE)\n\n    # Step 4: Training setup\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=1e-4)\n    best_acc = 0.0\n\n    print(f\"\\n🚀 Training classifier for {crop_folder} with {len(df)} samples\")\n\n    for epoch in range(1, EPOCHS + 1):\n        model.train()\n        correct, total, loss_sum = 0, 0, 0.0\n        for imgs, labels in train_loader:\n            imgs, labels = imgs.to(DEVICE), labels.to(DEVICE)\n            optimizer.zero_grad()\n            out = model(imgs)\n            loss = criterion(out, labels)\n            loss.backward()\n            optimizer.step()\n            _, preds = out.max(1)\n            correct += (preds == labels).sum().item()\n            total += labels.size(0)\n            loss_sum += loss.item() * imgs.size(0)\n        acc = correct / total\n        print(f\"Epoch {epoch:02d} - Train Acc: {acc:.4f} | Loss: {loss_sum/total:.4f}\")\n\n        # Validation\n        model.eval()\n        with torch.no_grad():\n            correct, total = 0, 0\n            for imgs, labels in val_loader:\n                imgs, labels = imgs.to(DEVICE), labels.to(DEVICE)\n                out = model(imgs)\n                _, preds = out.max(1)\n                correct += (preds == labels).sum().item()\n                total += labels.size(0)\n            val_acc = correct / total\n            print(f\"            Val Acc:   {val_acc:.4f}\")\n            if val_acc > best_acc:\n                best_acc = val_acc\n                save_path = Path(f\"runs/classify/{crop_folder}_best.pth\")\n                save_path.parent.mkdir(exist_ok=True, parents=True)\n                torch.save(model.state_dict(), save_path)\n\n    print(f\"✅ Best model saved for {crop_folder}: Val Acc = {best_acc:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T08:31:00.253454Z","iopub.execute_input":"2025-06-01T08:31:00.253777Z"}},"outputs":[],"execution_count":null}]}