{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":12249891,"sourceType":"datasetVersion","datasetId":7718540}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**CLASSIFER with test case**","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedShuffleSplit, StratifiedKFold\nfrom pathlib import Path\n\ndef split_train_val_test_5fold(\n    condition: str,\n    test_size: float = 0.2,\n    n_splits: int = 5,\n    random_state: int = 42\n):\n    \"\"\"\n    1) Reads /kaggle/input/csv-files/<condition>.csv, drops NaN scores\n    2) Splits off a stratified test set (test_size fraction)\n    3) Performs a stratified n_splits-fold CV on the remaining data\n    4) Writes out:\n       - ./<condition>/test/<condition>_test.csv\n       - ./<condition>/<condition>_trainval_<n_splits>folds.csv\n       - ./<condition>/fold_0/... fold_{n_splits-1}\n    \"\"\"\n    # 1) Load and clean\n    df = pd.read_csv(f'/kaggle/input/csv-files/{condition}.csv')\n    df = df.dropna(subset=['score']).reset_index(drop=True)\n\n    # 2) Stratified shuffle split → train_val vs test\n    sss = StratifiedShuffleSplit(n_splits=1, test_size=test_size, random_state=random_state)\n    train_val_idx, test_idx = next(sss.split(df, df['score']))\n    df_train_val = df.loc[train_val_idx].reset_index(drop=True)\n    df_test      = df.loc[test_idx].reset_index(drop=True)\n\n    # Save test set\n    condition_dir = Path(f'./{condition}')\n    (condition_dir / 'test').mkdir(parents=True, exist_ok=True)\n    df_test.to_csv(\n        condition_dir / 'test' / f'{condition}_test.csv', \n        index=False\n    )\n    print(f\"Saved test set ({len(df_test)} rows) to: {condition_dir/'test'/f'{condition}_test.csv'}\")\n\n    # 3) Stratified K-Fold on train_val\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=random_state)\n    df_train_val['fold'] = -1\n    for fold, (_, val_idx) in enumerate(skf.split(df_train_val, df_train_val['score'])):\n        df_train_val.loc[val_idx, 'fold'] = fold\n\n    # Save combined train_val folds CSV\n    df_train_val.to_csv(\n        condition_dir / f'{condition}_trainval_{n_splits}folds.csv',\n        index=False\n    )\n    print(f\"Saved train/val folds CSV to: {condition_dir/f'{condition}_trainval_{n_splits}folds.csv'}\")\n\n    # 4) Write out per-fold train/val splits\n    for fold in range(n_splits):\n        fold_dir = condition_dir / f'fold_{fold}'\n        fold_dir.mkdir(parents=True, exist_ok=True)\n\n        train_df = df_train_val[df_train_val['fold'] != fold].reset_index(drop=True)\n        val_df   = df_train_val[df_train_val['fold'] == fold].reset_index(drop=True)\n\n        train_df.to_csv(fold_dir / f'{condition}_train.csv', index=False)\n        val_df.to_csv(  fold_dir / f'{condition}_val.csv',   index=False)\n        print(f\"  Fold {fold}: {len(train_df)} train rows, {len(val_df)} val rows\")\n\n# Run for each condition\nfor cond in ['Neural_Foraminal_Narrowing', 'Spinal_Canal_Stenosis', 'Subarticular_Stenosis']:\n    split_train_val_test_5fold(cond)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-14T09:59:10.058529Z","iopub.execute_input":"2025-07-14T09:59:10.058800Z","iopub.status.idle":"2025-07-14T09:59:14.343112Z","shell.execute_reply.started":"2025-07-14T09:59:10.058777Z","shell.execute_reply":"2025-07-14T09:59:14.342378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport random\nimport shutil\n\n# Define augmentation methods\naugmentations = ['rotate', 'horizontal_flip', 'vertical_flip', 'gaussian_noise']\n\ndef augment_data(\n    df: pd.DataFrame,\n    augmentations: list,\n    seed: int = 42,\n    min_ops: int = 2,\n    max_ops: int = 4\n) -> pd.DataFrame:\n    \"\"\"\n    Oversample every class up to the majority count, reproducibly.\n    Each oversampled row gets a list of 2–4 random augmentations.\n    \"\"\"\n    # reproducible\n    np.random.seed(seed)\n    random.seed(seed)\n\n    counts = df['score'].value_counts()\n    max_n  = counts.max()\n\n    balanced_parts = []\n    for cls, grp in df.groupby('score'):\n        # sample up to max_n (includes original + duplicates)\n        reps = grp.sample(n=max_n, replace=True, random_state=seed).copy()\n        # assign between min_ops and max_ops augmentations per row\n        ops_list = []\n        for _ in range(len(reps)):\n            k = random.randint(min_ops, max_ops)\n            ops = random.sample(augmentations, k)\n            ops_list.append(ops)\n        reps['augmentation'] = ops_list\n        balanced_parts.append(reps)\n\n    # concatenate and shuffle\n    df_balanced = pd.concat(balanced_parts, ignore_index=True)\n    df_balanced = df_balanced.sample(frac=1, random_state=seed).reset_index(drop=True)\n    return df_balanced\n\n# --- Adjusted for train/val/test splits ---\nconditions = [\n    'Neural_Foraminal_Narrowing',\n    'Spinal_Canal_Stenosis',\n    'Subarticular_Stenosis'\n]\nfolds = list(range(5))  # 5-fold CV\n\nfor cond in conditions:\n    for fold in folds:\n        base_in  = f'./{cond}/fold_{fold}'\n        base_out = f'./augmented_output/{cond}/fold_{fold}'\n        os.makedirs(base_out, exist_ok=True)\n\n        # load train, augment, save\n        train_csv = os.path.join(base_in, f'{cond}_train.csv')\n        df_train  = pd.read_csv(train_csv)\n        df_aug    = augment_data(df_train, augmentations, seed=42)\n        df_aug.to_csv(os.path.join(base_out, f'{cond}_augmented_train.csv'), index=False)\n        print(f'→ Augmented train saved: {base_out}/{cond}_augmented_train.csv')\n\n        # copy val unchanged\n        val_csv = os.path.join(base_in, f'{cond}_val.csv')\n        shutil.copy(val_csv, os.path.join(base_out, f'{cond}_val.csv'))\n        print(f'→ Val copied:           {base_out}/{cond}_val.csv')\n\n    # Copy test set unchanged\n    test_in  = f'./{cond}/test/{cond}_test.csv'\n    test_out = f'./augmented_output/{cond}/test'\n    os.makedirs(test_out, exist_ok=True)\n    shutil.copy(test_in, os.path.join(test_out, f'{cond}_test.csv'))\n    print(f'→ Test copied:          {test_out}/{cond}_test.csv')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-14T09:59:14.344709Z","iopub.execute_input":"2025-07-14T09:59:14.344913Z","iopub.status.idle":"2025-07-14T09:59:18.924512Z","shell.execute_reply.started":"2025-07-14T09:59:14.344896Z","shell.execute_reply":"2025-07-14T09:59:18.923918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pydicom\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image, ImageOps\nfrom pathlib import Path\n\nclass DataPreparationImage:\n    def __init__(\n        self,\n        dataset_directory: str,\n        condition: str,\n        csv_directory: str,\n        num_folds: int = 5,\n        augmentation_list=None,\n    ):\n        self.dataset_directory  = Path(dataset_directory)\n        self.condition          = condition\n        self.csv_directory      = Path(csv_directory)\n        self.num_folds          = num_folds\n        self.augmentation_list  = augmentation_list or [\n            'rotate','horizontal_flip','vertical_flip','gaussian_noise'\n        ]\n\n        print(f\"Starting image data prep for {self.condition} ({self.num_folds} folds + test)\")\n        self._create_folders()\n        self._process_all_folds()\n        self._process_test()\n\n    def _create_folders(self):\n        base = Path(f\"./{self.condition}\")\n        base.mkdir(exist_ok=True)\n        for fold in range(self.num_folds):\n            (base/f\"fold_{fold}\"/\"train\").mkdir(parents=True, exist_ok=True)\n            (base/f\"fold_{fold}\"/\"val\").mkdir(parents=True, exist_ok=True)\n        (base/\"test\").mkdir(parents=True, exist_ok=True)\n\n    def _read_csv(self, split: str, fold: int = None) -> pd.DataFrame:\n        if split in ('train', 'val'):\n            suffix   = 'augmented_train' if split=='train' else 'val'\n            fn       = f\"{self.condition}_{suffix}.csv\"\n            path     = self.csv_directory/self.condition/f\"fold_{fold}\"/fn\n        elif split=='test':\n            fn       = f\"{self.condition}_test.csv\"\n            path     = self.csv_directory/self.condition/\"test\"/fn\n        else:\n            raise ValueError(f\"Unknown split: {split}\")\n        return pd.read_csv(path)\n\n    def _read_dicom(self, path: Path) -> np.ndarray:\n        ds  = pydicom.dcmread(str(path))\n        img = ds.pixel_array.astype(float)\n        img = (img - img.min())/(img.max()-img.min()+1e-6)*255.0\n        rgb = np.stack([img]*3, axis=-1)\n        return rgb.astype('uint8')\n\n    def _crop(self, image: np.ndarray, x: float, y: float, box: int = 16) -> Image.Image:\n        img = Image.fromarray(image)\n        return img.crop((int(x-box),int(y-box),int(x+box),int(y+box)))\n\n    def _apply_augmentation(self, img: Image.Image, aug: str) -> Image.Image:\n        if aug=='rotate':\n            return img.rotate(np.random.uniform(-20,20), expand=True)\n        if aug=='horizontal_flip':\n            return ImageOps.mirror(img)\n        if aug=='vertical_flip':\n            return ImageOps.flip(img)\n        if aug=='gaussian_noise':\n            arr   = np.array(img)\n            noise = np.random.normal(0,25,arr.shape)\n            return Image.fromarray(np.clip(arr+noise,0,255).astype('uint8'))\n        return img\n\n    def _process_all_folds(self):\n        for fold in range(self.num_folds):\n            print(f\" Processing fold {fold}…\")\n            # TRAIN\n            df_train = self._read_csv('train', fold)\n            for _, row in df_train.iterrows():\n                sid, seid, inst = row['study_id'], row['series_id'], row['instance_number']\n                x, y    = row['x'], row['y']\n                aug_op  = row.get('augmentation')\n                dcm     = self.dataset_directory/str(sid)/str(seid)/f\"{inst}.dcm\"\n                img     = self._read_dicom(dcm)\n                patch   = self._crop(img, x, y)\n                if aug_op in self.augmentation_list:\n                    out = self._apply_augmentation(patch, aug_op)\n                    suffix = f\"_{aug_op}\"\n                else:\n                    out = patch; suffix=\"\"\n                fname = f\"{sid}_{seid}_{inst}_{int(x)}_{int(y)}{suffix}.png\"\n                out.save(Path(self.condition)/f\"fold_{fold}\"/\"train\"/fname)\n\n            # VAL\n            df_val = self._read_csv('val', fold)\n            for _, row in df_val.iterrows():\n                sid, seid, inst = row['study_id'], row['series_id'], row['instance_number']\n                x, y    = row['x'], row['y']\n                dcm     = self.dataset_directory/str(sid)/str(seid)/f\"{inst}.dcm\"\n                img     = self._read_dicom(dcm)\n                patch   = self._crop(img, x, y)\n                fname   = f\"{sid}_{seid}_{inst}_{int(x)}_{int(y)}.png\"\n                patch.save(Path(self.condition)/f\"fold_{fold}\"/\"val\"/fname)\n\n    def _process_test(self):\n        print(\" Processing held-out test set…\")\n        df_test = self._read_csv('test')\n        for _, row in df_test.iterrows():\n            sid, seid, inst = row['study_id'], row['series_id'], row['instance_number']\n            x, y    = row['x'], row['y']\n            dcm     = self.dataset_directory/str(sid)/str(seid)/f\"{inst}.dcm\"\n            img     = self._read_dicom(dcm)\n            patch   = self._crop(img, x, y)\n            fname   = f\"{sid}_{seid}_{inst}_{int(x)}_{int(y)}.png\"\n            patch.save(Path(self.condition)/\"test\"/fname)\n        print(f\" Saved {len(df_test)} test images to ./{self.condition}/test\")\n\n\n# ─────────────────────────────────────────────────────────────────────────────\n# Run data prep for all three conditions\n# ─────────────────────────────────────────────────────────────────────────────\n\nDATASET_DIR = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images\"\nCSV_DIR     = \"/kaggle/working/augmented_output\"\n\nCONDITIONS = [\n    'Neural_Foraminal_Narrowing',\n    'Spinal_Canal_Stenosis',\n    'Subarticular_Stenosis'\n]\n\nfor cond in CONDITIONS:\n    DataPreparationImage(\n        dataset_directory=DATASET_DIR,\n        condition=cond,\n        csv_directory=CSV_DIR,\n        num_folds=5\n    )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-14T10:02:14.803152Z","iopub.execute_input":"2025-07-14T10:02:14.803756Z","iopub.status.idle":"2025-07-14T11:24:22.367015Z","shell.execute_reply.started":"2025-07-14T10:02:14.803732Z","shell.execute_reply":"2025-07-14T11:24:22.366318Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom pathlib import Path\n\n# 1) Parameters\nCSV_DIR    = Path('/kaggle/working/augmented_output')   # where your augmented CSVs live\nCONDITIONS = [\n    'Spinal_Canal_Stenosis',\n    'Neural_Foraminal_Narrowing',\n    'Subarticular_Stenosis'\n]\nFOLDS      = list(range(5))  # five folds: 0–4\nOUT_ROOT   = Path('/kaggle/working')    # base for label output folders\n\n# 2) Helper: build 'subject' and 'label' columns\ndef working_on_csv(csv_path: Path, split: str) -> pd.DataFrame:\n    df = pd.read_csv(csv_path)\n\n    # ensure we always have an 'augmentation' column\n    if 'augmentation' not in df.columns:\n        df['augmentation'] = None\n\n    def make_subject(r):\n        name = f\"{r.study_id}_{r.series_id}_{r.instance_number}_{int(r.x)}_{int(r.y)}.png\"\n        # only append \"_augmented\" on training samples that were actually augmented\n        if split == 'train' and pd.notna(r.augmentation) and r.augmentation:\n            name = name.replace('.png', '_augmented.png')\n        return name\n\n    def make_label(score):\n        # if your original CSV already used 1,2,3 as numeric scores, just cast\n        if pd.api.types.is_numeric_dtype(type(score)):\n            return int(score)\n        # otherwise map from the text labels you used\n        return {\n            'Normal/Mild': 1,\n            'Moderate':    2,\n            'Severe':      3\n        }.get(score, None)\n\n    df['subject'] = df.apply(make_subject, axis=1)\n    df['label']   = df['score'].apply(make_label)\n\n    return df[['subject', 'label']]\n\n# 3) Create output folder structure (including test)\nfor cond in CONDITIONS:\n    base = OUT_ROOT / f\"{cond}_label\"\n    for fold in FOLDS:\n        (base / f\"fold_{fold}\").mkdir(parents=True, exist_ok=True)\n    (base / \"test\").mkdir(parents=True, exist_ok=True)\n\n# 4) Process each condition: train/val and then test\nfor cond in CONDITIONS:\n    label_root = OUT_ROOT / f\"{cond}_label\"\n\n    # train & val\n    for fold in FOLDS:\n        train_csv = CSV_DIR / cond / f\"fold_{fold}\" / f\"{cond}_augmented_train.csv\"\n        val_csv   = CSV_DIR / cond / f\"fold_{fold}\" / f\"{cond}_val.csv\"\n\n        train_labels = working_on_csv(train_csv, 'train')\n        val_labels   = working_on_csv(val_csv,   'val')\n\n        out_base = label_root / f\"fold_{fold}\"\n        train_labels.to_csv(out_base / f\"{cond}_augmented_labels.csv\", index=False)\n        val_labels.to_csv(  out_base / f\"{cond}_val_labels.csv\",       index=False)\n\n        print(f\"→ {cond} fold {fold}: train={len(train_labels)}, val={len(val_labels)}\")\n\n    # test\n    test_csv    = CSV_DIR / cond / \"test\" / f\"{cond}_test.csv\"\n    test_labels = working_on_csv(test_csv, 'test')\n    out_test    = label_root / \"test\"\n    test_labels.to_csv(out_test / f\"{cond}_test_labels.csv\", index=False)\n\n    print(f\"→ {cond} test: {len(test_labels)} rows → {out_test}/{cond}_test_labels.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-14T16:01:00.341711Z","iopub.execute_input":"2025-07-14T16:01:00.342277Z","iopub.status.idle":"2025-07-14T16:01:00.771382Z","shell.execute_reply.started":"2025-07-14T16:01:00.342250Z","shell.execute_reply":"2025-07-14T16:01:00.770194Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────────────────────\n# Cell: 5-Fold CV + Held-Out Test Evaluation for 3 Conditions\n#           (50 layers frozen, Flatten-only head, L2 on hidden layers, no reg on final)\n# ─────────────────────────────────────────────────────────────────────────────\n\nimport os\nimport gc\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nimport tensorflow as tf\nfrom tensorflow.keras.applications import ResNet50\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.layers import Flatten, Dense, Dropout, BatchNormalization\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.regularizers import l2\nfrom sklearn.metrics import (\n    accuracy_score, precision_score, recall_score, f1_score,\n    confusion_matrix, roc_curve, auc\n)\nfrom sklearn.preprocessing import label_binarize\nimport matplotlib.pyplot as plt\n\n# ── Global Configuration ─────────────────────────────────────────────────────\nCONDITIONS     = [\n    'Neural_Foraminal_Narrowing',\n    'Spinal_Canal_Stenosis',\n    'Subarticular_Stenosis'\n]\nFOLDS          = list(range(5))\nBATCH_SIZE     = 16\nPATIENCE       = 5\nEPOCHS         = 10\nLEARNING_RATE  = 1e-4\nIMG_SIZE       = (224, 224)\nL2_WEIGHT      = 1e-4\nDROP_RATE      = 0.5\n\nfor CONDITION in CONDITIONS:\n    print(f\"\\n===== Processing condition: {CONDITION} =====\")\n    RESULTS_DIR = Path(f'./results/{CONDITION}')\n    RESULTS_DIR.mkdir(parents=True, exist_ok=True)\n    fold_results = []\n\n    # ── 5-Fold Cross-Validation ───────────────────────────────────────────\n    for fold in FOLDS:\n        print(f\"\\n▶ Training fold {fold}\")\n        IMAGE_ROOT = f'/kaggle/working/{CONDITION}/fold_{fold}'\n        LABEL_ROOT = f'/kaggle/working/{CONDITION}_label/fold_{fold}'\n\n        # — Load & adjust labels\n        train_df = pd.read_csv(Path(LABEL_ROOT)/f'{CONDITION}_augmented_labels.csv')\n        val_df   = pd.read_csv(Path(LABEL_ROOT)/f'{CONDITION}_val_labels.csv')\n        for df in (train_df, val_df):\n            df['subject'] = df['subject'].str.replace('_augmented','', regex=False)\n            df['label']   = df['label'].astype(str)\n\n        # — Generators (just rescale)\n        datagen = ImageDataGenerator(rescale=1./255)\n        train_gen = datagen.flow_from_dataframe(\n            train_df,\n            directory=os.path.join(IMAGE_ROOT, 'train'),\n            x_col='subject', y_col='label',\n            target_size=IMG_SIZE, batch_size=BATCH_SIZE,\n            class_mode='categorical', shuffle=True\n        )\n        val_gen = datagen.flow_from_dataframe(\n            val_df,\n            directory=os.path.join(IMAGE_ROOT, 'val'),\n            x_col='subject', y_col='label',\n            target_size=IMG_SIZE, batch_size=BATCH_SIZE,\n            class_mode='categorical', shuffle=False\n        )\n\n        # — Determine num_classes dynamically\n        num_classes = len(train_gen.class_indices)\n\n        # — Build & compile model\n        base = ResNet50(include_top=False, weights='imagenet', input_shape=IMG_SIZE+(3,))\n        for layer in base.layers[:50]:\n            layer.trainable = False\n        for layer in base.layers[50:]:\n            layer.trainable = True\n\n        x = Flatten()(base.output)\n        x = Dense(256, activation='relu', kernel_regularizer=l2(L2_WEIGHT))(x)\n        x = BatchNormalization()(x)\n        x = Dropout(DROP_RATE)(x)\n        x = Dense(128, activation='relu', kernel_regularizer=l2(L2_WEIGHT))(x)\n        x = Dropout(DROP_RATE)(x)\n        preds = Dense(num_classes, activation='softmax')(x)\n\n        model = Model(inputs=base.input, outputs=preds)\n        model.compile(\n            optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),\n            loss='categorical_crossentropy',\n            metrics=['accuracy']\n        )\n\n        # — Callbacks\n        out_dir = RESULTS_DIR / f'fold_{fold}'\n        out_dir.mkdir(parents=True, exist_ok=True)\n        ckpt = ModelCheckpoint(out_dir/'best_model.keras', monitor='val_loss',\n                               save_best_only=True, verbose=1)\n        es   = EarlyStopping(monitor='val_loss', patience=PATIENCE,\n                             restore_best_weights=True, verbose=1)\n        rl   = ReduceLROnPlateau(monitor='val_loss', factor=0.5,\n                                 patience=2, min_lr=1e-6, verbose=1)\n\n        # — Train\n        history = model.fit(\n            train_gen,\n            validation_data=val_gen,\n            epochs=EPOCHS,\n            callbacks=[es, ckpt, rl],\n            verbose=2\n        )\n        with open(out_dir/'history.json','w') as f:\n            json.dump(history.history, f)\n\n        # — Evaluate on validation\n        val_gen.reset()\n        y_pred = np.argmax(model.predict(val_gen), axis=1)\n        y_true = val_gen.classes\n\n        fm = {\n            'fold': fold,\n            'Accuracy':  accuracy_score(y_true, y_pred),\n            'Precision': precision_score(y_true, y_pred, average='weighted'),\n            'Recall':    recall_score(y_true, y_pred, average='weighted'),\n            'F1 Score':  f1_score(y_true, y_pred, average='weighted')\n        }\n        fold_results.append(fm)\n        with open(out_dir/'metrics.txt','w') as f:\n            for k, v in fm.items():\n                if k!='fold':\n                    f.write(f\"{k}: {v:.4f}\\n\")\n        print(f\"✅ Fold {fold} metrics:\", fm)\n\n        tf.keras.backend.clear_session()\n        gc.collect()\n\n    # ── Pick best fold & evaluate held-out test ────────────────────────────\n    best = max(fold_results, key=lambda x: x['F1 Score'])\n    best_fold = best['fold']\n    print(f\"\\n▶ Best fold for {CONDITION}: {best_fold} (F1 = {best['F1 Score']:.4f})\")\n\n    model = load_model(RESULTS_DIR/f'fold_{best_fold}'/'best_model.keras')\n\n    TEST_IMAGE_DIR = f'/kaggle/working/{CONDITION}/test'\n    TEST_LABEL_CSV = f'/kaggle/working/{CONDITION}_label/test/{CONDITION}_test_labels.csv'\n\n    test_df = pd.read_csv(TEST_LABEL_CSV)\n    test_df['label'] = test_df['label'].astype(str)\n    test_gen = datagen.flow_from_dataframe(\n        test_df,\n        directory=TEST_IMAGE_DIR,\n        x_col='subject', y_col='label',\n        target_size=IMG_SIZE, batch_size=BATCH_SIZE,\n        class_mode='categorical', shuffle=False\n    )\n\n    # — dynamic num_classes for ROC\n    num_classes = len(test_gen.class_indices)\n    probs       = model.predict(test_gen, verbose=1)\n    y_pred      = np.argmax(probs, axis=1)\n    y_true      = test_gen.classes\n    classes     = list(range(num_classes))\n\n    acc_test  = accuracy_score(y_true, y_pred)\n    prec_test = precision_score(y_true, y_pred, average='weighted')\n    rec_test  = recall_score(y_true, y_pred, average='weighted')\n    f1_test   = f1_score(y_true, y_pred, average='weighted')\n    cm        = confusion_matrix(y_true, y_pred)\n    y_true_bin = label_binarize(y_true, classes=classes)\n\n    test_out = RESULTS_DIR / 'test'\n    test_out.mkdir(exist_ok=True)\n\n    with open(test_out/'metrics_test.txt','w') as f:\n        f.write(\n            f\"Accuracy: {acc_test:.4f}\\n\"\n            f\"Precision: {prec_test:.4f}\\n\"\n            f\"Recall:    {rec_test:.4f}\\n\"\n            f\"F1 Score:  {f1_test:.4f}\\n\"\n        )\n    np.savetxt(test_out/'confusion_matrix.csv', cm, delimiter=',', fmt='%d')\n\n    with open(test_out/'roc_auc.txt','w') as f:\n        for i in classes:\n            f.write(f\"Class {i} AUC: {auc(*roc_curve(y_true_bin[:,i], probs[:,i])[:2]):.4f}\\n\")\n\n    plt.figure()\n    for i in classes:\n        fpr, tpr, _ = roc_curve(y_true_bin[:,i], probs[:,i])\n        plt.plot(fpr, tpr, label=f'Class {i} (AUC={auc(fpr,tpr):.2f})')\n    plt.plot([0,1],[0,1],'k--')\n    plt.xlabel('FPR'); plt.ylabel('TPR')\n    plt.title(f'ROC Curves on Held-Out Test Set for {CONDITION}')\n    plt.legend(loc='lower right')\n    plt.savefig(test_out/'roc_curves.png')\n    plt.close()\n\n    print(f\"✅ Test evaluation for {CONDITION} saved to {test_out}\")\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-14T12:04:21.781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import json\nimport os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\n\nfrom pathlib import Path\nfrom sklearn.metrics import (\n    accuracy_score, precision_score, recall_score, f1_score,\n    confusion_matrix, roc_curve, auc, ConfusionMatrixDisplay\n)\nfrom sklearn.preprocessing import label_binarize\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import load_model\n\n# ── Global Config ─────────────────────────────────────────────────────────────\nCONDITIONS      = [\n    'Neural_Foraminal_Narrowing',\n    'Spinal_Canal_Stenosis',\n    'Subarticular_Stenosis'\n]\nFOLDS           = list(range(5))\nBATCH_SIZE      = 16\nIMG_SIZE        = (224,224)\n\n# ── Loop over each condition ──────────────────────────────────────────────────\nfor CONDITION in CONDITIONS:\n    print(f\"\\n===== Summary for {CONDITION} =====\\n\")\n    RESULTS_DIR    = Path(f'./results/{CONDITION}')\n    LABEL_DIR      = Path(f'/kaggle/working/{CONDITION}_label')\n    IMG_DIR        = Path(f'/kaggle/working/{CONDITION}')\n    TEST_IMAGE_DIR = IMG_DIR / 'test'\n    TEST_LABEL_CSV = LABEL_DIR / 'test' / f'{CONDITION}_test_labels.csv'\n\n    # Collect per‐fold histories and metrics\n    fold_dirs    = sorted(RESULTS_DIR.glob('fold_*'),\n                          key=lambda p: int(p.name.split('_')[1]))\n    train_metrics, val_metrics = [], []\n    loss_curves = {}\n\n    # ── Gather CV info ────────────────────────────────────────────────────────\n    for fd in fold_dirs:\n        fold = int(fd.name.split('_')[1])\n        # load loss history\n        hist = json.load(open(fd/'history.json'))\n        loss_curves[fold] = (hist['loss'], hist['val_loss'])\n\n        # load best model\n        model = load_model(fd/'best_model.keras')\n\n        # prepare train generator\n        train_csv = LABEL_DIR/f'fold_{fold}'/f'{CONDITION}_augmented_labels.csv'\n        df_train  = (pd.read_csv(train_csv)\n                       .assign(subject=lambda d: d.subject.str.replace('_augmented','',regex=False),\n                               label=lambda d: (d.label.astype(int)-1).astype(str)))\n        train_gen = ImageDataGenerator(rescale=1./255).flow_from_dataframe(\n            df_train,\n            directory=IMG_DIR/f'fold_{fold}'/'train',\n            x_col='subject', y_col='label',\n            target_size=IMG_SIZE, batch_size=BATCH_SIZE,\n            class_mode='categorical', shuffle=False\n        )\n\n        # prepare val generator\n        val_csv = LABEL_DIR/f'fold_{fold}'/f'{CONDITION}_val_labels.csv'\n        df_val  = (pd.read_csv(val_csv)\n                       .assign(subject=lambda d: d.subject.str.replace('_augmented','',regex=False),\n                               label=lambda d: (d.label.astype(int)-1).astype(str)))\n        val_gen = ImageDataGenerator(rescale=1./255).flow_from_dataframe(\n            df_val,\n            directory=IMG_DIR/f'fold_{fold}'/'val',\n            x_col='subject', y_col='label',\n            target_size=IMG_SIZE, batch_size=BATCH_SIZE,\n            class_mode='categorical', shuffle=False\n        )\n\n        # predictions\n        y_tr_pred = np.argmax(model.predict(train_gen), axis=1)\n        y_tr_true = train_gen.classes\n        y_val_pred = np.argmax(model.predict(val_gen), axis=1)\n        y_val_true = val_gen.classes\n\n        # store metrics\n        train_metrics.append({\n            'Accuracy':  accuracy_score(y_tr_true, y_tr_pred),\n            'Precision': precision_score(y_tr_true, y_tr_pred, average='weighted'),\n            'Recall':    recall_score(y_tr_true, y_tr_pred, average='weighted'),\n            'F1 Score':  f1_score(y_tr_true, y_tr_pred, average='weighted')\n        })\n        val_metrics.append({\n            'Accuracy':  accuracy_score(y_val_true, y_val_pred),\n            'Precision': precision_score(y_val_true, y_val_pred, average='weighted'),\n            'Recall':    recall_score(y_val_true, y_val_pred, average='weighted'),\n            'F1 Score':  f1_score(y_val_true, y_val_pred, average='weighted')\n        })\n\n        tf.keras.backend.clear_session()\n\n    # ── 1) Plot CV Loss Curves ────────────────────────────────────────────────\n    plt.figure(figsize=(8,5))\n    for fold, (loss, vloss) in loss_curves.items():\n        epochs = range(1, len(loss)+1)\n        plt.plot(epochs, loss,  label=f'Fold {fold} Train', alpha=0.6)\n        plt.plot(epochs, vloss, '--', label=f'Fold {fold} Val', alpha=0.6)\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.title(f'{CONDITION}: Train & Val Loss Curves')\n    plt.legend(ncol=2, fontsize='small')\n    plt.tight_layout()\n    plt.show()\n\n    # ── 2) Print Avg CV Metrics ─────────────────────────────────────────────\n    def average(metrics):\n        return {k: np.mean([m[k] for m in metrics]) for k in metrics[0]}\n\n    avg_tr = average(train_metrics)\n    avg_val = average(val_metrics)\n    print(\"→ Average CV TRAIN metrics:\")\n    for k, v in avg_tr.items():\n        print(f\"   {k}: {v:.3f}\")\n    print(\"→ Average CV   VAL metrics:\")\n    for k, v in avg_val.items():\n        print(f\"   {k}: {v:.3f}\")\n\n    # ── 3) Held-Out Test Evaluation ────────────────────────────────────────\n    best_fold = int(np.argmax([m['F1 Score'] for m in val_metrics]))\n    print(f\"\\n✨ Best fold = {best_fold}\")\n\n    model = load_model(RESULTS_DIR/f'fold_{best_fold}'/'best_model.keras')\n    test_df = (pd.read_csv(TEST_LABEL_CSV)\n                 .assign(label=lambda d: (d.label.astype(int)-1).astype(str)))\n    test_gen = ImageDataGenerator(rescale=1./255).flow_from_dataframe(\n        test_df,\n        directory=TEST_IMAGE_DIR,\n        x_col='subject', y_col='label',\n        target_size=IMG_SIZE, batch_size=BATCH_SIZE,\n        class_mode='categorical', shuffle=False\n    )\n\n    probs   = model.predict(test_gen, verbose=0)\n    y_test  = test_gen.classes\n    y_pred  = np.argmax(probs, axis=1)\n    classes = list(range(len(test_gen.class_indices)))\n\n    metrics_test = {\n        'Accuracy':  accuracy_score(y_test, y_pred),\n        'Precision': precision_score(y_test, y_pred, average='weighted'),\n        'Recall':    recall_score(y_test, y_pred, average='weighted'),\n        'F1 Score':  f1_score(y_test, y_pred, average='weighted')\n    }\n    print(\"\\n→ Held-out TEST metrics:\")\n    for k, v in metrics_test.items():\n        print(f\"   {k}: {v:.3f}\")\n\n    # ── 4) Confusion & ROC ────────────────────────────────────────────────\n    cm = confusion_matrix(y_test, y_pred)\n    disp = ConfusionMatrixDisplay(cm, display_labels=list(test_gen.class_indices.keys()))\n    plt.figure(figsize=(4,4))\n    disp.plot(ax=plt.gca(), cmap='Blues', colorbar=False)\n    plt.title(f'{CONDITION}: Test Confusion Matrix')\n    plt.show()\n\n    y_bin = label_binarize(y_test, classes=classes)\n    plt.figure(figsize=(6,5))\n    for i, label in enumerate(test_gen.class_indices):\n        fpr, tpr, _ = roc_curve(y_bin[:,i], probs[:,i])\n        plt.plot(fpr, tpr, label=f\"{label} (AUC={auc(fpr,tpr):.2f})\")\n    plt.plot([0,1], [0,1], 'k--')\n    plt.xlabel('FPR'); plt.ylabel('TPR')\n    plt.title(f'{CONDITION}: Test ROC Curves')\n    plt.legend(loc='lower right')\n    plt.show()\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-14T12:04:21.781Z"}},"outputs":[],"execution_count":null}]}