{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":12249891,"sourceType":"datasetVersion","datasetId":7718540}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**CLASSIFER with test case**","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedShuffleSplit, StratifiedKFold\nfrom pathlib import Path\n\ndef split_train_val_test_5fold(\n    condition: str,\n    test_size: float = 0.2,\n    n_splits: int = 5,\n    random_state: int = 42\n):\n    \"\"\"\n    1) Reads /kaggle/input/csv-files/<condition>.csv, drops NaN scores\n    2) Splits off a stratified test set (test_size fraction)\n    3) Performs a stratified n_splits-fold CV on the remaining data\n    4) Writes out:\n       - ./<condition>/test/<condition>_test.csv\n       - ./<condition>/<condition>_trainval_<n_splits>folds.csv\n       - ./<condition>/fold_0/... fold_{n_splits-1}\n    \"\"\"\n    # 1) Load and clean\n    df = pd.read_csv(f'/kaggle/input/csv-files/{condition}.csv')\n    df = df.dropna(subset=['score']).reset_index(drop=True)\n\n    # 2) Stratified shuffle split → train_val vs test\n    sss = StratifiedShuffleSplit(n_splits=1, test_size=test_size, random_state=random_state)\n    train_val_idx, test_idx = next(sss.split(df, df['score']))\n    df_train_val = df.loc[train_val_idx].reset_index(drop=True)\n    df_test      = df.loc[test_idx].reset_index(drop=True)\n\n    # Save test set\n    condition_dir = Path(f'./{condition}')\n    (condition_dir / 'test').mkdir(parents=True, exist_ok=True)\n    df_test.to_csv(\n        condition_dir / 'test' / f'{condition}_test.csv', \n        index=False\n    )\n    print(f\"Saved test set ({len(df_test)} rows) to: {condition_dir/'test'/f'{condition}_test.csv'}\")\n\n    # 3) Stratified K-Fold on train_val\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=random_state)\n    df_train_val['fold'] = -1\n    for fold, (_, val_idx) in enumerate(skf.split(df_train_val, df_train_val['score'])):\n        df_train_val.loc[val_idx, 'fold'] = fold\n\n    # Save combined train_val folds CSV\n    df_train_val.to_csv(\n        condition_dir / f'{condition}_trainval_{n_splits}folds.csv',\n        index=False\n    )\n    print(f\"Saved train/val folds CSV to: {condition_dir/f'{condition}_trainval_{n_splits}folds.csv'}\")\n\n    # 4) Write out per-fold train/val splits\n    for fold in range(n_splits):\n        fold_dir = condition_dir / f'fold_{fold}'\n        fold_dir.mkdir(parents=True, exist_ok=True)\n\n        train_df = df_train_val[df_train_val['fold'] != fold].reset_index(drop=True)\n        val_df   = df_train_val[df_train_val['fold'] == fold].reset_index(drop=True)\n\n        train_df.to_csv(fold_dir / f'{condition}_train.csv', index=False)\n        val_df.to_csv(  fold_dir / f'{condition}_val.csv',   index=False)\n        print(f\"  Fold {fold}: {len(train_df)} train rows, {len(val_df)} val rows\")\n\n# Run for each condition\nfor cond in ['Neural_Foraminal_Narrowing', 'Spinal_Canal_Stenosis', 'Subarticular_Stenosis']:\n    split_train_val_test_5fold(cond)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T04:47:06.295657Z","iopub.execute_input":"2025-07-17T04:47:06.296327Z","iopub.status.idle":"2025-07-17T04:47:09.631492Z","shell.execute_reply.started":"2025-07-17T04:47:06.296299Z","shell.execute_reply":"2025-07-17T04:47:09.630681Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport random\nimport shutil\n\nimport numpy as np\nimport pandas as pd\n\n# ── Reproducibility ─────────────────────────────────────────────────────────\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\n\n# ── Augmentation operations ─────────────────────────────────────────────────\naugmentation_ops = [\n    'rotate',\n    'horizontal_flip',\n    'vertical_flip',\n    'gaussian_noise',\n    'brightness',\n    'zoom'\n]\n\ndef augment_data(df: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"\n    Oversample each class up to the majority count.\n    Adds an 'augmentation' column indicating which op to apply.\n    \"\"\"\n    counts = df['score'].value_counts()\n    target = counts.max()\n    \n    # original rows, no augmentation\n    base = df.copy()\n    base['augmentation'] = None\n    \n    aug_list = [base]\n    for cls, cnt in counts.items():\n        if cnt < target:\n            needed = int(target - cnt)\n            sampled = (\n                df[df.score == cls]\n                .sample(needed, replace=True, random_state=SEED)\n                .copy()\n            )\n            # assign one random augment op to each row\n            sampled['augmentation'] = [\n                random.choice(augmentation_ops) for _ in range(needed)\n            ]\n            aug_list.append(sampled)\n    \n    df_aug = pd.concat(aug_list, ignore_index=True)\n    return df_aug\n\n# --- Adjusted for train/val/test splits ---\nconditions = [\n    'Neural_Foraminal_Narrowing',\n    'Spinal_Canal_Stenosis',\n    'Subarticular_Stenosis'\n]\nfolds = range(5)  # 5-fold CV\n\nfor cond in conditions:\n    for fold in folds:\n        in_dir  = f'./{cond}/fold_{fold}'\n        out_dir = f'./augmented_output/{cond}/fold_{fold}'\n        os.makedirs(out_dir, exist_ok=True)\n\n        # 1) load train, augment, save\n        train_csv = os.path.join(in_dir, f'{cond}_train.csv')\n        df_train  = pd.read_csv(train_csv)\n        if df_train.empty:\n            raise RuntimeError(f\"No training data for {cond} fold {fold}\")\n\n        df_aug = augment_data(df_train)\n        df_aug.to_csv(\n            os.path.join(out_dir, f'{cond}_augmented_train.csv'),\n            index=False\n        )\n        print(f'→ Augmented train saved: {out_dir}/{cond}_augmented_train.csv')\n\n        # 2) copy val \n        val_csv = os.path.join(in_dir, f'{cond}_val.csv')\n        shutil.copy(val_csv, os.path.join(out_dir, f'{cond}_val.csv'))\n        print(f'→ Val copied:           {out_dir}/{cond}_val.csv')\n\n    # 3) copy test \n    test_in  = f'./{cond}/test/{cond}_test.csv'\n    test_out = f'./augmented_output/{cond}/test'\n    os.makedirs(test_out, exist_ok=True)\n    shutil.copy(test_in, os.path.join(test_out, f'{cond}_test.csv'))\n    print(f'→ Test copied:          {test_out}/{cond}_test.csv')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T04:47:09.632787Z","iopub.execute_input":"2025-07-17T04:47:09.633029Z","iopub.status.idle":"2025-07-17T04:47:12.5001Z","shell.execute_reply.started":"2025-07-17T04:47:09.633012Z","shell.execute_reply":"2025-07-17T04:47:12.499527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────────────────────\n# Cell: Image Data Preparation for 5-Fold CV + Held-Out Test\n#  (with brightness & zoom augmentations added)\n# ─────────────────────────────────────────────────────────────────────────────\n\n# ─────────────────────────────────────────────────────────────────────────────\n# Cell: Image Data Preparation for 5-Fold CV + Held-Out Test\n#  (with brightness & zoom augmentations added, 128×128 → 224×224 crops)\n# ─────────────────────────────────────────────────────────────────────────────\n\nimport os\nimport pydicom\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image, ImageOps, ImageEnhance\nfrom pathlib import Path\n\nclass DataPreparationImage:\n    def __init__(\n        self,\n        dataset_directory: str,\n        condition: str,\n        csv_directory: str,\n        num_folds: int = 5,\n        augmentation_list=None,\n    ):\n        self.dataset_directory = Path(dataset_directory)\n        self.condition       = condition\n        self.csv_directory   = Path(csv_directory)\n        self.num_folds       = num_folds\n        self.augmentation_list = augmentation_list or [\n            'rotate','horizontal_flip','vertical_flip',\n            'gaussian_noise','brightness','zoom'\n        ]\n\n        print(f\"Starting image data prep for {self.condition} ({self.num_folds} folds + test)\")\n        self._create_folders()\n        self._process_all_folds()\n        self._process_test()\n\n    def _create_folders(self):\n        base = Path(f\"./{self.condition}\")\n        base.mkdir(exist_ok=True)\n        for fold in range(self.num_folds):\n            (base / f\"fold_{fold}\" / \"train\").mkdir(parents=True, exist_ok=True)\n            (base / f\"fold_{fold}\" / \"val\").mkdir(parents=True, exist_ok=True)\n        (base / \"test\").mkdir(parents=True, exist_ok=True)\n\n    def _read_csv(self, split: str, fold: int = None) -> pd.DataFrame:\n        if split in ('train','val'):\n            if fold is None:\n                raise ValueError(\"Must provide fold for train/val\")\n            suffix = 'augmented_train' if split=='train' else 'val'\n            path = self.csv_directory / self.condition / f\"fold_{fold}\" / f\"{self.condition}_{suffix}.csv\"\n        elif split == 'test':\n            path = self.csv_directory / self.condition / \"test\" / f\"{self.condition}_test.csv\"\n        else:\n            raise ValueError(f\"Unknown split: {split}\")\n        return pd.read_csv(path)\n\n    def _read_dicom(self, path: Path) -> np.ndarray:\n        ds = pydicom.dcmread(str(path))\n        img = ds.pixel_array.astype(float)\n        img = (img - img.min()) / (img.max() - img.min() + 1e-6) * 255.0\n        return np.stack([img]*3, axis=-1).astype('uint8')\n\n    def _crop(self, image: np.ndarray, x: float, y: float, box: int = 64) -> Image.Image:\n        \"\"\"\n        Crop a 128×128 window centered at (x,y) then resize to 224×224 (bicubic).\n        \"\"\"\n        img = Image.fromarray(image)\n        left, top   = int(x - box), int(y - box)\n        right, bottom = int(x + box), int(y + box)\n        patch = img.crop((left, top, right, bottom))\n        # resize up to model input size\n        return patch.resize((224, 224), resample=Image.BICUBIC)\n    def _apply_augmentation(self, img: Image.Image, aug: str) -> Image.Image:\n        # rotate/flips/noise unchanged\n        if aug == 'rotate':\n            return img.rotate(np.random.uniform(-20,20), expand=True)\n        if aug == 'horizontal_flip':\n            return ImageOps.mirror(img)\n        if aug == 'vertical_flip':\n            return ImageOps.flip(img)\n        if aug == 'gaussian_noise':\n            arr = np.array(img)\n            noise = np.random.normal(0,25,arr.shape)\n            return Image.fromarray(np.clip(arr+noise,0,255).astype('uint8'))\n        # new brightness augmentation\n        if aug == 'brightness':\n            enhancer = ImageEnhance.Brightness(img)\n            factor = np.random.uniform(0.7, 1.3)\n            return enhancer.enhance(factor)\n        # new zoom augmentation\n        if aug == 'zoom':\n            w, h = img.size\n            factor = np.random.uniform(1.0, 1.2)\n            new_w, new_h = int(w*factor), int(h*factor)\n            zoomed = img.resize((new_w, new_h), Image.BILINEAR)\n            left = (new_w - w)//2\n            top  = (new_h - h)//2\n            return zoomed.crop((left, top, left + w, top + h))\n        # fallback: no op\n        return img\n\n    def _process_all_folds(self):\n        for fold in range(self.num_folds):\n            print(f\" Processing fold {fold}…\")\n            # TRAIN (with augmentation)\n            df_train = self._read_csv('train', fold)\n            for _, row in df_train.iterrows():\n                sid, seid, inst = row['study_id'], row['series_id'], row['instance_number']\n                x, y            = row['x'], row['y']\n                aug_op          = row.get('augmentation')\n                dcm_path        = self.dataset_directory / str(sid) / str(seid) / f\"{inst}.dcm\"\n                img             = self._read_dicom(dcm_path)\n                patch           = self._crop(img, x, y)\n                if aug_op in self.augmentation_list:\n                    out_img = self._apply_augmentation(patch, aug_op)\n                    suffix  = f\"_{aug_op}\"\n                else:\n                    out_img = patch\n                    suffix  = \"\"\n                fname = f\"{sid}_{seid}_{inst}_{int(x)}_{int(y)}{suffix}.png\"\n                out_path = Path(self.condition) / f\"fold_{fold}\" / \"train\" / fname\n                out_img.save(out_path)\n\n            # VAL (no augmentation)\n            df_val = self._read_csv('val', fold)\n            for _, row in df_val.iterrows():\n                sid, seid, inst = row['study_id'], row['series_id'], row['instance_number']\n                x, y            = row['x'], row['y']\n                img = self._read_dicom(self.dataset_directory / str(sid) / str(seid) / f\"{inst}.dcm\")\n                patch = self._crop(img, x, y)\n                fname = f\"{sid}_{seid}_{inst}_{int(x)}_{int(y)}.png\"\n                out_path = Path(self.condition) / f\"fold_{fold}\" / \"val\" / fname\n                patch.save(out_path)\n\n    def _process_test(self):\n        print(\" Processing held-out test set…\")\n        df_test = self._read_csv('test')\n        for _, row in df_test.iterrows():\n            sid, seid, inst = row['study_id'], row['series_id'], row['instance_number']\n            x, y            = row['x'], row['y']\n            img = self._read_dicom(self.dataset_directory / str(sid) / str(seid) / f\"{inst}.dcm\")\n            patch = self._crop(img, x, y)\n            fname = f\"{sid}_{seid}_{inst}_{int(x)}_{int(y)}.png\"\n            out_path = Path(self.condition) / \"test\" / fname\n            patch.save(out_path)\n        print(f\" Saved {len(df_test)} test images to ./{self.condition}/test\")\n\n\n# ─────────────────────────────────────────────────────────────────────────────\n# Run for your condition\n# ─────────────────────────────────────────────────────────────────────────────\nDATASET_DIR = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images\"\nCSV_DIR     = \"/kaggle/working/augmented_output\"\n\nDataPreparationImage(\n    dataset_directory=DATASET_DIR,\n    condition=\"Spinal_Canal_Stenosis\",\n    csv_directory=CSV_DIR,\n    num_folds=5\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T04:47:12.500797Z","iopub.execute_input":"2025-07-17T04:47:12.50099Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom pathlib import Path\n\n# 1) Parameters\nCSV_DIR    = Path('/kaggle/working/augmented_output')   # where your augmented CSVs live\nCONDITIONS = [\n    'Spinal_Canal_Stenosis',\n    'Neural_Foraminal_Narrowing',\n    'Subarticular_Stenosis'\n]\nFOLDS      = list(range(5))  # five folds: 0–4\nOUT_ROOT   = Path('/kaggle/working')    # base for label output folders\n\n# 2) Helper: build 'subject' and 'label' columns\ndef working_on_csv(csv_path: Path, split: str) -> pd.DataFrame:\n    df = pd.read_csv(csv_path)\n\n    # ensure we always have an 'augmentation' column\n    if 'augmentation' not in df.columns:\n        df['augmentation'] = None\n\n    def make_subject(r):\n        name = f\"{r.study_id}_{r.series_id}_{r.instance_number}_{int(r.x)}_{int(r.y)}.png\"\n        # only append \"_augmented\" on training samples that were actually augmented\n        if split == 'train' and pd.notna(r.augmentation) and r.augmentation:\n            name = name.replace('.png', '_augmented.png')\n        return name\n\n    def make_label(score):\n        # if your original CSV already used 1,2,3 as numeric scores, just cast\n        if pd.api.types.is_numeric_dtype(type(score)):\n            return int(score)\n        # otherwise map from the text labels you used\n        return {\n            'Normal/Mild': 1,\n            'Moderate':    2,\n            'Severe':      3\n        }.get(score, None)\n\n    df['subject'] = df.apply(make_subject, axis=1)\n    df['label']   = df['score'].apply(make_label)\n\n    return df[['subject', 'label']]\n\n# 3) Create output folder structure (including test)\nfor cond in CONDITIONS:\n    base = OUT_ROOT / f\"{cond}_label\"\n    for fold in FOLDS:\n        (base / f\"fold_{fold}\").mkdir(parents=True, exist_ok=True)\n    (base / \"test\").mkdir(parents=True, exist_ok=True)\n\n# 4) Process each condition: train/val and then test\nfor cond in CONDITIONS:\n    label_root = OUT_ROOT / f\"{cond}_label\"\n\n    # train & val\n    for fold in FOLDS:\n        train_csv = CSV_DIR / cond / f\"fold_{fold}\" / f\"{cond}_augmented_train.csv\"\n        val_csv   = CSV_DIR / cond / f\"fold_{fold}\" / f\"{cond}_val.csv\"\n\n        train_labels = working_on_csv(train_csv, 'train')\n        val_labels   = working_on_csv(val_csv,   'val')\n\n        out_base = label_root / f\"fold_{fold}\"\n        train_labels.to_csv(out_base / f\"{cond}_augmented_labels.csv\", index=False)\n        val_labels.to_csv(  out_base / f\"{cond}_val_labels.csv\",       index=False)\n\n        print(f\"→ {cond} fold {fold}: train={len(train_labels)}, val={len(val_labels)}\")\n\n    # test\n    test_csv    = CSV_DIR / cond / \"test\" / f\"{cond}_test.csv\"\n    test_labels = working_on_csv(test_csv, 'test')\n    out_test    = label_root / \"test\"\n    test_labels.to_csv(out_test / f\"{cond}_test_labels.csv\", index=False)\n\n    print(f\"→ {cond} test: {len(test_labels)} rows → {out_test}/{cond}_test_labels.csv\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────────────────────\n# Cell: 5-Fold CV + Held-Out Test Evaluation (VGG16 strong tune)\n#         - GAP head, label smoothing, class weights\n#         - Two-phase fine-tune: head warm-up → unfreeze block5+block4\n#         - Robust callbacks on PR-AUC, ReduceLROnPlateau, EarlyStopping\n# ─────────────────────────────────────────────────────────────────────────────\n\nimport os\nimport gc\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.applications import VGG16\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.layers import (\n    GlobalAveragePooling2D, Dense, Dropout, BatchNormalization, Input\n)\nfrom tensorflow.keras.callbacks import (\n    EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n)\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.regularizers import l2\nfrom sklearn.metrics import (\n    accuracy_score, precision_score, recall_score, f1_score,\n    confusion_matrix, roc_curve, auc, average_precision_score\n)\nfrom sklearn.preprocessing import label_binarize\nimport matplotlib.pyplot as plt\n\n# ── Configuration ────────────────────────────────────────────────────────────\nCONDITION       = 'Spinal_Canal_Stenosis'\nFOLDS           = list(range(5))\nIMAGE_ROOT_TMPL = '/kaggle/working/Spinal_Canal_Stenosis/fold_{fold}'\nLABEL_ROOT_TMPL = '/kaggle/working/Spinal_Canal_Stenosis_label/fold_{fold}'\nTEST_IMAGE_DIR  = f'/kaggle/working/{CONDITION}/test'\nTEST_LABEL_CSV  = f'/kaggle/working/{CONDITION}_label/test/{CONDITION}_test_labels.csv'\nRESULTS_DIR     = Path(f'./results/{CONDITION}')\n\nBATCH_SIZE      = 16\nPATIENCE        = 5\nPHASE_A_EPOCHS  = 8        # warm-up head\nPHASE_B_EPOCHS  = 20       # fine-tune unfreezed blocks\nLR_A            = 3e-4\nLR_B            = 3e-5\nIMG_SIZE        = (224, 224)\nL2_WEIGHT       = 1e-4\nDROP_RATE1      = 0.5\nDROP_RATE2      = 0.4\nLABEL_SMOOTH    = 0.05\nNUM_CLASSES     = 3\nUNFREEZE_BLOCKS = (\"block5\", \"block4\")  # progressive unfreezing\n\n# ── Metrics (Keras) ─────────────────────────────────────────────────────────\ndef make_metrics():\n    return [\n        tf.keras.metrics.AUC(name=\"auc_roc\", curve=\"ROC\", multi_label=True, num_labels=NUM_CLASSES),\n        tf.keras.metrics.AUC(name=\"auc_pr\",  curve=\"PR\",  multi_label=True, num_labels=NUM_CLASSES),\n        \"accuracy\",\n    ]\n\n# ── Loss ─────────────────────────────────────────────────────────────────────\nloss_ce = tf.keras.losses.CategoricalCrossentropy(label_smoothing=LABEL_SMOOTH)\n\n# ── Augmentation ─────────────────────────────────────────────────────────────\ntrain_datagen = ImageDataGenerator(\n    rescale=1./255,\n    rotation_range=5,\n    width_shift_range=0.05,\n    height_shift_range=0.05,\n    zoom_range=0.10,\n    horizontal_flip=True,\n    fill_mode='nearest'\n)\nval_datagen = ImageDataGenerator(rescale=1./255)\n\n# ── Helpers ──────────────────────────────────────────────────────────────────\ndef compute_class_weights(class_indices, num_classes=NUM_CLASSES):\n    \"\"\"Inverse-frequency weights normalized to mean=1.\"\"\"\n    counts = np.bincount(class_indices, minlength=num_classes).astype(np.float32)\n    inv = 1.0 / np.maximum(counts, 1.0)\n    inv /= inv.mean()\n    return {i: float(inv[i]) for i in range(num_classes)}\n\ndef build_vgg16_model():\n    base = VGG16(include_top=False, weights='imagenet', input_shape=IMG_SIZE+(3,))\n    base.trainable = False  # Phase A: freeze all\n\n    inp = Input(shape=IMG_SIZE+(3,))\n    x = base(inp, training=False)\n    x = GlobalAveragePooling2D()(x)\n    x = Dropout(DROP_RATE1)(x)\n    x = Dense(512, activation='relu', kernel_regularizer=l2(L2_WEIGHT))(x)\n    x = BatchNormalization()(x)\n    x = Dropout(DROP_RATE2)(x)\n    out = Dense(NUM_CLASSES, activation='softmax')(x)\n\n    model = Model(inputs=inp, outputs=out)\n    return model, base\n\ndef compile_with(model, lr):\n    opt = tf.keras.optimizers.AdamW(learning_rate=lr, weight_decay=L2_WEIGHT)\n    model.compile(optimizer=opt, loss=loss_ce, metrics=make_metrics())\n\ndef make_callbacks(out_dir):\n    ckpt = ModelCheckpoint(out_dir/'best_model.keras',\n                           monitor='val_auc_pr', mode='max',\n                           save_best_only=True, verbose=1)\n    es  = EarlyStopping(monitor='val_auc_pr', mode='max',\n                        patience=PATIENCE, restore_best_weights=True, verbose=1)\n    rl  = ReduceLROnPlateau(monitor='val_auc_pr', mode='max',\n                            factor=0.5, patience=2, min_lr=1e-6, verbose=1)\n    return [ckpt, rl, es]\n\nfold_results = []\n\nfor fold in FOLDS:\n    print(f\"\\n▶ Training fold {fold}\")\n    IMAGE_ROOT = IMAGE_ROOT_TMPL.format(fold=fold)\n    LABEL_ROOT = LABEL_ROOT_TMPL.format(fold=fold)\n\n    # — Load labels\n    train_df = pd.read_csv(Path(LABEL_ROOT)/f'{CONDITION}_augmented_labels.csv')\n    val_df   = pd.read_csv(Path(LABEL_ROOT)/f'{CONDITION}_val_labels.csv')\n    for df in (train_df, val_df):\n        df['subject'] = df['subject'].str.replace('_augmented','',regex=False)\n        df['label']   = (df['label'].astype(int) - 1).astype(str)\n\n    # — Generators (train with aug, val only rescale)\n    train_gen = train_datagen.flow_from_dataframe(\n        train_df,\n        directory=os.path.join(IMAGE_ROOT,'train'),\n        x_col='subject', y_col='label',\n        target_size=IMG_SIZE, batch_size=BATCH_SIZE,\n        class_mode='categorical', shuffle=True, seed=42\n    )\n    val_gen = val_datagen.flow_from_dataframe(\n        val_df,\n        directory=os.path.join(IMAGE_ROOT,'val'),\n        x_col='subject', y_col='label',\n        target_size=IMG_SIZE, batch_size=BATCH_SIZE,\n        class_mode='categorical', shuffle=False\n    )\n\n    # — Class weights (from training set distribution)\n    class_weights = compute_class_weights(train_gen.classes, NUM_CLASSES)\n    print(\"Class weights:\", class_weights)\n\n    # — Build & compile model\n    model, base = build_vgg16_model()\n    out_dir = RESULTS_DIR / f'fold_{fold}'\n    out_dir.mkdir(parents=True, exist_ok=True)\n\n    # — Phase A: warm-up head (base frozen)\n    compile_with(model, LR_A)\n    cbs = make_callbacks(out_dir)\n    history_a = model.fit(\n        train_gen,\n        validation_data=val_gen,\n        epochs=PHASE_A_EPOCHS,\n        class_weight=class_weights,\n        callbacks=cbs,\n        verbose=2\n    )\n    with open(out_dir/'history_phaseA.json','w') as f:\n        json.dump(history_a.history, f)\n\n    # — Phase B: progressive unfreezing (block5 + block4)\n    for layer in base.layers:\n        layer.trainable = False\n    for layer in base.layers:\n        if any(layer.name.startswith(b) for b in UNFREEZE_BLOCKS):\n            layer.trainable = True\n\n    compile_with(model, LR_B)\n    history_b = model.fit(\n        train_gen,\n        validation_data=val_gen,\n        epochs=PHASE_B_EPOCHS,\n        class_weight=class_weights,\n        callbacks=cbs,\n        verbose=2\n    )\n    with open(out_dir/'history_phaseB.json','w') as f:\n        json.dump(history_b.history, f)\n\n    # — Evaluate on validation\n    val_gen.reset()\n    probs_val = model.predict(val_gen, verbose=0)\n    y_pred = np.argmax(probs_val, axis=1)\n    y_true = val_gen.classes\n\n    # Weighted + Macro metrics\n    fm = {\n        'fold': fold,\n        'Accuracy_weighted':  accuracy_score(y_true, y_pred),\n        'Precision_weighted': precision_score(y_true, y_pred, average='weighted', zero_division=0),\n        'Recall_weighted':    recall_score(y_true, y_pred, average='weighted', zero_division=0),\n        'F1_weighted':        f1_score(y_true, y_pred, average='weighted', zero_division=0),\n        'Precision_macro':    precision_score(y_true, y_pred, average='macro', zero_division=0),\n        'Recall_macro':       recall_score(y_true, y_pred, average='macro', zero_division=0),\n        'F1_macro':           f1_score(y_true, y_pred, average='macro', zero_division=0),\n    }\n    # Macro ROC-AUC & PR-AUC on val\n    y_true_bin = label_binarize(y_true, classes=[0,1,2])\n    roc_aucs, pr_aucs = [], []\n    for i in range(NUM_CLASSES):\n        fpr, tpr, _ = roc_curve(y_true_bin[:, i], probs_val[:, i])\n        roc_aucs.append(auc(fpr, tpr))\n        pr_aucs.append(average_precision_score(y_true_bin[:, i], probs_val[:, i]))\n    fm['ROC_AUC_macro'] = float(np.mean(roc_aucs))\n    fm['PR_AUC_macro']  = float(np.mean(pr_aucs))\n\n    fold_results.append(fm)\n    with open(out_dir/'metrics_val.json','w') as f:\n        json.dump(fm, f, indent=2)\n    print(f\"✅ Fold {fold} metrics:\", fm)\n\n    tf.keras.backend.clear_session()\n    gc.collect()\n\n# ── Pick best fold by Macro F1 & evaluate held-out test ───────────────────────\nbest = max(fold_results, key=lambda x: x['F1_macro'])\nbest_fold = best['fold']\nprint(f\"\\n▶ Best fold selected: {best_fold} (Macro F1 = {best['F1_macro']:.4f})\")\n\nprint(\"▶ Evaluating held-out test set…\")\n# No custom_objects needed (using standard CE loss)\nmodel = load_model(RESULTS_DIR/f'fold_{best_fold}'/'best_model.keras')\n\ntest_df = pd.read_csv(TEST_LABEL_CSV)\ntest_df['label'] = (test_df['label'].astype(int)-1).astype(str)\ntest_gen = val_datagen.flow_from_dataframe(\n    test_df,\n    directory=TEST_IMAGE_DIR,\n    x_col='subject', y_col='label',\n    target_size=IMG_SIZE, batch_size=BATCH_SIZE,\n    class_mode='categorical', shuffle=False\n)\n\nprobs  = model.predict(test_gen, verbose=1)\ny_pred = np.argmax(probs, axis=1)\ny_true = test_gen.classes\ny_true_bin = label_binarize(y_true, classes=[0,1,2])\n\n# Weighted + Macro metrics on test\nacc_test   = accuracy_score(y_true, y_pred)\nprec_w     = precision_score(y_true, y_pred, average='weighted', zero_division=0)\nrec_w      = recall_score(y_true, y_pred, average='weighted', zero_division=0)\nf1_w       = f1_score(y_true, y_pred, average='weighted', zero_division=0)\nprec_m     = precision_score(y_true, y_pred, average='macro', zero_division=0)\nrec_m      = recall_score(y_true, y_pred, average='macro', zero_division=0)\nf1_m       = f1_score(y_true, y_pred, average='macro', zero_division=0)\n\n# Per-class ROC-AUC & PR-AUC\nroc_lines, pr_lines = [], []\nfor i in range(NUM_CLASSES):\n    fpr, tpr, _ = roc_curve(y_true_bin[:, i], probs[:, i])\n    roc_auc_i = auc(fpr, tpr)\n    ap_i      = average_precision_score(y_true_bin[:, i], probs[:, i])\n    roc_lines.append(f\"Class {i} ROC-AUC: {roc_auc_i:.4f}\")\n    pr_lines.append(f\"Class {i} PR-AUC (AP): {ap_i:.4f}\")\n\ncm = confusion_matrix(y_true, y_pred)\n\ntest_out = RESULTS_DIR / 'test'\ntest_out.mkdir(exist_ok=True)\nwith open(test_out/'metrics_test.txt','w') as f:\n    f.write(\n        \"Weighted:\\n\"\n        f\"  Accuracy:  {acc_test:.4f}\\n\"\n        f\"  Precision: {prec_w:.4f}\\n\"\n        f\"  Recall:    {rec_w:.4f}\\n\"\n        f\"  F1:        {f1_w:.4f}\\n\\n\"\n        \"Macro:\\n\"\n        f\"  Precision: {prec_m:.4f}\\n\"\n        f\"  Recall:    {rec_m:.4f}\\n\"\n        f\"  F1:        {f1_m:.4f}\\n\"\n    )\nwith open(test_out/'roc_auc.txt','w') as f:\n    f.write(\"\\n\".join(roc_lines) + \"\\n\")\nwith open(test_out/'pr_auc.txt','w') as f:\n    f.write(\"\\n\".join(pr_lines) + \"\\n\")\n\nnp.savetxt(test_out/'confusion_matrix.csv', cm, delimiter=',', fmt='%d')\n\n# ROC Curves\nplt.figure()\nfor i in range(NUM_CLASSES):\n    fpr, tpr, _ = roc_curve(y_true_bin[:,i], probs[:,i])\n    plt.plot(fpr, tpr, label=f'Class {i} (AUC={auc(fpr,tpr):.2f})')\nplt.plot([0,1],[0,1],'k--')\nplt.xlabel('FPR'); plt.ylabel('TPR')\nplt.title('ROC Curves on Held-Out Test Set')\nplt.legend(loc='lower right')\nplt.savefig(test_out/'roc_curves.png')\nplt.close()\n\n# PR Curves\nplt.figure()\nfor i in range(NUM_CLASSES):\n    # precision-recall curve via sklearn\n    # (Use average_precision_score for area; plot points using sklearn if desired)\n    # Simplified plotting using thresholds from roc_curve's probabilities is omitted for brevity.\n    pass\nplt.close()\n\nprint(f\"✅ Test evaluation saved to {test_out}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────────────────────\n# Cell: Summary for CV & Held-Out Test – Loss/PR Curves, Avg Metrics, Confusion & ROC/PR\n#   (aligned with tuned VGG16 training cell: CE+label smoothing, PR-AUC monitor,\n#    history_phaseA.json / history_phaseB.json, best fold by macro F1)\n# ─────────────────────────────────────────────────────────────────────────────\n\nimport json, os, gc\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\n\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import load_model\n\nfrom sklearn.metrics import (\n    accuracy_score, precision_score, recall_score, f1_score,\n    confusion_matrix, ConfusionMatrixDisplay,\n    roc_curve, auc, average_precision_score, precision_recall_curve\n)\nfrom sklearn.preprocessing import label_binarize\n\n# ── Config ────────────────────────────────────────────────────────────────────\nCONDITION      = 'Spinal_Canal_Stenosis'\nRESULTS_DIR    = Path(f'./results/{CONDITION}')\nLABEL_DIR      = Path(f'/kaggle/working/{CONDITION}_label')\nIMG_DIR        = Path(f'/kaggle/working/{CONDITION}')\nTEST_IMAGE_DIR = IMG_DIR / 'test'\nTEST_LABEL_CSV = LABEL_DIR / 'test' / f'{CONDITION}_test_labels.csv'\n\nBATCH_SIZE = 16\nIMG_SIZE   = (224, 224)\nNUM_CLASSES = 3\n\n# ── Find all fold directories ─────────────────────────────────────────────────\nfold_dirs = sorted(RESULTS_DIR.glob('fold_*'),\n                   key=lambda p: int(p.name.split('_')[1]))\nassert len(fold_dirs) > 0, \"No fold_* directories found under RESULTS_DIR.\"\n\n# ── Containers ────────────────────────────────────────────────────────────────\ntrain_metrics, val_metrics = [], []\nloss_curves, vloss_curves = {}, {}\nvpr_curves = {}   # per-epoch val PR-AUC if available\nvroc_curves = {}  # per-epoch val ROC-AUC if available\n\nval_macro_auc = []  # per-fold macro ROC-AUC (computed from predictions)\nval_macro_ap  = []  # per-fold macro PR-AUC (computed from predictions)\n\n# ── Generators (rescale only) ────────────────────────────────────────────────\nval_datagen  = ImageDataGenerator(rescale=1./255)\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\n# ── Loop over folds to collect histories & metrics ────────────────────────────\nfor fd in fold_dirs:\n    fold = int(fd.name.split('_')[1])\n\n    # --- load histories (phaseA + phaseB or fallback to single history.json)\n    hist_a_path = fd / 'history_phaseA.json'\n    hist_b_path = fd / 'history_phaseB.json'\n    if hist_a_path.exists() and hist_b_path.exists():\n        hist_a = json.load(open(hist_a_path))\n        hist_b = json.load(open(hist_b_path))\n        loss   = list(hist_a.get('loss', []))    + list(hist_b.get('loss', []))\n        vloss  = list(hist_a.get('val_loss', []))+ list(hist_b.get('val_loss', []))\n        vpr    = list(hist_a.get('val_auc_pr', []))  + list(hist_b.get('val_auc_pr', []))\n        vroc   = list(hist_a.get('val_auc_roc', [])) + list(hist_b.get('val_auc_roc', []))\n    else:\n        # backward-compatible fallback\n        hist = json.load(open(fd/'history.json'))\n        loss  = hist.get('loss', [])\n        vloss = hist.get('val_loss', [])\n        vpr   = hist.get('val_auc_pr', [])\n        vroc  = hist.get('val_auc_roc', [])\n\n    loss_curves[fold] = loss\n    vloss_curves[fold] = vloss\n    if vpr:  vpr_curves[fold]  = vpr\n    if vroc: vroc_curves[fold] = vroc\n\n    # --- load best model (no custom_objects needed)\n    model = load_model(fd/'best_model.keras')\n\n    # --- build train & val generators for metrics (no shuffling)\n    train_csv = LABEL_DIR/f'fold_{fold}'/f'{CONDITION}_augmented_labels.csv'\n    df_train  = pd.read_csv(train_csv)\n    df_train['subject'] = df_train['subject'].str.replace('_augmented', '', regex=False)\n    df_train['label']   = (df_train['label'].astype(int) - 1).astype(str)\n    train_gen = val_datagen.flow_from_dataframe(\n        df_train,\n        directory=IMG_DIR/f'fold_{fold}'/'train',\n        x_col='subject', y_col='label',\n        target_size=IMG_SIZE, batch_size=BATCH_SIZE,\n        class_mode='categorical', shuffle=False\n    )\n\n    val_csv = LABEL_DIR/f'fold_{fold}'/f'{CONDITION}_val_labels.csv'\n    df_val  = pd.read_csv(val_csv)\n    df_val['subject'] = df_val['subject'].str.replace('_augmented','', regex=False)\n    df_val['label']   = (df_val['label'].astype(int) - 1).astype(str)\n    val_gen = val_datagen.flow_from_dataframe(\n        df_val,\n        directory=IMG_DIR/f'fold_{fold}'/'val',\n        x_col='subject', y_col='label',\n        target_size=IMG_SIZE, batch_size=BATCH_SIZE,\n        class_mode='categorical', shuffle=False\n    )\n\n    # --- predictions\n    probs_train = model.predict(train_gen, verbose=0)\n    y_train_pred = np.argmax(probs_train, axis=1)\n    y_train_true = train_gen.classes\n\n    probs_val = model.predict(val_gen, verbose=0)\n    y_val_pred = np.argmax(probs_val, axis=1)\n    y_val_true = val_gen.classes\n\n    # --- metrics: weighted + macro\n    train_metrics.append({\n        'Accuracy':  accuracy_score(y_train_true, y_train_pred),\n        'Precision': precision_score(y_train_true, y_train_pred, average='weighted', zero_division=0),\n        'Recall':    recall_score(y_train_true, y_train_pred, average='weighted', zero_division=0),\n        'F1 Score':  f1_score(y_train_true, y_train_pred, average='weighted', zero_division=0),\n        'F1 Macro':  f1_score(y_train_true, y_train_pred, average='macro', zero_division=0),\n    })\n    val_metrics.append({\n        'Accuracy':  accuracy_score(y_val_true, y_val_pred),\n        'Precision': precision_score(y_val_true, y_val_pred, average='weighted', zero_division=0),\n        'Recall':    recall_score(y_val_true, y_val_pred, average='weighted', zero_division=0),\n        'F1 Score':  f1_score(y_val_true, y_val_pred, average='weighted', zero_division=0),\n        'F1 Macro':  f1_score(y_val_true, y_val_pred, average='macro', zero_division=0),\n    })\n\n    # --- macro ROC-AUC & PR-AUC on validation (from probs)\n    y_true_bin = label_binarize(y_val_true, classes=list(range(NUM_CLASSES)))\n    fold_roc_aucs, fold_pr_aucs = [], []\n    for i in range(NUM_CLASSES):\n        fpr, tpr, _ = roc_curve(y_true_bin[:, i], probs_val[:, i])\n        fold_roc_aucs.append(auc(fpr, tpr))\n        fold_pr_aucs.append(average_precision_score(y_true_bin[:, i], probs_val[:, i]))\n    val_macro_auc.append(np.mean(fold_roc_aucs))\n    val_macro_ap.append(np.mean(fold_pr_aucs))\n\n    tf.keras.backend.clear_session()\n    gc.collect()\n\n# ── 1) Plot train/val loss curves per fold ────────────────────────────────────\nplt.figure(figsize=(9,5))\nfor fold in sorted(loss_curves):\n    epochs = range(1, len(loss_curves[fold]) + 1)\n    plt.plot(epochs, loss_curves[fold],     label=f'Fold {fold} Train', alpha=0.7)\n    plt.plot(epochs, vloss_curves[fold], '--', label=f'Fold {fold} Val',   alpha=0.7)\nplt.xlabel('Epoch'); plt.ylabel('Loss')\nplt.title('Train & Val Loss Curves (All Folds)')\nplt.legend(ncol=2, fontsize='small')\nplt.tight_layout()\nplt.show()\n\n# (Optional) plot val PR-AUC curves if logged\nif len(vpr_curves) > 0:\n    plt.figure(figsize=(9,5))\n    for fold in sorted(vpr_curves):\n        plt.plot(range(1, len(vpr_curves[fold])+1), vpr_curves[fold], label=f'Fold {fold}')\n    plt.xlabel('Epoch'); plt.ylabel('Val PR-AUC')\n    plt.title('Validation PR-AUC across epochs')\n    plt.legend(ncol=2, fontsize='small')\n    plt.tight_layout()\n    plt.show()\n\n# ── 2) Average CV metrics (weighted + macro + AUCs) ──────────────────────────\ndef avg(dicts):\n    keys = dicts[0].keys()\n    return {k: float(np.mean([d[k] for d in dicts])) for k in keys}\n\navg_train = avg(train_metrics)\navg_val   = avg(val_metrics)\n\nprint(\"➡️  Average CV TRAIN metrics:\")\nfor k,v in avg_train.items():\n    print(f\"   {k}: {v:.3f}\")\nprint(\"➡️  Average CV   VAL metrics:\")\nfor k,v in avg_val.items():\n    print(f\"   {k}: {v:.3f}\")\nprint(f\"➡️  Average CV   VAL ROC-AUC (macro): {np.mean(val_macro_auc):.3f}\")\nprint(f\"➡️  Average CV   VAL PR-AUC  (macro): {np.mean(val_macro_ap):.3f}\")\n\n# ── 3) Evaluate held-out test on best fold (by Val Macro F1) ─────────────────\nbest_idx  = int(np.argmax([m['F1 Macro'] for m in val_metrics]))\nbest_fold = int(fold_dirs[best_idx].name.split('_')[1])\nprint(f\"\\n✨ Best fold = {best_fold} (Val Macro F1 = {val_metrics[best_idx]['F1 Macro']:.4f})\")\n\nmodel = load_model(RESULTS_DIR/f'fold_{best_fold}'/'best_model.keras')\n\ntest_df = pd.read_csv(TEST_LABEL_CSV)\ntest_df['label'] = (test_df['label'].astype(int)-1).astype(str)\ntest_gen = test_datagen.flow_from_dataframe(\n    test_df,\n    directory=TEST_IMAGE_DIR,\n    x_col='subject', y_col='label',\n    target_size=IMG_SIZE, batch_size=BATCH_SIZE,\n    class_mode='categorical', shuffle=False\n)\n\nprobs  = model.predict(test_gen, verbose=0)\ny_test = test_gen.classes\ny_pred = np.argmax(probs, axis=1)\n\n# Weighted + Macro metrics on test\nacc_w  = accuracy_score(y_test, y_pred)\nprec_w = precision_score(y_test, y_pred, average='weighted', zero_division=0)\nrec_w  = recall_score(y_test, y_pred, average='weighted', zero_division=0)\nf1_w   = f1_score(y_test, y_pred, average='weighted', zero_division=0)\nprec_m = precision_score(y_test, y_pred, average='macro', zero_division=0)\nrec_m  = recall_score(y_test, y_pred, average='macro', zero_division=0)\nf1_m   = f1_score(y_test, y_pred, average='macro', zero_division=0)\n\nprint(\"\\n➡️ Held-out TEST metrics:\")\nprint(f\"   Weighted  - Acc: {acc_w:.3f}  Prec: {prec_w:.3f}  Rec: {rec_w:.3f}  F1: {f1_w:.3f}\")\nprint(f\"   Macro     - Prec: {prec_m:.3f} Rec: {rec_m:.3f}  F1: {f1_m:.3f}\")\n\n# ── 4) Confusion Matrix ───────────────────────────────────────────────────────\ncm = confusion_matrix(y_test, y_pred)\ndisp = ConfusionMatrixDisplay(cm, display_labels=list(test_gen.class_indices.keys()))\nplt.figure(figsize=(4.2,4))\ndisp.plot(ax=plt.gca(), cmap='Blues', colorbar=False)\nplt.title('Held-out Test Confusion')\nplt.tight_layout()\nplt.show()\n\n# ── 5) ROC & PR Curves with per-class AUC/AP ─────────────────────────────────\ny_bin = label_binarize(y_test, classes=list(range(NUM_CLASSES)))\n\n# ROC\nplt.figure(figsize=(6.2,5))\nfor i, label in enumerate(test_gen.class_indices):\n    fpr, tpr, _ = roc_curve(y_bin[:, i], probs[:, i])\n    plt.plot(fpr, tpr, label=f\"{label} (AUC={auc(fpr,tpr):.2f})\")\nplt.plot([0,1],[0,1],'k--', linewidth=1)\nplt.xlabel('FPR'); plt.ylabel('TPR')\nplt.title('Held-out Test ROC Curves')\nplt.legend(loc='lower right', fontsize='small')\nplt.tight_layout()\nplt.show()\n\n# PR\nplt.figure(figsize=(6.2,5))\nfor i, label in enumerate(test_gen.class_indices):\n    precision, recall, _ = precision_recall_curve(y_bin[:, i], probs[:, i])\n    ap = average_precision_score(y_bin[:, i], probs[:, i])\n    plt.plot(recall, precision, label=f\"{label} (AP={ap:.2f})\")\nplt.xlabel('Recall'); plt.ylabel('Precision')\nplt.title('Held-out Test PR Curves')\nplt.legend(loc='lower left', fontsize='small')\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────────────────────\n# Cell: Per-Class Precision / Recall / F1 on Held-Out Test Set (fixed)\n# ─────────────────────────────────────────────────────────────────────────────\n\nimport pandas as pd\nfrom sklearn.metrics import precision_recall_fscore_support, classification_report\nfrom pathlib import Path\n\n# invert class_indices: idx -> name\ninv_map = {v: k for k, v in test_gen.class_indices.items()}\nlabels = sorted(inv_map)                 # e.g., [0,1,2]\nclass_names = [inv_map[i] for i in labels]\n\n# Use y_test from the previous cell (not y_true)\nprecisions, recalls, f1s, supports = precision_recall_fscore_support(\n    y_test, y_pred, labels=labels, zero_division=0\n)\n\ndf_metrics = pd.DataFrame({\n    'Class':     class_names,\n    'LabelIdx':  labels,\n    'Support':   supports,\n    'Precision': precisions,\n    'Recall':    recalls,\n    'F1 Score':  f1s\n}).sort_values('LabelIdx')\n\nprint(df_metrics.to_string(index=False))\n\nprint(\"\\nFull classification report:\\n\")\nprint(classification_report(\n    y_test, y_pred, labels=labels, target_names=class_names, zero_division=0\n))\n\n# (optional) save alongside other test artifacts\nout_dir = Path(f'./results/{CONDITION}/test')\nout_dir.mkdir(parents=True, exist_ok=True)\ndf_metrics.to_csv(out_dir / 'per_class_metrics.csv', index=False)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}