{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Cell 1: Install all dependencies once\n%pip install ultralytics==8.0.111 torch torchvision pandas numpy pydicom albumentations scikit-learn tqdm\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T09:41:45.810241Z","iopub.execute_input":"2025-06-07T09:41:45.810496Z","iopub.status.idle":"2025-06-07T09:41:49.180414Z","shell.execute_reply.started":"2025-06-07T09:41:45.810478Z","shell.execute_reply":"2025-06-07T09:41:49.179655Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Point to the RSNA CSVs in your Kaggle dataset\nData_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification'\n\nimport os\nimport pandas as pd\n\n# 1) Read train_label_coordinates.csv and train_series_descriptions.csv\ntrain_label_coordinates = pd.read_csv(os.path.join(Data_path, 'train_label_coordinates.csv'))\ntrain_series_descriptions = pd.read_csv(os.path.join(Data_path, 'train_series_descriptions.csv'))\n\n# 2) Merge on ['study_id', 'series_id'] to pull in series_description\nmerged_csv = pd.merge(\n    train_label_coordinates,\n    train_series_descriptions[['study_id', 'series_id', 'series_description']],\n    on=['study_id', 'series_id'],\n    how='left'\n)\n\n# 3) Read train.csv (holds the severity scores)\ntrain_df = pd.read_csv(os.path.join(Data_path, 'train.csv'))\n\n# 4) Helper to look up the correct column (e.g. 'neural_foraminal_narrowing_l1_l2') and fetch its value\ndef get_score(row):\n    study_id = row['study_id']\n    condition = row['condition']\n    level = row['level']  # like \"L1/L2\" or \"R3/R4\"\n\n    # Split \"L1/L2\" → \"L1\", \"L2\" so we can build a column name same as in train.csv\n    level_1, level_2 = level.split('/')\n    condition_level = f\"{condition}_{level_1}_{level_2}\".replace(' ', '_').lower()\n    # e.g., \"neural_foraminal_narrowing_l1_l2\"\n\n    if condition_level in train_df.columns and study_id in train_df['study_id'].values:\n        return train_df.loc[train_df['study_id'] == study_id, condition_level].values[0]\n    else:\n        return None\n\n# 5) Apply it to every row\nmerged_csv['score'] = merged_csv.apply(get_score, axis=1)\n\n# 6) Save the merged result into /kaggle/working so downstream steps can consume it\nout_path = '/kaggle/working/dataset_description.csv'\nmerged_csv.to_csv(out_path, index=False)\nprint(f\"✅ Wrote merged CSV with scores to: {out_path}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T09:41:49.181293Z","iopub.execute_input":"2025-06-07T09:41:49.181542Z","iopub.status.idle":"2025-06-07T09:42:02.484104Z","shell.execute_reply.started":"2025-06-07T09:41:49.181525Z","shell.execute_reply":"2025-06-07T09:42:02.483119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\n\n# 1) Path to the merged file you created earlier\ninput_csv = '/kaggle/working/dataset_description.csv'\n\n# 2) Load the full dataset_description.csv\ndf = pd.read_csv(input_csv)\n\n# 3) Define which 'condition' values belong to each output group\ncondition_groups = {\n    'Spinal Canal Stenosis': ['Spinal Canal Stenosis'],\n    'Neural Foraminal Narrowing': ['Right Neural Foraminal Narrowing', 'Left Neural Foraminal Narrowing'],\n    'Subarticular Stenosis': ['Right Subarticular Stenosis', 'Left Subarticular Stenosis']\n}\n\n# 4) For each group, filter and save a separate CSV\nfor group_name, conditions in condition_groups.items():\n    filtered_df = df[df['condition'].isin(conditions)].copy()\n    # Make a filesystem‐friendly name, e.g. \"Spinal_Canal_Stenosis.csv\"\n    out_name = group_name.replace(' ', '_') + '.csv'\n    out_path = os.path.join('/kaggle/working', out_name)\n    filtered_df.to_csv(out_path, index=False)\n    print(f\"→ Wrote {len(filtered_df)} rows to {out_name}\")\n\nprint(\"✅ Done splitting into three CSVs.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T09:42:02.485192Z","iopub.execute_input":"2025-06-07T09:42:02.485773Z","iopub.status.idle":"2025-06-07T09:42:02.972506Z","shell.execute_reply.started":"2025-06-07T09:42:02.485739Z","shell.execute_reply":"2025-06-07T09:42:02.971673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\n\ndef cross_validation_2fold(csv_path, output_name):\n    \"\"\"\n    Reads a condition‐specific CSV, creates 'class_id' from condition+level,\n    performs a 2‐fold stratified split, and writes out a new CSV with a 'fold' column.\n    \"\"\"\n    df = pd.read_csv(csv_path)\n\n    # Create a combined “condition_level” string\n    df['condition_level'] = df['condition'] + '_' + df['level']\n\n    # Convert to numeric class IDs for stratification\n    df['class_id'] = df['condition_level'].astype('category').cat.codes\n\n    # Use 2 splits instead of 5\n    skf = StratifiedKFold(n_splits=2, shuffle=True, random_state=42)\n\n    df['fold'] = -1\n    for fold_number, (_, val_idx) in enumerate(skf.split(df, df['class_id'])):\n        df.loc[val_idx, 'fold'] = fold_number\n\n    output_path = os.path.join('/kaggle/working', f\"{output_name}.csv\")\n    df.to_csv(output_path, index=False)\n    print(f\"Saved 2‐fold CSV to: {output_path}\")\n\n\n# Paths to the condition‐specific CSVs (from the “split by condition” step)\nspinal_csv = '/kaggle/working/Spinal_Canal_Stenosis.csv'\nneural_csv = '/kaggle/working/Neural_Foraminal_Narrowing.csv'\nsubart_csv = '/kaggle/working/Subarticular_Stenosis.csv'\n\n# Run 2‐fold stratified splitting for each condition\ncross_validation_2fold(spinal_csv, 'Spinal_Canal_Stenosis_2folds')\ncross_validation_2fold(neural_csv, 'Neural_Foraminal_Narrowing_2folds')\ncross_validation_2fold(subart_csv, 'Subarticular_Stenosis_2folds')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-07T09:42:43.135339Z","iopub.execute_input":"2025-06-07T09:42:43.135613Z","iopub.status.idle":"2025-06-07T09:42:44.272865Z","shell.execute_reply.started":"2025-06-07T09:42:43.135592Z","shell.execute_reply":"2025-06-07T09:42:44.271947Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nprint(os.listdir('/kaggle/working'))\n# You should see:\n# [\n#   'dataset_description.csv',\n#   'Spinal_Canal_Stenosis.csv',\n#   'Neural_Foraminal_Narrowing.csv',\n#   'Subarticular_Stenosis.csv',\n#   'Spinal_Canal_Stenosis_folds.csv',\n#   'Neural_Foraminal_Narrowing_folds.csv',\n#   'Subarticular_Stenosis_folds.csv',\n#   … (any other files)\n# ]\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport yaml\nimport csv\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nimport pydicom\n\n\nclass DetectorDataPreparation:\n    def __init__(\n        self,\n        dataset_directory='/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images',\n        csv_path='',                     # e.g. '/kaggle/working/Spinal_Canal_Stenosis_2folds.csv'\n        condition_level_classes={},\n        condition_name='',\n        val_fold=0,                      # will be 0 or 1\n        width_box=16,\n    ):\n        self.dataset_directory = dataset_directory\n        self.csv_path = csv_path\n        self.condition_level_classes = condition_level_classes\n        self.condition_name = condition_name\n        self.val_fold = val_fold\n        self.width_box = width_box\n\n        self.save_directory = Path(f'/kaggle/working/{self.condition_name}')\n        self._create_folders()\n        self._read_cross_validation()\n        self._dicom_to_png(self.training_data, self.train_image_path)\n        self._save_height_width_csv()\n        self._create_yolo_labels(self.training_data, self.train_labels_path)\n        self._dicom_to_png(self.validation_data, self.val_images_path)\n        self._save_height_width_csv()\n        self._create_yolo_labels(self.validation_data, self.val_labels_path)\n        self._create_yaml_file()\n\n\n    def _create_folders(self):\n        base = self.save_directory\n        base.mkdir(parents=True, exist_ok=True)\n        self.fold_path = base / f'fold_{self.val_fold}'\n        self.fold_path.mkdir(parents=True, exist_ok=True)\n        self.dataset_path = self.fold_path / 'datasets'\n        self.dataset_path.mkdir(parents=True, exist_ok=True)\n\n        (self.dataset_path / 'train' / 'images').mkdir(parents=True, exist_ok=True)\n        (self.dataset_path / 'train' / 'labels').mkdir(parents=True, exist_ok=True)\n        (self.dataset_path / 'val' / 'images').mkdir(parents=True, exist_ok=True)\n        (self.dataset_path / 'val' / 'labels').mkdir(parents=True, exist_ok=True)\n\n        self.train_image_path = self.dataset_path / 'train' / 'images'\n        self.train_labels_path = self.dataset_path / 'train' / 'labels'\n        self.val_images_path   = self.dataset_path / 'val' / 'images'\n        self.val_labels_path   = self.dataset_path / 'val' / 'labels'\n\n\n    def _read_cross_validation(self):\n        df = pd.read_csv(self.csv_path)\n        self.validation_data = df[df['fold'] == self.val_fold].reset_index(drop=True)\n        print(f\"[{self.condition_name}][fold {self.val_fold}]  Validation rows: {len(self.validation_data)}\")\n        self.training_data = df[df['fold'] != self.val_fold].reset_index(drop=True)\n        print(f\"[{self.condition_name}][fold {self.val_fold}]  Training rows:   {len(self.training_data)}\")\n\n\n    def _read_dicom(self, dicom_path):\n        ds = pydicom.dcmread(dicom_path)\n        img = ds.pixel_array.astype(float)\n        img = (img - img.min()) / (img.max() - img.min() + 1e-6) * 255.0\n        img = np.stack([img]*3, axis=-1).astype('uint8')\n        return img\n\n\n    def _dicom_to_png(self, df: pd.DataFrame, image_directory: Path):\n        self.height_width_info = []\n        for study_id, study_grp in df.groupby('study_id'):\n            for series_id, series_grp in study_grp.groupby('series_id'):\n                series_folder = os.path.join(\n                    self.dataset_directory,\n                    str(study_id),\n                    str(series_id)\n                )\n                if not os.path.isdir(series_folder):\n                    print(f\"⚠️ Missing folder: {series_folder}\")\n                    continue\n\n                inst_map = {}\n                for fname in os.listdir(series_folder):\n                    if not fname.lower().endswith('.dcm'):\n                        continue\n                    full_path = os.path.join(series_folder, fname)\n                    try:\n                        ds = pydicom.dcmread(full_path, stop_before_pixels=True)\n                        inst_num = int(ds.InstanceNumber)\n                        inst_map[inst_num] = full_path\n                    except Exception:\n                        continue\n\n                instance_list = series_grp['instance_number'].unique().tolist()\n                if len(inst_map) == 0:\n                    print(f\"⚠️ No .dcm files found in {series_folder}\")\n                    continue\n\n                first_inst = instance_list[0]\n                if first_inst not in inst_map:\n                    first_inst = next(iter(inst_map.keys()))\n                first_path = inst_map[first_inst]\n                img0 = self._read_dicom(first_path)\n                h, w, _ = img0.shape\n                self.height_width_info.append({\n                    'study_id': study_id,\n                    'series_id': series_id,\n                    'height': h,\n                    'width': w\n                })\n\n                for inst in instance_list:\n                    if inst not in inst_map:\n                        print(f\"⚠️ InstanceNumber {inst} not found in {series_folder}\")\n                        continue\n                    dcm_path = inst_map[inst]\n                    img = self._read_dicom(dcm_path)\n                    out_png = image_directory / f\"{study_id}_{series_id}_{inst}.png\"\n                    cv2.imwrite(str(out_png), img)\n\n        print(f\"[{self.condition_name}][fold {self.val_fold}]  Saved PNGs to {image_directory}\")\n\n\n    def _save_height_width_csv(self):\n        hw_path = self.fold_path / f\"{self.condition_name}_height_weight.csv\"\n        with open(hw_path, 'w', newline='') as f:\n            writer = csv.DictWriter(f, fieldnames=['study_id', 'series_id', 'height', 'width'])\n            writer.writeheader()\n            writer.writerows(self.height_width_info)\n\n        fold_df = pd.read_csv(self.csv_path)\n        hw_df   = pd.read_csv(hw_path)\n        merged_df = pd.merge(\n            fold_df,\n            hw_df[['study_id', 'series_id', 'height', 'width']],\n            on=['study_id', 'series_id'],\n            how='left'\n        )\n        merged_df.to_csv(hw_path, index=False)\n        print(f\"[{self.condition_name}][fold {self.val_fold}]  Height/width CSV saved to {hw_path}\")\n\n\n    def _find_class_label(self, condition, level):\n        cond_norm = condition.replace(' ', '_')\n        lvl_norm  = level.replace('/', '_')\n        key       = f\"{cond_norm}_{lvl_norm}\"\n        return self.condition_level_classes[key]\n\n\n    def _create_yolo_labels(self, df: pd.DataFrame, labels_directory: Path):\n        merged_csv = pd.read_csv(self.fold_path / f\"{self.condition_name}_height_weight.csv\")\n\n        for study_id, study_grp in merged_csv.groupby('study_id'):\n            for series_id, series_grp in study_grp.groupby('series_id'):\n                for inst_num, inst_grp in series_grp.groupby('instance_number'):\n                    labels = []\n                    h = inst_grp['height'].iloc[0]\n                    w = inst_grp['width'].iloc[0]\n\n                    for _, row in inst_grp.iterrows():\n                        cond = row['condition']\n                        lvl  = row['level']\n                        x    = row['x']\n                        y    = row['y']\n                        class_id = self._find_class_label(cond, lvl)\n\n                        x_norm = float(x) / w\n                        y_norm = float(y) / h\n                        box_w  = float(self.width_box) / w\n                        box_h  = float(self.width_box) / h\n\n                        labels.append((class_id, x_norm, y_norm, box_w, box_h))\n\n                    txt_path = labels_directory / f\"{study_id}_{series_id}_{inst_num}.txt\"\n                    with open(txt_path, 'w') as f:\n                        for (cid, xn, yn, bw, bh) in labels:\n                            f.write(f\"{cid} {xn:.6f} {yn:.6f} {bw:.6f} {bh:.6f}\\n\")\n\n        print(f\"[{self.condition_name}][fold {self.val_fold}]  Wrote YOLO labels to {labels_directory}\")\n\n\n    def _create_yaml_file(self):\n        yaml_path = self.dataset_path / 'yolo_config.yaml'\n        num_classes = len(self.condition_level_classes)\n        names_list = list(self.condition_level_classes.keys())\n\n        data = {\n            'train': './train',\n            'val': './val',\n            'nc': num_classes,\n            'names': names_list\n        }\n\n        with open(yaml_path, 'w') as f:\n            yaml.dump(data, f, default_flow_style=False)\n\n        print(f\"[{self.condition_name}][fold {self.val_fold}]  Created YAML at {yaml_path}\")\n\n\n############################################\n# Run for each condition, but only 2 folds (0 and 1):\n\n# 1) Spinal Canal Stenosis\nfor fold in range(2):   # <— range(2) instead of range(5)\n    spinal_classes = {\n        'Spinal_Canal_Stenosis_L1_L2': 0,\n        'Spinal_Canal_Stenosis_L2_L3': 1,\n        'Spinal_Canal_Stenosis_L3_L4': 2,\n        'Spinal_Canal_Stenosis_L4_L5': 3,\n        'Spinal_Canal_Stenosis_L5_S1': 4,\n    }\n\n    DetectorDataPreparation(\n        dataset_directory='/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images',\n        csv_path='/kaggle/working/Spinal_Canal_Stenosis_2folds.csv',  # new 2‐folds file\n        condition_level_classes=spinal_classes,\n        condition_name='Spinal_Canal_Stenosis',\n        val_fold=fold,\n        width_box=16,\n    )\nprint(\"✅ Spinal Canal Stenosis data prep DONE for both folds\\n\")\n\n\n# 2) Subarticular Stenosis\nfor fold in range(2):   # <— range(2)\n    subart_classes = {\n        'Left_Subarticular_Stenosis_L1_L2': 0,\n        'Left_Subarticular_Stenosis_L2_L3': 1,\n        'Left_Subarticular_Stenosis_L3_L4': 2,\n        'Left_Subarticular_Stenosis_L4_L5': 3,\n        'Left_Subarticular_Stenosis_L5_S1': 4,\n        'Right_Subarticular_Stenosis_L1_L2': 5,\n        'Right_Subarticular_Stenosis_L2_L3': 6,\n        'Right_Subarticular_Stenosis_L3_L4': 7,\n        'Right_Subarticular_Stenosis_L4_L5': 8,\n        'Right_Subarticular_Stenosis_L5_S1': 9,\n    }\n\n    DetectorDataPreparation(\n        dataset_directory='/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images',\n        csv_path='/kaggle/working/Subarticular_Stenosis_2folds.csv',\n        condition_level_classes=subart_classes,\n        condition_name='Subarticular_Stenosis',\n        val_fold=fold,\n        width_box=16,\n    )\nprint(\"✅ Subarticular Stenosis data prep DONE for both folds\\n\")\n\n\n# 3) Neural Foraminal Narrowing\nfor fold in range(2):   # <— range(2)\n    neural_classes = {\n        'Left_Neural_Foraminal_Narrowing_L1_L2': 0,\n        'Left_Neural_Foraminal_Narrowing_L2_L3': 1,\n        'Left_Neural_Foraminal_Narrowing_L3_L4': 2,\n        'Left_Neural_Foraminal_Narrowing_L4_L5': 3,\n        'Left_Neural_Foraminal_Narrowing_L5_S1': 4,\n        'Right_Neural_Foraminal_Narrowing_L1_L2': 5,\n        'Right_Neural_Foraminal_Narrowing_L2_L3': 6,\n        'Right_Neural_Foraminal_Narrowing_L3_L4': 7,\n        'Right_Neural_Foraminal_Narrowing_L4_L5': 8,\n        'Right_Neural_Foraminal_Narrowing_L5_S1': 9,\n    }\n\n    DetectorDataPreparation(\n        dataset_directory='/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images',\n        csv_path='/kaggle/working/Neural_Foraminal_Narrowing_2folds.csv',\n        condition_level_classes=neural_classes,\n        condition_name='Neural_Foraminal_Narrowing',\n        val_fold=fold,\n        width_box=16,\n    )\nprint(\"✅ Neural Foraminal Narrowing data prep DONE for both folds\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-13T11:17:06.097752Z","iopub.execute_input":"2025-06-13T11:17:06.098446Z","iopub.status.idle":"2025-06-13T11:17:06.138942Z","shell.execute_reply.started":"2025-06-13T11:17:06.098421Z","shell.execute_reply":"2025-06-13T11:17:06.137909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────────────────────\n# Cell: Zip & Clean Up Each Fold’s Detection Output\n# ─────────────────────────────────────────────────────────────────────────────\nimport shutil\nfrom pathlib import Path\n\nBASE = Path('/kaggle/working')\nCONDITIONS = [\"Spinal_Canal_Stenosis\", \"Subarticular_Stenosis\", \"Neural_Foraminal_Narrowing\"]\nFOLDS      = [0, 1]\n\nfor cond in CONDITIONS:\n    for f in FOLDS:\n        src = BASE / cond / f'fold_{f}' / 'datasets'\n        dst = BASE / f'{cond}_fold_{f}_datasets'\n        if src.exists():\n            # 1) create ZIP: /kaggle/working/<cond>_fold_<f>_datasets.zip\n            shutil.make_archive(str(dst), 'zip', root_dir=src)\n            print(f\"✓ Zipped {src} → {dst}.zip\")\n            # 2) delete the uncompressed data\n            shutil.rmtree(src)\n            print(f\"✓ Removed {src} to reclaim space\\n\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import zipfile\nfrom pathlib import Path\n\nBASE       = Path('/kaggle/working')\nCONDITIONS = [\"Spinal_Canal_Stenosis\",\"Subarticular_Stenosis\",\"Neural_Foraminal_Narrowing\"]\nFOLDS      = [0,1]\n\nfor cond in CONDITIONS:\n    for f in FOLDS:\n        zip_fp = BASE/f'{cond}_fold_{f}_datasets.zip'\n        out_dir = BASE/cond/f'fold_{f}'/'datasets'\n        if zip_fp.exists() and not out_dir.exists():\n            out_dir.mkdir(parents=True, exist_ok=True)\n            with zipfile.ZipFile(zip_fp, 'r') as zp:\n                zp.extractall(out_dir)\n            print(f\"✓ Unzipped {zip_fp} → {out_dir}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────────────────────\n# Cell: 2-Fold YOLOv8 Training with WandB Disabled\n# ─────────────────────────────────────────────────────────────────────────────\n\nimport os\n# Disable Weights & Biases logging\nos.environ['WANDB_MODE']    = 'disabled'\nos.environ['WANDB_PROJECT'] = 'yolo_training'\n\nfrom ultralytics import YOLO\nfrom pathlib import Path\n\n# CONFIGURATION\nDATA_ROOT    = \"/kaggle/working\"\nRESULTS_ROOT = \"/kaggle/working/yolo_results\"\nCONDITIONS   = [\"Spinal_Canal_Stenosis\", \"Subarticular_Stenosis\", \"Neural_Foraminal_Narrowing\"]\nFOLDS        = [0, 1]\nEPOCHS       = 20\nPATIENCE     = 5\nBATCH_SIZE   = 8\n\n# TRAINING LOOP\nfor condition in CONDITIONS:\n    for fold in FOLDS:\n        data_yaml  = f\"{DATA_ROOT}/{condition}/fold_{fold}/datasets/yolo_config.yaml\"\n        project_dir = Path(RESULTS_ROOT) / condition / f\"fold_{fold}\"\n        project_dir.mkdir(parents=True, exist_ok=True)\n\n        print(f\"\\n▶ Training {condition}, fold {fold}\")\n        # Initialize from YAML (no .pt unpickle)\n        model = YOLO(\"yolov8n.yaml\")\n        # Train with WandB disabled\n        model.train(\n            data     = data_yaml,\n            project  = str(project_dir),\n            name     = \"exp\",\n            epochs   = EPOCHS,\n            patience = PATIENCE,\n            batch    = BATCH_SIZE\n        )\n        print(f\"✅ Done {condition} fold {fold}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"CLASSIFER ","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom pathlib import Path\n\ndef splitted_data_2fold(condition):\n    \"\"\"\n    Reads ./condition_csv/<condition>.csv, drops NaN scores,\n    performs a 2-fold stratified split on 'score', and writes out:\n      - ./<condition>/<condition>_2folds.csv\n      - ./<condition>/fold_0/<condition>_train.csv, <condition>_val.csv\n      - ./<condition>/fold_1/<condition>_train.csv, <condition>_val.csv\n    \"\"\"\n    # 1) Load and clean\n    df = pd.read_csv(f'./condition_csv/{condition}.csv')\n    df = df.dropna(subset=['score']).reset_index(drop=True)\n\n    # 2) Stratified 2-fold split\n    skf = StratifiedKFold(n_splits=2, shuffle=True, random_state=42)\n    df['fold'] = -1\n    for fold, (_, val_idx) in enumerate(skf.split(df, df['score'])):\n        df.loc[val_idx, 'fold'] = fold\n\n    # 3) Save the combined 2-folds CSV\n    condition_dir = Path(f'./{condition}')\n    condition_dir.mkdir(exist_ok=True)\n    df.to_csv(condition_dir / f'{condition}_2folds.csv', index=False)\n    print(f\"Saved 2-folds CSV to: {condition_dir / f'{condition}_2folds.csv'}\")\n\n    # 4) Write out per-fold train/val splits\n    for fold in range(2):\n        fold_dir = condition_dir / f'fold_{fold}'\n        fold_dir.mkdir(exist_ok=True)\n        train_df = df[df['fold'] != fold]\n        val_df   = df[df['fold'] == fold]\n        train_df.to_csv(fold_dir / f'{condition}_train.csv', index=False)\n        val_df.to_csv(  fold_dir / f'{condition}_val.csv',   index=False)\n        print(f\"  Fold {fold}: {len(train_df)} train rows → {fold_dir / f'{condition}_train.csv'}\")\n        print(f\"  Fold {fold}: {len(val_df)}   val rows → {fold_dir / f'{condition}_val.csv'}\")\n\n# Run for each condition\nfor cond in ['Neural_Foraminal_Narrowing', 'Spinal_Canal_Stenosis', 'Subarticular_Stenosis']:\n    splitted_data_2fold(cond)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport random\nimport shutil\n\n# Define augmentation methods\naugmentations = ['rotate', 'horizontal_flip', 'vertical_flip', 'gaussian_noise']\n\n# Function to balance minority classes\ndef augment_data(df: pd.DataFrame, augmentations: list) -> pd.DataFrame:\n    \"\"\"\n    Oversample minority \"score\" classes to balance against the majority class.\n    Adds an 'augmentation' column to each augmented sample.\n    \"\"\"\n    counts = df['score'].value_counts()\n    maj_cls = counts.idxmax()\n    target1 = counts[maj_cls] // 3\n    target2 = counts[maj_cls] // 2\n\n    minors = [cls for cls, c in counts.items() if c < counts[maj_cls]]\n    if len(minors) < 2:\n        minors = minors * 2\n\n    def sample_and_label(min_cls, target):\n        existing = df[df['score'] == min_cls]\n        needed = max(0, target - len(existing))\n        if needed <= 0:\n            return pd.DataFrame(columns=list(df.columns) + ['augmentation'])\n        sampled = existing.sample(needed, replace=True).copy()\n        sampled['augmentation'] = [random.choice(augmentations) for _ in range(needed)]\n        return sampled\n\n    aug1 = sample_and_label(minors[0], target1)\n    aug2 = sample_and_label(minors[1], target2)\n\n    df['augmentation'] = None\n    return pd.concat([df, aug1, aug2], ignore_index=True)\n\n# Two-fold augmentation for each condition and fold\nconditions = ['Neural_Foraminal_Narrowing', 'Spinal_Canal_Stenosis', 'Subarticular_Stenosis']\nfolds = [0, 1]\n\nfor cond in conditions:\n    for fold in folds:\n        train_csv = f'./{cond}/fold_{fold}/{cond}_train.csv'\n        val_csv   = f'./{cond}/fold_{fold}/{cond}_val.csv'\n\n        df_train = pd.read_csv(train_csv)\n        df_aug   = augment_data(df_train, augmentations)\n\n        out_dir = os.path.join('augmented_output', cond, f'fold_{fold}')\n        os.makedirs(out_dir, exist_ok=True)\n\n        aug_train_path = os.path.join(out_dir, f'{cond}_augmented_train.csv')\n        df_aug.to_csv(aug_train_path, index=False)\n        print(f'Augmented train data saved to: {aug_train_path}')\n\n        # Copy validation set unchanged\n        aug_val_path = os.path.join(out_dir, f'{cond}_val.csv')\n        shutil.copy(val_csv, aug_val_path)\n        print(f'Validation data copied to: {aug_val_path}')\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────────────────────\n# Cell: Image Data Preparation for 2-Fold Splits\n# ─────────────────────────────────────────────────────────────────────────────\n\nimport os\nimport pydicom\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image, ImageOps\nfrom pathlib import Path\n\nclass DataPreparationImage:\n    def __init__(\n        self,\n        dataset_directory: str,\n        condition: str,\n        csv_directory: str,\n        num_folds: int = 2,\n        augmentation_list=None,\n    ):\n        self.dataset_directory = dataset_directory\n        self.condition = condition\n        self.csv_directory = csv_directory\n        self.num_folds = num_folds\n        self.augmentation_list = augmentation_list or ['rotate','horizontal_flip','vertical_flip','gaussian_noise']\n\n        print(f\"Starting image data prep for {self.condition} ({self.num_folds} folds)\")\n        self._create_folders()\n        self._process_all_folds()\n\n    def _create_folders(self):\n        base = Path(f\"./{self.condition}\")\n        base.mkdir(exist_ok=True)\n        for fold in range(self.num_folds):\n            (base / f\"fold_{fold}\" / \"train\").mkdir(parents=True, exist_ok=True)\n            (base / f\"fold_{fold}\" / \"val\").mkdir(parents=True, exist_ok=True)\n\n    def _read_csv(self, fold: int, split: str) -> pd.DataFrame:\n        filename = f\"{self.condition}_{'augmented_train' if split=='train' else 'val'}.csv\"\n        path = Path(self.csv_directory) / self.condition / f\"fold_{fold}\" / filename\n        return pd.read_csv(path)\n\n    def _read_dicom(self, path: str) -> np.ndarray:\n        ds = pydicom.dcmread(path)\n        img = ds.pixel_array.astype(float)\n        img = (img - img.min())/(img.max()-img.min()+1e-6)*255.0\n        return np.stack([img]*3,axis=-1).astype('uint8')\n\n    def _crop(self, image: np.ndarray, x: float, y: float, box: int = 16) -> Image.Image:\n        img = Image.fromarray(image)\n        left, top = int(x-box), int(y-box)\n        right, bottom = int(x+box), int(y+box)\n        return img.crop((left, top, right, bottom))\n\n    def _apply_augmentation(self, img: Image.Image, aug: str) -> Image.Image:\n        if aug=='rotate':\n            return img.rotate(np.random.uniform(-20,20), expand=True)\n        if aug=='horizontal_flip':\n            return ImageOps.mirror(img)\n        if aug=='vertical_flip':\n            return ImageOps.flip(img)\n        if aug=='gaussian_noise':\n            arr = np.array(img)\n            noise = np.random.normal(0,25,arr.shape)\n            return Image.fromarray(np.clip(arr+noise,0,255).astype('uint8'))\n        return img\n\n    def _process_all_folds(self):\n        for fold in range(self.num_folds):\n            # TRAIN SPLIT\n            df_train = self._read_csv(fold, 'train')\n            for _, row in df_train.iterrows():\n                sid, seid, inst = row['study_id'], row['series_id'], row['instance_number']\n                x, y = row['x'], row['y']\n                aug = row.get('augmentation')\n                dcm_path = os.path.join(self.dataset_directory, str(sid), str(seid), f\"{inst}.dcm\")\n                img = self._read_dicom(dcm_path)\n                cropped = self._crop(img, x, y)\n                if aug in self.augmentation_list:\n                    out_img = self._apply_augmentation(cropped, aug)\n                    suffix = f\"_{aug}\"\n                else:\n                    out_img = cropped\n                    suffix = \"\"\n                fname = f\"{sid}_{seid}_{inst}_{int(x)}_{int(y)}{suffix}.png\"\n                out_path = Path(self.condition) / f\"fold_{fold}\" / \"train\" / fname\n                out_img.save(out_path)\n\n            # VAL SPLIT (no augmentation)\n            df_val = self._read_csv(fold, 'val')\n            for _, row in df_val.iterrows():\n                sid, seid, inst = row['study_id'], row['series_id'], row['instance_number']\n                x, y = row['x'], row['y']\n                dcm_path = os.path.join(self.dataset_directory, str(sid), str(seid), f\"{inst}.dcm\")\n                img = self._read_dicom(dcm_path)\n                cropped = self._crop(img, x, y)\n                fname = f\"{sid}_{seid}_{inst}_{int(x)}_{int(y)}.png\"\n                out_path = Path(self.condition) / f\"fold_{fold}\" / \"val\" / fname\n                cropped.save(out_path)\n\n# ─────────────────────────────────────────────────────────────────────────────\n# Run for your three conditions:\n# ─────────────────────────────────────────────────────────────────────────────\n\nDATASET_DIR = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train\"\nCSV_DIR     = \"/kaggle/working/augmented_output\"\n\nfor cond in [\"Subarticular_Stenosis\", \"Spinal_Canal_Stenosis\", \"Neural_Foraminal_Narrowing\"]:\n    DataPreparationImage(\n        dataset_directory=DATASET_DIR,\n        condition=cond,\n        csv_directory=CSV_DIR,\n        num_folds=2\n    )\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────────────────────\n# Cell: Classification Label Preparation for 2-Fold Splits\n# ─────────────────────────────────────────────────────────────────────────────\n\nimport os\nimport pandas as pd\nfrom pathlib import Path\n\n# 1) Parameters\nCSV_DIR    = '../data_augmentation/augmented_output'   # where your augmented CSVs live\nCONDITIONS = ['Spinal_Canal_Stenosis',\n              'Neural_Foraminal_Narrowing',\n              'Subarticular_Stenosis']\nFOLDS      = [0, 1]  # only two folds\nOUT_ROOT   = './'    # base for label output folders\n\n# 2) Helper: build 'subject' and 'label' columns\ndef working_on_csv(csv_path: str, split: str) -> pd.DataFrame:\n    df = pd.read_csv(csv_path)\n    # subject PNG filename\n    def make_subject(r):\n        name = f\"{r.study_id}_{r.series_id}_{r.instance_number}_{int(r.x)}_{int(r.y)}.png\"\n        if split=='train' and pd.notna(r.augmentation) and r.augmentation:\n            return name.replace('.png', '_augmented.png')\n        return name\n    # numeric label mapping\n    def make_label(score):\n        return {'Normal/Mild':1, 'Moderate':2, 'Severe':3}.get(score, None)\n\n    df['subject'] = df.apply(make_subject, axis=1)\n    df['label']   = df['score'].apply(make_label)\n    return df[['subject','label']]\n\n# 3) Create output folder structure\nfor cond in CONDITIONS:\n    base = Path(OUT_ROOT) / f\"{cond}_label\"\n    for fold in FOLDS:\n        (base / f\"fold_{fold}\").mkdir(parents=True, exist_ok=True)\n\n# 4) Process each condition & fold\nfor cond in CONDITIONS:\n    for fold in FOLDS:\n        # ←— **NOTE**: train file ends in `_augmented_train.csv`\n        train_csv = f\"{CSV_DIR}/{cond}/fold_{fold}/{cond}_augmented_train.csv\"\n        val_csv   = f\"{CSV_DIR}/{cond}/fold_{fold}/{cond}_val.csv\"\n\n        # build label DataFrames\n        train_labels = working_on_csv(train_csv, 'train')\n        val_labels   = working_on_csv(val_csv,   'val')\n\n        # write out\n        out_base = Path(OUT_ROOT) / f\"{cond}_label\" / f\"fold_{fold}\"\n        train_labels.to_csv(out_base / f\"{cond}_augmented_labels.csv\", index=False)\n        val_labels.to_csv(  out_base / f\"{cond}_val_labels.csv\",       index=False)\n\n        print(f\"→ {cond} fold {fold}:\")\n        print(f\"   • train labels → {out_base / f'{cond}_augmented_labels.csv'} ({len(train_labels)} rows)\")\n        print(f\"   • val   labels → {out_base / f'{cond}_val_labels.csv'} ({len(val_labels)} rows)\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────────────────────\n# Cell: Fast Severity Classification Training on 2 Folds\n# ─────────────────────────────────────────────────────────────────────────────\n\nimport os\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom pathlib import Path\n\n# CONFIGURATION\nCONDITION     = 'Subarticular_Stenosis'    # or change to each condition\nFOLDS         = [0, 1]                     # two folds\nIMAGE_ROOT_TMPL = './{cond}/fold_{fold}'\nLABEL_ROOT_TMPL = './{cond}_label/fold_{fold}'\nEPOCHS        = 10                         # fewer epochs for quick results\nBATCH_SIZE    = 16\nPATIENCE      = 3\nLEARNING_RATE = 1e-3\n\ndef load_data(df, image_root, split):\n    X, y = [], []\n    for _, row in df.iterrows():\n        img_path = os.path.join(image_root, split, row['subject'])\n        img = tf.keras.preprocessing.image.load_img(img_path, target_size=(32,32))\n        arr = tf.keras.preprocessing.image.img_to_array(img) / 255.0\n        X.append(arr)\n        y.append(int(row['label']) - 1)\n    return np.array(X), np.array(y)\n\nfor fold in FOLDS:\n    print(f\"\\n▶ Training {CONDITION}, fold {fold}\")\n    IMAGE_ROOT = IMAGE_ROOT_TMPL.format(cond=CONDITION, fold=fold)\n    LABEL_ROOT = LABEL_ROOT_TMPL.format(cond=CONDITION, fold=fold)\n\n    # Load labels\n    train_df = pd.read_csv(os.path.join(LABEL_ROOT, f'{CONDITION}_augmented_labels.csv'))\n    val_df   = pd.read_csv(os.path.join(LABEL_ROOT, f'{CONDITION}_val_labels.csv'))\n\n    # Load image arrays and labels\n    X_train, y_train = load_data(train_df, IMAGE_ROOT, 'train')\n    X_val,   y_val   = load_data(val_df,   IMAGE_ROOT, 'val')\n\n    # Build a small CNN\n    model = Sequential([\n        Conv2D(16, 3, activation='relu', input_shape=(32,32,3)),\n        MaxPooling2D(),\n        Conv2D(32, 3, activation='relu'),\n        MaxPooling2D(),\n        Conv2D(64, 3, activation='relu'),\n        MaxPooling2D(),\n        Flatten(),\n        Dense(64, activation='relu'),\n        Dropout(0.3),\n        Dense(3, activation='softmax'),\n    ])\n    model.compile(\n        optimizer=Adam(learning_rate=LEARNING_RATE),\n        loss='sparse_categorical_crossentropy',\n        metrics=['accuracy']\n    )\n\n    # Early stopping\n    es = EarlyStopping(\n        monitor='val_loss',\n        patience=PATIENCE,\n        restore_best_weights=True,\n        verbose=1\n    )\n\n    # Train\n    model.fit(\n        X_train, y_train,\n        validation_data=(X_val, y_val),\n        epochs=EPOCHS,\n        batch_size=BATCH_SIZE,\n        callbacks=[es],\n        verbose=2\n    )\n\n    # Save model\n    out_dir = Path(f'./results/{CONDITION}/fold_{fold}')\n    out_dir.mkdir(parents=True, exist_ok=True)\n    model.save(out_dir / 'quick_model.h5')\n    print(f\"✅ Saved quick model for fold {fold} to: {out_dir/'quick_model.h5'}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────────────────────\n# Cell: Training + Evaluation with Confusion Matrix for 2 Folds\n# ─────────────────────────────────────────────────────────────────────────────\n\nimport os\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom sklearn.metrics import (\n    accuracy_score, precision_score, recall_score, f1_score,\n    classification_report, confusion_matrix\n)\nfrom pathlib import Path\n\n# CONFIGURATION\nCONDITION       = 'Subarticular_Stenosis'\nFOLDS           = [0, 1]\nIMAGE_ROOT_TMPL = './{cond}/fold_{fold}'\nLABEL_ROOT_TMPL = './{cond}_label/fold_{fold}'\nEPOCHS          = 10\nBATCH_SIZE      = 16\nPATIENCE        = 3\nLEARNING_RATE   = 1e-3\nCLASS_NAMES     = ['Normal/Mild', 'Moderate', 'Severe']\n\ndef load_data(df, image_root, split):\n    X, y = [], []\n    for _, row in df.iterrows():\n        img_path = os.path.join(image_root, split, row['subject'])\n        img = tf.keras.preprocessing.image.load_img(img_path, target_size=(32,32))\n        arr = tf.keras.preprocessing.image.img_to_array(img) / 255.0\n        X.append(arr)\n        y.append(int(row['label']) - 1)\n    return np.array(X), np.array(y)\n\nfor fold in FOLDS:\n    print(f\"\\n▶ Training {CONDITION}, fold {fold}\")\n    IMAGE_ROOT = IMAGE_ROOT_TMPL.format(cond=CONDITION, fold=fold)\n    LABEL_ROOT = LABEL_ROOT_TMPL.format(cond=CONDITION, fold=fold)\n\n    # Load labels\n    train_df = pd.read_csv(os.path.join(LABEL_ROOT, f'{CONDITION}_augmented_labels.csv'))\n    val_df   = pd.read_csv(os.path.join(LABEL_ROOT, f'{CONDITION}_val_labels.csv'))\n\n    # Load data arrays\n    X_train, y_train = load_data(train_df, IMAGE_ROOT, 'train')\n    X_val,   y_val   = load_data(val_df,   IMAGE_ROOT, 'val')\n\n    # Build model\n    model = Sequential([\n        Conv2D(16, 3, activation='relu', input_shape=(32,32,3)),\n        MaxPooling2D(),\n        Conv2D(32, 3, activation='relu'),\n        MaxPooling2D(),\n        Conv2D(64, 3, activation='relu'),\n        MaxPooling2D(),\n        Flatten(),\n        Dense(64, activation='relu'),\n        Dropout(0.3),\n        Dense(3, activation='softmax'),\n    ])\n    model.compile(\n        optimizer=Adam(learning_rate=LEARNING_RATE),\n        loss='sparse_categorical_crossentropy',\n        metrics=['accuracy']\n    )\n\n    # Early stopping\n    es = EarlyStopping(monitor='val_loss', patience=PATIENCE, restore_best_weights=True, verbose=1)\n\n    # Train\n    model.fit(X_train, y_train, validation_data=(X_val, y_val),\n              epochs=EPOCHS, batch_size=BATCH_SIZE, callbacks=[es], verbose=2)\n\n    # Predict on validation set\n    y_pred_probs = model.predict(X_val, batch_size=BATCH_SIZE)\n    y_pred       = np.argmax(y_pred_probs, axis=1)\n\n    # Compute metrics\n    acc   = accuracy_score(y_val, y_pred)\n    prec  = precision_score(y_val, y_pred, average='weighted', zero_division=0)\n    rec   = recall_score(y_val, y_pred, average='weighted', zero_division=0)\n    f1    = f1_score(y_val, y_pred, average='weighted', zero_division=0)\n    cm    = confusion_matrix(y_val, y_pred)\n\n    # Display results\n    print(f\"\\nFold {fold} Validation Metrics:\")\n    print(f\"  Accuracy : {acc:.4f}\")\n    print(f\"  Precision: {prec:.4f}\")\n    print(f\"  Recall   : {rec:.4f}\")\n    print(f\"  F1 score : {f1:.4f}\\n\")\n    print(\"Classification Report:\")\n    print(classification_report(y_val, y_pred, target_names=CLASS_NAMES, zero_division=0))\n    print(\"Confusion Matrix:\")\n    print(pd.DataFrame(cm, index=CLASS_NAMES, columns=CLASS_NAMES))\n\n    # Save model and metrics\n    out_dir = Path(f'./results/{CONDITION}/fold_{fold}')\n    out_dir.mkdir(parents=True, exist_ok=True)\n    model.save(out_dir / 'quick_model.h5')\n\n    # Save metrics + confusion matrix to CSV\n    metrics_df = pd.DataFrame({\n        'metric':      ['accuracy','precision','recall','f1_score'],\n        'value':       [acc,prec,rec,f1]\n    })\n    metrics_df.to_csv(out_dir / 'validation_metrics.csv', index=False)\n    cm_df = pd.DataFrame(cm, index=CLASS_NAMES, columns=CLASS_NAMES)\n    cm_df.to_csv(out_dir / 'confusion_matrix.csv')\n    print(f\"✅ Saved metrics and confusion matrix to {out_dir}\\n\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}