{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":4104,"databundleVersionId":46661,"isSourceIdPinned":false},{"sourceType":"datasetVersion","sourceId":16042206,"datasetId":10288595,"databundleVersionId":17008720},{"sourceType":"datasetVersion","sourceId":16042420,"datasetId":10288726,"databundleVersionId":17008954},{"sourceType":"datasetVersion","sourceId":16072316,"datasetId":10306500,"databundleVersionId":17041140}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip -q install timm albumentations lightgbm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:35:48.885837Z","iopub.execute_input":"2026-05-03T14:35:48.886544Z","iopub.status.idle":"2026-05-03T14:35:53.428733Z","shell.execute_reply.started":"2026-05-03T14:35:48.886511Z","shell.execute_reply":"2026-05-03T14:35:53.427699Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport io\nimport glob\nimport json\nimport random\nimport zipfile\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit, StratifiedKFold\nfrom sklearn.metrics import (\n    accuracy_score, f1_score, confusion_matrix, classification_report,\n    roc_auc_score, precision_score, recall_score\n)\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, datasets\nimport timm\n\nfrom lightgbm import LGBMClassifier","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:36:13.170594Z","iopub.execute_input":"2026-05-03T14:36:13.17125Z","iopub.status.idle":"2026-05-03T14:36:30.313379Z","shell.execute_reply.started":"2026-05-03T14:36:13.171214Z","shell.execute_reply":"2026-05-03T14:36:30.312507Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\nset_seed(42)\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:36:44.835069Z","iopub.execute_input":"2026-05-03T14:36:44.836712Z","iopub.status.idle":"2026-05-03T14:36:45.098104Z","shell.execute_reply.started":"2026-05-03T14:36:44.836659Z","shell.execute_reply":"2026-05-03T14:36:45.097429Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for root, dirs, files in os.walk(\"/kaggle/input\"):\n    print(root)\n    for f in files[:5]:\n        print(\"   \", f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:37:03.175303Z","iopub.execute_input":"2026-05-03T14:37:03.176103Z","iopub.status.idle":"2026-05-03T14:37:06.92756Z","shell.execute_reply.started":"2026-05-03T14:37:03.176069Z","shell.execute_reply":"2026-05-03T14:37:06.926978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"CSV files:\")\nfor p in glob.glob(\"/kaggle/input/**/*.csv\", recursive=True):\n    print(p)\n\nprint(\"\\nZIP files:\")\nfor p in glob.glob(\"/kaggle/input/**/*.zip\", recursive=True):\n    print(p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:37:21.311156Z","iopub.execute_input":"2026-05-03T14:37:21.31208Z","iopub.status.idle":"2026-05-03T14:37:22.595999Z","shell.execute_reply.started":"2026-05-03T14:37:21.312049Z","shell.execute_reply":"2026-05-03T14:37:22.5954Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ---------- EYEPACS ----------\nEYEPACS_LABELS = glob.glob(\"/kaggle/input/**/trainLabels.csv\", recursive=True)[0]\n\n# If images are zipped\neyepacs_zip_list = glob.glob(\"/kaggle/input/**/train.zip\", recursive=True)\nEYEPACS_TRAIN_ZIP = eyepacs_zip_list[0] if len(eyepacs_zip_list) > 0 else None\n\n# If images are already extracted in a folder\neyepacs_train_dirs = glob.glob(\"/kaggle/input/**/train\", recursive=True)\nEYEPACS_TRAIN_DIR = eyepacs_train_dirs[0] if len(eyepacs_train_dirs) > 0 else None\n\n# ---------- UCI EARLY STAGE ----------\nUCI_CSV = glob.glob(\"/kaggle/input/**/diabetes_data_upload.csv\", recursive=True)[0]\n\n# ---------- IDRiD ----------\nIDRID_ROOT = glob.glob(\"/kaggle/input/**/IDRID*\", recursive=True)\nprint(\"IDRID candidates:\", IDRID_ROOT[:10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:37:42.605697Z","iopub.execute_input":"2026-05-03T14:37:42.606365Z","iopub.status.idle":"2026-05-03T14:37:43.520374Z","shell.execute_reply.started":"2026-05-03T14:37:42.606334Z","shell.execute_reply":"2026-05-03T14:37:43.519383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, glob, zipfile, random, warnings, json\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit, StratifiedKFold\nfrom sklearn.metrics import (\n    accuracy_score, f1_score, confusion_matrix, classification_report,\n    roc_auc_score, recall_score\n)\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, datasets\n\nimport timm\nfrom lightgbm import LGBMClassifier\nimport joblib","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:51:17.820782Z","iopub.execute_input":"2026-05-03T14:51:17.82129Z","iopub.status.idle":"2026-05-03T14:51:17.826818Z","shell.execute_reply.started":"2026-05-03T14:51:17.821261Z","shell.execute_reply":"2026-05-03T14:51:17.826013Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# EXACT PATHS FROM YOUR NOTEBOOK\n# =========================\nUCI_CSV = \"/kaggle/input/datasets/mahirratul/early-stage/diabetes_data_upload.csv\"\n\nIDRID_BASE = \"/kaggle/input/datasets/mahirratul/idrid-dataset\"\n\nEYEPACS_BASE = \"/kaggle/input/competitions/diabetic-retinopathy-detection\"\nEYEPACS_LABELS_ZIP = f\"{EYEPACS_BASE}/trainLabels.csv.zip\"\nEYEPACS_SAMPLE_ZIP = f\"{EYEPACS_BASE}/sample.zip\"\nEYEPACS_TRAIN_ZIP = f\"{EYEPACS_BASE}/train.zip\"\n\nprint(\"UCI exists:\", os.path.exists(UCI_CSV))\nprint(\"IDRiD base exists:\", os.path.exists(IDRID_BASE))\nprint(\"EyePACS labels zip exists:\", os.path.exists(EYEPACS_LABELS_ZIP))\nprint(\"EyePACS sample zip exists:\", os.path.exists(EYEPACS_SAMPLE_ZIP))\nprint(\"EyePACS train zip exists:\", os.path.exists(EYEPACS_TRAIN_ZIP))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:51:30.82175Z","iopub.execute_input":"2026-05-03T14:51:30.822437Z","iopub.status.idle":"2026-05-03T14:51:30.836566Z","shell.execute_reply.started":"2026-05-03T14:51:30.822408Z","shell.execute_reply":"2026-05-03T14:51:30.835789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.makedirs(\"/kaggle/working/data\", exist_ok=True)\nos.makedirs(\"/kaggle/working/models\", exist_ok=True)\nos.makedirs(\"/kaggle/working/reports\", exist_ok=True)\nos.makedirs(\"/kaggle/working/eyepacs_small/train\", exist_ok=True)\nos.makedirs(\"/kaggle/working/eyepacs_small/val\", exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:52:01.680233Z","iopub.execute_input":"2026-05-03T14:52:01.6808Z","iopub.status.idle":"2026-05-03T14:52:01.685847Z","shell.execute_reply.started":"2026-05-03T14:52:01.680773Z","shell.execute_reply":"2026-05-03T14:52:01.685102Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.makedirs(\"/kaggle/working/data/eyepacs_labels\", exist_ok=True)\n\nwith zipfile.ZipFile(EYEPACS_LABELS_ZIP, \"r\") as zf:\n    zf.extractall(\"/kaggle/working/data/eyepacs_labels\")\n\nEYEPACS_LABELS = \"/kaggle/working/data/eyepacs_labels/trainLabels.csv\"\nprint(\"Labels file exists:\", os.path.exists(EYEPACS_LABELS))\n\ndr = pd.read_csv(EYEPACS_LABELS)\nprint(dr.head())\nprint(dr.shape)\nprint(dr[\"level\"].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:53:41.746306Z","iopub.execute_input":"2026-05-03T14:53:41.747029Z","iopub.status.idle":"2026-05-03T14:53:41.812055Z","shell.execute_reply.started":"2026-05-03T14:53:41.746999Z","shell.execute_reply":"2026-05-03T14:53:41.811468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dr[\"patient_id\"] = dr[\"image\"].apply(lambda x: str(x).rsplit(\"_\", 1)[0])\nprint(dr.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:53:59.420772Z","iopub.execute_input":"2026-05-03T14:53:59.421615Z","iopub.status.idle":"2026-05-03T14:53:59.440054Z","shell.execute_reply.started":"2026-05-03T14:53:59.421585Z","shell.execute_reply":"2026-05-03T14:53:59.439168Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 4 — Finish the UCI symptom branch first\n\nThis one is ready right now, so complete it fully.\n\n4.1 Load and inspect","metadata":{}},{"cell_type":"code","source":"uci = pd.read_csv(UCI_CSV)\nprint(uci.head())\nprint(uci.shape)\nprint(uci.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:54:12.459115Z","iopub.execute_input":"2026-05-03T14:54:12.45986Z","iopub.status.idle":"2026-05-03T14:54:12.47671Z","shell.execute_reply.started":"2026-05-03T14:54:12.459831Z","shell.execute_reply":"2026-05-03T14:54:12.476077Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"4.2 Clean columns","metadata":{}},{"cell_type":"code","source":"uci.columns = [c.strip().lower().replace(\" \", \"_\").replace(\"-\", \"_\") for c in uci.columns]\n\ndef normalize_val(x):\n    if isinstance(x, str):\n        return x.strip().lower()\n    return x\n\nfor c in uci.columns:\n    uci[c] = uci[c].apply(normalize_val)\n\nprint(uci.head())\nprint(uci.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:55:18.274804Z","iopub.execute_input":"2026-05-03T14:55:18.275349Z","iopub.status.idle":"2026-05-03T14:55:18.291018Z","shell.execute_reply.started":"2026-05-03T14:55:18.275322Z","shell.execute_reply":"2026-05-03T14:55:18.290369Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"4.3 Encode values","metadata":{}},{"cell_type":"code","source":"map_binary = {\n    \"yes\": 1, \"no\": 0,\n    \"male\": 1, \"female\": 0,\n    \"positive\": 1, \"negative\": 0\n}\n\nfor c in uci.columns:\n    if uci[c].dtype == \"object\":\n        vals = set(uci[c].dropna().unique())\n        if vals.issubset(set(map_binary.keys())):\n            uci[c] = uci[c].map(map_binary)\n\nprint(uci.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:55:40.055881Z","iopub.execute_input":"2026-05-03T14:55:40.056442Z","iopub.status.idle":"2026-05-03T14:55:40.082916Z","shell.execute_reply.started":"2026-05-03T14:55:40.056415Z","shell.execute_reply":"2026-05-03T14:55:40.082282Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_col = \"class\"\nX = uci.drop(columns=[target_col])\ny = uci[target_col].astype(int)\n\nprint(X.head())\nprint(y.value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:55:53.728912Z","iopub.execute_input":"2026-05-03T14:55:53.729616Z","iopub.status.idle":"2026-05-03T14:55:53.738487Z","shell.execute_reply.started":"2026-05-03T14:55:53.729589Z","shell.execute_reply":"2026-05-03T14:55:53.737769Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\nfold_scores = []\n\nfor fold, (tr_idx, va_idx) in enumerate(skf.split(X, y), 1):\n    X_tr, X_va = X.iloc[tr_idx], X.iloc[va_idx]\n    y_tr, y_va = y.iloc[tr_idx], y.iloc[va_idx]\n\n    model = LGBMClassifier(\n        objective=\"binary\",\n        n_estimators=300,\n        learning_rate=0.05,\n        num_leaves=31,\n        subsample=0.8,\n        colsample_bytree=0.8,\n        random_state=42\n    )\n\n    model.fit(X_tr, y_tr)\n    pred_prob = model.predict_proba(X_va)[:, 1]\n    pred = (pred_prob >= 0.5).astype(int)\n\n    auc = roc_auc_score(y_va, pred_prob)\n    f1 = f1_score(y_va, pred)\n    rec = recall_score(y_va, pred)\n\n    fold_scores.append([fold, auc, f1, rec])\n\nscores_df = pd.DataFrame(fold_scores, columns=[\"fold\", \"auc\", \"f1\", \"recall\"])\nprint(scores_df)\nprint(\"\\nMean:\")\nprint(scores_df.mean(numeric_only=True))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:56:09.689494Z","iopub.execute_input":"2026-05-03T14:56:09.690061Z","iopub.status.idle":"2026-05-03T14:56:10.260021Z","shell.execute_reply.started":"2026-05-03T14:56:09.690031Z","shell.execute_reply":"2026-05-03T14:56:10.259425Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"symptom_model = LGBMClassifier(\n    objective=\"binary\",\n    n_estimators=300,\n    learning_rate=0.05,\n    num_leaves=31,\n    subsample=0.8,\n    colsample_bytree=0.8,\n    random_state=42\n)\n\nsymptom_model.fit(X, y)\njoblib.dump(symptom_model, \"/kaggle/working/models/lightgbm_symptom.pkl\")\nprint(\"Saved: /kaggle/working/models/lightgbm_symptom.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:56:38.801521Z","iopub.execute_input":"2026-05-03T14:56:38.802084Z","iopub.status.idle":"2026-05-03T14:56:38.938507Z","shell.execute_reply.started":"2026-05-03T14:56:38.802058Z","shell.execute_reply":"2026-05-03T14:56:38.937698Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 5 — Now prepare IDRiD properly","metadata":{}},{"cell_type":"code","source":"for p in glob.glob(f\"{IDRID_BASE}/**\", recursive=True):\n    if os.path.isdir(p):\n        low = p.lower()\n        if (\n            \"original images\" in low\n            or \"microaneurysms\" in low\n            or \"haemorrhages\" in low\n            or \"hard exudates\" in low\n            or \"soft exudates\" in low\n        ):\n            print(p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:58:05.119107Z","iopub.execute_input":"2026-05-03T14:58:05.119888Z","iopub.status.idle":"2026-05-03T14:58:07.209097Z","shell.execute_reply.started":"2026-05-03T14:58:05.119858Z","shell.execute_reply":"2026-05-03T14:58:07.20842Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"IDRID_IMG_DIR = glob.glob(f\"{IDRID_BASE}/**/1. Original Images/a. Training Set\", recursive=True)[0]\n\nMA_MASK_DIR = glob.glob(f\"{IDRID_BASE}/**/2. All Segmentation Groundtruths/a. Training Set/1. Microaneurysms\", recursive=True)[0]\nHE_MASK_DIR = glob.glob(f\"{IDRID_BASE}/**/2. All Segmentation Groundtruths/a. Training Set/2. Haemorrhages\", recursive=True)[0]\nEX_MASK_DIR = glob.glob(f\"{IDRID_BASE}/**/2. All Segmentation Groundtruths/a. Training Set/3. Hard Exudates\", recursive=True)[0]\nSE_MASK_DIR = glob.glob(f\"{IDRID_BASE}/**/2. All Segmentation Groundtruths/a. Training Set/4. Soft Exudates\", recursive=True)[0]\n\nprint(\"IDRID_IMG_DIR =\", IDRID_IMG_DIR)\nprint(\"MA_MASK_DIR   =\", MA_MASK_DIR)\nprint(\"HE_MASK_DIR   =\", HE_MASK_DIR)\nprint(\"EX_MASK_DIR   =\", EX_MASK_DIR)\nprint(\"SE_MASK_DIR   =\", SE_MASK_DIR)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:58:32.317186Z","iopub.execute_input":"2026-05-03T14:58:32.317893Z","iopub.status.idle":"2026-05-03T14:58:33.086331Z","shell.execute_reply.started":"2026-05-03T14:58:32.31786Z","shell.execute_reply":"2026-05-03T14:58:33.085705Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"step-06— Build image-level lesion labels from IDRiD masks","metadata":{}},{"cell_type":"code","source":"img_files = sorted(glob.glob(f\"{IDRID_IMG_DIR}/*\"))\n\nrows = []\nfor img_path in img_files:\n    img_name = os.path.splitext(os.path.basename(img_path))[0]\n\n    ma = len(glob.glob(f\"{MA_MASK_DIR}/{img_name}*\")) > 0\n    he = len(glob.glob(f\"{HE_MASK_DIR}/{img_name}*\")) > 0\n    ex = len(glob.glob(f\"{EX_MASK_DIR}/{img_name}*\")) > 0\n    se = len(glob.glob(f\"{SE_MASK_DIR}/{img_name}*\")) > 0\n\n    rows.append({\n        \"image_path\": img_path,\n        \"image_id\": img_name,\n        \"ma\": int(ma),\n        \"he\": int(he),\n        \"ex\": int(ex),\n        \"se\": int(se),\n    })\n\nidrid_df = pd.DataFrame(rows)\nprint(idrid_df.head())\nprint(idrid_df.shape)\nprint(idrid_df[[\"ma\", \"he\", \"ex\", \"se\"]].sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:59:14.121947Z","iopub.execute_input":"2026-05-03T14:59:14.122599Z","iopub.status.idle":"2026-05-03T14:59:15.181327Z","shell.execute_reply.started":"2026-05-03T14:59:14.12257Z","shell.execute_reply":"2026-05-03T14:59:15.180687Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 7 — Split IDRiD","metadata":{}},{"cell_type":"code","source":"train_idrid, val_idrid = train_test_split(\n    idrid_df,\n    test_size=0.2,\n    random_state=42\n)\n\nprint(train_idrid.shape, val_idrid.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T14:59:38.273885Z","iopub.execute_input":"2026-05-03T14:59:38.274148Z","iopub.status.idle":"2026-05-03T14:59:38.280281Z","shell.execute_reply.started":"2026-05-03T14:59:38.274129Z","shell.execute_reply":"2026-05-03T14:59:38.27954Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 8 — Create transforms","metadata":{}},{"cell_type":"code","source":"train_tfms = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.RandomHorizontalFlip(),\n    transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1),\n    transforms.ToTensor(),\n])\n\nval_tfms = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T15:00:17.675925Z","iopub.execute_input":"2026-05-03T15:00:17.67648Z","iopub.status.idle":"2026-05-03T15:00:17.680891Z","shell.execute_reply.started":"2026-05-03T15:00:17.676448Z","shell.execute_reply":"2026-05-03T15:00:17.680091Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 9 — Create IDRiD dataset class","metadata":{}},{"cell_type":"code","source":"class IDRiDLesionDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img = Image.open(row[\"image_path\"]).convert(\"RGB\")\n        if self.transform:\n            img = self.transform(img)\n\n        label = torch.tensor(\n            [row[\"ma\"], row[\"he\"], row[\"ex\"], row[\"se\"]],\n            dtype=torch.float32\n        )\n        return img, label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T15:00:45.154535Z","iopub.execute_input":"2026-05-03T15:00:45.154936Z","iopub.status.idle":"2026-05-03T15:00:45.159958Z","shell.execute_reply.started":"2026-05-03T15:00:45.154909Z","shell.execute_reply":"2026-05-03T15:00:45.159321Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 10 — IDRiD loaders","metadata":{}},{"cell_type":"code","source":"idrid_train_ds = IDRiDLesionDataset(train_idrid, transform=train_tfms)\nidrid_val_ds = IDRiDLesionDataset(val_idrid, transform=val_tfms)\n\nidrid_train_loader = DataLoader(idrid_train_ds, batch_size=16, shuffle=True, num_workers=2)\nidrid_val_loader = DataLoader(idrid_val_ds, batch_size=16, shuffle=False, num_workers=2)\n\nprint(\"Train:\", len(idrid_train_ds))\nprint(\"Val:\", len(idrid_val_ds))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T15:01:06.409538Z","iopub.execute_input":"2026-05-03T15:01:06.409831Z","iopub.status.idle":"2026-05-03T15:01:06.415868Z","shell.execute_reply.started":"2026-05-03T15:01:06.409808Z","shell.execute_reply":"2026-05-03T15:01:06.41521Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 11 — Train the lesion model","metadata":{}},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T15:01:30.386386Z","iopub.execute_input":"2026-05-03T15:01:30.386845Z","iopub.status.idle":"2026-05-03T15:01:30.391459Z","shell.execute_reply.started":"2026-05-03T15:01:30.386817Z","shell.execute_reply":"2026-05-03T15:01:30.39059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lesion_model = timm.create_model(\"efficientnet_b0\", pretrained=True, num_classes=4)\nlesion_model = lesion_model.to(device)\n\nlesion_criterion = nn.BCEWithLogitsLoss()\nlesion_optimizer = torch.optim.AdamW(lesion_model.parameters(), lr=1e-4, weight_decay=1e-4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T15:01:43.137342Z","iopub.execute_input":"2026-05-03T15:01:43.13806Z","iopub.status.idle":"2026-05-03T15:01:45.104686Z","shell.execute_reply.started":"2026-05-03T15:01:43.138031Z","shell.execute_reply":"2026-05-03T15:01:45.103798Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_lesion_epoch(model, loader, criterion, optimizer, device):\n    model.train()\n    total_loss = 0\n\n    for imgs, labels in loader:\n        imgs, labels = imgs.to(device), labels.to(device)\n        optimizer.zero_grad()\n        logits = model(imgs)\n        loss = criterion(logits, labels)\n        loss.backward()\n        optimizer.step()\n        total_loss += loss.item() * imgs.size(0)\n\n    return total_loss / len(loader.dataset)\n\n@torch.no_grad()\ndef eval_lesion_epoch(model, loader, criterion, device):\n    model.eval()\n    total_loss = 0\n    all_true, all_pred = [], []\n\n    for imgs, labels in loader:\n        imgs, labels = imgs.to(device), labels.to(device)\n        logits = model(imgs)\n        loss = criterion(logits, labels)\n\n        total_loss += loss.item() * imgs.size(0)\n        probs = torch.sigmoid(logits)\n        preds = (probs >= 0.5).float()\n\n        all_true.append(labels.cpu().numpy())\n        all_pred.append(preds.cpu().numpy())\n\n    all_true = np.vstack(all_true)\n    all_pred = np.vstack(all_pred)\n\n    f1s = []\n    for i in range(4):\n        f1s.append(f1_score(all_true[:, i], all_pred[:, i], zero_division=0))\n\n    return total_loss / len(loader.dataset), np.mean(f1s)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T15:02:03.279978Z","iopub.execute_input":"2026-05-03T15:02:03.280804Z","iopub.status.idle":"2026-05-03T15:02:03.288267Z","shell.execute_reply.started":"2026-05-03T15:02:03.280773Z","shell.execute_reply":"2026-05-03T15:02:03.287384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_lesion_f1 = -1\n\nfor epoch in range(1, 11):\n    tr_loss = train_lesion_epoch(lesion_model, idrid_train_loader, lesion_criterion, lesion_optimizer, device)\n    va_loss, va_f1 = eval_lesion_epoch(lesion_model, idrid_val_loader, lesion_criterion, device)\n\n    print(f\"Epoch {epoch} | Train Loss: {tr_loss:.4f} | Val Loss: {va_loss:.4f} | Macro Lesion F1: {va_f1:.4f}\")\n\n    if va_f1 > best_lesion_f1:\n        best_lesion_f1 = va_f1\n        torch.save(lesion_model.state_dict(), \"/kaggle/working/models/lesion_model_best.pt\")\n        print(\"Saved best lesion model\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T15:02:22.368566Z","iopub.execute_input":"2026-05-03T15:02:22.369277Z","iopub.status.idle":"2026-05-03T15:08:07.84495Z","shell.execute_reply.started":"2026-05-03T15:02:22.369246Z","shell.execute_reply":"2026-05-03T15:08:07.843812Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"SAVING FIRST Day work","metadata":{}},{"cell_type":"code","source":"import os, json, joblib, torch\nimport pandas as pd\n\nSAVE_DIR = \"/kaggle/working/scope_a_checkpoint_v1\"\nos.makedirs(SAVE_DIR, exist_ok=True)\n\n# -----------------------------\n# 1. Save symptom model\n# -----------------------------\njoblib.dump(symptom_model, f\"{SAVE_DIR}/lightgbm_symptom.pkl\")\n\n# -----------------------------\n# 2. Save lesion model checkpoint\n#    includes weights + metadata\n# -----------------------------\ntorch.save({\n    \"model_name\": \"efficientnet_b0\",\n    \"num_classes\": 4,\n    \"best_lesion_f1\": float(best_lesion_f1) if \"best_lesion_f1\" in globals() else None,\n    \"model_state_dict\": lesion_model.state_dict(),\n    \"optimizer_state_dict\": lesion_optimizer.state_dict() if \"lesion_optimizer\" in globals() else None,\n}, f\"{SAVE_DIR}/lesion_model_checkpoint.pt\")\n\n# -----------------------------\n# 3. Save processed dataframes\n# -----------------------------\nif \"uci\" in globals():\n    uci.to_csv(f\"{SAVE_DIR}/uci_clean.csv\", index=False)\n\nif \"idrid_df\" in globals():\n    idrid_df.to_csv(f\"{SAVE_DIR}/idrid_df.csv\", index=False)\n\nif \"train_idrid\" in globals():\n    train_idrid.to_csv(f\"{SAVE_DIR}/train_idrid.csv\", index=False)\n\nif \"val_idrid\" in globals():\n    val_idrid.to_csv(f\"{SAVE_DIR}/val_idrid.csv\", index=False)\n\n# -----------------------------\n# 4. Save transforms / feature column order info\n# -----------------------------\nmeta = {\n    \"uci_target_col\": \"class\",\n    \"symptom_feature_columns\": list(X.columns) if \"X\" in globals() else None,\n    \"lesion_labels\": [\"ma\", \"he\", \"ex\", \"se\"],\n    \"stage_completed\": \"uci_done__idrid_lesion_done\",\n    \"next_stage\": \"eyepacs_swin_training\",\n}\n\nwith open(f\"{SAVE_DIR}/progress_meta.json\", \"w\") as f:\n    json.dump(meta, f, indent=2)\n\nprint(\"Saved checkpoint files:\")\nprint(os.listdir(SAVE_DIR))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T15:15:19.29238Z","iopub.execute_input":"2026-05-03T15:15:19.293141Z","iopub.status.idle":"2026-05-03T15:15:19.445825Z","shell.execute_reply.started":"2026-05-03T15:15:19.293106Z","shell.execute_reply":"2026-05-03T15:15:19.445138Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"readme = \"\"\"\nCheckpoint: Scope A prototype progress\n\nCompleted:\n- UCI symptom model trained and saved\n- IDRiD lesion presence model trained and saved\n\nNext:\n- Attach original EyePACS competition data\n- Build EyePACS patient-level split\n- Train Swin-T baseline\n- Then connect fusion + bilingual reporting\n\nMain files:\n- lightgbm_symptom.pkl\n- lesion_model_checkpoint.pt\n- uci_clean.csv\n- idrid_df.csv\n- train_idrid.csv\n- val_idrid.csv\n- progress_meta.json\n\"\"\"\n\nwith open(\"/kaggle/working/scope_a_checkpoint_v1/README.txt\", \"w\") as f:\n    f.write(readme)\n\nprint(\"README saved\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T15:15:36.187915Z","iopub.execute_input":"2026-05-03T15:15:36.188704Z","iopub.status.idle":"2026-05-03T15:15:36.193722Z","shell.execute_reply.started":"2026-05-03T15:15:36.188672Z","shell.execute_reply":"2026-05-03T15:15:36.193051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\n\nshutil.make_archive(\n    \"/kaggle/working/scope_a_checkpoint_v1\",\n    'zip',\n    \"/kaggle/working/scope_a_checkpoint_v1\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-03T15:27:44.450876Z","iopub.execute_input":"2026-05-03T15:27:44.451733Z","iopub.status.idle":"2026-05-03T15:27:46.892983Z","shell.execute_reply.started":"2026-05-03T15:27:44.451666Z","shell.execute_reply":"2026-05-03T15:27:46.892295Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Check GPU and EyePACS paths****","metadata":{}},{"cell_type":"code","source":"import os, glob, zipfile, random, warnings, subprocess, json\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\nimport timm\n\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom sklearn.metrics import (\n    accuracy_score,\n    f1_score,\n    classification_report,\n    confusion_matrix,\n    cohen_kappa_score\n)\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Device:\", device)\n\nEYEPACS_BASE = \"/kaggle/input/competitions/diabetic-retinopathy-detection\"\nEYEPACS_LABELS_ZIP = f\"{EYEPACS_BASE}/trainLabels.csv.zip\"\nEYEPACS_TRAIN_ZIP_001 = f\"{EYEPACS_BASE}/train.zip.001\"\n\nprint(\"EyePACS base exists:\", os.path.exists(EYEPACS_BASE))\nprint(\"Labels zip exists:\", os.path.exists(EYEPACS_LABELS_ZIP))\nprint(\"Train split zip exists:\", os.path.exists(EYEPACS_TRAIN_ZIP_001))\nprint(os.listdir(EYEPACS_BASE))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:41:09.137706Z","iopub.execute_input":"2026-05-04T13:41:09.138033Z","iopub.status.idle":"2026-05-04T13:41:28.07881Z","shell.execute_reply.started":"2026-05-04T13:41:09.138001Z","shell.execute_reply":"2026-05-04T13:41:28.078031Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"os.makedirs(\"/kaggle/working/eyepacs_labels\", exist_ok=True)\n\nwith zipfile.ZipFile(EYEPACS_LABELS_ZIP, \"r\") as zf:\n    zf.extractall(\"/kaggle/working/eyepacs_labels\")\n\nlabels_path = \"/kaggle/working/eyepacs_labels/trainLabels.csv\"\n\ndr = pd.read_csv(labels_path)\n\nprint(dr.head())\nprint(\"Shape:\", dr.shape)\nprint(\"Class distribution:\")\nprint(dr[\"level\"].value_counts().sort_index())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:42:16.028133Z","iopub.execute_input":"2026-05-04T13:42:16.029082Z","iopub.status.idle":"2026-05-04T13:42:16.120732Z","shell.execute_reply.started":"2026-05-04T13:42:16.029048Z","shell.execute_reply":"2026-05-04T13:42:16.120059Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"****Step 3 — Create patient-level split\n\nThis is important because EyePACS has left/right eye images. We should avoid putting one eye in train and the other eye in validation.","metadata":{}},{"cell_type":"code","source":"dr[\"patient_id\"] = dr[\"image\"].apply(lambda x: str(x).rsplit(\"_\", 1)[0])\n\ngss = GroupShuffleSplit(n_splits=1, test_size=0.15, random_state=42)\ntrain_idx, val_idx = next(\n    gss.split(dr, dr[\"level\"], groups=dr[\"patient_id\"])\n)\n\ntrain_df = dr.iloc[train_idx].reset_index(drop=True)\nval_df = dr.iloc[val_idx].reset_index(drop=True)\n\nprint(\"Train:\", train_df.shape)\nprint(\"Val:\", val_df.shape)\n\nprint(\"\\nTrain distribution:\")\nprint(train_df[\"level\"].value_counts().sort_index())\n\nprint(\"\\nVal distribution:\")\nprint(val_df[\"level\"].value_counts().sort_index())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:42:51.815296Z","iopub.execute_input":"2026-05-04T13:42:51.815697Z","iopub.status.idle":"2026-05-04T13:42:51.874738Z","shell.execute_reply.started":"2026-05-04T13:42:51.815665Z","shell.execute_reply":"2026-05-04T13:42:51.874117Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 4 — Create a small balanced subset first","metadata":{}},{"cell_type":"code","source":"def sample_per_class(df, per_class_dict, seed=42):\n    parts = []\n    for cls, n in per_class_dict.items():\n        sub = df[df[\"level\"] == cls]\n        n = min(len(sub), n)\n        parts.append(sub.sample(n=n, random_state=seed))\n    return pd.concat(parts).sample(frac=1, random_state=seed).reset_index(drop=True)\n\ntrain_small = sample_per_class(train_df, {\n    0: 1000,\n    1: 300,\n    2: 600,\n    3: 250,\n    4: 250\n})\n\nval_small = sample_per_class(val_df, {\n    0: 250,\n    1: 75,\n    2: 150,\n    3: 50,\n    4: 50\n})\n\nprint(\"Train small distribution:\")\nprint(train_small[\"level\"].value_counts().sort_index())\n\nprint(\"\\nVal small distribution:\")\nprint(val_small[\"level\"].value_counts().sort_index())\n\nos.makedirs(\"/kaggle/working/scope_a_checkpoint_v2\", exist_ok=True)\ntrain_small.to_csv(\"/kaggle/working/scope_a_checkpoint_v2/eyepacs_train_small.csv\", index=False)\nval_small.to_csv(\"/kaggle/working/scope_a_checkpoint_v2/eyepacs_val_small.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:43:19.272885Z","iopub.execute_input":"2026-05-04T13:43:19.273474Z","iopub.status.idle":"2026-05-04T13:43:19.307459Z","shell.execute_reply.started":"2026-05-04T13:43:19.273445Z","shell.execute_reply":"2026-05-04T13:43:19.306865Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!7z l /kaggle/input/competitions/diabetic-retinopathy-detection/train.zip.001 | head -80","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:43:46.697859Z","iopub.execute_input":"2026-05-04T13:43:46.698397Z","iopub.status.idle":"2026-05-04T13:43:47.599589Z","shell.execute_reply.started":"2026-05-04T13:43:46.698361Z","shell.execute_reply":"2026-05-04T13:43:47.598663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"archive_list_output = subprocess.check_output(\n    [\"7z\", \"l\", EYEPACS_TRAIN_ZIP_001],\n    text=True,\n    errors=\"ignore\"\n)\n\nimage_paths_in_archive = []\nfor line in archive_list_output.splitlines():\n    line = line.strip()\n    if line.lower().endswith((\".jpeg\", \".jpg\", \".png\")):\n        image_paths_in_archive.append(line.split()[-1])\n\nprint(\"Number of image paths found in archive listing:\", len(image_paths_in_archive))\nprint(\"First 10 archive image paths:\")\nprint(image_paths_in_archive[:10])\n\nsample_archive_path = image_paths_in_archive[0]\narchive_prefix = \"\"\n\nif \"/\" in sample_archive_path:\n    archive_prefix = sample_archive_path.rsplit(\"/\", 1)[0] + \"/\"\n\narchive_ext = os.path.splitext(sample_archive_path)[1]\n\nprint(\"Detected prefix:\", archive_prefix)\nprint(\"Detected extension:\", archive_ext)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:44:08.984154Z","iopub.execute_input":"2026-05-04T13:44:08.984527Z","iopub.status.idle":"2026-05-04T13:44:09.684792Z","shell.execute_reply.started":"2026-05-04T13:44:08.984489Z","shell.execute_reply":"2026-05-04T13:44:09.683939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_small[\"archive_path\"] = archive_prefix + train_small[\"image\"].astype(str) + archive_ext\nval_small[\"archive_path\"] = archive_prefix + val_small[\"image\"].astype(str) + archive_ext\n\nos.makedirs(\"/kaggle/working/eyepacs_extract_lists\", exist_ok=True)\n\ntrain_list_file = \"/kaggle/working/eyepacs_extract_lists/train_files.txt\"\nval_list_file = \"/kaggle/working/eyepacs_extract_lists/val_files.txt\"\n\ntrain_small[\"archive_path\"].to_csv(train_list_file, index=False, header=False)\nval_small[\"archive_path\"].to_csv(val_list_file, index=False, header=False)\n\nprint(\"Example train paths:\")\nprint(train_small[\"archive_path\"].head())\n\nprint(\"Saved:\")\nprint(train_list_file)\nprint(val_list_file)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:44:29.473276Z","iopub.execute_input":"2026-05-04T13:44:29.474038Z","iopub.status.idle":"2026-05-04T13:44:29.490068Z","shell.execute_reply.started":"2026-05-04T13:44:29.474007Z","shell.execute_reply":"2026-05-04T13:44:29.489362Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.makedirs(\"/kaggle/working/eyepacs_subset_raw/train\", exist_ok=True)\nos.makedirs(\"/kaggle/working/eyepacs_subset_raw/val\", exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:44:43.836022Z","iopub.execute_input":"2026-05-04T13:44:43.836735Z","iopub.status.idle":"2026-05-04T13:44:43.840428Z","shell.execute_reply.started":"2026-05-04T13:44:43.836702Z","shell.execute_reply":"2026-05-04T13:44:43.839767Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!7z x /kaggle/input/competitions/diabetic-retinopathy-detection/train.zip.001 \\\n-o/kaggle/working/eyepacs_subset_raw/train \\\n@/kaggle/working/eyepacs_extract_lists/train_files.txt -y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:44:58.852289Z","iopub.execute_input":"2026-05-04T13:44:58.8531Z","iopub.status.idle":"2026-05-04T13:46:17.812053Z","shell.execute_reply.started":"2026-05-04T13:44:58.853066Z","shell.execute_reply":"2026-05-04T13:46:17.811052Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!7z x /kaggle/input/competitions/diabetic-retinopathy-detection/train.zip.001 \\\n-o/kaggle/working/eyepacs_subset_raw/val \\\n@/kaggle/working/eyepacs_extract_lists/val_files.txt -y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:48:10.933499Z","iopub.execute_input":"2026-05-04T13:48:10.934073Z","iopub.status.idle":"2026-05-04T13:48:18.512291Z","shell.execute_reply.started":"2026-05-04T13:48:10.934039Z","shell.execute_reply":"2026-05-04T13:48:18.511567Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 8 — Verify extraction","metadata":{}},{"cell_type":"code","source":"for root, dirs, files in os.walk(\"/kaggle/working/eyepacs_subset_raw\"):\n    if len(files) > 0:\n        print(root, \"->\", len(files), \"files\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:48:45.805741Z","iopub.execute_input":"2026-05-04T13:48:45.806129Z","iopub.status.idle":"2026-05-04T13:48:45.8169Z","shell.execute_reply.started":"2026-05-04T13:48:45.806082Z","shell.execute_reply":"2026-05-04T13:48:45.816421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def find_image_path(base_dir, image_id):\n    matches = glob.glob(\n        os.path.join(base_dir, \"**\", f\"{image_id}.*\"),\n        recursive=True\n    )\n    matches = [m for m in matches if m.lower().endswith((\".jpeg\", \".jpg\", \".png\"))]\n    return matches[0] if len(matches) > 0 else None\n\ntrain_small[\"image_path\"] = train_small[\"image\"].apply(\n    lambda x: find_image_path(\"/kaggle/working/eyepacs_subset_raw/train\", x)\n)\n\nval_small[\"image_path\"] = val_small[\"image\"].apply(\n    lambda x: find_image_path(\"/kaggle/working/eyepacs_subset_raw/val\", x)\n)\n\nprint(\"Missing train images:\", train_small[\"image_path\"].isna().sum())\nprint(\"Missing val images:\", val_small[\"image_path\"].isna().sum())\n\ntrain_small = train_small.dropna(subset=[\"image_path\"]).reset_index(drop=True)\nval_small = val_small.dropna(subset=[\"image_path\"]).reset_index(drop=True)\n\nprint(\"Final train:\", train_small.shape)\nprint(\"Final val:\", val_small.shape)\n\ntrain_small.to_csv(\"/kaggle/working/scope_a_checkpoint_v2/eyepacs_train_small_with_paths.csv\", index=False)\nval_small.to_csv(\"/kaggle/working/scope_a_checkpoint_v2/eyepacs_val_small_with_paths.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:49:00.564679Z","iopub.execute_input":"2026-05-04T13:49:00.56498Z","iopub.status.idle":"2026-05-04T13:49:10.463359Z","shell.execute_reply.started":"2026-05-04T13:49:00.564953Z","shell.execute_reply":"2026-05-04T13:49:10.462581Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 9 — Create EyePACS dataset class","metadata":{}},{"cell_type":"code","source":"class EyePACSDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        image_path = row[\"image_path\"]\n        label = int(row[\"level\"])\n\n        img = Image.open(image_path).convert(\"RGB\")\n\n        if self.transform:\n            img = self.transform(img)\n\n        return img, label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:50:02.756002Z","iopub.execute_input":"2026-05-04T13:50:02.75679Z","iopub.status.idle":"2026-05-04T13:50:02.76212Z","shell.execute_reply.started":"2026-05-04T13:50:02.756756Z","shell.execute_reply":"2026-05-04T13:50:02.761163Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 10 — Image transforms","metadata":{}},{"cell_type":"code","source":"train_tfms = transforms.Compose([\n    transforms.Resize((256, 256)),\n    transforms.CenterCrop((224, 224)),\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.ColorJitter(\n        brightness=0.12,\n        contrast=0.12,\n        saturation=0.08\n    ),\n    transforms.ToTensor(),\n    transforms.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225]\n    )\n])\n\nval_tfms = transforms.Compose([\n    transforms.Resize((256, 256)),\n    transforms.CenterCrop((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225]\n    )\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:50:29.743195Z","iopub.execute_input":"2026-05-04T13:50:29.743527Z","iopub.status.idle":"2026-05-04T13:50:29.749798Z","shell.execute_reply.started":"2026-05-04T13:50:29.743497Z","shell.execute_reply":"2026-05-04T13:50:29.74894Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 11 — Dataloaders","metadata":{}},{"cell_type":"code","source":"train_ds = EyePACSDataset(train_small, transform=train_tfms)\nval_ds = EyePACSDataset(val_small, transform=val_tfms)\n\ntrain_loader = DataLoader(\n    train_ds,\n    batch_size=16,\n    shuffle=True,\n    num_workers=2,\n    pin_memory=True\n)\n\nval_loader = DataLoader(\n    val_ds,\n    batch_size=16,\n    shuffle=False,\n    num_workers=2,\n    pin_memory=True\n)\n\nprint(\"Train size:\", len(train_ds))\nprint(\"Val size:\", len(val_ds))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:50:55.744714Z","iopub.execute_input":"2026-05-04T13:50:55.74545Z","iopub.status.idle":"2026-05-04T13:50:55.752252Z","shell.execute_reply.started":"2026-05-04T13:50:55.745416Z","shell.execute_reply":"2026-05-04T13:50:55.751413Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 12 — Build Swin-T model","metadata":{}},{"cell_type":"code","source":"model = timm.create_model(\n    \"swin_tiny_patch4_window7_224\",\n    pretrained=True,\n    num_classes=5\n)\n\nmodel = model.to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:51:20.649662Z","iopub.execute_input":"2026-05-04T13:51:20.649963Z","iopub.status.idle":"2026-05-04T13:51:23.956558Z","shell.execute_reply.started":"2026-05-04T13:51:20.649936Z","shell.execute_reply":"2026-05-04T13:51:23.955873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class_counts = train_small[\"level\"].value_counts().sort_index().values\nclass_weights = 1.0 / torch.tensor(class_counts, dtype=torch.float32)\nclass_weights = class_weights / class_weights.sum() * len(class_counts)\nclass_weights = class_weights.to(device)\n\nprint(\"Class counts:\", class_counts)\nprint(\"Class weights:\", class_weights)\n\ncriterion = nn.CrossEntropyLoss(weight=class_weights)\n\noptimizer = torch.optim.AdamW(\n    model.parameters(),\n    lr=1e-4,\n    weight_decay=1e-4\n)\n\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n    optimizer,\n    T_max=8\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:51:40.005854Z","iopub.execute_input":"2026-05-04T13:51:40.006577Z","iopub.status.idle":"2026-05-04T13:51:40.534932Z","shell.execute_reply.started":"2026-05-04T13:51:40.006546Z","shell.execute_reply":"2026-05-04T13:51:40.534181Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 13 — Training and validation functions","metadata":{}},{"cell_type":"code","source":"def compute_metrics(y_true, y_pred):\n    acc = accuracy_score(y_true, y_pred)\n    macro_f1 = f1_score(y_true, y_pred, average=\"macro\")\n    qwk = cohen_kappa_score(y_true, y_pred, weights=\"quadratic\")\n    return acc, macro_f1, qwk\n\n\ndef train_one_epoch(model, loader, criterion, optimizer, device):\n    model.train()\n\n    total_loss = 0.0\n    all_preds = []\n    all_labels = []\n\n    for imgs, labels in loader:\n        imgs = imgs.to(device, non_blocking=True)\n        labels = labels.to(device, non_blocking=True)\n\n        optimizer.zero_grad()\n\n        logits = model(imgs)\n        loss = criterion(logits, labels)\n\n        loss.backward()\n        optimizer.step()\n\n        total_loss += loss.item() * imgs.size(0)\n\n        preds = torch.argmax(logits, dim=1)\n        all_preds.extend(preds.detach().cpu().numpy())\n        all_labels.extend(labels.detach().cpu().numpy())\n\n    avg_loss = total_loss / len(loader.dataset)\n    acc, macro_f1, qwk = compute_metrics(all_labels, all_preds)\n\n    return avg_loss, acc, macro_f1, qwk\n\n\n@torch.no_grad()\ndef eval_one_epoch(model, loader, criterion, device):\n    model.eval()\n\n    total_loss = 0.0\n    all_preds = []\n    all_labels = []\n    all_probs = []\n\n    for imgs, labels in loader:\n        imgs = imgs.to(device, non_blocking=True)\n        labels = labels.to(device, non_blocking=True)\n\n        logits = model(imgs)\n        loss = criterion(logits, labels)\n\n        probs = torch.softmax(logits, dim=1)\n        preds = torch.argmax(probs, dim=1)\n\n        total_loss += loss.item() * imgs.size(0)\n\n        all_preds.extend(preds.detach().cpu().numpy())\n        all_labels.extend(labels.detach().cpu().numpy())\n        all_probs.extend(probs.detach().cpu().numpy())\n\n    avg_loss = total_loss / len(loader.dataset)\n    acc, macro_f1, qwk = compute_metrics(all_labels, all_preds)\n\n    return avg_loss, acc, macro_f1, qwk, all_labels, all_preds, np.array(all_probs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:52:10.84466Z","iopub.execute_input":"2026-05-04T13:52:10.845426Z","iopub.status.idle":"2026-05-04T13:52:10.854882Z","shell.execute_reply.started":"2026-05-04T13:52:10.845393Z","shell.execute_reply":"2026-05-04T13:52:10.854026Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 14 — Train Swin-T","metadata":{}},{"cell_type":"code","source":"best_f1 = -1\nhistory = []\n\nnum_epochs = 8\n\nfor epoch in range(1, num_epochs + 1):\n    train_loss, train_acc, train_f1, train_qwk = train_one_epoch(\n        model, train_loader, criterion, optimizer, device\n    )\n\n    val_loss, val_acc, val_f1, val_qwk, y_true, y_pred, y_probs = eval_one_epoch(\n        model, val_loader, criterion, device\n    )\n\n    scheduler.step()\n\n    row = {\n        \"epoch\": epoch,\n        \"train_loss\": train_loss,\n        \"train_acc\": train_acc,\n        \"train_f1\": train_f1,\n        \"train_qwk\": train_qwk,\n        \"val_loss\": val_loss,\n        \"val_acc\": val_acc,\n        \"val_f1\": val_f1,\n        \"val_qwk\": val_qwk\n    }\n\n    history.append(row)\n\n    print(f\"\\nEpoch {epoch}/{num_epochs}\")\n    print(f\"Train | Loss {train_loss:.4f} | Acc {train_acc:.4f} | F1 {train_f1:.4f} | QWK {train_qwk:.4f}\")\n    print(f\"Val   | Loss {val_loss:.4f} | Acc {val_acc:.4f} | F1 {val_f1:.4f} | QWK {val_qwk:.4f}\")\n\n    if val_f1 > best_f1:\n        best_f1 = val_f1\n\n        torch.save({\n            \"epoch\": epoch,\n            \"best_f1\": float(best_f1),\n            \"model_name\": \"swin_tiny_patch4_window7_224\",\n            \"num_classes\": 5,\n            \"model_state_dict\": model.state_dict(),\n            \"optimizer_state_dict\": optimizer.state_dict(),\n            \"class_weights\": class_weights.detach().cpu().numpy().tolist(),\n            \"label_map\": {\n                0: \"No DR\",\n                1: \"Mild DR\",\n                2: \"Moderate DR\",\n                3: \"Severe DR\",\n                4: \"Proliferative DR\"\n            }\n        }, \"/kaggle/working/scope_a_checkpoint_v2/swin_dr_checkpoint.pt\")\n\n        print(\"Saved best Swin-T checkpoint\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T13:52:36.263184Z","iopub.execute_input":"2026-05-04T13:52:36.2639Z","iopub.status.idle":"2026-05-04T14:19:24.730486Z","shell.execute_reply.started":"2026-05-04T13:52:36.263868Z","shell.execute_reply":"2026-05-04T14:19:24.729542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, json\n\nV2_DIR = \"/kaggle/working/scope_a_checkpoint_v2\"\n\nmeta = {\n    \"stage_completed\": \"eyepacs_swin_t_done\",\n    \"checkpoint_version\": \"scope_a_checkpoint_v2\",\n    \"dr_model\": {\n        \"file\": \"swin_dr_checkpoint.pt\",\n        \"model_name\": \"swin_tiny_patch4_window7_224\",\n        \"num_classes\": 5,\n        \"label_map\": {\n            \"0\": \"No DR\",\n            \"1\": \"Mild DR\",\n            \"2\": \"Moderate DR\",\n            \"3\": \"Severe DR\",\n            \"4\": \"Proliferative DR\"\n        }\n    },\n    \"next_stage\": \"fusion_and_bilingual_reporting\"\n}\n\nwith open(f\"{V2_DIR}/progress_meta.json\", \"w\") as f:\n    json.dump(meta, f, indent=2)\n\nprint(os.listdir(V2_DIR))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T14:22:46.40148Z","iopub.execute_input":"2026-05-04T14:22:46.402144Z","iopub.status.idle":"2026-05-04T14:22:46.408602Z","shell.execute_reply.started":"2026-05-04T14:22:46.402105Z","shell.execute_reply":"2026-05-04T14:22:46.407938Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\nimport os\n\nSRC_DIR = \"/kaggle/working/scope_a_checkpoint_v2\"\nZIP_PATH = \"/kaggle/working/scope_a_checkpoint_v2.zip\"\n\n# Check folder exists\nprint(\"Folder exists:\", os.path.exists(SRC_DIR))\nprint(\"Files inside:\")\nprint(os.listdir(SRC_DIR))\n\n# Remove old zip if already exists\nif os.path.exists(ZIP_PATH):\n    os.remove(ZIP_PATH)\n\n# Create zip\nshutil.make_archive(\n    base_name=\"/kaggle/working/scope_a_checkpoint_v2\",\n    format=\"zip\",\n    root_dir=\"/kaggle/working\",\n    base_dir=\"scope_a_checkpoint_v2\"\n)\n\nprint(\"Zip created:\", os.path.exists(ZIP_PATH))\nprint(\"Zip path:\", ZIP_PATH)\nprint(\"Zip size MB:\", os.path.getsize(ZIP_PATH) / (1024 * 1024))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T14:24:22.334035Z","iopub.execute_input":"2026-05-04T14:24:22.334731Z","iopub.status.idle":"2026-05-04T14:24:38.492502Z","shell.execute_reply.started":"2026-05-04T14:24:22.334697Z","shell.execute_reply":"2026-05-04T14:24:38.491661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import zipfile\n\nZIP_PATH = \"/kaggle/working/scope_a_checkpoint_v2.zip\"\n\nwith zipfile.ZipFile(ZIP_PATH, \"r\") as zf:\n    files = zf.namelist()\n    print(\"Total files:\", len(files))\n    for f in files:\n        print(f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-04T14:24:51.335381Z","iopub.execute_input":"2026-05-04T14:24:51.336107Z","iopub.status.idle":"2026-05-04T14:24:51.342112Z","shell.execute_reply.started":"2026-05-04T14:24:51.336075Z","shell.execute_reply":"2026-05-04T14:24:51.341556Z"}},"outputs":[],"execution_count":null}]}