{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":10338,"databundleVersionId":862042},{"sourceType":"datasetVersion","sourceId":15353548,"datasetId":9820170,"databundleVersionId":16263555},{"sourceType":"datasetVersion","sourceId":23812,"datasetId":17810,"databundleVersionId":23851},{"sourceType":"datasetVersion","sourceId":1493513,"datasetId":876960,"databundleVersionId":1527499},{"sourceType":"datasetVersion","sourceId":6717213,"datasetId":1317048,"databundleVersionId":6801677},{"sourceType":"datasetVersion","sourceId":18613,"datasetId":5839,"databundleVersionId":18613},{"sourceType":"datasetVersion","sourceId":15336015,"datasetId":9809316,"databundleVersionId":16244170}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🧠 Medical Imaging — Master Dataset Merge Pipeline (RELATIVE PATHS)\n**Platform:** Kaggle Notebook\n#\nThis notebook:\n1. Loads all 5 dataset parquets\n2. Filters to 3 target classes only (Normal, Pneumonia, COVID-19)\n3. Converts RSNA DICOMs to PNG\n4. Produces ONE merged parquet with RELATIVE (portable) paths:\n   - master_merged_relative.parquet\n5. Validates output\n#\nUse this version when you plan to use the parquet on Colab / local.\nPaths are relative to each dataset's base directory.\nTo resolve a path:  base_dir[dataset] + row[\"path\"]\nRSNA paths are relative to /kaggle/working/  (i.e. \"rsna_png/xxxx.png\")\n#\nAdd the following datasets via Add Data before running:\n  - Your parquet files dataset (kaushik2005/parquetfiles)\n  - NIH:       nih-chest-xrays/data\n  - CheXpert:  willarevalo/chexpert-v10-small\n  - COVIDx:    andyczhao/covidx-cxr2\n  - Pediatric: paultimothymooney/chest-xray-pneumonia\n  - RSNA:      rsna-pneumonia-detection-challenge (competition)","metadata":{"_uuid":"905740ee-46e7-4144-a25f-b028ab0936eb","_cell_guid":"55404edf-b0c0-46fa-803a-8181038d0544","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 1 — Install dependencies\n# ─────────────────────────────────────────────────────────────\n\n!pip install -q pyarrow pydicom","metadata":{"_uuid":"53f69148-28c7-42ec-8259-01069122b773","_cell_guid":"98760483-5157-4dda-b5b0-2ddb2b03db0f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-25T03:24:15.864638Z","iopub.execute_input":"2026-03-25T03:24:15.864812Z","iopub.status.idle":"2026-03-25T03:24:20.44132Z","shell.execute_reply.started":"2026-03-25T03:24:15.864792Z","shell.execute_reply":"2026-03-25T03:24:20.440342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 2 — Imports\n# ─────────────────────────────────────────────────────────────\n\nimport os\nimport json\nimport random\nimport hashlib\nimport logging\nimport warnings\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nfrom PIL import Image\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8e644c19-2aa8-4333-9cfb-1eac633261ec","_cell_guid":"c5ce991b-df44-4fa9-abb6-b5ab561f0085","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-25T03:24:24.78703Z","iopub.execute_input":"2026-03-25T03:24:24.787352Z","iopub.status.idle":"2026-03-25T03:24:25.666859Z","shell.execute_reply.started":"2026-03-25T03:24:24.787318Z","shell.execute_reply":"2026-03-25T03:24:25.666313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 3 — Paths and constants\n# ─────────────────────────────────────────────────────────────\n\n# ── Input parquet locations ───────────────────────────────────\nPARQUET_PATHS = {\n    \"NIH\":       \"/kaggle/input/datasets/kaushik2005/relativeparquet/nih_master_index.parquet\",\n    \"CheXpert\":  \"/kaggle/input/datasets/kaushik2005/relativeparquet/chexpert_master_index.parquet\",\n    \"COVIDx\":    \"/kaggle/input/datasets/kaushik2005/relativeparquet/covidx_master_index.parquet\",\n    \"Pediatric\": \"/kaggle/input/datasets/kaushik2005/relativeparquet/pediatric_master_index.parquet\",\n    \"RSNA\":      \"/kaggle/input/datasets/kaushik2005/relativeparquet/rsna_master_index.parquet\",\n}\n\n# ── Absolute base dirs — used ONLY for RSNA DICOM conversion\n# and for the path-resolution validation check at the end.\n# The saved parquet will NOT contain these prefixes.\nABS_BASE_DIRS = {\n    \"NIH\":       \"/kaggle/input/datasets/organizations/nih-chest-xrays/data/\",\n    \"CheXpert\":  \"/kaggle/input/datasets/willarevalo/chexpert-v10-small/CheXpert-v1.0-small/\",\n    \"COVIDx\":    \"/kaggle/input/datasets/andyczhao/covidx-cxr2/\",\n    \"Pediatric\": \"/kaggle/input/datasets/paultimothymooney/chest-xray-pneumonia/chest_xray/\",\n    \"RSNA\":      \"/kaggle/input/competitions/rsna-pneumonia-detection-challenge/\",\n}\n\n# ── Output paths ──────────────────────────────────────────────\nOUTPUT_DIR   = Path(\"/kaggle/working\")\nRSNA_PNG_DIR = OUTPUT_DIR / \"rsna_png\"\nRSNA_PNG_DIR.mkdir(parents=True, exist_ok=True)\n\nOUT_RELATIVE = OUTPUT_DIR / \"master_merged_relative.parquet\"\nOUT_REPORT   = OUTPUT_DIR / \"merge_report_relative.json\"\n\n# ── Target classes only ───────────────────────────────────────\nTARGET_CLASSES = {\"No Finding\", \"Pneumonia\", \"COVID-19\"}\n\n# ── Logging ───────────────────────────────────────────────────\nLOG_PATH = OUTPUT_DIR / \"merge_log_relative.txt\"\nfor handler in logging.root.handlers[:]:\n    logging.root.removeHandler(handler)\nlogging.basicConfig(\n    level=logging.INFO,\n    format=\"%(asctime)s [%(levelname)s] %(message)s\",\n    handlers=[\n        logging.FileHandler(LOG_PATH),\n        logging.StreamHandler(),\n    ],\n)\nlog = logging.getLogger(__name__)\nlog.info(\"Merge pipeline (RELATIVE) initialised.\")\n\nprint(\"Paths configured.\")\nprint(f\"  Output dir    : {OUTPUT_DIR}\")\nprint(f\"  RSNA PNG dir  : {RSNA_PNG_DIR}\")\nprint(f\"  Target classes: {TARGET_CLASSES}\")\nprint(f\"  Mode          : RELATIVE PATHS\")","metadata":{"_uuid":"2dd96fea-2c3b-41d5-bb2a-88c7a78d22b9","_cell_guid":"ada0009c-6c72-425b-a8c5-c331514adfdc","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-25T03:24:28.069375Z","iopub.execute_input":"2026-03-25T03:24:28.070199Z","iopub.status.idle":"2026-03-25T03:24:28.079548Z","shell.execute_reply.started":"2026-03-25T03:24:28.070174Z","shell.execute_reply":"2026-03-25T03:24:28.078659Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 4 — Load and inspect all parquets\n# ─────────────────────────────────────────────────────────────\n\nframes = {}\nfor name, path in PARQUET_PATHS.items():\n    df = pd.read_parquet(path)\n    frames[name] = df\n    total   = len(df)\n    kept    = df[\"disease\"].isin(TARGET_CLASSES).sum()\n    dropped = total - kept\n    print(f\"{name}\")\n    print(f\"  Total rows     : {total:,}\")\n    print(f\"  Target classes : {kept:,}  ({kept/total*100:.1f}%)\")\n    print(f\"  Will drop      : {dropped:,}  ({dropped/total*100:.1f}%)\")\n    print(f\"  Disease counts : {df['disease'].value_counts().to_dict()}\")\n    print()","metadata":{"_uuid":"262a3378-3eb6-496d-9ea9-4c8672a936c5","_cell_guid":"5f69888d-aef7-4f0f-b636-63e3c590f6c4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-25T03:24:35.674996Z","iopub.execute_input":"2026-03-25T03:24:35.675278Z","iopub.status.idle":"2026-03-25T03:24:37.290559Z","shell.execute_reply.started":"2026-03-25T03:24:35.675254Z","shell.execute_reply":"2026-03-25T03:24:37.289851Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 5 — RSNA DICOM → PNG conversion\n#\n# Converts all RSNA .dcm files that belong to target classes.\n# Skips already-converted files so re-runs are safe.\n# ─────────────────────────────────────────────────────────────\n\ndef convert_dcm_to_png(dcm_path: str, png_path: str) -> bool:\n    \"\"\"\n    Read a DICOM file, normalise pixel values to 0–255,\n    save as RGB PNG. Returns True on success.\n    \"\"\"\n    try:\n        dcm    = pydicom.dcmread(dcm_path)\n        pixels = dcm.pixel_array.astype(np.float32)\n        p_min, p_max = pixels.min(), pixels.max()\n        if p_max > p_min:\n            pixels = (pixels - p_min) / (p_max - p_min) * 255.0\n        else:\n            pixels = np.zeros_like(pixels)\n        img = Image.fromarray(pixels.astype(np.uint8)).convert(\"RGB\")\n        img.save(png_path, \"PNG\")\n        return True\n    except Exception as e:\n        log.warning(f\"DICOM conversion failed: {dcm_path} — {e}\")\n        return False\n\n\nrsna_df     = frames[\"RSNA\"]\nrsna_target = rsna_df[rsna_df[\"disease\"].isin(TARGET_CLASSES)].copy()\nrsna_base   = ABS_BASE_DIRS[\"RSNA\"]\n\nconverted = 0\nskipped   = 0\nfailed    = 0\n\nlog.info(f\"Converting {len(rsna_target):,} RSNA DICOMs to PNG ...\")\nprint(f\"Converting {len(rsna_target):,} RSNA DICOMs to PNG ...\")\nprint(\"(this takes ~10-15 minutes — progress shown every 1,000 files)\")\n\nfor idx, (_, row) in enumerate(rsna_target.iterrows()):\n    rel_path = row[\"path\"]                        # e.g. stage_2_train_images/xxxx.dcm\n    dcm_path = rsna_base + rel_path\n    png_name = Path(rel_path).stem + \".png\"\n    png_path = str(RSNA_PNG_DIR / png_name)\n\n    if Path(png_path).exists():\n        skipped += 1\n    else:\n        success = convert_dcm_to_png(dcm_path, png_path)\n        if success:\n            converted += 1\n        else:\n            failed += 1\n\n    if (idx + 1) % 1000 == 0:\n        print(f\"  {idx+1:,} / {len(rsna_target):,}  \"\n              f\"(converted: {converted:,}  skipped: {skipped:,}  failed: {failed:,})\")\n\nlog.info(f\"DICOM conversion done — converted: {converted}, skipped: {skipped}, failed: {failed}\")\nprint(f\"\\nDone. Converted: {converted:,}  Skipped: {skipped:,}  Failed: {failed:,}\")","metadata":{"_uuid":"ff790c91-10af-4ecc-b98b-73c5e7ebd524","_cell_guid":"17d36789-edff-4087-bc1c-e0f6ee3ac3b2","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-25T03:24:54.077935Z","iopub.execute_input":"2026-03-25T03:24:54.078639Z","iopub.status.idle":"2026-03-25T04:21:43.609943Z","shell.execute_reply.started":"2026-03-25T03:24:54.078612Z","shell.execute_reply":"2026-03-25T04:21:43.609337Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 6 — Unified schema builder (RELATIVE path mode)\n#\n# For each dataset:\n#   1. Filter to target classes\n#   2. Keep path as-is (already relative in your parquets)\n#   3. For RSNA: set path = \"rsna_png/<stem>.png\"\n#      (relative to /kaggle/working — portable)\n#   4. Drop dataset-specific columns not in unified schema\n#   5. Keep only UNIFIED_COLS\n# ─────────────────────────────────────────────────────────────\n\n# Final unified columns — \"path\" will hold the relative path\nUNIFIED_COLS = [\n    \"image_id\", \"dataset\", \"dataset_id\",\n    \"file_name\", \"patient_id\", \"study_id\",\n    \"view\", \"original_view\", \"width\", \"height\", \"aspect_ratio\",\n    \"disease\", \"raw_labels\", \"num_findings\",\n    \"age\", \"age_group\", \"gender\", \"has_age\", \"has_gender\",\n]\n\nunified_frames = []\n\nfor name, df in frames.items():\n    log.info(f\"Processing {name} ...\")\n\n    # 1. Filter to target classes\n    df_filtered = df[df[\"disease\"].isin(TARGET_CLASSES)].copy()\n    log.info(f\"  {name}: {len(df):,} → {len(df_filtered):,} rows after class filter\")\n\n    # 2. Build relative path\n    if name == \"RSNA\":\n        # Relative path: rsna_png/<stem>.png\n        # Resolves to: /kaggle/working/rsna_png/<stem>.png\n        df_filtered[\"path\"] = df_filtered[\"path\"].apply(\n            lambda p: \"rsna_png/\" + Path(p).stem + \".png\"\n        )\n        df_filtered[\"file_name\"] = df_filtered[\"path\"].apply(\n            lambda p: Path(p).name\n        )\n    else:\n        # Path is already relative in the parquets — keep as-is\n        # (no base dir prefix added)\n        pass\n\n    # 3. Drop columns not in unified schema (fixed — no exclusion bug)\n    cols_to_drop = [\"split\", \"binary_target\", \"path_absolute\",\n                    \"path_relative\", \"path_absolute_old\"]\n    df_filtered = df_filtered.drop(columns=cols_to_drop, errors=\"ignore\")\n\n    # 4. Keep only unified columns + path\n    cols_present = [c for c in UNIFIED_COLS if c in df_filtered.columns]\n    df_filtered  = df_filtered[cols_present + [\"path\"]]\n\n    unified_frames.append(df_filtered)\n    print(f\"  {name}: {len(df_filtered):,} rows  |  sample path: {df_filtered['path'].iloc[0]}\")","metadata":{"_uuid":"05f57bdd-6b9a-47d3-a524-4d7f0f12fc46","_cell_guid":"94cee531-a45c-4d92-9f11-a92f2b71fd65","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-24T16:21:54.055179Z","iopub.execute_input":"2026-03-24T16:21:54.055484Z","iopub.status.idle":"2026-03-24T16:21:54.445033Z","shell.execute_reply.started":"2026-03-24T16:21:54.055457Z","shell.execute_reply":"2026-03-24T16:21:54.444423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 7 — Concatenate all datasets\n# ─────────────────────────────────────────────────────────────\n\nmaster = pd.concat(unified_frames, ignore_index=True)\nlog.info(f\"Concatenated: {len(master):,} total rows\")\n\nprint(f\"Total rows after merge : {len(master):,}\")\nprint(f\"\\nDataset breakdown:\")\nprint(master[\"dataset\"].value_counts().to_string())\nprint(f\"\\nDisease breakdown:\")\nprint(master[\"disease\"].value_counts().to_string())\nprint(f\"\\nColumns in merged dataset:\")\nprint(list(master.columns))","metadata":{"_uuid":"9358fd82-caf8-4cb0-9046-38c282c02405","_cell_guid":"ac92f918-e5a0-4005-925e-8002193b18b7","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-24T16:21:55.442159Z","iopub.execute_input":"2026-03-24T16:21:55.442853Z","iopub.status.idle":"2026-03-24T16:21:55.558742Z","shell.execute_reply.started":"2026-03-24T16:21:55.442817Z","shell.execute_reply":"2026-03-24T16:21:55.558127Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 8 — Cross-dataset duplicate image_id check\n# ─────────────────────────────────────────────────────────────\n\ntotal_ids  = len(master)\nunique_ids = master[\"image_id\"].nunique()\ndupes      = total_ids - unique_ids\n\nprint(f\"Total rows       : {total_ids:,}\")\nprint(f\"Unique image_ids : {unique_ids:,}\")\nprint(f\"Duplicate ids    : {dupes}\")\n\nif dupes > 0:\n    log.warning(f\"Found {dupes} duplicate image_ids — dropping\")\n    master = master.drop_duplicates(subset=[\"image_id\"], keep=\"first\")\n    print(f\"After dedup      : {len(master):,} rows\")\nelse:\n    print(\"No duplicate image_ids — clean.\")","metadata":{"_uuid":"ced14c3b-8534-4f3a-a61b-169bce97680f","_cell_guid":"3ecb8565-e6e2-486f-a6b4-2b2caede7eb4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-24T16:21:58.335295Z","iopub.execute_input":"2026-03-24T16:21:58.335574Z","iopub.status.idle":"2026-03-24T16:21:58.380099Z","shell.execute_reply.started":"2026-03-24T16:21:58.335548Z","shell.execute_reply":"2026-03-24T16:21:58.379425Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 9 — Corruption spot-check (500 random images)\n#\n# For relative paths we resolve using ABS_BASE_DIRS before opening.\n# RSNA paths resolve to OUTPUT_DIR / path.\n# ─────────────────────────────────────────────────────────────\n\ndef resolve_path(row: pd.Series) -> str:\n    \"\"\"Resolve a relative path to absolute for file-open check.\"\"\"\n    ds   = row[\"dataset\"]\n    path = row[\"path\"]\n    if ds == \"RSNA\":\n        return str(OUTPUT_DIR / path)\n    else:\n        return ABS_BASE_DIRS[ds] + path\n\n\nsample_size  = 500\nsample_rows  = master.sample(n=min(sample_size, len(master)), random_state=42)\nfailed_paths = []\nchecked      = 0\n\nprint(f\"Spot-checking {len(sample_rows):,} random images ...\")\n\nfor _, row in sample_rows.iterrows():\n    abs_path = resolve_path(row)\n    try:\n        with Image.open(abs_path) as img:\n            img.verify()\n        checked += 1\n    except Exception as e:\n        failed_paths.append((abs_path, str(e)))\n\nprint(f\"  Checked : {checked:,}\")\nprint(f\"  Failed  : {len(failed_paths):,}\")\nif failed_paths:\n    print(f\"\\n  Failed paths (first 5):\")\n    for p, err in failed_paths[:5]:\n        print(f\"    {p} — {err}\")\nelse:\n    print(\"  All sampled images opened successfully.\")","metadata":{"_uuid":"b43c9412-b4ad-4d44-bad4-796c53640f6e","_cell_guid":"dd3c647e-4cd9-4951-a2bf-632467fc419d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-24T16:21:59.750987Z","iopub.execute_input":"2026-03-24T16:21:59.751652Z","iopub.status.idle":"2026-03-24T16:22:03.902105Z","shell.execute_reply.started":"2026-03-24T16:21:59.751627Z","shell.execute_reply":"2026-03-24T16:22:03.901387Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 10 — Save relative path parquet\n# ─────────────────────────────────────────────────────────────\n\nmaster.to_parquet(OUT_RELATIVE, index=False)\nlog.info(f\"Saved → {OUT_RELATIVE}\")\nprint(f\"Saved → {OUT_RELATIVE}\")\nprint(f\"  Rows    : {len(master):,}\")\nprint(f\"  Columns : {list(master.columns)}\")\nprint(f\"  Sample path: {master['path'].iloc[0]}\")","metadata":{"_uuid":"ff1d7b39-3b4a-4ff2-9f32-f80dbf2a39f5","_cell_guid":"0421f2a8-7995-4c42-aebe-a268928ffee3","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-24T16:22:08.289301Z","iopub.execute_input":"2026-03-24T16:22:08.289596Z","iopub.status.idle":"2026-03-24T16:22:08.639542Z","shell.execute_reply.started":"2026-03-24T16:22:08.28957Z","shell.execute_reply":"2026-03-24T16:22:08.638925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 11 — Generate merge report\n# ─────────────────────────────────────────────────────────────\n\nhas_age    = master[\"has_age\"]\nhas_gender = master[\"has_gender\"]\n\nreport = {\n    \"mode\": \"relative\",\n    \"total_images\": int(len(master)),\n    \"dataset_counts\": master[\"dataset\"].value_counts().to_dict(),\n    \"disease_counts\": master[\"disease\"].value_counts().to_dict(),\n    \"demographics\": {\n        \"has_age_count\":    int(has_age.sum()),\n        \"has_age_pct\":      round(has_age.mean() * 100, 1),\n        \"has_gender_count\": int(has_gender.sum()),\n        \"has_gender_pct\":   round(has_gender.mean() * 100, 1),\n        \"age_groups\":       master[\"age_group\"].value_counts().to_dict(),\n        \"gender\":           master[\"gender\"].value_counts().to_dict(),\n    },\n    \"class_balance\": {\n        cls: {\n            \"count\": int((master[\"disease\"] == cls).sum()),\n            \"pct\":   round((master[\"disease\"] == cls).mean() * 100, 1),\n        }\n        for cls in TARGET_CLASSES\n    },\n    \"per_dataset_disease\": {\n        ds: master[master[\"dataset\"] == ds][\"disease\"].value_counts().to_dict()\n        for ds in master[\"dataset\"].unique()\n    },\n    \"columns\": list(master.columns),\n    \"duplicate_image_ids\": dupes,\n    \"corruption_check\": {\n        \"sample_size\":  len(sample_rows),\n        \"failed_count\": len(failed_paths),\n    },\n}\n\nwith open(OUT_REPORT, \"w\") as f:\n    json.dump(report, f, indent=2)\nlog.info(f\"Saved merge report → {OUT_REPORT}\")\nprint(f\"Saved merge report → {OUT_REPORT}\")","metadata":{"_uuid":"b0d9566d-39b1-42c8-8db2-7bec1df23b95","_cell_guid":"a332dbab-3a0b-41d4-a837-c05026adf840","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-24T16:22:11.942876Z","iopub.execute_input":"2026-03-24T16:22:11.943534Z","iopub.status.idle":"2026-03-24T16:22:12.195184Z","shell.execute_reply.started":"2026-03-24T16:22:11.943504Z","shell.execute_reply":"2026-03-24T16:22:12.194543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 12 — VALIDATION\n# ─────────────────────────────────────────────────────────────\n\ndf_val = pd.read_parquet(OUT_RELATIVE)\n\nprint(\"=\" * 60)\nprint(\"VALIDATION — master_merged_relative.parquet\")\nprint(\"=\" * 60)\n\n# 1. Shape\nprint(f\"\\n1. Shape\")\nprint(f\"   Rows    : {len(df_val):,}\")\nprint(f\"   Columns : {len(df_val.columns)}\")\nexpected = 175883\nprint(f\"   Expected: ~{expected:,}\")\nprint(f\"   Match   : {abs(len(df_val) - expected) < 1000}\")\n\n# 2. Required columns\nrequired_cols = [\n    \"image_id\", \"dataset\", \"dataset_id\", \"path\", \"file_name\",\n    \"patient_id\", \"view\", \"original_view\", \"width\", \"height\",\n    \"aspect_ratio\", \"disease\", \"raw_labels\", \"num_findings\",\n    \"age\", \"age_group\", \"gender\", \"has_age\", \"has_gender\"\n]\nmissing = [c for c in required_cols if c not in df_val.columns]\nprint(f\"\\n2. Required columns\")\nprint(f\"   Missing : {missing if missing else 'None — all present'}\")\n\n# 3. Duplicate image_ids\nprint(f\"\\n3. Duplicate image_ids : {df_val['image_id'].duplicated().sum()} (expected 0)\")\n\n# 4. Disease distribution\nprint(f\"\\n4. Disease distribution\")\nprint(f\"   {df_val['disease'].value_counts().to_dict()}\")\ninvalid = (~df_val[\"disease\"].isin(TARGET_CLASSES)).sum()\nprint(f\"   Invalid classes : {invalid} (expected 0)\")\n\n# 5. All frontal\nprint(f\"\\n5. Non-frontal rows : {(df_val['view'] != 'Frontal').sum()} (expected 0)\")\n\n# 6. Dataset breakdown\nprint(f\"\\n6. Dataset breakdown\")\nprint(f\"   {df_val['dataset'].value_counts().to_dict()}\")\n\n# 7. Demographics\nprint(f\"\\n7. Demographics coverage\")\nprint(f\"   has_age    : {df_val['has_age'].sum():,} / {len(df_val):,}  ({df_val['has_age'].mean()*100:.1f}%)\")\nprint(f\"   has_gender : {df_val['has_gender'].sum():,} / {len(df_val):,}  ({df_val['has_gender'].mean()*100:.1f}%)\")\n\n# 8. Path format — relative paths must NOT start with /\nrel_ok = (~df_val[\"path\"].str.startswith(\"/\")).all()\nprint(f\"\\n8. Path format (relative)\")\nprint(f\"   No paths start with /  : {rel_ok}\")\nprint(f\"   Sample NIH path        : {df_val[df_val['dataset']=='NIH']['path'].iloc[0]}\")\nprint(f\"   Sample CheXpert path   : {df_val[df_val['dataset']=='CheXpert']['path'].iloc[0]}\")\nprint(f\"   Sample COVIDx path     : {df_val[df_val['dataset']=='COVIDx']['path'].iloc[0]}\")\nprint(f\"   Sample Pediatric path  : {df_val[df_val['dataset']=='Pediatric']['path'].iloc[0]}\")\nprint(f\"   Sample RSNA path       : {df_val[df_val['dataset']=='RSNA']['path'].iloc[0]}\")\n\n# 9. RSNA paths point to PNG not DCM\nrsna_rows   = df_val[df_val[\"dataset\"] == \"RSNA\"]\nrsna_png_ok = rsna_rows[\"path\"].str.endswith(\".png\").all()\nprint(f\"\\n9. RSNA paths are PNG (not DCM)\")\nprint(f\"   All RSNA paths end with .png : {rsna_png_ok}\")\n\n# 10. Null image_ids\nprint(f\"\\n10. Null image_ids : {df_val['image_id'].isna().sum()} (expected 0)\")\n\n# 11. Class balance\nprint(f\"\\n11. Class balance\")\nfor cls in TARGET_CLASSES:\n    count = (df_val[\"disease\"] == cls).sum()\n    pct   = count / len(df_val) * 100\n    print(f\"    {cls:<15} {count:>7,}  ({pct:.1f}%)\")\n\n# 12. Path resolution check — spot-verify relative paths resolve correctly\nprint(f\"\\n12. Path resolution check (relative → absolute)\")\nfor ds_name in df_val[\"dataset\"].unique():\n    sample = df_val[df_val[\"dataset\"] == ds_name].iloc[0]\n    if ds_name == \"RSNA\":\n        resolved = str(OUTPUT_DIR / sample[\"path\"])\n    else:\n        resolved = ABS_BASE_DIRS[ds_name] + sample[\"path\"]\n    exists = Path(resolved).exists()\n    print(f\"   {ds_name:<12} exists={exists}  →  {resolved[:80]}\")\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"Validation complete.\")\nprint(\"=\" * 60)","metadata":{"_uuid":"6be0d926-3801-4fcc-91dc-4f79ad62fc8b","_cell_guid":"31b49369-23ff-465d-bee4-d4b10f5461dc","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-24T16:22:14.111959Z","iopub.execute_input":"2026-03-24T16:22:14.112235Z","iopub.status.idle":"2026-03-24T16:22:14.771423Z","shell.execute_reply.started":"2026-03-24T16:22:14.112212Z","shell.execute_reply":"2026-03-24T16:22:14.770595Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 13 — Summary\n# ─────────────────────────────────────────────────────────────\n\nprint(\"=\" * 60)\nprint(\"MERGE PIPELINE (RELATIVE) COMPLETE\")\nprint(\"=\" * 60)\nprint(f\"\\nTotal images : {len(master):,}\")\nprint(f\"\\nDisease breakdown:\")\nfor cls in TARGET_CLASSES:\n    count = (master[\"disease\"] == cls).sum()\n    pct   = count / len(master) * 100\n    print(f\"  {cls:<15} {count:>7,}  ({pct:.1f}%)\")\n\nprint(f\"\\nDataset breakdown:\")\nfor ds, count in master[\"dataset\"].value_counts().items():\n    pct = count / len(master) * 100\n    print(f\"  {ds:<12} {count:>7,}  ({pct:.1f}%)\")\n\nprint(f\"\\nDemographic coverage:\")\nprint(f\"  Age    : {master['has_age'].sum():,} / {len(master):,}  ({master['has_age'].mean()*100:.1f}%)\")\nprint(f\"  Gender : {master['has_gender'].sum():,} / {len(master):,}  ({master['has_gender'].mean()*100:.1f}%)\")\n\nprint(f\"\\nOutput file:\")\nprint(f\"  {OUT_RELATIVE}  ← use on Colab / local\")\nprint(f\"  {OUT_REPORT}\")\nprint(f\"  {LOG_PATH}\")\nprint(f\"\\nRSNA PNGs saved to : {RSNA_PNG_DIR}\")\nprint(f\"  ({len(list(RSNA_PNG_DIR.glob('*.png'))):,} PNG files)\")\nprint(f\"\\nTo resolve paths on Colab/local, use:\")\nprint(f\"  base_dirs = {{\")\nfor ds, base in ABS_BASE_DIRS.items():\n    print(f\"      '{ds}': '<your_local_path>/',\")\nprint(f\"  }}\")\nprint(f\"  resolved = base_dirs[row['dataset']] + row['path']\")","metadata":{"_uuid":"4c248b29-c93c-4fd0-9329-3c83bde13fd2","_cell_guid":"0a251440-f309-46e7-a29f-93a7dc411992","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-24T16:22:17.237829Z","iopub.execute_input":"2026-03-24T16:22:17.238117Z","iopub.status.idle":"2026-03-24T16:22:17.504254Z","shell.execute_reply.started":"2026-03-24T16:22:17.238092Z","shell.execute_reply":"2026-03-24T16:22:17.503506Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 15 — Install pHash dependency\n# ─────────────────────────────────────────────────────────────\n!pip install -q imagehash","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T16:22:19.736439Z","iopub.execute_input":"2026-03-24T16:22:19.737083Z","iopub.status.idle":"2026-03-24T16:22:22.974393Z","shell.execute_reply.started":"2026-03-24T16:22:19.737051Z","shell.execute_reply":"2026-03-24T16:22:22.973459Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 16 — Compute pHashes for all images\n#\n# Computes a 64-bit perceptual hash for every image.\n# Stores results back into the DataFrame.\n# ~175k images at ~2ms each = ~6 minutes on T4.\n#\n# NOTE (relative version): paths in the parquet are relative.\n# We resolve them to absolute before opening using resolve_path().\n# ─────────────────────────────────────────────────────────────\nimport imagehash\nfrom PIL import Image\nfrom pathlib import Path\n \ndf = pd.read_parquet(\"/kaggle/working/master_merged_relative.parquet\")\n \n# ── Path resolution helpers ───────────────────────────────────\n# Must match the base dirs used when the relative parquet was built.\nOUTPUT_DIR   = Path(\"/kaggle/working\")\nABS_BASE_DIRS = {\n    \"NIH\":       \"/kaggle/input/datasets/organizations/nih-chest-xrays/data/\",\n    \"CheXpert\":  \"/kaggle/input/datasets/willarevalo/chexpert-v10-small/CheXpert-v1.0-small/\",\n    \"COVIDx\":    \"/kaggle/input/datasets/andyczhao/covidx-cxr2/\",\n    \"Pediatric\": \"/kaggle/input/datasets/paultimothymooney/chest-xray-pneumonia/chest_xray/\",\n    \"RSNA\":      None,   # RSNA PNGs live in OUTPUT_DIR/rsna_png/\n}\n \ndef resolve_path(dataset: str, rel_path: str) -> str:\n    \"\"\"Convert a relative path to absolute for file I/O.\"\"\"\n    if dataset == \"RSNA\":\n        return str(OUTPUT_DIR / rel_path)   # rsna_png/<stem>.png\n    return ABS_BASE_DIRS[dataset] + rel_path\n \n# ─────────────────────────────────────────────────────────────\nprint(f\"Computing pHashes for {len(df):,} images ...\")\nprint(\"Progress shown every 10,000 images.\")\n \nphashes     = []\nhash_failed = []\n \nfor idx, (_, row) in enumerate(df.iterrows()):\n    abs_path = resolve_path(row[\"dataset\"], row[\"path\"])   # ← only change vs absolute version\n    try:\n        with Image.open(abs_path) as img:\n            h = str(imagehash.phash(img.convert(\"L\")))     # grayscale for medical images\n        phashes.append(h)\n    except Exception as e:\n        phashes.append(None)\n        hash_failed.append((abs_path, str(e)))\n \n    if (idx + 1) % 10000 == 0:\n        print(f\"  {idx+1:,} / {len(df):,}\")\n \ndf[\"phash\"] = phashes\n \nprint(f\"\\nDone.\")\nprint(f\"  Hashed successfully : {df['phash'].notna().sum():,}\")\nprint(f\"  Failed              : {len(hash_failed):,}\")\nif hash_failed:\n    print(f\"  First failure: {hash_failed[0]}\")\n\n# Save hashes so you don't recompute after every reload\ndf.to_parquet(\"/kaggle/working/master_with_phashes.parquet\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T16:22:26.508981Z","iopub.execute_input":"2026-03-24T16:22:26.509784Z","iopub.status.idle":"2026-03-24T16:23:11.316213Z","shell.execute_reply.started":"2026-03-24T16:22:26.509749Z","shell.execute_reply":"2026-03-24T16:23:11.315246Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 17 — Exact pHash duplicate detection\n#\n# Step 1: Find exact pHash matches (hamming distance = 0)\n# These are guaranteed duplicates — same image, possibly\n# different filename or dataset.\n#\n# Step 2: For near-duplicates (hamming distance ≤ 10),\n# only check COVIDx vs NIH since that's the known overlap.\n# Full cross-dataset near-dup search would be O(n²).\n#\n# NOTE: No changes needed here — operates on the DataFrame\n# in memory, not on file paths directly.\n# ─────────────────────────────────────────────────────────────\nfrom collections import defaultdict\n\ndf_valid = pd.read_parquet(\"/kaggle/input/datasets/kaushik2005/balanced/master_with_phashes.parquet\")\n \nprint(\"=\" * 55)\nprint(\"STEP 1 — Exact duplicate detection (hamming = 0)\")\nprint(\"=\" * 55)\n \n# Group by exact phash value\nhash_groups = defaultdict(list)\nfor idx, row in df_valid.iterrows():\n    hash_groups[row[\"phash\"]].append({\n        \"idx\":       idx,\n        \"image_id\":  row[\"image_id\"],\n        \"dataset\":   row[\"dataset\"],\n        \"file_name\": row[\"file_name\"],\n        \"disease\":   row[\"disease\"],\n    })\n \n# Find groups with more than one image (exact duplicates)\nexact_dup_groups = {h: g for h, g in hash_groups.items() if len(g) > 1}\nexact_dup_count  = sum(len(g) - 1 for g in exact_dup_groups.values())\n \nprint(f\"Unique pHash values      : {len(hash_groups):,}\")\nprint(f\"Duplicate groups found   : {len(exact_dup_groups):,}\")\nprint(f\"Duplicate images to drop : {exact_dup_count:,}\")\n \n# Show first 5 duplicate groups\nif exact_dup_groups:\n    print(f\"\\nSample duplicate groups:\")\n    for i, (h, group) in enumerate(list(exact_dup_groups.items())[:5]):\n        print(f\"\\n  Group {i+1} (phash={h}):\")\n        for item in group:\n            print(f\"    [{item['dataset']}] {item['file_name']} — {item['disease']}\")\n \n# Identify indices to drop — keep first occurrence, drop rest\n# Priority: keep NIH/CheXpert/RSNA over COVIDx\n# (COVIDx sourced from them, so original is the canonical copy)\nDATASET_PRIORITY = {\"NIH\": 0, \"CheXpert\": 1, \"RSNA\": 2, \"Pediatric\": 3, \"COVIDx\": 4}\n \nindices_to_drop = set()\nfor h, group in exact_dup_groups.items():\n    sorted_group = sorted(group, key=lambda x: DATASET_PRIORITY.get(x[\"dataset\"], 99))\n    for item in sorted_group[1:]:\n        indices_to_drop.add(item[\"idx\"])\n \nprint(f\"\\nIndices marked for removal : {len(indices_to_drop):,}\")\n ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T16:33:28.683721Z","iopub.execute_input":"2026-03-24T16:33:28.684278Z","iopub.status.idle":"2026-03-24T16:33:37.057189Z","shell.execute_reply.started":"2026-03-24T16:33:28.684248Z","shell.execute_reply":"2026-03-24T16:33:37.056469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 18 — Near-duplicate detection (COVIDx vs NIH only)\n# Tightened threshold to avoid false positives on similar-looking\n# but distinct chest X-rays (especially No Finding class).\n# ─────────────────────────────────────────────────────────────\nimport imagehash as ih\n\nHAMMING_THRESHOLD = 4   # tightened from 10 — eliminates false positives\nnear_dup_pairs    = []\n\ncovidx_rows = df_valid[df_valid[\"dataset\"] == \"COVIDx\"][[\"image_id\", \"phash\", \"file_name\", \"dataset\"]].copy()\nnih_rows    = df_valid[df_valid[\"dataset\"] == \"NIH\"][[\"image_id\", \"phash\", \"file_name\", \"dataset\"]].copy()\n\nprint(f\"Comparing {len(covidx_rows):,} COVIDx vs {len(nih_rows):,} NIH images ...\")\nprint(f\"Hamming threshold : ≤ {HAMMING_THRESHOLD}\")\n\n# Bucket by first 4 chars (16 bits) instead of 8 to widen\n# the candidate window — avoids missing near-dups at boundaries\nnih_buckets = defaultdict(list)\nfor _, row in nih_rows.iterrows():\n    prefix = row[\"phash\"][:4]\n    nih_buckets[prefix].append(row)\n\ncompared  = 0\nnear_dups = 0\n\nfor _, cov_row in covidx_rows.iterrows():\n    prefix     = cov_row[\"phash\"][:4]\n    candidates = nih_buckets.get(prefix, [])\n    for nih_row in candidates:\n        try:\n            dist = ih.hex_to_hash(cov_row[\"phash\"]) - ih.hex_to_hash(nih_row[\"phash\"])\n            compared += 1\n            if dist <= HAMMING_THRESHOLD:\n                near_dups += 1\n                near_dup_pairs.append({\n                    \"covidx_id\":   cov_row[\"image_id\"],\n                    \"covidx_file\": cov_row[\"file_name\"],\n                    \"nih_id\":      nih_row[\"image_id\"],\n                    \"nih_file\":    nih_row[\"file_name\"],\n                    \"distance\":    dist,\n                })\n        except Exception:\n            pass\n\nprint(f\"\\nComparisons made : {compared:,}\")\nprint(f\"Near-dup pairs   : {near_dups:,}\")\n\n# One COVIDx image may match multiple NIH images — only drop the COVIDx once\ncovidx_near_dup_ids = {p[\"covidx_id\"] for p in near_dup_pairs}\nnear_dup_indices    = set(\n    df_valid[df_valid[\"image_id\"].isin(covidx_near_dup_ids)].index.tolist()\n)\nindices_to_drop.update(near_dup_indices)\n\nprint(f\"\\nTotal indices to drop (exact + near-dup) : {len(indices_to_drop):,}\")\nif near_dup_pairs:\n    print(f\"\\nSample near-duplicate pairs:\")\n    for pair in near_dup_pairs[:5]:\n        print(f\"  COVIDx: {pair['covidx_file']}  ↔  NIH: {pair['nih_file']}  (dist={pair['distance']})\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T16:33:41.911557Z","iopub.execute_input":"2026-03-24T16:33:41.912152Z","iopub.status.idle":"2026-03-24T16:44:02.835842Z","shell.execute_reply.started":"2026-03-24T16:33:41.91212Z","shell.execute_reply":"2026-03-24T16:44:02.834972Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 19 — Apply deduplication and save clean parquet\n#\n# NOTE (relative version): output file is master_deduped_relative.parquet\n# Relative paths are preserved as-is — no path changes needed.\n# ─────────────────────────────────────────────────────────────\nn_before  = len(df)\ndf_clean  = df.drop(index=list(indices_to_drop)).reset_index(drop=True)\n \nn_after   = len(df_clean)\nn_dropped = n_before - n_after\n \nprint(f\"Before dedup : {n_before:,}\")\nprint(f\"After dedup  : {n_after:,}\")\nprint(f\"Dropped      : {n_dropped:,}\")\n \nprint(f\"\\nDisease distribution after dedup:\")\nprint(df_clean[\"disease\"].value_counts().to_string())\n \nprint(f\"\\nDataset breakdown after dedup:\")\nprint(df_clean[\"dataset\"].value_counts().to_string())\n \n# Save deduplicated parquet\nOUT_DEDUP = Path(\"/kaggle/working/master_deduped_relative.parquet\")   # ← only naming change vs absolute version\ndf_clean.to_parquet(OUT_DEDUP, index=False)\nprint(f\"\\nSaved → {OUT_DEDUP}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 20 — Verify COVID-19 drops are legitimate\n# ─────────────────────────────────────────────────────────────\n\ncovid_dropped_ids = set(df[df[\"disease\"] == \"COVID-19\"][\"image_id\"]) - \\\n                    set(df_clean[df_clean[\"disease\"] == \"COVID-19\"][\"image_id\"])\n\nprint(f\"COVID-19 images dropped : {len(covid_dropped_ids):,}\")\n\n# Which datasets did the dropped COVID images come from?\ndropped_covid = df[df[\"image_id\"].isin(covid_dropped_ids)]\nprint(\"\\nDropped COVID-19 by dataset:\")\nprint(dropped_covid[\"dataset\"].value_counts())\n\n# Sample to inspect\nprint(\"\\nSample dropped COVID-19 images:\")\nprint(dropped_covid[[\"file_name\", \"dataset\", \"disease\"]].head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T09:30:13.558588Z","iopub.execute_input":"2026-03-24T09:30:13.558929Z","iopub.status.idle":"2026-03-24T09:30:13.719559Z","shell.execute_reply.started":"2026-03-24T09:30:13.558901Z","shell.execute_reply":"2026-03-24T09:30:13.718803Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 21 — Inspect COVIDx COVID image filename patterns\n# to separate clinical vs web-scraped sources\n# ─────────────────────────────────────────────────────────────\n\ncovidx_covid = df[\n    (df[\"dataset\"] == \"COVIDx\") & \n    (df[\"disease\"] == \"COVID-19\")\n].copy()\n\nprint(f\"Total COVIDx COVID-19 images : {len(covidx_covid):,}\")\n\n# Flag suspicious filenames\nimport re\n\ndef is_suspicious(fname):\n    fname = str(fname).lower()\n    return any([\n        \"_jumbo.jpg\"  in fname,\n        \"_jumbo.jpeg\" in fname,\n        \"10.1016\"     in fname,   # DOI\n        \".pdf\"        in fname,\n        \"slash\"       in fname,   # mangled DOI\n        \"article_\"    in fname,   # scraped article\n    ])\n\ncovidx_covid[\"suspicious\"] = covidx_covid[\"file_name\"].apply(is_suspicious)\n\nprint(f\"\\nSuspicious (web-scraped) : {covidx_covid['suspicious'].sum():,}\")\nprint(f\"Clean (clinical)         : {(~covidx_covid['suspicious']).sum():,}\")\n\nprint(f\"\\nSample suspicious filenames:\")\nprint(covidx_covid[covidx_covid[\"suspicious\"]][\"file_name\"].head(10).tolist())\n\nprint(f\"\\nSample clean filenames:\")\nprint(covidx_covid[~covidx_covid[\"suspicious\"]][\"file_name\"].head(10).tolist())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T09:31:10.043273Z","iopub.execute_input":"2026-03-24T09:31:10.04395Z","iopub.status.idle":"2026-03-24T09:31:10.179448Z","shell.execute_reply.started":"2026-03-24T09:31:10.043918Z","shell.execute_reply":"2026-03-24T09:31:10.178529Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 22 — Remove the 70 suspicious COVIDx images from\n# df_clean and re-save. Don't touch the near-dup logic.\n# ─────────────────────────────────────────────────────────────\n\ndef is_suspicious(fname):\n    fname = str(fname).lower()\n    return any([\n        \"_jumbo.jpg\"  in fname,\n        \"_jumbo.jpeg\" in fname,\n        \"10.1016\"     in fname,\n        \".pdf\"        in fname,\n        \"slash\"       in fname,\n        \"article_\"    in fname,\n    ])\n\n# Find suspicious rows in df_clean\nsuspicious_mask = (\n    (df_clean[\"dataset\"] == \"COVIDx\") &\n    (df_clean[\"disease\"] == \"COVID-19\") &\n    (df_clean[\"file_name\"].apply(is_suspicious))\n)\n\nprint(f\"Suspicious images still in df_clean : {suspicious_mask.sum()}\")\n\ndf_clean = df_clean[~suspicious_mask].reset_index(drop=True)\nprint(f\"Rows after removing suspicious      : {len(df_clean):,}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T09:31:50.587946Z","iopub.execute_input":"2026-03-24T09:31:50.588244Z","iopub.status.idle":"2026-03-24T09:31:50.772204Z","shell.execute_reply.started":"2026-03-24T09:31:50.588219Z","shell.execute_reply":"2026-03-24T09:31:50.771573Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 23 — Restore the 4,777 incorrectly dropped COVID images\n#\n# The near-dup check flagged real COVID paper figures as\n# duplicates of NIH normals due to hashing artifacts.\n# We restore them since they are not true duplicates.\n# ─────────────────────────────────────────────────────────────\n\n# These are the COVID images that were in df but not in df_clean\n# (excluding the 70 suspicious ones we just intentionally removed)\nlegitimately_removed = set(\n    df[df[\"image_id\"].isin(covid_dropped_ids)][\"image_id\"]\n) \n\n# Re-add everything from legitimately_removed that isn't suspicious\nrows_to_restore = df[\n    (df[\"image_id\"].isin(legitimately_removed)) &\n    (~df[\"file_name\"].apply(is_suspicious))\n]\n\nprint(f\"COVID images to restore : {len(rows_to_restore):,}\")\n\ndf_clean = pd.concat([df_clean, rows_to_restore], ignore_index=True)\nprint(f\"Rows after restore      : {len(df_clean):,}\")\n\nprint(f\"\\nDisease distribution:\")\nprint(df_clean[\"disease\"].value_counts())\nprint(f\"\\nDataset breakdown:\")\nprint(df_clean[\"dataset\"].value_counts())\n\n# Drop phash if it snuck back in\nif \"phash\" in df_clean.columns:\n    df_clean = df_clean.drop(columns=[\"phash\"])\n\n# Overwrite the saved parquet\nOUT_DEDUP = Path(\"/kaggle/working/master_deduped_relative.parquet\")\ndf_clean.to_parquet(OUT_DEDUP, index=False)\nprint(f\"\\nSaved → {OUT_DEDUP}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T09:32:12.349082Z","iopub.execute_input":"2026-03-24T09:32:12.349944Z","iopub.status.idle":"2026-03-24T09:32:12.901611Z","shell.execute_reply.started":"2026-03-24T09:32:12.34991Z","shell.execute_reply":"2026-03-24T09:32:12.900746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# Disease breakdown per dataset\n# ─────────────────────────────────────────────────────────────\n\nbreakdown = df_clean.groupby([\"dataset\", \"disease\"]).size().unstack(fill_value=0)\n\n# Add a total column\nbreakdown[\"TOTAL\"] = breakdown.sum(axis=1)\n\n# Add a total row\nbreakdown.loc[\"TOTAL\"] = breakdown.sum()\n\nprint(breakdown.to_string())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T09:55:16.988837Z","iopub.execute_input":"2026-03-24T09:55:16.989368Z","iopub.status.idle":"2026-03-24T09:55:17.057361Z","shell.execute_reply.started":"2026-03-24T09:55:16.989312Z","shell.execute_reply":"2026-03-24T09:55:17.056711Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 25 — Inspect the \"other\" category\n# ─────────────────────────────────────────────────────────────\n\nother = covidx[covidx[\"source_type\"] == \"other\"]\n\nprint(f\"Total 'other' : {len(other):,}\")\nprint(f\"\\nDisease breakdown:\")\nprint(other[\"disease\"].value_counts())\n\nprint(f\"\\nSample filenames (first 20):\")\nfor f in other[\"file_name\"].head(20).tolist():\n    print(f\"  {f}\")\n\nprint(f\"\\nSample filenames (last 20):\")\nfor f in other[\"file_name\"].tail(20).tolist():\n    print(f\"  {f}\")\n\n# Check file extensions\nextensions = other[\"file_name\"].apply(lambda x: Path(x).suffix.lower())\nprint(f\"\\nFile extensions:\")\nprint(extensions.value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T10:01:54.794142Z","iopub.execute_input":"2026-03-24T10:01:54.79444Z","iopub.status.idle":"2026-03-24T10:01:54.871663Z","shell.execute_reply.started":"2026-03-24T10:01:54.794415Z","shell.execute_reply":"2026-03-24T10:01:54.870874Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 26 — Drop low quality COVIDx sources and re-save\n# ─────────────────────────────────────────────────────────────\n\nbefore = len(df_clean)\n\ndrop_sources = {\"journal_figure\", \"github_collection\"}\n\n# Get image_ids to drop\ncovidx_drop_ids = set(\n    covidx[covidx[\"source_type\"].isin(drop_sources)][\"image_id\"]\n)\n\nprint(f\"Images to drop : {len(covidx_drop_ids):,}\")\nprint(f\"  journal_figure    : {(covidx['source_type'] == 'journal_figure').sum():,}\")\nprint(f\"  github_collection : {(covidx['source_type'] == 'github_collection').sum():,}\")\n\ndf_clean = df_clean[~df_clean[\"image_id\"].isin(covidx_drop_ids)].reset_index(drop=True)\n\nprint(f\"\\nBefore : {before:,}\")\nprint(f\"After  : {len(df_clean):,}\")\nprint(f\"Dropped: {before - len(df_clean):,}\")\n\nprint(f\"\\nFinal disease distribution:\")\nprint(df_clean[\"disease\"].value_counts())\nprint(f\"\\nFinal dataset breakdown:\")\nprint(df_clean[\"dataset\"].value_counts())\n\nprint(f\"\\nFinal COVIDx source breakdown:\")\ncovidx_final = df_clean[df_clean[\"dataset\"] == \"COVIDx\"]\ncovidx_final[\"source_type\"] = covidx_final[\"file_name\"].apply(classify_source)\nprint(covidx_final[\"source_type\"].value_counts())\n\n# Overwrite saved parquet\nOUT_DEDUP = Path(\"/kaggle/working/master_deduped_relative.parquet\")\ndf_clean.to_parquet(OUT_DEDUP, index=False)\nprint(f\"\\nSaved → {OUT_DEDUP}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T10:02:33.147278Z","iopub.execute_input":"2026-03-24T10:02:33.147953Z","iopub.status.idle":"2026-03-24T10:02:33.733295Z","shell.execute_reply.started":"2026-03-24T10:02:33.147922Z","shell.execute_reply":"2026-03-24T10:02:33.732516Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 27 — Drop remaining unknown_web and tumblr_scraped\n# ─────────────────────────────────────────────────────────────\n\nbefore = len(df_clean)\n\ndrop_sources_2 = {\"unknown_web\", \"tumblr_scraped\"}\n\n# Need to recompute source_type on df_clean since it's not a column there\ndf_clean[\"source_type\"] = df_clean.apply(\n    lambda row: classify_source(row[\"file_name\"]) if row[\"dataset\"] == \"COVIDx\" else \"non_covidx\",\n    axis=1\n)\n\ncovidx_drop_ids_2 = set(\n    df_clean[df_clean[\"source_type\"].isin(drop_sources_2)][\"image_id\"]\n)\n\nprint(f\"Images to drop : {len(covidx_drop_ids_2):,}\")\n\ndf_clean = df_clean[~df_clean[\"image_id\"].isin(covidx_drop_ids_2)].reset_index(drop=True)\n\n# Drop the temporary source_type column\ndf_clean = df_clean.drop(columns=[\"source_type\"])\n\nprint(f\"Before : {before:,}\")\nprint(f\"After  : {len(df_clean):,}\")\nprint(f\"Dropped: {before - len(df_clean):,}\")\n\nprint(f\"\\nFinal disease distribution:\")\nprint(df_clean[\"disease\"].value_counts())\n\nprint(f\"\\nFinal dataset breakdown:\")\nprint(df_clean[\"dataset\"].value_counts())\n\nprint(f\"\\nFinal COVIDx source breakdown:\")\ncovidx_final = df_clean[df_clean[\"dataset\"] == \"COVIDx\"].copy()\ncovidx_final[\"source_type\"] = covidx_final[\"file_name\"].apply(classify_source)\nprint(covidx_final[\"source_type\"].value_counts())\n\n# Overwrite saved parquet\nOUT_DEDUP = Path(\"/kaggle/working/master_deduped_relative.parquet\")\ndf_clean.to_parquet(OUT_DEDUP, index=False)\nprint(f\"\\nSaved → {OUT_DEDUP}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T10:03:12.452853Z","iopub.execute_input":"2026-03-24T10:03:12.453161Z","iopub.status.idle":"2026-03-24T10:03:14.017578Z","shell.execute_reply.started":"2026-03-24T10:03:12.453135Z","shell.execute_reply":"2026-03-24T10:03:14.016817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 28 — Phase 3.5 : Identity Grouping\n#\n# Goal: assign a unified identity_id to each row so that\n# the same patient is never split across train/val/test.\n#\n# Strategy per dataset:\n#   NIH       → patient_id is reliable (00000001 format)\n#   CheXpert  → patient_id is reliable (patient00001 format)\n#   RSNA      → patient_id is reliable (UUID format)\n#   Pediatric → no patient_id — each image is treated as\n#               a unique patient (small dataset, no metadata)\n#   COVIDx    → extract from filename:\n#               bimcv   : sub-SXXXXX → use SXXXXX as patient\n#               midrc   : MIDRC-RICORD-1C-XXXXXX → use XXXXXX\n#               other   : no reliable id → use image_id\n# ─────────────────────────────────────────────────────────────\n\nimport hashlib\nimport re\n\ndef extract_covidx_patient(fname):\n    fname = str(fname)\n    # BIMCV: sub-S09531_ses-E20968_... → patient = S09531\n    m = re.match(r\"sub-(S\\d+)_\", fname)\n    if m:\n        return f\"bimcv_{m.group(1)}\"\n    # MIDRC: MIDRC-RICORD-1C-419639-... → patient = 419639\n    m = re.match(r\"MIDRC-RICORD-1C-(\\d+)-\", fname)\n    if m:\n        return f\"midrc_{m.group(1)}\"\n    m = re.match(r\"MIDRC-RICORD-1C-SITE\\d+-(\\d+)-\", fname)\n    if m:\n        return f\"midrc_site_{m.group(1)}\"\n    # Fallback — treat as unique patient\n    return f\"covidx_unique_{fname}\"\n\ndef make_identity_id(dataset, patient_id, fname):\n    \"\"\"\n    Returns a stable identity_id string.\n    Format: <dataset>_<patient_key>\n    \"\"\"\n    if dataset == \"NIH\":\n        # patient_id already clean e.g. \"00000001\"\n        key = f\"nih_{str(patient_id)}\"\n\n    elif dataset == \"CheXpert\":\n        # patient_id e.g. \"patient00001\"\n        key = f\"chexpert_{str(patient_id)}\"\n\n    elif dataset == \"RSNA\":\n        # patient_id is UUID\n        key = f\"rsna_{str(patient_id)}\"\n\n    elif dataset == \"Pediatric\":\n        # No reliable patient_id — each image unique\n        key = f\"pediatric_unique_{fname}\"\n\n    elif dataset == \"COVIDx\":\n        key = extract_covidx_patient(fname)\n\n    else:\n        key = f\"unknown_{fname}\"\n\n    return key\n\ndf_clean[\"identity_id\"] = df_clean.apply(\n    lambda row: make_identity_id(\n        row[\"dataset\"],\n        row.get(\"patient_id\", None),\n        row[\"file_name\"]\n    ),\n    axis=1\n)\n\n# ── Summary ───────────────────────────────────────────────────\ntotal_images    = len(df_clean)\ntotal_identities = df_clean[\"identity_id\"].nunique()\n\nprint(f\"Total images     : {total_images:,}\")\nprint(f\"Total identities : {total_identities:,}\")\nprint(f\"Avg images/identity : {total_images/total_identities:.2f}\")\n\nprint(f\"\\nIdentity coverage per dataset:\")\nfor ds in df_clean[\"dataset\"].unique():\n    sub = df_clean[df_clean[\"dataset\"] == ds]\n    n_images     = len(sub)\n    n_identities = sub[\"identity_id\"].nunique()\n    print(f\"  {ds:<12} {n_images:>7,} images  |  {n_identities:>7,} identities  |  \"\n          f\"{n_images/n_identities:.2f} imgs/identity\")\n\n# ── Sanity check — no identity spans multiple datasets ────────\n# (identities are prefixed by dataset so this should always pass)\nidentity_dataset_counts = (\n    df_clean.groupby(\"identity_id\")[\"dataset\"].nunique()\n)\ncross_dataset = (identity_dataset_counts > 1).sum()\nprint(f\"\\nIdentities spanning multiple datasets : {cross_dataset} (expected 0)\")\n\n# ── Save ──────────────────────────────────────────────────────\nOUT_IDENTITY = Path(\"/kaggle/working/master_with_identity.parquet\")\ndf_clean.to_parquet(OUT_IDENTITY, index=False)\nprint(f\"\\nSaved → {OUT_IDENTITY}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T10:24:06.207255Z","iopub.execute_input":"2026-03-24T10:24:06.207828Z","iopub.status.idle":"2026-03-24T10:24:08.023434Z","shell.execute_reply.started":"2026-03-24T10:24:06.2078Z","shell.execute_reply":"2026-03-24T10:24:08.02272Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 29 — Phase 4 : Global Train / Val / Test Split\n#\n# Rules:\n#   - Split at IDENTITY level (not image level)\n#   - 70 / 15 / 15 ratio\n#   - Stratified by disease AND dataset\n#   - Reproducible (random_state=42)\n#   - Verified: no identity leakage across splits\n# ─────────────────────────────────────────────────────────────\n\nfrom sklearn.model_selection import train_test_split\n\ndf = pd.read_parquet(\"/kaggle/working/master_with_identity.parquet\")\n\n# ── Step 1: Build identity-level summary ─────────────────────\n# Each identity gets one row with its dominant disease + dataset\n# This is what we stratify on\n\nidentity_df = (\n    df.groupby(\"identity_id\")\n    .agg(\n        dataset=(\"dataset\", \"first\"),\n        disease=(\"disease\", lambda x: x.value_counts().index[0]),  # dominant label\n        n_images=(\"image_id\", \"count\"),\n    )\n    .reset_index()\n)\n\nprint(f\"Total identities : {len(identity_df):,}\")\nprint(f\"\\nIdentity-level disease distribution:\")\nprint(identity_df[\"disease\"].value_counts())\nprint(f\"\\nIdentity-level dataset distribution:\")\nprint(identity_df[\"dataset\"].value_counts())\n\n# ── Step 2: Create stratification key ────────────────────────\n# Combine disease + dataset into one column for stratification\nidentity_df[\"strat_key\"] = (\n    identity_df[\"disease\"] + \"_\" + identity_df[\"dataset\"]\n)\n\nprint(f\"\\nStratification keys:\")\nprint(identity_df[\"strat_key\"].value_counts().to_string())\n\n# ── Step 3: Check minimum class size for stratification ──────\n# train_test_split needs at least 2 samples per strat_key\nmin_count = identity_df[\"strat_key\"].value_counts().min()\nprint(f\"\\nMin strat_key count : {min_count}  (need ≥ 2)\")\n\n# Merge rare keys (< 2 samples) into a catch-all\nrare_keys = identity_df[\"strat_key\"].value_counts()\nrare_keys = rare_keys[rare_keys < 2].index.tolist()\nif rare_keys:\n    print(f\"Rare keys (will be merged): {rare_keys}\")\n    identity_df.loc[identity_df[\"strat_key\"].isin(rare_keys), \"strat_key\"] = \"rare\"\n\n# ── Step 4: Split identities 70 / 15 / 15 ───────────────────\n\n# First split: 70% train, 30% temp\ntrain_ids, temp_ids = train_test_split(\n    identity_df,\n    test_size=0.30,\n    random_state=42,\n    stratify=identity_df[\"strat_key\"],\n)\n\n# Second split: 50% of temp = 15% val, 50% of temp = 15% test\nval_ids, test_ids = train_test_split(\n    temp_ids,\n    test_size=0.50,\n    random_state=42,\n    stratify=temp_ids[\"strat_key\"],\n)\n\nprint(f\"\\nIdentity split:\")\nprint(f\"  Train : {len(train_ids):,}  ({len(train_ids)/len(identity_df)*100:.1f}%)\")\nprint(f\"  Val   : {len(val_ids):,}  ({len(val_ids)/len(identity_df)*100:.1f}%)\")\nprint(f\"  Test  : {len(test_ids):,}  ({len(test_ids)/len(identity_df)*100:.1f}%)\")\n\n# ── Step 5: Map splits back to image level ───────────────────\n\ntrain_identity_set = set(train_ids[\"identity_id\"])\nval_identity_set   = set(val_ids[\"identity_id\"])\ntest_identity_set  = set(test_ids[\"identity_id\"])\n\ndef assign_split(identity_id):\n    if identity_id in train_identity_set:\n        return \"train\"\n    elif identity_id in val_identity_set:\n        return \"val\"\n    else:\n        return \"test\"\n\ndf[\"split\"] = df[\"identity_id\"].apply(assign_split)\n\nprint(f\"\\nImage split:\")\nprint(f\"  Train : {(df['split']=='train').sum():,}  ({(df['split']=='train').mean()*100:.1f}%)\")\nprint(f\"  Val   : {(df['split']=='val').sum():,}  ({(df['split']=='val').mean()*100:.1f}%)\")\nprint(f\"  Test  : {(df['split']=='test').sum():,}  ({(df['split']=='test').mean()*100:.1f}%)\")\n\n# ── Step 6: Leakage verification ─────────────────────────────\n\ntrain_identities = set(df[df[\"split\"] == \"train\"][\"identity_id\"])\nval_identities   = set(df[df[\"split\"] == \"val\"][\"identity_id\"])\ntest_identities  = set(df[df[\"split\"] == \"test\"][\"identity_id\"])\n\ntrain_val_leak  = len(train_identities & val_identities)\ntrain_test_leak = len(train_identities & test_identities)\nval_test_leak   = len(val_identities & test_identities)\n\nprint(f\"\\nLeakage check (all must be 0):\")\nprint(f\"  train ∩ val  : {train_val_leak}\")\nprint(f\"  train ∩ test : {train_test_leak}\")\nprint(f\"  val   ∩ test : {val_test_leak}\")\n\n# ── Step 7: Disease distribution per split ───────────────────\n\nprint(f\"\\nDisease distribution per split:\")\nfor split in [\"train\", \"val\", \"test\"]:\n    sub = df[df[\"split\"] == split]\n    print(f\"\\n  {split.upper()} ({len(sub):,} images)\")\n    for disease in [\"No Finding\", \"COVID-19\", \"Pneumonia\"]:\n        count = (sub[\"disease\"] == disease).sum()\n        pct   = count / len(sub) * 100\n        print(f\"    {disease:<15} {count:>7,}  ({pct:.1f}%)\")\n\n# ── Step 8: Dataset distribution per split ───────────────────\n\nprint(f\"\\nDataset distribution per split:\")\nsplit_dataset = df.groupby([\"split\", \"dataset\"]).size().unstack(fill_value=0)\nprint(split_dataset.to_string())\n\n# ── Step 9: Save splits ──────────────────────────────────────\n\ntrain_df = df[df[\"split\"] == \"train\"].drop(columns=[\"split\"]).reset_index(drop=True)\nval_df   = df[df[\"split\"] == \"val\"].drop(columns=[\"split\"]).reset_index(drop=True)\ntest_df  = df[df[\"split\"] == \"test\"].drop(columns=[\"split\"]).reset_index(drop=True)\n\ntrain_df.to_parquet(\"/kaggle/working/train.parquet\", index=False)\nval_df.to_parquet(\"/kaggle/working/val.parquet\",     index=False)\ntest_df.to_parquet(\"/kaggle/working/test.parquet\",   index=False)\n\n# Also save full df with split column for reference\ndf.to_parquet(\"/kaggle/working/master_split.parquet\", index=False)\n\nprint(f\"\\nSaved:\")\nprint(f\"  /kaggle/working/train.parquet  ({len(train_df):,} rows)\")\nprint(f\"  /kaggle/working/val.parquet    ({len(val_df):,} rows)\")\nprint(f\"  /kaggle/working/test.parquet   ({len(test_df):,} rows)\")\nprint(f\"  /kaggle/working/master_split.parquet  (full, with split column)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T10:26:36.718646Z","iopub.execute_input":"2026-03-24T10:26:36.719071Z","iopub.status.idle":"2026-03-24T10:26:57.131044Z","shell.execute_reply.started":"2026-03-24T10:26:36.719028Z","shell.execute_reply":"2026-03-24T10:26:57.130174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 30 — Phase 5 : Dataset Balancing\n#\n# Constraints:\n#   max_dataset_ratio ≤ 0.40  (no dataset dominates)\n#   min_class_ratio   ≥ 0.20  (every class well represented)\n#\n# Strategy:\n#   1. Cap each dataset at 40% of train\n#   2. Oversample minority classes to hit 20% minimum\n#   3. Verify both constraints are satisfied\n#   4. Save balanced_train.parquet\n#\n# NOTE: Val and Test are NOT balanced — they must reflect\n# real-world distribution for honest evaluation.\n# ─────────────────────────────────────────────────────────────\n\nimport numpy as np\nfrom pathlib import Path\n\ntrain_df = pd.read_parquet(\"/kaggle/input/datasets/kaushik2005/balanced/train.parquet\")\n\nprint(f\"Original train size : {len(train_df):,}\")\nprint(f\"\\nOriginal disease distribution:\")\nfor disease in [\"No Finding\", \"COVID-19\", \"Pneumonia\"]:\n    count = (train_df[\"disease\"] == disease).sum()\n    pct   = count / len(train_df) * 100\n    print(f\"  {disease:<15} {count:>7,}  ({pct:.1f}%)\")\n\nprint(f\"\\nOriginal dataset distribution:\")\nfor ds, count in train_df[\"dataset\"].value_counts().items():\n    pct = count / len(train_df) * 100\n    print(f\"  {ds:<12} {count:>7,}  ({pct:.1f}%)\")\n\n# ─────────────────────────────────────────────────────────────\n# STEP 1 — Cap each dataset at 40% of current train size\n# ─────────────────────────────────────────────────────────────\n\nMAX_DATASET_RATIO = 0.40\ntotal_train       = len(train_df)\nmax_per_dataset   = int(total_train * MAX_DATASET_RATIO)\n\nprint(f\"\\n{'='*55}\")\nprint(f\"STEP 1 — Dataset capping (max {MAX_DATASET_RATIO*100:.0f}% each)\")\nprint(f\"{'='*55}\")\nprint(f\"Max images per dataset : {max_per_dataset:,}\")\n\ncapped_frames = []\nfor ds in train_df[\"dataset\"].unique():\n    ds_df = train_df[train_df[\"dataset\"] == ds]\n    if len(ds_df) > max_per_dataset:\n        # Sample down — stratified by disease to preserve class ratios\n        ds_df = ds_df.groupby(\"disease\", group_keys=False).apply(\n            lambda x: x.sample(\n                n=max(1, int(max_per_dataset * len(x) / len(ds_df))),\n                random_state=42\n            )\n        )\n        print(f\"  {ds:<12} capped  → {len(ds_df):,}\")\n    else:\n        print(f\"  {ds:<12} ok      → {len(ds_df):,}\")\n    capped_frames.append(ds_df)\n\ntrain_capped = pd.concat(capped_frames, ignore_index=True)\n\nprint(f\"\\nAfter capping : {len(train_capped):,}\")\nprint(f\"Dataset ratios after capping:\")\nfor ds, count in train_capped[\"dataset\"].value_counts().items():\n    pct = count / len(train_capped) * 100\n    flag = \"⚠️\" if pct > 40 else \"✅\"\n    print(f\"  {flag} {ds:<12} {count:>7,}  ({pct:.1f}%)\")\n\n# ─────────────────────────────────────────────────────────────\n# STEP 2 — Oversample minority classes to hit 20% minimum\n#\n# Target: each class ≥ 20% of balanced train\n# We oversample with replacement (repeat images) for minority\n# classes. This is standard for GAN training.\n# ─────────────────────────────────────────────────────────────\n\nMIN_CLASS_RATIO = 0.20\n\nprint(f\"\\n{'='*55}\")\nprint(f\"STEP 2 — Class balancing (min {MIN_CLASS_RATIO*100:.0f}% each)\")\nprint(f\"{'='*55}\")\n\n# Calculate target size per class\n# Target total = max(current total, what we need for 20% min)\ncurrent_total  = len(train_capped)\nclass_counts   = train_capped[\"disease\"].value_counts()\nmax_class      = class_counts.max()\n\n# Each class needs at least 20% of total\n# If Pneumonia = 20%, then total = Pneumonia_count / 0.20\nmin_total_needed = int(class_counts.min() / MIN_CLASS_RATIO)\ntarget_total     = max(current_total, min_total_needed)\n\nprint(f\"Current total    : {current_total:,}\")\nprint(f\"Min total needed : {min_total_needed:,}\")\nprint(f\"Target total     : {target_total:,}\")\n\ntarget_per_class = int(target_total * MIN_CLASS_RATIO)\nprint(f\"Target per class : {target_per_class:,}  (20% of {target_total:,})\")\n\nbalanced_frames = []\nfor disease in [\"No Finding\", \"COVID-19\", \"Pneumonia\"]:\n    disease_df    = train_capped[train_capped[\"disease\"] == disease]\n    current_count = len(disease_df)\n\n    if current_count < target_per_class:\n        # Oversample with replacement\n        n_extra  = target_per_class - current_count\n        extra_df = disease_df.sample(n=n_extra, replace=True, random_state=42)\n        disease_df = pd.concat([disease_df, extra_df], ignore_index=True)\n        print(f\"  {disease:<15} {current_count:>7,} → {len(disease_df):>7,}  \"\n              f\"(+{n_extra:,} oversampled)\")\n    else:\n        print(f\"  {disease:<15} {current_count:>7,} → {len(disease_df):>7,}  \"\n              f\"(no change needed)\")\n\n    balanced_frames.append(disease_df)\n\ntrain_balanced = pd.concat(balanced_frames, ignore_index=True).sample(\n    frac=1, random_state=42   # shuffle so classes aren't in blocks\n).reset_index(drop=True)\n\n# ─────────────────────────────────────────────────────────────\n# STEP 3 — Verify both constraints\n# ─────────────────────────────────────────────────────────────\n\nprint(f\"\\n{'='*55}\")\nprint(f\"STEP 3 — Constraint verification\")\nprint(f\"{'='*55}\")\n\nprint(f\"\\nFinal balanced train size : {len(train_balanced):,}\")\n\nprint(f\"\\nDisease distribution (constraint: each ≥ 20%):\")\nall_class_ok = True\nfor disease in [\"No Finding\", \"COVID-19\", \"Pneumonia\"]:\n    count = (train_balanced[\"disease\"] == disease).sum()\n    pct   = count / len(train_balanced) * 100\n    ok    = pct >= 20.0\n    flag  = \"✅\" if ok else \"❌\"\n    if not ok:\n        all_class_ok = False\n    print(f\"  {flag} {disease:<15} {count:>7,}  ({pct:.1f}%)\")\n\nprint(f\"\\nDataset distribution (constraint: each ≤ 40%):\")\nall_dataset_ok = True\nfor ds, count in train_balanced[\"dataset\"].value_counts().items():\n    pct  = count / len(train_balanced) * 100\n    ok   = pct <= 40.0\n    flag = \"✅\" if ok else \"❌\"\n    if not ok:\n        all_dataset_ok = False\n    print(f\"  {flag} {ds:<12} {count:>7,}  ({pct:.1f}%)\")\n\nprint(f\"\\nConstraint summary:\")\nprint(f\"  min_class_ratio  ≥ 20% : {'✅ PASSED' if all_class_ok else '❌ FAILED'}\")\nprint(f\"  max_dataset_ratio ≤ 40% : {'✅ PASSED' if all_dataset_ok else '❌ FAILED'}\")\n\n# ─────────────────────────────────────────────────────────────\n# STEP 4 — Disease × Dataset cross table\n# ─────────────────────────────────────────────────────────────\n\nprint(f\"\\nDisease × Dataset breakdown:\")\nbreakdown = train_balanced.groupby([\"dataset\", \"disease\"]).size().unstack(fill_value=0)\nbreakdown[\"TOTAL\"] = breakdown.sum(axis=1)\nbreakdown.loc[\"TOTAL\"] = breakdown.sum()\nprint(breakdown.to_string())\n\n# ─────────────────────────────────────────────────────────────\n# STEP 5 — Save\n# ─────────────────────────────────────────────────────────────\n\nOUT_BALANCED = Path(\"/kaggle/working/balanced_train.parquet\")\ntrain_balanced.to_parquet(OUT_BALANCED, index=False)\nprint(f\"\\nSaved → {OUT_BALANCED}\")\nprint(f\"  Rows    : {len(train_balanced):,}\")\nprint(f\"  Columns : {list(train_balanced.columns)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T17:06:40.044535Z","iopub.execute_input":"2026-03-24T17:06:40.045129Z","iopub.status.idle":"2026-03-24T17:06:41.087631Z","shell.execute_reply.started":"2026-03-24T17:06:40.045099Z","shell.execute_reply":"2026-03-24T17:06:41.086968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# CELL 31 — Fix Phase 5 : Iterative balancing\n#\n# The one-shot approach under-shoots because oversampling\n# increases the total, which lowers the percentage.\n# We iterate until all class ratios hit ≥ 20%.\n# ─────────────────────────────────────────────────────────────\n\n# Start from the capped dataframe (before oversampling)\n# Re-derive it cleanly\nMAX_DATASET_RATIO = 0.40\nMIN_CLASS_RATIO   = 0.20\ntotal_train       = len(train_df)\nmax_per_dataset   = int(total_train * MAX_DATASET_RATIO)\n\ncapped_frames = []\nfor ds in train_df[\"dataset\"].unique():\n    ds_df = train_df[train_df[\"dataset\"] == ds]\n    if len(ds_df) > max_per_dataset:\n        ds_df = ds_df.groupby(\"disease\", group_keys=False).apply(\n            lambda x: x.sample(\n                n=max(1, int(max_per_dataset * len(x) / len(ds_df))),\n                random_state=42\n            )\n        )\n    capped_frames.append(ds_df)\n\ntrain_capped = pd.concat(capped_frames, ignore_index=True)\nprint(f\"Starting from capped train : {len(train_capped):,}\")\n\n# ── Iterative oversampling ────────────────────────────────────\ntrain_balanced = train_capped.copy()\nMAX_ITER       = 10\n\nfor iteration in range(MAX_ITER):\n    current_total  = len(train_balanced)\n    class_counts   = train_balanced[\"disease\"].value_counts()\n    class_ratios   = class_counts / current_total\n\n    # Check if all classes already meet the minimum\n    if (class_ratios >= MIN_CLASS_RATIO).all():\n        print(f\"Converged at iteration {iteration} ✅\")\n        break\n\n    print(f\"\\nIteration {iteration + 1}:\")\n    print(f\"  Total : {current_total:,}\")\n    for d, r in class_ratios.items():\n        flag = \"✅\" if r >= MIN_CLASS_RATIO else \"❌\"\n        print(f\"  {flag} {d:<15} {class_counts[d]:>7,}  ({r*100:.1f}%)\")\n\n    # For each class below minimum, oversample to reach\n    # 20% of the NEW total after adding samples\n    # Formula: target_count = (MIN_RATIO * sum_of_other_classes) / (1 - MIN_RATIO)\n    frames = [train_balanced]\n    for disease in class_counts.index:\n        ratio = class_ratios[disease]\n        if ratio < MIN_CLASS_RATIO:\n            disease_df   = train_balanced[train_balanced[\"disease\"] == disease]\n            other_total  = current_total - len(disease_df)\n            target_count = int(MIN_CLASS_RATIO * other_total / (1 - MIN_CLASS_RATIO))\n            n_extra      = max(0, target_count - len(disease_df))\n            if n_extra > 0:\n                extra_df = disease_df.sample(n=n_extra, replace=True, random_state=42 + iteration)\n                frames.append(extra_df)\n                print(f\"  Oversampling {disease}: +{n_extra:,} → {target_count:,}\")\n\n    train_balanced = pd.concat(frames, ignore_index=True)\n\n# ── Final shuffle ─────────────────────────────────────────────\ntrain_balanced = train_balanced.sample(frac=1, random_state=42).reset_index(drop=True)\n\n# ── Verify constraints ────────────────────────────────────────\nprint(f\"\\n{'='*55}\")\nprint(f\"FINAL VERIFICATION\")\nprint(f\"{'='*55}\")\nprint(f\"Final balanced train size : {len(train_balanced):,}\")\n\nprint(f\"\\nDisease distribution (constraint: each ≥ 20%):\")\nall_class_ok = True\nfor disease in [\"No Finding\", \"COVID-19\", \"Pneumonia\"]:\n    count = (train_balanced[\"disease\"] == disease).sum()\n    pct   = count / len(train_balanced) * 100\n    ok    = pct >= 20.0\n    flag  = \"✅\" if ok else \"❌\"\n    if not ok: all_class_ok = False\n    print(f\"  {flag} {disease:<15} {count:>7,}  ({pct:.1f}%)\")\n\nprint(f\"\\nDataset distribution (constraint: each ≤ 40%):\")\nall_dataset_ok = True\nfor ds, count in train_balanced[\"dataset\"].value_counts().items():\n    pct  = count / len(train_balanced) * 100\n    ok   = pct <= 40.0\n    flag = \"✅\" if ok else \"❌\"\n    if not ok: all_dataset_ok = False\n    print(f\"  {flag} {ds:<12} {count:>7,}  ({pct:.1f}%)\")\n\nprint(f\"\\nConstraint summary:\")\nprint(f\"  min_class_ratio  ≥ 20% : {'✅ PASSED' if all_class_ok else '❌ FAILED'}\")\nprint(f\"  max_dataset_ratio ≤ 40% : {'✅ PASSED' if all_dataset_ok else '❌ FAILED'}\")\n\nprint(f\"\\nDisease × Dataset breakdown:\")\nbreakdown = train_balanced.groupby([\"dataset\", \"disease\"]).size().unstack(fill_value=0)\nbreakdown[\"TOTAL\"] = breakdown.sum(axis=1)\nbreakdown.loc[\"TOTAL\"] = breakdown.sum()\nprint(breakdown.to_string())\n\n# ── Save ──────────────────────────────────────────────────────\nOUT_BALANCED = Path(\"/kaggle/working/balanced_train.parquet\")\ntrain_balanced.to_parquet(OUT_BALANCED, index=False)\nprint(f\"\\nSaved → {OUT_BALANCED}  ({len(train_balanced):,} rows)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T16:20:51.762962Z","iopub.execute_input":"2026-03-24T16:20:51.763272Z","iopub.status.idle":"2026-03-24T16:20:51.778464Z","shell.execute_reply.started":"2026-03-24T16:20:51.763245Z","shell.execute_reply":"2026-03-24T16:20:51.777508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nValidation script for balanced_train.parquet\nChecks class balance across the three disease classes:\n  - No Finding\n  - COVID-19\n  - Pneumonia\n\"\"\"\n\nimport pandas as pd\nfrom pathlib import Path\n\n# ── Load ──────────────────────────────────────────────────────\nBALANCED_PATH = \"/kaggle/input/datasets/kaushik2005/balanced/balanced_train.parquet\"   # update path if needed\ndf = pd.read_parquet(BALANCED_PATH)\n\ntotal = len(df)\nTARGET_CLASSES  = [\"No Finding\", \"COVID-19\", \"Pneumonia\"]\nMIN_CLASS_RATIO = 0.20   # 20% minimum per class\nMAX_DS_RATIO    = 0.40   # 40% maximum per dataset\n\nprint(\"=\" * 60)\nprint(\"BALANCED PARQUET — VALIDATION REPORT\")\nprint(\"=\" * 60)\nprint(f\"\\nTotal rows    : {total:,}\")\nprint(f\"Total columns : {df.shape[1]}\")\nprint(f\"Columns       : {list(df.columns)}\\n\")\n\n# ── 1. Disease class distribution ─────────────────────────────\nprint(\"=\" * 60)\nprint(\"1. DISEASE CLASS DISTRIBUTION  (constraint: each ≥ 20%)\")\nprint(\"=\" * 60)\n\nclass_counts = df[\"disease\"].value_counts()\nall_class_ok = True\n\nfor disease in TARGET_CLASSES:\n    count = class_counts.get(disease, 0)\n    pct   = count / total * 100\n    ok    = pct >= MIN_CLASS_RATIO * 100\n    flag  = \"✅\" if ok else \"❌\"\n    if not ok:\n        all_class_ok = False\n    print(f\"  {flag}  {disease:<15}  {count:>8,}  ({pct:.2f}%)\")\n\n# Any unexpected classes?\nextra = set(class_counts.index) - set(TARGET_CLASSES)\nif extra:\n    print(f\"\\n  ⚠️  Unexpected disease values found: {extra}\")\n    for e in extra:\n        print(f\"       {e}: {class_counts[e]:,}\")\n\nprint(f\"\\n  Result → {'✅ ALL CLASSES PASS' if all_class_ok else '❌ SOME CLASSES FAILED'}\")\n\n# ── 2. Dataset distribution ────────────────────────────────────\nprint(\"\\n\" + \"=\" * 60)\nprint(\"2. DATASET DISTRIBUTION  (constraint: each ≤ 40%)\")\nprint(\"=\" * 60)\n\nds_counts    = df[\"dataset\"].value_counts()\nall_ds_ok    = True\n\nfor ds, count in ds_counts.items():\n    pct  = count / total * 100\n    ok   = pct <= MAX_DS_RATIO * 100\n    flag = \"✅\" if ok else \"❌\"\n    if not ok:\n        all_ds_ok = False\n    print(f\"  {flag}  {ds:<12}  {count:>8,}  ({pct:.2f}%)\")\n\nprint(f\"\\n  Result → {'✅ ALL DATASETS PASS' if all_ds_ok else '❌ SOME DATASETS FAILED'}\")\n\n# ── 3. Disease × Dataset cross-tabulation ─────────────────────\nprint(\"\\n\" + \"=\" * 60)\nprint(\"3. DISEASE × DATASET BREAKDOWN\")\nprint(\"=\" * 60)\n\nxtab = df.groupby([\"dataset\", \"disease\"]).size().unstack(fill_value=0)\n\n# Ensure all three target columns are present\nfor c in TARGET_CLASSES:\n    if c not in xtab.columns:\n        xtab[c] = 0\n\nxtab = xtab[TARGET_CLASSES]          # consistent column order\nxtab[\"TOTAL\"] = xtab.sum(axis=1)\nxtab.loc[\"TOTAL\"] = xtab.sum()\n\nprint(xtab.to_string())\n\n# ── 4. Duplicate / null checks ────────────────────────────────\nprint(\"\\n\" + \"=\" * 60)\nprint(\"4. DATA QUALITY CHECKS\")\nprint(\"=\" * 60)\n\nnull_counts = df[[\"disease\", \"dataset\", \"path\"]].isnull().sum()\nprint(f\"\\n  Null values in key columns:\")\nfor col, n in null_counts.items():\n    flag = \"✅\" if n == 0 else \"⚠️ \"\n    print(f\"    {flag} {col}: {n}\")\n\nn_dups = df.duplicated(subset=[\"image_id\"]).sum() if \"image_id\" in df.columns else df.duplicated().sum()\nflag   = \"✅\" if n_dups == 0 else \"⚠️ \"\nprint(f\"\\n  {flag} Duplicate rows (by image_id): {n_dups:,}\")\n\n# ── 5. Summary ─────────────────────────────────────────────────\nprint(\"\\n\" + \"=\" * 60)\nprint(\"5. CONSTRAINT SUMMARY\")\nprint(\"=\" * 60)\nprint(f\"  min_class_ratio   ≥ 20%  : {'✅ PASSED' if all_class_ok else '❌ FAILED'}\")\nprint(f\"  max_dataset_ratio ≤ 40%  : {'✅ PASSED' if all_ds_ok    else '❌ FAILED'}\")\noverall = all_class_ok and all_ds_ok\nprint(f\"\\n  Overall           : {'✅ ALL CONSTRAINTS MET' if overall else '❌ CONSTRAINTS NOT FULLY MET'}\")\nprint(\"=\" * 60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-24T16:13:01.86778Z","iopub.execute_input":"2026-03-24T16:13:01.868394Z","iopub.status.idle":"2026-03-24T16:13:02.2959Z","shell.execute_reply.started":"2026-03-24T16:13:01.868365Z","shell.execute_reply":"2026-03-24T16:13:02.295169Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ─────────────────────────────────────────────────────────────\n# Phase 5 — Analysis ONLY (No Balancing Applied)\n#\n# Strategy:\n#   1. DO NOT modify dataset\n#   2. Print statistics\n#   3. Verify constraints (informational only)\n#   4. Save as unchanged_train.parquet (unchanged)\n#   5. Generate demographic graphs\n# ─────────────────────────────────────────────────────────────\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as mpatches\nimport matplotlib.gridspec as gridspec\nimport numpy as np\nfrom pathlib import Path\nfrom collections import Counter\n\ntrain_df = pd.read_parquet(\"/kaggle/input/datasets/kaushik2005/balanced/train.parquet\")\n\nprint(f\"Total train size : {len(train_df):,}\")\n\n# ─────────────────────────────────────────────────────────────\n# STEP 1 — Disease distribution\n# ─────────────────────────────────────────────────────────────\n\nprint(f\"\\n{'='*55}\")\nprint(\"Disease distribution\")\nprint(f\"{'='*55}\")\n\nfor disease in [\"No Finding\", \"COVID-19\", \"Pneumonia\"]:\n    count = (train_df[\"disease\"] == disease).sum()\n    pct   = count / len(train_df) * 100\n    print(f\"  {disease:<15} {count:>7,} ({pct:.2f}%)\")\n\n# ─────────────────────────────────────────────────────────────\n# STEP 2 — Dataset distribution\n# ─────────────────────────────────────────────────────────────\n\nprint(f\"\\n{'='*55}\")\nprint(\"Dataset distribution\")\nprint(f\"{'='*55}\")\n\nfor ds, count in train_df[\"dataset\"].value_counts().items():\n    pct = count / len(train_df) * 100\n    print(f\"  {ds:<12} {count:>7,} ({pct:.2f}%)\")\n\n# ─────────────────────────────────────────────────────────────\n# STEP 3 — Constraint check (informational only)\n# ─────────────────────────────────────────────────────────────\n\nprint(f\"\\n{'='*55}\")\nprint(\"Constraint check (NO enforcement)\")\nprint(f\"{'='*55}\")\n\nMIN_CLASS_RATIO   = 0.20\nMAX_DATASET_RATIO = 0.40\n\nprint(\"\\nClass constraint (≥ 20%):\")\nfor disease in [\"No Finding\", \"COVID-19\", \"Pneumonia\"]:\n    count = (train_df[\"disease\"] == disease).sum()\n    pct   = count / len(train_df)\n    flag  = \"✅\" if pct >= MIN_CLASS_RATIO else \"⚠️\"\n    print(f\"  {flag} {disease:<15} {pct*100:.2f}%\")\n\nprint(\"\\nDataset constraint (≤ 40%):\")\nfor ds, count in train_df[\"dataset\"].value_counts().items():\n    pct  = count / len(train_df)\n    flag = \"✅\" if pct <= MAX_DATASET_RATIO else \"⚠️\"\n    print(f\"  {flag} {ds:<12} {pct*100:.2f}%\")\n\n# ─────────────────────────────────────────────────────────────\n# STEP 4 — Disease × Dataset breakdown\n# ─────────────────────────────────────────────────────────────\n\nprint(f\"\\n{'='*55}\")\nprint(\"Disease × Dataset breakdown\")\nprint(f\"{'='*55}\")\n\nbreakdown = train_df.groupby([\"dataset\", \"disease\"]).size().unstack(fill_value=0)\nbreakdown[\"TOTAL\"] = breakdown.sum(axis=1)\nbreakdown.loc[\"TOTAL\"] = breakdown.sum()\n\nprint(breakdown.to_string())\n\n# ─────────────────────────────────────────────────────────────\n# STEP 5 — Save (UNCHANGED DATA)\n# ─────────────────────────────────────────────────────────────\n\nOUT_PATH = Path(\"/kaggle/working/unchanged_train.parquet\")\ntrain_df.to_parquet(OUT_PATH, index=False)\n\nprint(f\"\\nSaved → {OUT_PATH}\")\nprint(f\"Rows  : {len(train_df):,}\")\nprint(\"NOTE: No balancing applied — dataset unchanged\")\n\n# ─────────────────────────────────────────────────────────────\n# STEP 6 — Demographic graphs\n# ─────────────────────────────────────────────────────────────\n\nBG_DARK        = \"#0F172A\"\nBG_CARD        = \"#1E293B\"\nTEXT_PRIMARY   = \"#F1F5F9\"\nTEXT_SECONDARY = \"#94A3B8\"\nBORDER         = \"#334155\"\n\nDISEASE_COLORS = {\n    \"No Finding\": \"#3B82F6\",\n    \"COVID-19\":   \"#A855F7\",\n    \"Pneumonia\":  \"#EF4444\",\n}\nDATASET_COLORS = {\n    \"NIH\":       \"#3B82F6\",\n    \"CheXpert\":  \"#F97316\",\n    \"COVIDx\":    \"#A855F7\",\n    \"Pediatric\": \"#22C55E\",\n    \"RSNA\":      \"#EF4444\",\n}\nGENDER_COLORS  = {\"Male\": \"#3B82F6\", \"Female\": \"#EC4899\", \"Unknown\": \"#64748B\"}\nAGE_GROUP_ORDER  = [\"Infant\", \"Child\", \"Adolescent\", \"Adult\", \"Senior\", \"Unknown\"]\nAGE_GROUP_COLORS = [\"#FCD34D\", \"#6EE7B7\", \"#93C5FD\", \"#818CF8\", \"#F9A8D4\", \"#475569\"]\n\ndef style_ax(ax):\n    ax.set_facecolor(BG_CARD)\n    ax.tick_params(colors=TEXT_SECONDARY, labelsize=9)\n    for spine in [\"top\", \"right\"]:\n        ax.spines[spine].set_visible(False)\n    for spine in [\"bottom\", \"left\"]:\n        ax.spines[spine].set_color(BORDER)\n    ax.title.set_color(TEXT_SECONDARY)\n    ax.title.set_fontsize(11)\n\ntotal = len(train_df)\n\nfig = plt.figure(figsize=(22, 28), facecolor=BG_DARK)\ngs  = gridspec.GridSpec(4, 3, figure=fig,\n                         hspace=0.5, wspace=0.35,\n                         height_ratios=[0.12, 1, 1, 1])\n\n# ── Row 0 — Summary cards ──────────────────────────────────────\nax_stats = fig.add_subplot(gs[0, :])\nax_stats.set_facecolor(BG_DARK)\nax_stats.axis(\"off\")\n\nstats = [\n    (\"Total images\",      f\"{total:,}\"),\n    (\"Datasets\",          f\"{train_df['dataset'].nunique()}\"),\n    (\"Disease classes\",   f\"{train_df['disease'].nunique()}\"),\n    (\"Age coverage\",      f\"{train_df['has_age'].mean()*100:.1f}%\"),\n    (\"Gender coverage\",   f\"{train_df['has_gender'].mean()*100:.1f}%\"),\n]\ncard_w, card_gap = 0.17, 0.04\nstart_x = 0.025\ncard_h, card_y = 0.72, 0.15\n\nfor i, (label, value) in enumerate(stats):\n    x = start_x + i * (card_w + card_gap)\n    rect = mpatches.FancyBboxPatch(\n        (x, card_y), card_w, card_h,\n        boxstyle=\"round,pad=0.01\",\n        transform=ax_stats.transAxes,\n        facecolor=BG_CARD, edgecolor=\"#3B82F6\",\n        linewidth=1.2, clip_on=False\n    )\n    ax_stats.add_patch(rect)\n    ax_stats.text(x + card_w/2, card_y + card_h*0.72, label,\n                  transform=ax_stats.transAxes,\n                  ha=\"center\", va=\"center\", fontsize=9, color=TEXT_SECONDARY)\n    ax_stats.text(x + card_w/2, card_y + card_h*0.28, value,\n                  transform=ax_stats.transAxes,\n                  ha=\"center\", va=\"center\",\n                  fontsize=12, fontweight=\"bold\", color=TEXT_PRIMARY)\n\n# ── Row 1 Col 0 — Disease distribution bar ────────────────────\nax = fig.add_subplot(gs[1, 0])\nstyle_ax(ax)\n\ndiseases = [\"No Finding\", \"COVID-19\", \"Pneumonia\"]\nd_counts = [(train_df[\"disease\"] == d).sum() for d in diseases]\nd_colors = [DISEASE_COLORS[d] for d in diseases]\nbars = ax.bar(diseases, d_counts, color=d_colors,\n              edgecolor=BG_DARK, linewidth=0.5, width=0.6)\nax.set_title(\"Disease distribution\")\nax.yaxis.set_major_formatter(plt.FuncFormatter(lambda x, _: f\"{int(x):,}\"))\nfor bar, val in zip(bars, d_counts):\n    ax.text(bar.get_x() + bar.get_width()/2,\n            bar.get_height() + max(d_counts)*0.01,\n            f\"{val:,}\\n({val/total*100:.1f}%)\",\n            ha=\"center\", va=\"bottom\", fontsize=8.5, color=TEXT_SECONDARY)\nax.set_ylim(0, max(d_counts) * 1.25)\nax.tick_params(axis=\"x\", rotation=10)\n\n# ── Row 1 Col 1 — Dataset distribution donut ─────────────────\nax = fig.add_subplot(gs[1, 1])\nstyle_ax(ax)\nax.set_facecolor(BG_CARD)\n\nds_counts = train_df[\"dataset\"].value_counts()\nds_vals   = [int(ds_counts.get(d, 0)) for d in ds_counts.index]\nds_cols   = [DATASET_COLORS.get(d, \"#64748B\") for d in ds_counts.index]\nwedges, _ = ax.pie(ds_vals, colors=ds_cols, startangle=90,\n                    wedgeprops={\"width\": 0.52, \"edgecolor\": BG_DARK, \"linewidth\": 1.5},\n                    radius=0.9)\nax.text(0, 0, f\"{total:,}\\ntotal\", ha=\"center\", va=\"center\",\n        fontsize=10, color=TEXT_PRIMARY, fontweight=\"bold\")\nax.set_title(\"Dataset distribution\")\nlegend_patches = [\n    mpatches.Patch(color=DATASET_COLORS.get(d, \"#64748B\"),\n                   label=f\"{d}  {int(ds_counts[d]):,}  ({ds_counts[d]/total*100:.1f}%)\")\n    for d in ds_counts.index\n]\nax.legend(handles=legend_patches, loc=\"lower center\",\n          bbox_to_anchor=(0.5, -0.18), ncol=1, fontsize=8,\n          frameon=False, labelcolor=TEXT_SECONDARY)\n\n# ── Row 1 Col 2 — Disease × Dataset stacked bar ───────────────\nax = fig.add_subplot(gs[1, 2])\nstyle_ax(ax)\n\nds_order  = train_df[\"dataset\"].value_counts().index.tolist()\nx         = np.arange(len(ds_order))\nw         = 0.55\nbottoms   = np.zeros(len(ds_order))\n\nfor disease, color in DISEASE_COLORS.items():\n    vals = [(train_df[train_df[\"dataset\"] == ds][\"disease\"] == disease).sum()\n            for ds in ds_order]\n    ax.bar(x, vals, bottom=bottoms, width=w, label=disease,\n           color=color, edgecolor=BG_DARK, linewidth=0.4)\n    bottoms += np.array(vals)\n\nax.set_xticks(x)\nax.set_xticklabels(ds_order, rotation=20, fontsize=8)\nax.set_title(\"Disease × dataset (stacked)\")\nax.yaxis.set_major_formatter(plt.FuncFormatter(lambda x, _: f\"{int(x):,}\"))\nax.legend(fontsize=8, frameon=False, labelcolor=TEXT_SECONDARY,\n          loc=\"upper right\")\n\n# ── Row 2 Col 0 — Age group bar ───────────────────────────────\nax = fig.add_subplot(gs[2, 0])\nstyle_ax(ax)\n\nage_counts  = train_df[\"age_group\"].value_counts()\nage_ordered = {g: int(age_counts.get(g, 0)) for g in AGE_GROUP_ORDER}\nbars = ax.bar(list(age_ordered.keys()), list(age_ordered.values()),\n              color=AGE_GROUP_COLORS, edgecolor=BG_DARK, linewidth=0.5, width=0.65)\nax.set_title(\"Age group distribution\")\nax.tick_params(axis=\"x\", rotation=30)\nax.yaxis.set_major_formatter(plt.FuncFormatter(lambda x, _: f\"{int(x):,}\"))\ntotal_ag = sum(age_ordered.values())\nfor bar, val in zip(bars, age_ordered.values()):\n    if val > 0:\n        ax.text(bar.get_x() + bar.get_width()/2,\n                bar.get_height() + total_ag*0.008,\n                f\"{val/total_ag*100:.1f}%\",\n                ha=\"center\", va=\"bottom\", fontsize=7.5, color=TEXT_SECONDARY)\nax.set_ylim(0, max(age_ordered.values()) * 1.2)\n\n# ── Row 2 Col 1 — Gender donut ────────────────────────────────\nax = fig.add_subplot(gs[2, 1])\nstyle_ax(ax)\nax.set_facecolor(BG_CARD)\n\ngender_counts = train_df[\"gender\"].value_counts()\ngenders = [g for g in [\"Male\", \"Female\", \"Unknown\"] if g in gender_counts.index]\ng_vals  = [int(gender_counts[g]) for g in genders]\ng_cols  = [GENDER_COLORS[g] for g in genders]\nwedges, _ = ax.pie(g_vals, colors=g_cols, startangle=90,\n                    wedgeprops={\"width\": 0.52, \"edgecolor\": BG_DARK, \"linewidth\": 1.5},\n                    radius=0.9)\nax.text(0, 0, f\"{sum(g_vals):,}\\ntotal\", ha=\"center\", va=\"center\",\n        fontsize=10, color=TEXT_PRIMARY, fontweight=\"bold\")\nax.set_title(\"Gender distribution\")\ng_patches = [\n    mpatches.Patch(color=GENDER_COLORS[g],\n                   label=f\"{g}  {int(gender_counts.get(g,0)):,}  ({int(gender_counts.get(g,0))/sum(g_vals)*100:.1f}%)\")\n    for g in genders\n]\nax.legend(handles=g_patches, loc=\"lower center\",\n          bbox_to_anchor=(0.5, -0.14), ncol=1, fontsize=8.5,\n          frameon=False, labelcolor=TEXT_SECONDARY)\n\n# ── Row 2 Col 2 — Age group × disease ────────────────────────\nax = fig.add_subplot(gs[2, 2])\nstyle_ax(ax)\n\nage_groups_present = [g for g in AGE_GROUP_ORDER if g in train_df[\"age_group\"].unique()]\nx  = np.arange(len(age_groups_present))\nw  = 0.8 / len(DISEASE_COLORS)\nfor i, (disease, color) in enumerate(DISEASE_COLORS.items()):\n    vals = [(train_df[train_df[\"age_group\"] == ag][\"disease\"] == disease).sum()\n            for ag in age_groups_present]\n    ax.bar(x + i*w - w, vals, width=w*0.85, label=disease,\n           color=color, edgecolor=BG_DARK, linewidth=0.4)\nax.set_xticks(x)\nax.set_xticklabels(age_groups_present, rotation=30, fontsize=8)\nax.set_title(\"Disease × age group\")\nax.yaxis.set_major_formatter(plt.FuncFormatter(lambda x, _: f\"{int(x):,}\"))\nax.legend(fontsize=8, frameon=False, labelcolor=TEXT_SECONDARY, loc=\"upper right\")\n\n# ── Row 3 Col 0 — Age histogram (has_age only) ────────────────\nax = fig.add_subplot(gs[3, 0])\nstyle_ax(ax)\n\nage_data = train_df.loc[train_df[\"has_age\"], \"age\"].dropna()\nif len(age_data) > 0:\n    n, bins, patches_hist = ax.hist(age_data, bins=40,\n                                     edgecolor=BG_DARK, linewidth=0.4, alpha=0.9)\n    group_bin_colors = {\n        (0, 2):   \"#FCD34D\",\n        (2, 12):  \"#6EE7B7\",\n        (12, 18): \"#93C5FD\",\n        (18, 65): \"#818CF8\",\n        (65, 200):\"#F9A8D4\",\n    }\n    for patch, left in zip(patches_hist, bins[:-1]):\n        for (lo, hi), col in group_bin_colors.items():\n            if lo <= left < hi:\n                patch.set_facecolor(col)\n                break\n    mean_age   = float(age_data.mean())\n    median_age = float(age_data.median())\n    y_max = ax.get_ylim()[1]\n    ax.axvline(mean_age,   color=\"#F59E0B\", linewidth=1.5, linestyle=\"--\")\n    ax.axvline(median_age, color=\"#34D399\", linewidth=1.5, linestyle=\"--\")\n    ax.text(mean_age+0.5,   y_max*0.92, f\"Mean {mean_age:.1f}\",   color=\"#F59E0B\", fontsize=8)\n    ax.text(median_age+0.5, y_max*0.82, f\"Median {median_age:.1f}\", color=\"#34D399\", fontsize=8)\n    ax.set_xlabel(\"Age (years)\", color=TEXT_SECONDARY, fontsize=9)\n    ax.yaxis.set_major_formatter(plt.FuncFormatter(lambda x, _: f\"{int(x):,}\"))\nax.set_title(\"Age distribution (has_age only)\")\n\n# ── Row 3 Col 1 — Gender × disease ───────────────────────────\nax = fig.add_subplot(gs[3, 1])\nstyle_ax(ax)\n\ngender_list = [g for g in [\"Male\", \"Female\", \"Unknown\"]\n               if g in train_df[\"gender\"].unique()]\nx  = np.arange(len(gender_list))\nw  = 0.8 / len(DISEASE_COLORS)\nfor i, (disease, color) in enumerate(DISEASE_COLORS.items()):\n    vals = [(train_df[train_df[\"gender\"] == g][\"disease\"] == disease).sum()\n            for g in gender_list]\n    ax.bar(x + i*w - w, vals, width=w*0.85, label=disease,\n           color=color, edgecolor=BG_DARK, linewidth=0.4)\nax.set_xticks(x)\nax.set_xticklabels(gender_list, fontsize=9)\nax.set_title(\"Disease × gender\")\nax.yaxis.set_major_formatter(plt.FuncFormatter(lambda x, _: f\"{int(x):,}\"))\nax.legend(fontsize=8, frameon=False, labelcolor=TEXT_SECONDARY, loc=\"upper right\")\n\n# ── Row 3 Col 2 — Constraint status ──────────────────────────\nax = fig.add_subplot(gs[3, 2])\nstyle_ax(ax)\nax.axis(\"off\")\n\nlines = [\n    \"Constraint check\",\n    \"\",\n    \"Class:\",\n]\nfor disease in [\"No Finding\", \"COVID-19\", \"Pneumonia\"]:\n    count = (train_df[\"disease\"] == disease).sum()\n    pct   = count / total\n    flag  = \"✅\" if pct >= MIN_CLASS_RATIO else \"⚠️\"\n    lines.append(f\"  {flag} {disease:<15} {pct*100:.1f}%\")\n\nlines += [\"\", \"Dataset:\"]\nfor ds, count in train_df[\"dataset\"].value_counts().items():\n    pct  = count / total\n    flag = \"✅\" if pct <= MAX_DATASET_RATIO else \"⚠️\"\n    lines.append(f\"  {flag} {ds:<12} {pct*100:.1f}%\")\n\nax.text(0.05, 0.95, \"\\n\".join(lines),\n        transform=ax.transAxes,\n        va=\"top\", ha=\"left\",\n        fontsize=9.5, color=TEXT_SECONDARY,\n        linespacing=1.8,\n        bbox=dict(boxstyle=\"round,pad=0.6\",\n                  facecolor=BG_CARD,\n                  edgecolor=\"#3B82F6\",\n                  linewidth=1.0))\n\nfig.suptitle(\"Merged Train Dataset — Demographic Analysis (Unchanged)\",\n             color=TEXT_PRIMARY, fontsize=17, fontweight=\"bold\", y=1.005)\n\nout_path = \"/kaggle/working/train_demographics.png\"\nplt.savefig(out_path, dpi=150, bbox_inches=\"tight\", facecolor=BG_DARK)\nplt.show()\nprint(f\"\\nSaved → {out_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-27T04:42:04.088388Z","iopub.execute_input":"2026-03-27T04:42:04.088753Z","iopub.status.idle":"2026-03-27T04:42:07.94146Z","shell.execute_reply.started":"2026-03-27T04:42:04.088726Z","shell.execute_reply":"2026-03-27T04:42:07.94085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}