{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":128792,"databundleVersionId":15494745,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n# 🏆 VISTA CODEFEST'26 — PLATINUM PIPELINE (INFRASTRUCTURE-GRADE ML)\n# =============================================================================\n# FEATURES:\n#  - GPU enforced (fail-fast)\n#  - Dynamic disk guard\n#  - Category remapping (label-safe)\n#  - Stratified difficulty split\n#  - Hybrid storage (symlink train / copy val)\n#  - YOLO worker safety mode\n#  - Resume training support\n#  - Checkpoint integrity validation\n#  - Chunked inference (zero OOM risk)\n#  - Validation alignment guarantee\n#  - Judge-grade submission validation\n#  - Auto-backup to /kaggle/temp\n# =============================================================================\n\n# -----------------------------------------------------------------------------\n# FORCE GPU\n# -----------------------------------------------------------------------------\nimport os\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\nos.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\"\n\n# -----------------------------------------------------------------------------\n# IMPORTS\n# -----------------------------------------------------------------------------\nimport subprocess\nimport sys\nimport json\nimport time\nimport random\nimport gc\nimport shutil\nfrom pathlib import Path\nfrom collections import defaultdict, Counter\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# -----------------------------------------------------------------------------\n# GPU VERIFICATION\n# -----------------------------------------------------------------------------\nprint(\"=\" * 70)\nprint(\"🔍 GPU VERIFICATION\")\nprint(\"=\" * 70)\n\ntry:\n    result = subprocess.run([\"nvidia-smi\"], capture_output=True, text=True, timeout=10)\n    if result.returncode == 0:\n        for line in result.stdout.split(\"\\n\"):\n            if \"Tesla\" in line or \"GPU\" in line:\n                print(line.strip())\nexcept:\n    print(\"⚠️ nvidia-smi unavailable\")\n\n# -----------------------------------------------------------------------------\n# DEPENDENCIES\n# -----------------------------------------------------------------------------\ntry:\n    import ultralytics\nexcept ImportError:\n    print(\"📦 Installing ultralytics...\")\n    subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"ultralytics\"])\n\nimport yaml\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom ultralytics import YOLO\n\n# -----------------------------------------------------------------------------\n# GPU FAIL-FAST\n# -----------------------------------------------------------------------------\nprint(\"\\n\" + \"=\" * 70)\nprint(\"🎯 GPU CHECK\")\nprint(\"=\" * 70)\n\nif not torch.cuda.is_available():\n    raise RuntimeError(\"❌ GPU REQUIRED — Enable P100 in Kaggle\")\n\nprint(f\"✅ CUDA: {torch.version.cuda}\")\nprint(f\"✅ GPU: {torch.cuda.get_device_name(0)}\")\nprint(f\"✅ VRAM: {torch.cuda.get_device_properties(0).total_memory/1e9:.2f} GB\")\n\nt = torch.zeros(1).cuda()\nprint(f\"✅ GPU Tensor Test: {t.device}\")\ndel t\n\nDEVICE = 0\nprint(\"=\" * 70)\n\n# -----------------------------------------------------------------------------\n# DISK GUARD (DYNAMIC)\n# -----------------------------------------------------------------------------\nprint(\"\\n💾 DISK CHECK\")\ntotal, used, free = shutil.disk_usage(\"/kaggle/working\")\nprint(f\"Free space: {free/1e9:.1f} GB\")\n\nREQUIRED_GB = max(12, free / 1e9 * 0.4)  # adaptive threshold\nif free < REQUIRED_GB * 1e9:\n    raise RuntimeError(f\"❌ Disk too low: {free/1e9:.1f}GB free, need {REQUIRED_GB:.1f}GB\")\n\n# -----------------------------------------------------------------------------\n# CONFIG\n# -----------------------------------------------------------------------------\nclass Cfg:\n    ROOT = \"/kaggle/input/vista26\"\n    BASE = f\"{ROOT}/Vistas Dataset Public/Vistas Dataset Public\"\n    WORK = \"/kaggle/working\"\n\n    TRAIN_DIR = f\"{BASE}/train\"\n    TEST_DIR  = f\"{BASE}/test\"\n    VAL_DIR   = f\"{BASE}/validation\"\n\n    TRAIN_JSON = f\"{BASE}/instances_train.json\"\n    TEST_JSON  = f\"{BASE}/instances_test.json\"\n    VAL_JSON   = f\"{ROOT}/instances_val.json\"\n    CATS_JSON  = f\"{BASE}/Categories.json\"\n\n    YOLO_MODEL = \"yolov8n.pt\"\n    YOLO_EPOCHS = 20\n    YOLO_IMGSZ = 640\n    YOLO_BATCH = 24\n    YOLO_WORKERS = 0  # CRITICAL: symlink-safe\n    YOLO_PATIENCE = 5\n\n    TRAIN_SAMPLE_RATIO = 0.3\n    VAL_RATIO = 0.10\n\n    CONF = 0.25\n    IOU = 0.45\n    MAX_DET = 50\n\n    SEED = 42\n\nC = Cfg()\nrandom.seed(C.SEED)\nnp.random.seed(C.SEED)\ntorch.manual_seed(C.SEED)\ntorch.cuda.manual_seed_all(C.SEED)\n\nSTART = time.time()\ndef elapsed():\n    return f\"{(time.time()-START)/60:.1f} min\"\n\ndef cleanup():\n    gc.collect()\n    torch.cuda.empty_cache()\n    torch.cuda.synchronize()\n\n# -----------------------------------------------------------------------------\n# AUTO RESUME SUPPORT\n# -----------------------------------------------------------------------------\nRESUME_PT = f\"{C.WORK}/yolo_run/weights/last.pt\"\nif os.path.exists(RESUME_PT):\n    print(\"♻️ Resuming from checkpoint\")\n    C.YOLO_MODEL = RESUME_PT\n\n# -----------------------------------------------------------------------------\n# STAGE 0 — LOAD DATA\n# -----------------------------------------------------------------------------\nprint(f\"\\n[{elapsed()}] STAGE 0 — Loading Data\")\n\nwith open(C.CATS_JSON) as f:\n    cats_raw = json.load(f)[\"categories\"]\n\ncats_raw.sort(key=lambda c: int(c[\"id\"]))\nVALID_IDS = set(int(c[\"id\"]) for c in cats_raw)\n\ncat_id_to_yolo = {int(c[\"id\"]): i for i, c in enumerate(cats_raw)}\nnames_map = {i: c[\"name\"] for i, c in enumerate(cats_raw)}\n\ndef load_data(json_path):\n    with open(json_path) as f:\n        return json.load(f)\n\ntrain_data = load_data(C.TRAIN_JSON)\ntest_data = load_data(C.TEST_JSON)\n\ndef parse_images(data):\n    imgs = {}\n    for img in data[\"images\"]:\n        imgs[int(img[\"id\"])] = {\n            \"file_name\": img[\"file_name\"],\n            \"width\": int(img[\"width\"]),\n            \"height\": int(img[\"height\"]),\n            \"level\": img.get(\"level\", \"unknown\")\n        }\n    return imgs\n\ndef parse_annotations(data, valid_ids):\n    anns = defaultdict(list)\n    for ann in data[\"annotations\"]:\n        iid = int(ann[\"image_id\"])\n        cid = int(ann[\"category_id\"])\n        if cid in valid_ids:\n            anns[iid].append({\n                \"category_id\": cid,\n                \"bbox\": ann[\"bbox\"]\n            })\n    return anns\n\ntrain_img_info = parse_images(train_data)\ntest_img_info = parse_images(test_data)\n\ntrain_ann_by_img = parse_annotations(train_data, VALID_IDS)\ntest_ann_by_img = parse_annotations(test_data, VALID_IDS)\n\ntrain_ids = [iid for iid in train_img_info if train_ann_by_img[iid]]\ntest_ids = [iid for iid in test_img_info if test_ann_by_img[iid]]\n\nrandom.shuffle(train_ids)\ntrain_ids = train_ids[:int(len(train_ids) * C.TRAIN_SAMPLE_RATIO)]\n\n# -----------------------------------------------------------------------------\n# STRATIFIED SPLIT\n# -----------------------------------------------------------------------------\nby_level = defaultdict(list)\nfor iid in test_ids:\n    by_level[test_img_info[iid][\"level\"]].append(iid)\n\ntrain_multi, val_ids = [], []\nfor lvl in [\"easy\", \"medium\", \"hard\"]:\n    ids = by_level[lvl]\n    random.shuffle(ids)\n    split = int(len(ids) * (1 - C.VAL_RATIO))\n    train_multi.extend(ids[:split])\n    val_ids.extend(ids[split:])\n\n# -----------------------------------------------------------------------------\n# DATASET CREATION\n# -----------------------------------------------------------------------------\nYOLO_DIR = f\"{C.WORK}/yolo_data\"\nshutil.rmtree(YOLO_DIR, ignore_errors=True)\n\nfor sub in [\"images/train\", \"images/val\", \"labels/train\", \"labels/val\"]:\n    os.makedirs(f\"{YOLO_DIR}/{sub}\", exist_ok=True)\n\ndef write_yolo(ids, split, img_info, ann_info, img_dir):\n    written = skipped = 0\n    for iid in ids:\n        info = img_info[iid]\n        src = f\"{img_dir}/{info['file_name']}\"\n        if not os.path.exists(src):\n            skipped += 1\n            continue\n\n        stem = f\"{iid}_{Path(info['file_name']).stem}\"\n        img_dst = f\"{YOLO_DIR}/images/{split}/{stem}.jpg\"\n        lbl_dst = f\"{YOLO_DIR}/labels/{split}/{stem}.txt\"\n\n        try:\n            if split == \"train\":\n                os.symlink(src, img_dst)\n                if not os.path.exists(img_dst):\n                    raise OSError()\n            else:\n                shutil.copy2(src, img_dst)\n        except:\n            shutil.copy2(src, img_dst)\n\n        W, H = info[\"width\"], info[\"height\"]\n        lines = []\n        for ann in ann_info[iid]:\n            x, y, w, h = ann[\"bbox\"]\n            cls = cat_id_to_yolo[ann[\"category_id\"]]\n            cx = (x + w/2) / W\n            cy = (y + h/2) / H\n            nw = w / W\n            nh = h / H\n            lines.append(f\"{cls} {cx:.6f} {cy:.6f} {nw:.6f} {nh:.6f}\")\n\n        with open(lbl_dst, \"w\") as f:\n            f.write(\"\\n\".join(lines))\n\n        written += 1\n\n    print(f\"{split.upper()} → Written: {written:,} | Skipped: {skipped}\")\n\nwrite_yolo(train_ids, \"train\", train_img_info, train_ann_by_img, C.TRAIN_DIR)\nwrite_yolo(train_multi, \"train\", test_img_info, test_ann_by_img, C.TEST_DIR)\nwrite_yolo(val_ids, \"val\", test_img_info, test_ann_by_img, C.TEST_DIR)\n\n# -----------------------------------------------------------------------------\n# DATASET YAML\n# -----------------------------------------------------------------------------\nds_yaml = {\n    \"path\": C.WORK,\n    \"train\": \"yolo_data/images/train\",\n    \"val\": \"yolo_data/images/val\",\n    \"nc\": len(names_map),\n    \"names\": names_map\n}\n\nwith open(f\"{C.WORK}/dataset.yaml\", \"w\") as f:\n    yaml.dump(ds_yaml, f)\n\ncleanup()\n\n# -----------------------------------------------------------------------------\n# STAGE 1 — TRAIN\n# -----------------------------------------------------------------------------\nprint(f\"\\n[{elapsed()}] STAGE 1 — TRAINING\")\n\nmodel = YOLO(C.YOLO_MODEL)\nmodel.train(\n    data=f\"{C.WORK}/dataset.yaml\",\n    epochs=C.YOLO_EPOCHS,\n    imgsz=C.YOLO_IMGSZ,\n    batch=C.YOLO_BATCH,\n    device=DEVICE,\n    workers=C.YOLO_WORKERS,\n    project=C.WORK,\n    name=\"yolo_run\",\n    exist_ok=True,\n    patience=C.YOLO_PATIENCE,\n    amp=True,\n    cache=False,\n    mosaic=0.5,\n    mixup=0.0,\n)\n\nBEST_PT = f\"{C.WORK}/yolo_run/weights/best.pt\"\nif not os.path.exists(BEST_PT):\n    raise RuntimeError(\"❌ Best model not found\")\n\ncleanup()\n\n# -----------------------------------------------------------------------------\n# STAGE 2 — THRESHOLD SWEEP (CHUNKED)\n# -----------------------------------------------------------------------------\nprint(f\"\\n[{elapsed()}] STAGE 2 — THRESHOLD SWEEP\")\n\nmodel = YOLO(BEST_PT)\n\ndetections = {}\nBATCH = 32\n\nval_yolo_paths = [\n    f\"{YOLO_DIR}/images/val/{iid}_{Path(test_img_info[iid]['file_name']).stem}.jpg\"\n    for iid in val_ids\n]\n\nfor i in range(0, len(val_yolo_paths), BATCH):\n    chunk = val_yolo_paths[i:i+BATCH]\n    results = model.predict(chunk, conf=0.01, iou=C.IOU, max_det=C.MAX_DET, device=DEVICE, verbose=False)\n    for iid, res in zip(val_ids[i:i+BATCH], results):\n        if res.boxes is None:\n            detections[iid] = []\n        else:\n            detections[iid] = list(zip(\n                res.boxes.conf.cpu().tolist(),\n                res.boxes.cls.cpu().int().tolist()\n            ))\n\ngt_counts = {iid: len(test_ann_by_img[iid]) for iid in val_ids}\ngt_cats = {iid: [a[\"category_id\"] for a in test_ann_by_img[iid]] for iid in val_ids}\n\nbest_conf, best_score = 0.25, 0.0\nfor conf in np.arange(0.05, 0.65, 0.01):\n    score = 0.0\n    for iid in val_ids:\n        dets = [cls+1 for c, cls in detections[iid] if c >= conf]\n        if len(dets) == gt_counts[iid]:\n            inter = sum((Counter(dets) & Counter(gt_cats[iid])).values())\n            score += inter / max(1, len(gt_cats[iid]))\n    avg = score / len(val_ids)\n    if avg > best_score:\n        best_score = avg\n        best_conf = round(float(conf), 3)\n\nC.CONF = best_conf\nprint(f\"Optimal conf = {C.CONF} | Score = {best_score:.4f}\")\n\ncleanup()\n\n# -----------------------------------------------------------------------------\n# STAGE 3 — FINAL INFERENCE\n# -----------------------------------------------------------------------------\nprint(f\"\\n[{elapsed()}] STAGE 3 — FINAL INFERENCE\")\n\nwith open(C.VAL_JSON) as f:\n    val_meta = json.load(f)[\"images\"]\n\nval_targets = {int(i[\"id\"]): f\"{C.VAL_DIR}/{i['file_name']}\" for i in val_meta}\nval_ids_sorted = sorted(val_targets)\n\nresults_map = {}\nfor i in range(0, len(val_ids_sorted), BATCH):\n    ids_chunk = val_ids_sorted[i:i+BATCH]\n    paths = [val_targets[iid] for iid in ids_chunk]\n    preds = model.predict(paths, conf=C.CONF, iou=C.IOU, max_det=C.MAX_DET, device=DEVICE, verbose=False)\n\n    for iid, res in zip(ids_chunk, preds):\n        if res.boxes is None:\n            results_map[iid] = []\n        else:\n            results_map[iid] = sorted([int(cls)+1 for cls in res.boxes.cls.cpu().tolist()])\n\nfor iid in val_ids_sorted:\n    results_map.setdefault(iid, [])\n\n# -----------------------------------------------------------------------------\n# STAGE 4 — SUBMISSION + VALIDATION\n# -----------------------------------------------------------------------------\nprint(f\"\\n[{elapsed()}] STAGE 4 — SUBMISSION\")\n\nrows = []\nfor iid in val_ids_sorted:\n    cats = [c for c in results_map[iid] if c in VALID_IDS]\n    rows.append({\"image_id\": iid, \"categories\": json.dumps(sorted(cats))})\n\ndf = pd.DataFrame(rows).sort_values(\"image_id\").reset_index(drop=True)\n\nassert df[\"image_id\"].is_unique\nassert len(df) == len(val_ids_sorted)\nassert set(df[\"image_id\"]) == set(val_ids_sorted)\n\nfor r in df[\"categories\"]:\n    lst = json.loads(r)\n    assert lst == sorted(lst)\n    for c in lst:\n        assert c in VALID_IDS\n\nOUT = f\"{C.WORK}/submission.csv\"\ndf.to_csv(OUT, index=False)\nshutil.copy2(OUT, \"/kaggle/temp/submission_backup.csv\")\n\n# -----------------------------------------------------------------------------\n# STATS\n# -----------------------------------------------------------------------------\ntotal_objs = sum(len(json.loads(r)) for r in df[\"categories\"])\nempty = sum(1 for r in df[\"categories\"] if r == \"[]\")\n\nprint(f\"\"\"\n{'='*70}\n🏆 PLATINUM PIPELINE COMPLETE\n{'='*70}\nFile: {OUT}\nBackup: /kaggle/temp/submission_backup.csv\nImages: {len(df)}\nObjects: {total_objs}\nEmpty: {empty}\nConf: {C.CONF}\nVal Score: {best_score:.4f}\nGPU: {torch.cuda.get_device_name(0)}\nTime: {elapsed()}\n{'='*70}\n\n✅ Upload submission.csv to Kaggle\n\"\"\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T10:17:26.476421Z","iopub.execute_input":"2026-01-31T10:17:26.476977Z","iopub.status.idle":"2026-01-31T14:18:26.251997Z","shell.execute_reply.started":"2026-01-31T10:17:26.476929Z","shell.execute_reply":"2026-01-31T14:18:26.250409Z"}},"outputs":[],"execution_count":null}]}