{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":9988,"databundleVersionId":868324,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =========================\n# 1) IMPORTS & CONFIG\n# =========================\n\nimport os, sys, re, csv, json, time, math, random, shutil, glob\nfrom typing import Dict, List, Tuple, Optional\n\nimport numpy as np\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nimport torchvision\nimport torchvision.transforms.functional as TF\n\n# OpenCV (required for fast CC stats)\nimport cv2\n\n# sklearn (required; do NOT bypass)\nfrom sklearn.ensemble import HistGradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier\n\nimport matplotlib.pyplot as plt\n\n\n# -------------------------\n# KAGGLE PATHS (fixed)\n# -------------------------\nDATA_DIR = \"/kaggle/input/airbus-ship-detection\"\nTRAIN_IMG_DIR = os.path.join(DATA_DIR, \"train_v2\")\nTEST_IMG_DIR  = os.path.join(DATA_DIR, \"test_v2\")\nTRAIN_CSV     = os.path.join(DATA_DIR, \"train_ship_segmentations_v2.csv\")\nSAMPLE_SUB_CSV= os.path.join(DATA_DIR, \"sample_submission_v2.csv\")\n\nassert os.path.isdir(TRAIN_IMG_DIR), f\"Missing: {TRAIN_IMG_DIR}\"\nassert os.path.isdir(TEST_IMG_DIR),  f\"Missing: {TEST_IMG_DIR}\"\nassert os.path.isfile(TRAIN_CSV),    f\"Missing: {TRAIN_CSV}\"\nassert os.path.isfile(SAMPLE_SUB_CSV), f\"Missing: {SAMPLE_SUB_CSV}\"\n\nWORK_DIR = \"/kaggle/working\"\nos.makedirs(WORK_DIR, exist_ok=True)\n\n# -------------------------\n# FIXED REQUIREMENT\n# -------------------------\nTRAIN_N = 20000   # <-- 要件：train は 20000 枚（valとは別）\nEPOCHS_MAX = 20   # 以前の方針に合わせて上限20\nEARLY_PATIENCE = 5\n\n# -------------------------\n# Pipeline is fixed: SEG -> TREE\n# Only changeable parts: SEG_MODEL, TREE_MODEL\n# -------------------------\nSEG_MODEL  = \"deeplabv3_resnet50\"   # changeable: \"deeplabv3_resnet50\" | \"deeplabv3_resnet101\"\nTREE_MODEL = \"hgbdt\"               # changeable: \"hgbdt\" | \"rf\" | \"extratrees\"\n\n# -------------------------\n# Image sizes\n# -------------------------\nTRAIN_SIZE = 384    # training resize\nINFER_SIZE = 512    # inference resize (Kaggle runtime friendly)\n\nBATCH_SIZE = 8\nNUM_WORKERS = 2  # Kaggle: 2~4程度が安定しやすい（環境で調整）\nAMP = True\n\n# -------------------------\n# Loss (recall-biased)\n# -------------------------\nCE_WEIGHT_BG = 1.0\nCE_WEIGHT_SHIP = 50.0  # 強めに背景優位を補正（小物体・不均衡対策）\n\nTV_ALPHA = 0.4\nTV_BETA  = 0.6\nEPS = 1e-6\n\n# -------------------------\n# Threshold tuning (pixel)\n# -------------------------\nPIX_BINS = 512\nF_BETA = 1.225  # Kaggle/Recall寄りのバランス（あなたの既存設定に合わせる）\n\n# -------------------------\n# Component / tree settings\n# -------------------------\nMIN_COMP_AREA = 25\nPROP_MULT_INIT = 0.85\nCOMP_POS_OVERLAP = 0.30  # componentがGTとどれくらい重なるとpos扱いか（要調整）\n\n# Tree threshold tuning\nTREE_BETA = 1.225\n\n# probe safety (avoid all-empty submission)\nPROBE_N = 250\nPROBE_MIN_NONEMPTY_RATIO = 0.01\nPROBE_MAX_ATTEMPTS = 4\n\n# Random seed\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"DEVICE:\", DEVICE)\nprint(\"GPU:\", torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"CPU\")\n\n# Normalization (ImageNet)\nMEAN = [0.485, 0.456, 0.406]\nSTD  = [0.229, 0.224, 0.225]\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# 2) HELPERS: RLE/CSV/DATASET/UTILS\n# =========================\n\ndef rle_decode(rle: str, shape: Tuple[int, int] = (768, 768)) -> np.ndarray:\n    if rle is None:\n        return np.zeros(shape, dtype=np.uint8)\n    rle = str(rle).strip()\n    if len(rle) == 0:\n        return np.zeros(shape, dtype=np.uint8)\n\n    s = rle.split()\n    starts = np.asarray(s[0::2], dtype=np.int64) - 1\n    lengths = np.asarray(s[1::2], dtype=np.int64)\n    ends = starts + lengths\n\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n\n    img = img.reshape((shape[1], shape[0]), order=\"F\").T\n    return img\n\n\ndef rle_encode(mask: np.ndarray) -> str:\n    if mask is None:\n        return \"\"\n    m = mask.astype(np.uint8)\n    if m.max() == 0:\n        return \"\"\n    pixels = m.T.flatten(order=\"F\")\n    pixels = np.concatenate([[0], pixels, [0]])\n    changes = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs = changes[0::2]\n    ends = changes[1::2]\n    lengths = ends - runs\n    return \" \".join(str(x) for pair in zip(runs, lengths) for x in pair)\n\n\ndef read_train_csv_rles(train_csv: str) -> Dict[str, List[str]]:\n    \"\"\"\n    ImageId -> list of RLE strings (pos only; negatives may be missing)\n    \"\"\"\n    id2rles: Dict[str, List[str]] = {}\n    with open(train_csv, \"r\", encoding=\"utf-8\", newline=\"\") as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            iid = str(row[\"ImageId\"]).strip()\n            rle = row.get(\"EncodedPixels\", \"\")\n            rle = \"\" if rle is None else str(rle).strip()\n            if iid not in id2rles:\n                id2rles[iid] = []\n            if len(rle) > 0:\n                id2rles[iid].append(rle)\n    return id2rles\n\n\ndef list_image_ids_from_dir(img_dir: str) -> List[str]:\n    \"\"\"\n    IMPORTANT: build ALL_TRAIN_IDS from directory, not CSV.\n    \"\"\"\n    exts = (\"*.jpg\", \"*.jpeg\", \"*.png\")\n    files = []\n    for e in exts:\n        files += glob.glob(os.path.join(img_dir, e))\n    ids = [os.path.basename(p) for p in files]\n    ids.sort()\n    return ids\n\n\ndef union_mask(inst_masks: List[np.ndarray], size: int) -> np.ndarray:\n    if not inst_masks:\n        return np.zeros((size, size), dtype=np.uint8)\n    m = np.zeros((size, size), dtype=np.uint8)\n    for cm in inst_masks:\n        m = np.maximum(m, cm.astype(np.uint8))\n    return m\n\n\ndef tversky_loss_from_logits(logits: torch.Tensor, targets: torch.Tensor,\n                             alpha: float = 0.4, beta: float = 0.6, eps: float = 1e-6) -> torch.Tensor:\n    probs = torch.softmax(logits, dim=1)[:, 1]\n    tgt = targets.float()\n    tp = (probs * tgt).sum(dim=(1, 2))\n    fp = (probs * (1.0 - tgt)).sum(dim=(1, 2))\n    fn = ((1.0 - probs) * tgt).sum(dim=(1, 2))\n    tversky = (tp + eps) / (tp + alpha * fp + beta * fn + eps)\n    return 1.0 - tversky.mean()\n\n\n@torch.no_grad()\ndef tune_pixel_threshold_from_hist(hist_pos: torch.Tensor,\n                                   hist_neg: torch.Tensor,\n                                   beta: float = 1.225,\n                                   eps: float = 1e-6) -> Tuple[float, Dict[str, float]]:\n    pos_cum = torch.flip(torch.cumsum(torch.flip(hist_pos, dims=[0]), dim=0), dims=[0])\n    neg_cum = torch.flip(torch.cumsum(torch.flip(hist_neg, dims=[0]), dim=0), dims=[0])\n\n    total_pos = float(hist_pos.sum().item())\n    total_neg = float(hist_neg.sum().item())\n    b2 = beta * beta\n\n    best_i, best_f = 0, -1.0\n    best_p, best_r, best_iou = 0.0, 0.0, 0.0\n\n    for i in range(hist_pos.numel()):\n        tp = float(pos_cum[i].item())\n        fp = float(neg_cum[i].item())\n        fn = total_pos - tp\n\n        p = tp / (tp + fp + eps)\n        r = tp / (tp + fn + eps)\n        f = (1.0 + b2) * p * r / (b2 * p + r + eps)\n        iou = tp / (tp + fp + fn + eps)\n\n        if f > best_f:\n            best_f = f\n            best_i = i\n            best_p = p\n            best_r = r\n            best_iou = iou\n\n    thr = best_i / float(hist_pos.numel())\n    return thr, {\"precision\": best_p, \"recall\": best_r, \"fbeta\": best_f, \"iou\": best_iou, \"beta\": beta}\n\n\nclass ShipDataset(Dataset):\n    \"\"\"\n    Kaggle mode: on-the-fly RLE->mask decode (no caching).\n    Returns:\n      x: float tensor [3,H,W]\n      y: uint8 tensor [H,W] in {0,1}\n    \"\"\"\n    def __init__(self, img_dir: str, ids: List[str], id2rles: Dict[str, List[str]],\n                 size: int, is_train: bool):\n        self.img_dir = img_dir\n        self.ids = ids\n        self.id2rles = id2rles\n        self.size = int(size)\n        self.is_train = bool(is_train)\n\n    def __len__(self):\n        return len(self.ids)\n\n    def _load_mask(self, iid: str, orig_hw=(768,768)) -> np.ndarray:\n        rles = self.id2rles.get(iid, [])\n        if not rles:\n            return np.zeros(orig_hw, dtype=np.uint8)\n        m = np.zeros(orig_hw, dtype=np.uint8)\n        for r in rles:\n            m = np.maximum(m, rle_decode(r, shape=orig_hw))\n        return m\n\n    def __getitem__(self, idx: int):\n        iid = self.ids[idx]\n        path = os.path.join(self.img_dir, iid)\n        img = Image.open(path).convert(\"RGB\")\n        img_r = img.resize((self.size, self.size), resample=Image.BILINEAR)\n        x = TF.to_tensor(img_r)\n        x = TF.normalize(x, MEAN, STD)\n\n        # mask\n        m = self._load_mask(iid, orig_hw=(img.height, img.width))\n        m_img = Image.fromarray(m * 255).resize((self.size, self.size), resample=Image.NEAREST)\n        y = (np.array(m_img) > 127).astype(np.uint8)\n\n        y_t = torch.from_numpy(y).long()\n        return x, y_t, iid\n\n\ndef build_seg_model(name: str, num_classes: int = 2) -> nn.Module:\n    name = name.lower().strip()\n    if name == \"deeplabv3_resnet50\":\n        return torchvision.models.segmentation.deeplabv3_resnet50(weights=None, weights_backbone=None, num_classes=num_classes)\n    if name == \"deeplabv3_resnet101\":\n        return torchvision.models.segmentation.deeplabv3_resnet101(weights=None, weights_backbone=None, num_classes=num_classes)\n    raise ValueError(f\"Unknown SEG_MODEL={name}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# 3) SPLITS & DATALOADERS (train=20000 fixed, val separate)\n# =========================\n\nid2rles = read_train_csv_rles(TRAIN_CSV)\n\nall_train_ids = list_image_ids_from_dir(TRAIN_IMG_DIR)  # directory-based (IMPORTANT)\nprint(\"ALL_TRAIN_IDS:\", len(all_train_ids))\n\npos_ids = [iid for iid in all_train_ids if (iid in id2rles and len(id2rles[iid]) > 0)]\nneg_ids = [iid for iid in all_train_ids if iid not in set(pos_ids)]\nprint(\"pos_ids:\", len(pos_ids), \"neg_ids:\", len(neg_ids), \"pos_ratio:\", len(pos_ids)/max(1,len(all_train_ids)))\n\n# ---- helper: stratified sampling for stable training ----\ndef sample_ids_stratified(pos: List[str], neg: List[str], n: int, pos_frac: float, seed: int) -> List[str]:\n    rng = random.Random(seed)\n    npos = min(len(pos), int(round(n * pos_frac)))\n    nneg = n - npos\n    pos_s = rng.sample(pos, npos) if npos <= len(pos) else list(pos)\n    neg_s = rng.sample(neg, nneg) if nneg <= len(neg) else list(neg)\n    ids = pos_s + neg_s\n    rng.shuffle(ids)\n    return ids\n\n# ---- splits (keep disjoint) ----\n# You can adjust pos_frac for training; small ship task often benefits from a bit higher than natural.\nTRAIN_POS_FRAC = 0.35\n\ntrain_ids = sample_ids_stratified(pos_ids, neg_ids, TRAIN_N, TRAIN_POS_FRAC, SEED)\n\nremaining = [iid for iid in all_train_ids if iid not in set(train_ids)]\nrem_pos = [iid for iid in remaining if iid in id2rles and len(id2rles[iid]) > 0]\nrem_neg = [iid for iid in remaining if iid not in set(rem_pos)]\n\n# validation pools (distinct from train)\nVAL_SEG_N   = 4000\nVAL_TREE_TR = 4000\nVAL_TREE_CA = 4000\n\nval_seg_ids   = sample_ids_stratified(rem_pos, rem_neg, VAL_SEG_N, 0.30, SEED + 1)\nremaining2 = [iid for iid in remaining if iid not in set(val_seg_ids)]\nrem2_pos = [iid for iid in remaining2 if iid in id2rles and len(id2rles[iid]) > 0]\nrem2_neg = [iid for iid in remaining2 if iid not in set(rem2_pos)]\n\nval_tree_train_ids = sample_ids_stratified(rem2_pos, rem2_neg, VAL_TREE_TR, 0.30, SEED + 2)\nremaining3 = [iid for iid in remaining2 if iid not in set(val_tree_train_ids)]\nrem3_pos = [iid for iid in remaining3 if iid in id2rles and len(id2rles[iid]) > 0]\nrem3_neg = [iid for iid in remaining3 if iid not in set(rem3_pos)]\n\nval_tree_calib_ids = sample_ids_stratified(rem3_pos, rem3_neg, VAL_TREE_CA, 0.30, SEED + 3)\n\nprint(\"train_ids:\", len(train_ids), \"pos:\", sum([1 for x in train_ids if x in id2rles and len(id2rles[x])>0]))\nprint(\"val_seg_ids:\", len(val_seg_ids), \"pos:\", sum([1 for x in val_seg_ids if x in id2rles and len(id2rles[x])>0]))\nprint(\"val_tree_train_ids:\", len(val_tree_train_ids), \"pos:\", sum([1 for x in val_tree_train_ids if x in id2rles and len(id2rles[x])>0]))\nprint(\"val_tree_calib_ids:\", len(val_tree_calib_ids), \"pos:\", sum([1 for x in val_tree_calib_ids if x in id2rles and len(id2rles[x])>0]))\n\n# Datasets\ntrain_ds = ShipDataset(TRAIN_IMG_DIR, train_ids, id2rles, size=TRAIN_SIZE, is_train=True)\nval_seg_ds = ShipDataset(TRAIN_IMG_DIR, val_seg_ids, id2rles, size=TRAIN_SIZE, is_train=False)\nval_tree_train_ds = ShipDataset(TRAIN_IMG_DIR, val_tree_train_ids, id2rles, size=INFER_SIZE, is_train=False)  # tree uses INFER_SIZE prob\nval_tree_calib_ds = ShipDataset(TRAIN_IMG_DIR, val_tree_calib_ids, id2rles, size=INFER_SIZE, is_train=False)\n\n# DataLoaders (big flow 유지)\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True,\n                          num_workers=NUM_WORKERS, pin_memory=True, drop_last=True)\n\nval_seg_loader = DataLoader(val_seg_ds, batch_size=BATCH_SIZE, shuffle=False,\n                            num_workers=NUM_WORKERS, pin_memory=True)\n\nval_tree_train_loader = DataLoader(val_tree_train_ds, batch_size=1, shuffle=False,\n                                   num_workers=NUM_WORKERS, pin_memory=True)\n\nval_tree_calib_loader = DataLoader(val_tree_calib_ds, batch_size=1, shuffle=False,\n                                   num_workers=NUM_WORKERS, pin_memory=True)\n\nprint(\"steps/epoch train:\", len(train_loader), \"val_seg:\", len(val_seg_loader),\n      \"tree_train:\", len(val_tree_train_loader), \"tree_calib:\", len(val_tree_calib_loader))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# 4) TRAIN SEG + EVAL (best_thr via hist)\n# =========================\n\nmodel = build_seg_model(SEG_MODEL, num_classes=2).to(DEVICE)\nmodel.train()\n\nopt = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=1e-4)\n\n# class-weighted CE\nce_loss = nn.CrossEntropyLoss(weight=torch.tensor([CE_WEIGHT_BG, CE_WEIGHT_SHIP], device=DEVICE, dtype=torch.float32))\n\n# AMP\nuse_amp = (AMP and DEVICE.type == \"cuda\")\nscaler = torch.cuda.amp.GradScaler(enabled=use_amp)\n\ndef forward_logits(x: torch.Tensor) -> torch.Tensor:\n    out = model(x)\n    # torchvision segmentation returns dict: {\"out\": logits}\n    if isinstance(out, dict) and \"out\" in out:\n        return out[\"out\"]\n    return out\n\n@torch.no_grad()\ndef eval_on_val_seg_and_tune_thr() -> Tuple[float, Dict[str, float], float]:\n    model.eval()\n    bins = PIX_BINS\n    hist_pos = torch.zeros(bins, device=\"cpu\", dtype=torch.int64)\n    hist_neg = torch.zeros(bins, device=\"cpu\", dtype=torch.int64)\n\n    total_loss = 0.0\n    n_batches = 0\n\n    for x, y, _ in val_seg_loader:\n        x = x.to(DEVICE, non_blocking=True)\n        y = y.to(DEVICE, non_blocking=True)\n\n        with torch.cuda.amp.autocast(enabled=use_amp):\n            logits = forward_logits(x)\n            logits = F.interpolate(logits, size=(y.shape[-2], y.shape[-1]), mode=\"bilinear\", align_corners=False)\n            loss = ce_loss(logits, y) + tversky_loss_from_logits(logits, y, alpha=TV_ALPHA, beta=TV_BETA, eps=EPS)\n\n        total_loss += float(loss.item())\n        n_batches += 1\n\n        prob = torch.softmax(logits, dim=1)[:, 1]  # [B,H,W]\n        # hist bins\n        idx = torch.clamp((prob * bins).long(), 0, bins - 1).view(-1)\n        yy = y.view(-1)\n\n        pos_mask = (yy == 1)\n        neg_mask = ~pos_mask\n\n        if pos_mask.any():\n            hist_pos += torch.bincount(idx[pos_mask].detach().cpu(), minlength=bins)\n        if neg_mask.any():\n            hist_neg += torch.bincount(idx[neg_mask].detach().cpu(), minlength=bins)\n\n    thr, metrics = tune_pixel_threshold_from_hist(hist_pos, hist_neg, beta=F_BETA, eps=EPS)\n    val_loss = total_loss / max(1, n_batches)\n    model.train()\n    return thr, metrics, val_loss\n\n\n# Training loop with early stopping on val_loss (and keep best by val_fbeta too)\nbest_epoch = -1\nbest_thr = 0.5\nbest_val_f = -1.0\nbest_val_loss = 1e18\npat = 0\n\nCKPT_PATH = os.path.join(WORK_DIR, f\"ckpt_{SEG_MODEL}_train{TRAIN_N}.pt\")\nMETA_PATH = os.path.join(WORK_DIR, f\"meta_{SEG_MODEL}_train{TRAIN_N}.json\")\n\nfor epoch in range(1, EPOCHS_MAX + 1):\n    t0 = time.time()\n    model.train()\n    running = 0.0\n    nb = 0\n\n    for x, y, _ in train_loader:\n        x = x.to(DEVICE, non_blocking=True)\n        y = y.to(DEVICE, non_blocking=True)\n\n        opt.zero_grad(set_to_none=True)\n        with torch.cuda.amp.autocast(enabled=use_amp):\n            logits = forward_logits(x)\n            logits = F.interpolate(logits, size=(y.shape[-2], y.shape[-1]), mode=\"bilinear\", align_corners=False)\n            loss = ce_loss(logits, y) + tversky_loss_from_logits(logits, y, alpha=TV_ALPHA, beta=TV_BETA, eps=EPS)\n\n        scaler.scale(loss).backward()\n        scaler.step(opt)\n        scaler.update()\n\n        running += float(loss.item())\n        nb += 1\n\n    train_loss = running / max(1, nb)\n    thr, m, val_loss = eval_on_val_seg_and_tune_thr()\n\n    dt = time.time() - t0\n    print(f\"[Epoch {epoch:03d}/{EPOCHS_MAX}] time={dt:.1f}s train_loss={train_loss:.4f} val_loss={val_loss:.4f} \"\n          f\"best_thr={thr:.3f} | pixel(Fβ={m['fbeta']:.4f} P={m['precision']:.4f} R={m['recall']:.4f} IoU={m['iou']:.4f})\")\n\n    # choose best by val_fbeta primarily; tie-break by loss\n    improved = (m[\"fbeta\"] > best_val_f + 1e-6) or (abs(m[\"fbeta\"] - best_val_f) <= 1e-6 and val_loss < best_val_loss - 1e-6)\n    if improved:\n        best_val_f = m[\"fbeta\"]\n        best_val_loss = val_loss\n        best_thr = thr\n        best_epoch = epoch\n        pat = 0\n\n        torch.save(model.state_dict(), CKPT_PATH)\n        with open(META_PATH, \"w\", encoding=\"utf-8\") as f:\n            json.dump({\n                \"seg_model\": SEG_MODEL,\n                \"train_n\": TRAIN_N,\n                \"train_size\": TRAIN_SIZE,\n                \"infer_size\": INFER_SIZE,\n                \"best_epoch\": best_epoch,\n                \"best_thr\": best_thr,\n                \"best_val_fbeta\": best_val_f,\n                \"best_val_loss\": best_val_loss,\n                \"pixel_metrics\": m,\n                \"seed\": SEED\n            }, f, ensure_ascii=False, indent=2)\n        print(f\"  [SAVE] ckpt -> {CKPT_PATH}\")\n    else:\n        pat += 1\n        if pat >= EARLY_PATIENCE:\n            print(f\"[EARLY STOP] patience={EARLY_PATIENCE} reached.\")\n            break\n\nprint(\"BEST:\", {\"epoch\": best_epoch, \"thr\": best_thr, \"val_fbeta\": best_val_f, \"val_loss\": best_val_loss})\nassert os.path.exists(CKPT_PATH), \"Best checkpoint not saved?\"\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# 5) TREE: build dataset -> train -> calibrate threshold\n# =========================\n\n# load best weights for consistent prob generation\nmodel.load_state_dict(torch.load(CKPT_PATH, map_location=DEVICE))\nmodel.eval()\n\n@torch.no_grad()\ndef forward_prob_tensor(x_tensor: torch.Tensor) -> np.ndarray:\n    with torch.cuda.amp.autocast(enabled=use_amp):\n        logits = forward_logits(x_tensor)\n    logits = F.interpolate(logits, size=(x_tensor.shape[-2], x_tensor.shape[-1]),\n                           mode=\"bilinear\", align_corners=False)\n    prob = torch.softmax(logits, dim=1)[:, 1]\n    return prob.detach().cpu().numpy()\n\ndef component_features_from_bbox(prob: np.ndarray, labels: np.ndarray, cid: int, x: int, y: int, w: int, h: int) -> Optional[np.ndarray]:\n    labels_roi = labels[y:y+h, x:x+w]\n    cm_roi = (labels_roi == cid).astype(np.uint8)\n    area = int(cm_roi.sum())\n    if area < MIN_COMP_AREA:\n        return None\n\n    # create full mask only for kept component (keeps semantics; avoids Θ(ncomp*H*W) explosions)\n    H, W = prob.shape\n    cm = np.zeros((H, W), dtype=np.uint8)\n    cm[y:y+h, x:x+w] = cm_roi\n    return component_features(prob, cm)\n\ndef build_tree_dataset_from_loader(loader: DataLoader, seg_thr: float, prop_mult: float, overlap_thr: float) -> Tuple[List[np.ndarray], np.ndarray]:\n    \"\"\"\n    Returns:\n      X: list of feature vectors\n      Y: numpy array 0/1 for component-level label based on overlap with GT\n    \"\"\"\n    X: List[np.ndarray] = []\n    Y: List[int] = []\n\n    for x, y, _ in loader:\n        x = x.to(DEVICE, non_blocking=True)\n        y_np = y.squeeze(0).numpy().astype(np.uint8)  # [H,W] in {0,1}\n\n        prob = forward_prob_tensor(x)[0].astype(np.float32)  # [H,W]\n\n        proposal_thr = max(0.01, min(0.99, float(seg_thr) * float(prop_mult)))\n        mask = (prob >= proposal_thr).astype(np.uint8)\n\n        # CC with stats\n        num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(mask, connectivity=8)\n        if num_labels <= 1:\n            continue\n\n        # for each component\n        for cid in range(1, num_labels):\n            x0, y0, ww, hh, area = stats[cid]\n            if int(area) < MIN_COMP_AREA:\n                continue\n\n            feat = component_features_from_bbox(prob, labels, cid, x0, y0, ww, hh)\n            if feat is None:\n                continue\n\n            # component mask in bbox for overlap (efficient)\n            labels_roi = labels[y0:y0+hh, x0:x0+ww]\n            cm_roi = (labels_roi == cid).astype(np.uint8)\n            gt_roi = y_np[y0:y0+hh, x0:x0+ww]\n            inter = int((cm_roi * gt_roi).sum())\n            comp_area = int(cm_roi.sum())\n            # overlap ratio (component coverage by GT)\n            ov = inter / max(1, comp_area)\n\n            lab = 1 if ov >= float(overlap_thr) else 0\n\n            X.append(feat)\n            Y.append(lab)\n\n    if len(Y) == 0:\n        return [], np.zeros((0,), dtype=np.int64)\n    return X, np.asarray(Y, dtype=np.int64)\n\ndef train_tree(X: List[np.ndarray], Y: np.ndarray):\n    if len(X) == 0:\n        raise RuntimeError(\"Tree training data is empty.\")\n    if Y.sum() < 10:\n        raise RuntimeError(f\"Too few positive components for tree: pos={int(Y.sum())}\")\n\n    Xn = np.stack(X, axis=0)\n    model_name = TREE_MODEL.lower().strip()\n\n    if model_name == \"hgbdt\":\n        clf = HistGradientBoostingClassifier(\n            max_depth=6,\n            learning_rate=0.06,\n            max_iter=200,\n            random_state=SEED\n        )\n    elif model_name == \"rf\":\n        clf = RandomForestClassifier(\n            n_estimators=400,\n            max_depth=None,\n            min_samples_split=2,\n            min_samples_leaf=1,\n            n_jobs=-1,\n            random_state=SEED\n        )\n    elif model_name == \"extratrees\":\n        clf = ExtraTreesClassifier(\n            n_estimators=600,\n            max_depth=None,\n            min_samples_split=2,\n            min_samples_leaf=1,\n            n_jobs=-1,\n            random_state=SEED\n        )\n    else:\n        raise ValueError(f\"Unknown TREE_MODEL={TREE_MODEL}\")\n\n    clf.fit(Xn, Y)\n    return clf, {\"model_name\": model_name, \"n\": int(len(Y)), \"pos\": int(Y.sum())}\n\ndef tree_predict_proba(clf, X: List[np.ndarray]) -> np.ndarray:\n    Xn = np.stack(X, axis=0)\n    if hasattr(clf, \"predict_proba\"):\n        p = clf.predict_proba(Xn)[:, 1]\n        return p.astype(np.float32)\n    # fallback for estimators without predict_proba (shouldn't happen here)\n    s = clf.decision_function(Xn)\n    s = (s - s.min()) / (s.max() - s.min() + 1e-6)\n    return s.astype(np.float32)\n\ndef tune_tree_thr(probs: np.ndarray, Y: np.ndarray, beta: float = 1.225) -> Tuple[float, Dict[str, float]]:\n    b2 = beta * beta\n    best_thr, best_f = 0.5, -1.0\n    best_p, best_r = 0.0, 0.0\n\n    # candidate thresholds: quantiles + fixed grid\n    qs = np.unique(np.quantile(probs, np.linspace(0.02, 0.98, 25))).tolist()\n    grid = np.linspace(0.05, 0.95, 37).tolist()\n    cand = sorted(set([float(x) for x in qs + grid]))\n\n    for t in cand:\n        pred = (probs >= t).astype(np.uint8)\n        tp = int(((pred == 1) & (Y == 1)).sum())\n        fp = int(((pred == 1) & (Y == 0)).sum())\n        fn = int(((pred == 0) & (Y == 1)).sum())\n\n        p = tp / (tp + fp + 1e-6)\n        r = tp / (tp + fn + 1e-6)\n        f = (1.0 + b2) * p * r / (b2 * p + r + 1e-6)\n\n        if f > best_f:\n            best_f = f\n            best_thr = t\n            best_p = p\n            best_r = r\n\n    return float(best_thr), {\"precision\": float(best_p), \"recall\": float(best_r), \"fbeta\": float(best_f), \"beta\": float(beta)}\n\n# ---- build and train ----\nprop_mult = PROP_MULT_INIT\noverlap_thr = COMP_POS_OVERLAP\n\nprint(\"[TREE] building train set...\")\nXtr, Ytr = build_tree_dataset_from_loader(val_tree_train_loader, best_thr, prop_mult=prop_mult, overlap_thr=overlap_thr)\nprint(f\"[TREE TRAIN DATA] X={len(Xtr)} pos={int(Ytr.sum())} overlap_thr={overlap_thr} prop_mult={prop_mult}\")\n\ntree_clf, tree_info = train_tree(Xtr, Ytr)\nprint(\"[TREE] trained:\", tree_info)\n\n# ---- calib ----\nprint(\"[TREE] building calib set...\")\nXc, Yc = build_tree_dataset_from_loader(val_tree_calib_loader, best_thr, prop_mult=prop_mult, overlap_thr=overlap_thr)\nprint(f\"[TREE CALIB DATA] X={len(Xc)} pos={int(Yc.sum())}\")\n\nif len(Xc) < 200 or int(Yc.sum()) < 30:\n    print(\"[TREE] calib too small -> use train for threshold\")\n    probs_tr = tree_predict_proba(tree_clf, Xtr)\n    tree_thr, tm = tune_tree_thr(probs_tr, Ytr, beta=TREE_BETA)\n    print(f\"[TREE] thr={tree_thr:.3f} (train-thr) P={tm['precision']:.4f} R={tm['recall']:.4f} Fβ={tm['fbeta']:.4f}\")\nelse:\n    probs_c = tree_predict_proba(tree_clf, Xc)\n    tree_thr, tm = tune_tree_thr(probs_c, Yc, beta=TREE_BETA)\n    print(f\"[TREE] thr={tree_thr:.3f} P={tm['precision']:.4f} R={tm['recall']:.4f} Fβ={tm['fbeta']:.4f}\")\n\nTREE_JSON = os.path.join(WORK_DIR, f\"tree_{TREE_MODEL}_train{TRAIN_N}.json\")\nwith open(TREE_JSON, \"w\", encoding=\"utf-8\") as f:\n    json.dump({\n        \"tree_model\": tree_info[\"model_name\"],\n        \"tree_thr\": tree_thr,\n        \"overlap_thr\": overlap_thr,\n        \"prop_mult\": prop_mult,\n        \"tree_calib_metrics\": tm,\n        \"best_thr\": best_thr,\n        \"best_epoch\": best_epoch,\n        \"seg_model\": SEG_MODEL,\n        \"train_n\": TRAIN_N,\n        \"infer_size\": INFER_SIZE\n    }, f, ensure_ascii=False, indent=2)\nprint(\"[TREE] saved:\", TREE_JSON)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# [NEXT CELL] TEST INFERENCE -> TEMPLATE-AWARE FILL -> submission.csv\n# (Kaggle mode: no pre-cache; on-the-fly)\n# ============================================================\n\nimport os, json, time\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\n\nimport torch\nimport torch.nn.functional as F\nimport torchvision.transforms.functional as TF\n\n# ---- REQUIRED VARS CHECK (fail-fast) ----\n_required = [\n    \"model\", \"best_thr\", \"tree_clf\", \"tree_thr\", \"prop_mult\",\n    \"WORK_DIR\", \"INFER_SIZE\", \"TRAIN_N\", \"SEG_MODEL\", \"TREE_MODEL\"\n]\n_missing = [k for k in _required if k not in globals()]\nif _missing:\n    raise RuntimeError(f\"Missing required variables from previous cells: {_missing}\")\n\n# resolve DATA_DIR / TEST_IMG_DIR / SAMPLE_SUB_CSV if not provided explicitly\nif \"DATA_DIR\" not in globals():\n    # Kaggle default\n    DATA_DIR = \"/kaggle/input/airbus-ship-detection\"\nif \"TEST_IMG_DIR\" not in globals():\n    TEST_IMG_DIR = os.path.join(DATA_DIR, \"test_v2\")\nif \"SAMPLE_SUB_CSV\" not in globals():\n    SAMPLE_SUB_CSV = os.path.join(DATA_DIR, \"sample_submission_v2.csv\")\n\nassert os.path.isdir(TEST_IMG_DIR), f\"TEST_IMG_DIR not found: {TEST_IMG_DIR}\"\nassert os.path.isfile(SAMPLE_SUB_CSV), f\"SAMPLE_SUB_CSV not found: {SAMPLE_SUB_CSV}\"\n\n# ---- device ----\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = model.to(DEVICE)\nmodel.eval()\n\n# ---- autocast (new/old API safe) ----\ntry:\n    from torch.amp import autocast\n    _AMP_NEW = True\nexcept Exception:\n    from torch.cuda.amp import autocast\n    _AMP_NEW = False\n\n# ---- Normalization (ImageNet; keep consistent with training) ----\nMEAN = [0.485, 0.456, 0.406]\nSTD  = [0.229, 0.224, 0.225]\n\n# ---- controls ----\nUSE_TTA = False   # Kaggle: keep cheap unless you explicitly want to enable\nMIN_COMP_AREA = 20  # tune; too small -> component explosion, too big -> miss tiny ships\nPROP_MULT_INIT = float(prop_mult)  # from training cell\nCOMP_CONNECTIVITY = 8\n\n# ---- RLE encode (Airbus) ----\ndef rle_encode(mask: np.ndarray) -> str:\n    if mask is None:\n        return \"\"\n    m = mask.astype(np.uint8)\n    if m.max() == 0:\n        return \"\"\n    # flatten in Fortran order (column-major) with transpose trick\n    pixels = m.T.flatten(order=\"F\")\n    pixels = np.concatenate([[0], pixels, [0]])\n    changes = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs = changes[0::2]\n    ends = changes[1::2]\n    lengths = ends - runs\n    return \" \".join(str(x) for pair in zip(runs, lengths) for x in pair)\n\n# ---- tree proba helper (generic sklearn) ----\ndef tree_predict_proba(clf, X: np.ndarray) -> np.ndarray:\n    X = np.asarray(X, dtype=np.float32)\n    if hasattr(clf, \"predict_proba\"):\n        p = clf.predict_proba(X)\n        if p.ndim == 2 and p.shape[1] >= 2:\n            return p[:, 1]\n        return p.reshape(-1)\n    if hasattr(clf, \"decision_function\"):\n        z = clf.decision_function(X)\n        z = np.asarray(z).reshape(-1)\n        return 1.0 / (1.0 + np.exp(-z))\n    # last resort\n    y = clf.predict(X)\n    return np.asarray(y, dtype=np.float32).reshape(-1)\n\n# ---- connected components with stats (fast, avoids Θ(ncomp·H·W) explosion) ----\ntry:\n    import cv2\nexcept Exception as e:\n    raise RuntimeError(\"cv2 (opencv-python) is required in this Kaggle pipeline for connectedComponentsWithStats.\") from e\n\ndef connected_components_stats(mask_u8: np.ndarray, connectivity: int = 8):\n    # returns labels, num_components, stats\n    num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(mask_u8, connectivity=connectivity)\n    return labels, (num_labels - 1), stats  # stats: [num_labels,5] => x,y,w,h,area\n\n# ---- IMPORTANT: feature function must match training ----\n# We assume component_features(prob, cm) is defined in earlier cells (used by build_tree_dataset_from_loader).\nif \"component_features\" not in globals():\n    raise RuntimeError(\"component_features(prob, cm) is not defined. It must match the training-time features.\")\n\n# ---- forward prob helpers ----\n@torch.no_grad()\ndef forward_prob_tensor(x_tensor: torch.Tensor) -> np.ndarray:\n    with autocast(device_type=\"cuda\", enabled=(DEVICE.type == \"cuda\")):\n        out = model(x_tensor)\n        logits = out[\"out\"] if isinstance(out, dict) and \"out\" in out else out\n    # ensure output matches input spatial size\n    logits = F.interpolate(\n        logits, size=(x_tensor.shape[-2], x_tensor.shape[-1]),\n        mode=\"bilinear\", align_corners=False\n    )\n    prob = torch.softmax(logits, dim=1)[:, 1]  # ship prob\n    return prob.detach().cpu().numpy()\n\n@torch.no_grad()\ndef predict_prob_resized(img: Image.Image, size: int, use_tta: bool) -> np.ndarray:\n    img = img.convert(\"RGB\")\n    img_r = img.resize((size, size), resample=Image.BILINEAR)\n    x = TF.normalize(TF.to_tensor(img_r), MEAN, STD).unsqueeze(0).to(DEVICE)\n\n    if not use_tta:\n        return forward_prob_tensor(x)[0]\n\n    # light TTA: orig + hflip (cheap)\n    x2 = torch.flip(x, dims=[3])  # horizontal flip\n    p1 = forward_prob_tensor(x)[0]\n    p2 = forward_prob_tensor(x2)[0]\n    p2 = np.flip(p2, axis=1)  # unflip back (width axis)\n    return 0.5 * (p1 + p2)\n\ndef union_mask(instances: List[np.ndarray], size: int) -> np.ndarray:\n    if not instances:\n        return np.zeros((size, size), dtype=np.uint8)\n    m = np.zeros((size, size), dtype=np.uint8)\n    for ins in instances:\n        if ins is None:\n            continue\n        m |= (ins.astype(np.uint8) > 0)\n    return m.astype(np.uint8)\n\ndef instances_from_prob_fast(prob: np.ndarray,\n                             seg_thr: float,\n                             prop_mult: float,\n                             tree_clf,\n                             tree_thr: float) -> List[np.ndarray]:\n    \"\"\"\n    prob: [S,S] float in [0,1]\n    1) proposal mask by proposal_thr = seg_thr * prop_mult\n    2) connected components with stats (bbox/area)\n    3) feature per component (must match training)\n    4) tree gate by tree_thr\n    returns: list of component masks (uint8 {0,1}) in resized space\n    \"\"\"\n    S = prob.shape[0]\n    proposal_thr = float(np.clip(seg_thr * prop_mult, 0.01, 0.99))\n    mask = (prob >= proposal_thr).astype(np.uint8)\n\n    labels, ncomp, stats = connected_components_stats(mask, connectivity=COMP_CONNECTIVITY)\n    if ncomp <= 0:\n        return []\n\n    feats = []\n    comps = []\n\n    # iterate labels 1..ncomp (0 is background)\n    # stats rows: [x,y,w,h,area]\n    for cid in range(1, ncomp + 1):\n        x, y, w, h, area = stats[cid]\n        if int(area) < int(MIN_COMP_AREA):\n            continue\n\n        # ROI mask: only bbox area\n        roi = labels[y:y+h, x:x+w]\n        cm_roi = (roi == cid).astype(np.uint8)\n        if int(cm_roi.sum()) < int(MIN_COMP_AREA):\n            continue\n\n        # full-size component mask (kept masks only; avoids lab==cid full scan per component)\n        cm = np.zeros((S, S), dtype=np.uint8)\n        cm[y:y+h, x:x+w] = cm_roi\n\n        feat = component_features(prob, cm)\n        if feat is None:\n            continue\n        feats.append(feat)\n        comps.append(cm)\n\n    if not comps:\n        return []\n\n    # tree gate (must exist in this pipeline; no fallback-to-seg-only)\n    X = np.asarray(feats, dtype=np.float32)\n    p = tree_predict_proba(tree_clf, X)\n    keep = (p >= float(tree_thr))\n    kept = [cm for cm, k in zip(comps, keep) if bool(k)]\n    return kept\n\n# ---- template load ----\ntemplate = pd.read_csv(SAMPLE_SUB_CSV)\ntemplate_ids = template[\"ImageId\"].astype(str).tolist()\n\nid2rows: Dict[str, List[int]] = {}\nfor i, iid in enumerate(template_ids):\n    id2rows.setdefault(iid, []).append(i)\n\nunique_ids = list(id2rows.keys())\nprint(f\"[TEMPLATE] rows={len(template)} unique ImageId={len(unique_ids)}\")\nprint(f\"[INFER] start: seg_thr(best)={best_thr:.3f} prop_mult={PROP_MULT_INIT:.3f} tree_thr={tree_thr:.3f} infer_size={INFER_SIZE} TTA={USE_TTA}\")\n\n# ---- adaptive safety (avoid all-empty submission) ----\n# NOTE: prop_mult direction is IMPORTANT:\n# proposal_thr = seg_thr * prop_mult\n# To make proposals \"more permissive\" -> lower proposal_thr -> decrease seg_thr and/or decrease prop_mult.\ndef relax_thresholds(seg_thr0, tree_thr0, prop_mult0, attempt: int):\n    seg_thr2   = max(0.05, float(seg_thr0) - 0.04 * attempt)\n    tree_thr2  = max(0.0,  float(tree_thr0) - 0.10 * attempt)\n    prop_mult2 = max(0.50, float(prop_mult0) - 0.05 * attempt)  # permissive -> decrease\n    return seg_thr2, tree_thr2, prop_mult2\n\nout_encoded = [\"\"] * len(template)\n\n# ---- probe ----\nprobe_n = min(250, len(unique_ids))\nseg_thr_run  = float(best_thr)\ntree_thr_run = float(tree_thr)\nprop_mult_run = float(PROP_MULT_INIT)\n\nprobe_attempts = 0\nwhile probe_attempts <= 4:\n    probe_nonempty = 0\n    t0 = time.time()\n    for img_id in unique_ids[:probe_n]:\n        img_path = os.path.join(TEST_IMG_DIR, img_id)\n        img = Image.open(img_path).convert(\"RGB\")\n        prob = predict_prob_resized(img, INFER_SIZE, USE_TTA)\n        inst = instances_from_prob_fast(prob, seg_thr_run, prop_mult_run, tree_clf, tree_thr_run)\n        # template may have multiple rows; if only 1 row -> union else any instance\n        if len(id2rows[img_id]) == 1:\n            m = union_mask(inst, INFER_SIZE)\n            probe_nonempty += 1 if m.max() > 0 else 0\n        else:\n            probe_nonempty += 1 if len(inst) > 0 else 0\n\n    ratio = probe_nonempty / max(1, probe_n)\n    dt = time.time() - t0\n    print(f\"[PROBE] attempt={probe_attempts} seg_thr={seg_thr_run:.3f} tree_thr={tree_thr_run:.3f} prop_mult={prop_mult_run:.3f} nonempty_ratio={ratio:.3f} time={dt:.1f}s\")\n    if ratio >= 0.01:\n        break\n    probe_attempts += 1\n    seg_thr_run, tree_thr_run, prop_mult_run = relax_thresholds(best_thr, tree_thr, PROP_MULT_INIT, probe_attempts)\n\n# ---- full inference ----\nnon_empty_images = 0\nt_start = time.time()\n\nfor idx, img_id in enumerate(unique_ids):\n    img_path = os.path.join(TEST_IMG_DIR, img_id)\n    img = Image.open(img_path).convert(\"RGB\")\n    orig_w, orig_h = img.size\n\n    prob = predict_prob_resized(img, INFER_SIZE, USE_TTA)\n    inst_resized = instances_from_prob_fast(prob, seg_thr_run, prop_mult_run, tree_clf, tree_thr_run)\n\n    rows = id2rows[img_id]\n    if len(rows) == 1:\n        # single-row: union all instances\n        m = union_mask(inst_resized, INFER_SIZE)\n        if m.max() > 0:\n            non_empty_images += 1\n        # resize back to original 768x768 (or whatever)\n        m_img = Image.fromarray(m * 255).resize((orig_w, orig_h), resample=Image.NEAREST)\n        m0 = (np.array(m_img) > 127).astype(np.uint8)\n        out_encoded[rows[0]] = rle_encode(m0)\n    else:\n        # multi-row: each instance in separate row; if too many, merge remainder to last row\n        rles = []\n        for ins in inst_resized:\n            ins_img = Image.fromarray(ins * 255).resize((orig_w, orig_h), resample=Image.NEAREST)\n            ins0 = (np.array(ins_img) > 127).astype(np.uint8)\n            rr = rle_encode(ins0)\n            if isinstance(rr, str) and len(rr) > 0:\n                rles.append(rr)\n\n        if len(rles) > 0:\n            non_empty_images += 1\n\n        k = len(rows)\n        if len(rles) <= k:\n            for j, ridx in enumerate(rows):\n                out_encoded[ridx] = rles[j] if j < len(rles) else \"\"\n        else:\n            # fill first k-1 with individual, last row union of rest\n            for j in range(k - 1):\n                out_encoded[rows[j]] = rles[j]\n            # union remainder in resized space, then encode\n            rest = inst_resized[(k - 1):]\n            m = union_mask(rest, INFER_SIZE)\n            m_img = Image.fromarray(m * 255).resize((orig_w, orig_h), resample=Image.NEAREST)\n            m0 = (np.array(m_img) > 127).astype(np.uint8)\n            out_encoded[rows[-1]] = rle_encode(m0)\n\n    if (idx + 1) % 1000 == 0:\n        dt = time.time() - t_start\n        ips = (idx + 1) / max(1e-6, dt)\n        print(f\"[INFER] {idx+1}/{len(unique_ids)} images done | {ips:.2f} img/s | non_empty_images={non_empty_images}\")\n\n# ---- save submission ----\nos.makedirs(WORK_DIR, exist_ok=True)\nSUB_PATH = os.path.join(WORK_DIR, f\"submission_{SEG_MODEL}_tree{TREE_MODEL}_train{TRAIN_N}.csv\")\n\nout = template.copy()\nout[\"EncodedPixels\"] = out_encoded\nout.to_csv(SUB_PATH, index=False)\n\nsize = os.path.getsize(SUB_PATH) if os.path.exists(SUB_PATH) else 0\nnon_empty_rows = int((out[\"EncodedPixels\"].astype(str).str.len() > 0).sum())\n\nprint(\"============================================================\")\nprint(\"[DONE] submission:\", SUB_PATH)\nprint(\" rows:\", len(out), \"bytes:\", size, \"non_empty_rows:\", non_empty_rows, \"non_empty_images:\", non_empty_images)\nprint(\" SEG thr used:\", seg_thr_run, \"TREE thr used:\", tree_thr_run, \"PROP_MULT used:\", prop_mult_run)\nprint(\" TREE_JSON:\", TREE_JSON if \"TREE_JSON\" in globals() else \"N/A\")\nprint(\"============================================================\")\n\nwith open(SUB_PATH, \"rb\") as f:\n    print(\"[FILE HEAD BYTES]\", f.read(64))\nprint(out.head(10))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}