{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":113558,"databundleVersionId":14174843,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Install dependencies (quiet)\nimport sys, subprocess, importlib\n\ndef pip_install(pkg):\n    try:\n        importlib.import_module(pkg.split('==')[0].split('[')[0])\n    except Exception:\n        subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', pkg])\n\nfor pkg in [\n    'albumentations==1.4.7',\n    'segmentation-models-pytorch==0.3.3',\n    'timm==1.0.9',\n    'torchmetrics==1.4.0.post0',\n    'opencv-python-headless==4.10.0.84',\n]:\n    pip_install(pkg)\n\nprint('Setup complete')\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T05:24:55.098831Z","iopub.execute_input":"2025-11-08T05:24:55.099143Z","iopub.status.idle":"2025-11-08T05:25:07.608275Z","shell.execute_reply.started":"2025-11-08T05:24:55.099112Z","shell.execute_reply":"2025-11-08T05:25:07.607386Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Imports & config\nimport os\nfrom pathlib import Path\nimport random\nimport json\nimport math\nimport gc\nfrom typing import Tuple, List, Dict\n\nimport numpy as np\nimport pandas as pd\nimport cv2\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import AdamW\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport segmentation_models_pytorch as smp\n\nimport matplotlib.pyplot as plt\n\nSEED = 42\nIMG_SIZE = 768  # resize longer side for efficiency; maintains aspect via padding\nBATCH_SIZE = 4\nEPOCHS = 12\nLR = 3e-4\nWEIGHT_DECAY = 1e-4\nENCODER = 'timm-efficientnet-b0'\nENCODER_WEIGHTS = 'imagenet'\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)\n\nprint('Device:', DEVICE)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T05:25:29.274613Z","iopub.execute_input":"2025-11-08T05:25:29.275323Z","iopub.status.idle":"2025-11-08T05:25:30.672213Z","shell.execute_reply.started":"2025-11-08T05:25:29.275297Z","shell.execute_reply":"2025-11-08T05:25:30.671399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Locate dataset root under /kaggle/input\n\ndef find_dataset_root() -> Path:\n    base = Path('/kaggle/input')\n    if not base.exists():\n        # fallback for local previews; user can set DATA_DIR manually\n        return Path('.')\n    # choose the first directory that contains 'train_images'\n    candidates = []\n    for name in os.listdir(base):\n        p = base / name\n        if (p / 'train_images').exists():\n            candidates.append(p)\n        else:\n            # sometimes competition zip nests one more level\n            for sub in p.glob('*/'):\n                if (sub / 'train_images').exists():\n                    candidates.append(sub)\n    if not candidates:\n        raise FileNotFoundError('Could not find dataset directory with train_images under /kaggle/input')\n    # pick the largest by size just in case multiple\n    return sorted(candidates, key=lambda p: sum(f.stat().st_size for f in p.rglob('*') if f.is_file()), reverse=True)[0]\n\nDATA_DIR = find_dataset_root()\nTRAIN_IMG_DIR = DATA_DIR / 'train_images'\nTEST_IMG_DIR = DATA_DIR / 'test_images'\nTRAIN_MASK_DIR = DATA_DIR / 'train_masks'\nSAMPLE_SUB_PATH = DATA_DIR / 'sample_submission.csv'\n\nprint('DATA_DIR =', DATA_DIR)\nprint('TRAIN_IMG_DIR exists:', TRAIN_IMG_DIR.exists())\nprint('TEST_IMG_DIR exists:', TEST_IMG_DIR.exists())\nprint('TRAIN_MASK_DIR exists:', TRAIN_MASK_DIR.exists())\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T05:25:31.599603Z","iopub.execute_input":"2025-11-08T05:25:31.600111Z","iopub.status.idle":"2025-11-08T05:25:35.393043Z","shell.execute_reply.started":"2025-11-08T05:25:31.600081Z","shell.execute_reply":"2025-11-08T05:25:35.392265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# RLE utilities (row-major, 1-based start)\n# Matches common Kaggle conventions and serializes as space-separated pairs inside brackets\n\ndef rle_encode(mask: np.ndarray) -> str:\n    \"\"\"\n    Encode binary mask to RLE string like \"[start length start length ...]\".\n    mask: 2D array of 0/1.\n    \"\"\"\n    assert mask.ndim == 2\n    pixels = mask.flatten(order='C')\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] = runs[1::2] - runs[::2]\n    if len(runs) == 0:\n        return 'authentic'\n    return '[' + ' '.join(map(str, runs.tolist())) + ']'\n\n\ndef rle_decode(rle: str, shape: Tuple[int, int]) -> np.ndarray:\n    if rle == 'authentic' or rle.strip() == '' or rle.strip() == '[]':\n        return np.zeros(shape, dtype=np.uint8)\n    s = rle.strip().strip('[]').split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0::2], s[1::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape, order='C')\n\n\n# Small helper to load .npy masks that may contain multiple regions\n\ndef load_mask_npy(path: Path) -> np.ndarray:\n    arr = np.load(str(path), allow_pickle=True)\n    if isinstance(arr, np.ndarray) and arr.dtype == object:\n        # stored as list of masks\n        masks = [np.asarray(m, dtype=np.uint8) for m in arr]\n        m = np.zeros_like(masks[0], dtype=np.uint8)\n        for mm in masks:\n            m = np.maximum(m, (mm > 0).astype(np.uint8))\n        return m\n    if arr.ndim == 2:\n        return (arr > 0).astype(np.uint8)\n    if arr.ndim == 3:\n        return (arr.max(axis=0) > 0).astype(np.uint8)\n    return (arr.squeeze() > 0).astype(np.uint8)\n\nprint('RLE utils ready')\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T05:25:37.824125Z","iopub.execute_input":"2025-11-08T05:25:37.82442Z","iopub.status.idle":"2025-11-08T05:25:37.834772Z","shell.execute_reply.started":"2025-11-08T05:25:37.824397Z","shell.execute_reply":"2025-11-08T05:25:37.834155Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Index all images and masks\n\ndef list_images_and_labels(train_img_dir: Path, train_mask_dir: Path) -> pd.DataFrame:\n    forged_dir = train_img_dir / 'forged'\n    authentic_dir = train_img_dir / 'authentic'\n    rows = []\n    # forged images should have masks\n    for p in sorted(forged_dir.glob('*.png')):\n        case_id = p.stem  # numeric as string\n        mask_path = train_mask_dir / f'{case_id}.npy'\n        rows.append({'case_id': case_id, 'img_path': str(p), 'mask_path': str(mask_path), 'label': 'forged'})\n    # authentic images have no mask\n    for p in sorted(authentic_dir.glob('*.png')):\n        case_id = p.stem\n        rows.append({'case_id': case_id, 'img_path': str(p), 'mask_path': None, 'label': 'authentic'})\n    df = pd.DataFrame(rows)\n    return df\n\ntrain_df = list_images_and_labels(TRAIN_IMG_DIR, TRAIN_MASK_DIR)\nprint('Train samples:', len(train_df), '| forged:', (train_df.label=='forged').sum(), '| authentic:', (train_df.label=='authentic').sum())\ntrain_df.head()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T05:25:39.989424Z","iopub.execute_input":"2025-11-08T05:25:39.989735Z","iopub.status.idle":"2025-11-08T05:25:40.052925Z","shell.execute_reply.started":"2025-11-08T05:25:39.989712Z","shell.execute_reply":"2025-11-08T05:25:40.052214Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dataset and transforms\n\nMEAN = (0.485, 0.456, 0.406)\nSTD = (0.229, 0.224, 0.225)\n\n\ndef make_train_aug(size: int):\n    return A.Compose([\n        A.LongestMaxSize(max_size=size),\n        A.PadIfNeeded(min_height=size, min_width=size, border_mode=cv2.BORDER_CONSTANT, value=(0, 0, 0), mask_value=0),\n        A.HorizontalFlip(p=0.5),\n        A.RandomRotate90(p=0.5),\n        A.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.1, rotate_limit=15, border_mode=cv2.BORDER_CONSTANT, value=0, p=0.5),\n        A.GaussNoise(p=0.15),\n        A.ColorJitter(p=0.2),\n        A.Normalize(mean=MEAN, std=STD),\n        ToTensorV2(),\n    ])\n\n\ndef make_valid_aug(size: int):\n    return A.Compose([\n        A.LongestMaxSize(max_size=size),\n        A.PadIfNeeded(min_height=size, min_width=size, border_mode=cv2.BORDER_CONSTANT, value=(0, 0, 0), mask_value=0),\n        A.Normalize(mean=MEAN, std=STD),\n        ToTensorV2(),\n    ])\n\n\nclass ForgeryDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def _read_image(self, path: str) -> np.ndarray:\n        img = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n        if img is None:\n            raise FileNotFoundError(path)\n        if img.ndim == 2:\n            # grayscale -> 3 channels\n            img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)\n        else:\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        return img\n\n    def _read_mask(self, row) -> np.ndarray:\n        if row['label'] == 'authentic' or not row['mask_path'] or not os.path.exists(row['mask_path']):\n            return np.zeros(self._read_image(row['img_path']).shape[:2], dtype=np.uint8)\n        return load_mask_npy(Path(row['mask_path']))\n\n    def __getitem__(self, idx: int):\n        r = self.df.iloc[idx]\n        img = self._read_image(r['img_path'])\n        mask = self._read_mask(r)\n        if self.transform is not None:\n            aug = self.transform(image=img, mask=mask)\n            img, mask = aug['image'], aug['mask']  # img: Tensor CxHxW, mask: Tensor or ndarray HxW\n            if isinstance(mask, np.ndarray):\n                mask = torch.from_numpy(mask)\n            mask = mask.float().unsqueeze(0)\n            return img, mask\n        # Fallback (no transform)\n        img = torch.from_numpy(img).permute(2, 0, 1).float() / 255.0\n        mask = torch.from_numpy(mask).float().unsqueeze(0)\n        return img, mask\n\n\nprint('Dataset class ready')\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T05:25:42.133926Z","iopub.execute_input":"2025-11-08T05:25:42.134527Z","iopub.status.idle":"2025-11-08T05:25:42.145689Z","shell.execute_reply.started":"2025-11-08T05:25:42.134501Z","shell.execute_reply":"2025-11-08T05:25:42.144867Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train/valid split (stratified by label)\nfrom sklearn.model_selection import StratifiedKFold\n\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\ntrain_idx, valid_idx = next(skf.split(train_df, train_df['label']))\n\ntr_df = train_df.iloc[train_idx].reset_index(drop=True)\nval_df = train_df.iloc[valid_idx].reset_index(drop=True)\n\ntrain_ds = ForgeryDataset(tr_df, transform=make_train_aug(IMG_SIZE))\nvalid_ds = ForgeryDataset(val_df, transform=make_valid_aug(IMG_SIZE))\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=2, pin_memory=True, drop_last=True)\nvalid_loader = DataLoader(valid_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=2, pin_memory=True)\n\nlen(tr_df), len(val_df)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T05:25:44.879266Z","iopub.execute_input":"2025-11-08T05:25:44.879524Z","iopub.status.idle":"2025-11-08T05:25:44.904856Z","shell.execute_reply.started":"2025-11-08T05:25:44.879507Z","shell.execute_reply":"2025-11-08T05:25:44.904217Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model, loss, optimizer\n\nmodel = smp.Unet(encoder_name=ENCODER, encoder_weights=ENCODER_WEIGHTS, in_channels=3, classes=1, activation=None)\nmodel.to(DEVICE)\n\n# Loss = BCEWithLogits + soft Dice\nclass DiceLoss(nn.Module):\n    def __init__(self, eps: float = 1e-6):\n        super().__init__()\n        self.eps = eps\n    def forward(self, logits, targets):\n        probs = torch.sigmoid(logits)\n        targets = targets.float()\n        dims = (0, 2, 3)\n        intersection = (probs * targets).sum(dims)\n        denom = probs.sum(dims) + targets.sum(dims) + self.eps\n        dice = (2. * intersection + self.eps) / denom\n        return 1 - dice.mean()\n\nbce = nn.BCEWithLogitsLoss()\ndice = DiceLoss()\n\ndef criterion(logits, targets):\n    return 0.5 * bce(logits, targets) + 0.5 * dice(logits, targets)\n\noptimizer = AdamW(model.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)\n\nscaler = GradScaler(enabled=(DEVICE=='cuda'))\n\nprint('Model params (M):', sum(p.numel() for p in model.parameters())/1e6)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T05:25:52.84007Z","iopub.execute_input":"2025-11-08T05:25:52.840796Z","iopub.status.idle":"2025-11-08T05:25:53.413718Z","shell.execute_reply.started":"2025-11-08T05:25:52.840769Z","shell.execute_reply":"2025-11-08T05:25:53.413077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training & validation loops\nfrom torchmetrics.classification import BinaryF1Score\n\nf1_metric = BinaryF1Score(threshold=0.5).to(DEVICE)\n\n\ndef train_one_epoch(model, loader, optimizer, scaler):\n    model.train()\n    total_loss = 0.0\n    f1_running = 0.0\n    num_batches = 0\n    for imgs, masks in loader:\n        imgs = imgs.to(DEVICE, non_blocking=True)\n        masks = masks.to(DEVICE, non_blocking=True)\n        optimizer.zero_grad(set_to_none=True)\n        with autocast(enabled=(DEVICE=='cuda')):\n            logits = model(imgs)\n            loss = criterion(logits, masks)\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        total_loss += loss.item()\n        with torch.no_grad():\n            probs = torch.sigmoid(logits)\n            f1_running += f1_metric((probs>0.5).float(), masks.int()).item()\n        num_batches += 1\n    return total_loss/num_batches, f1_running/num_batches\n\n\ndef valid_one_epoch(model, loader):\n    model.eval()\n    total_loss = 0.0\n    f1_running = 0.0\n    num_batches = 0\n    with torch.no_grad():\n        for imgs, masks in loader:\n            imgs = imgs.to(DEVICE, non_blocking=True)\n            masks = masks.to(DEVICE, non_blocking=True)\n            logits = model(imgs)\n            loss = criterion(logits, masks)\n            total_loss += loss.item()\n            probs = torch.sigmoid(logits)\n            f1_running += f1_metric((probs>0.5).float(), masks.int()).item()\n            num_batches += 1\n    return total_loss/num_batches, f1_running/num_batches\n\nprint('Train/valid functions ready')\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T05:25:56.664444Z","iopub.execute_input":"2025-11-08T05:25:56.665112Z","iopub.status.idle":"2025-11-08T05:25:56.676462Z","shell.execute_reply.started":"2025-11-08T05:25:56.665086Z","shell.execute_reply":"2025-11-08T05:25:56.675686Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Run training\nbest_f1 = -1.0\nbest_path = Path('/kaggle/working/best_model.pt')\n\nfor epoch in range(1, EPOCHS+1):\n    tr_loss, tr_f1 = train_one_epoch(model, train_loader, optimizer, scaler)\n    va_loss, va_f1 = valid_one_epoch(model, valid_loader)\n    scheduler.step()\n    print(f'Epoch {epoch:02d}/{EPOCHS} | train loss {tr_loss:.4f} f1 {tr_f1:.4f} | valid loss {va_loss:.4f} f1 {va_f1:.4f}')\n    if va_f1 > best_f1:\n        best_f1 = va_f1\n        torch.save({'state_dict': model.state_dict(), 'epoch': epoch}, best_path)\n        print('  Saved new best ->', best_path, 'F1=', best_f1)\n\n# Load best for inference\nckpt = torch.load(best_path, map_location=DEVICE)\nmodel.load_state_dict(ckpt['state_dict'])\nmodel.eval()\nprint('Loaded best epoch:', ckpt.get('epoch'))\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T05:25:59.454638Z","iopub.execute_input":"2025-11-08T05:25:59.455472Z","iopub.status.idle":"2025-11-08T06:46:42.356486Z","shell.execute_reply.started":"2025-11-08T05:25:59.45544Z","shell.execute_reply":"2025-11-08T06:46:42.355548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inference utilities and submission\n\ndef sigmoid(x):\n    return 1/(1+np.exp(-x))\n\n@torch.no_grad()\ndef predict_mask(img: np.ndarray, tta: bool = False) -> np.ndarray:\n    aug = make_valid_aug(IMG_SIZE)\n    out = aug(image=img)\n    tensor = out['image'].unsqueeze(0).to(DEVICE)\n    logits = model(tensor)\n    probs = torch.sigmoid(logits)[0,0].cpu().numpy()\n    return probs\n\n\ndef postprocess(probs: np.ndarray, min_area: int = 50, thr: float = 0.5) -> np.ndarray:\n    mask = (probs >= thr).astype(np.uint8)\n    # remove small objects\n    num_labels, labels, stats, centroids = cv2.connectedComponentsWithStats(mask, connectivity=8)\n    cleaned = np.zeros_like(mask)\n    for i in range(1, num_labels):\n        if stats[i, cv2.CC_STAT_AREA] >= min_area:\n            cleaned[labels == i] = 1\n    return cleaned\n\n\n# Build test dataframe and a callable to generate submissions\n\ndef list_test_images(test_img_dir: Path) -> pd.DataFrame:\n    rows = []\n    for p in sorted(test_img_dir.glob('*.png')):\n        rows.append({'case_id': p.stem, 'img_path': str(p)})\n    return pd.DataFrame(rows)\n\n\ntest_df = list_test_images(TEST_IMG_DIR)\nprint('Test images:', len(test_df))\n\n@torch.no_grad()\ndef generate_submission(output_csv: str = '/kaggle/working/submission.csv', min_area: int = 100, thr: float = 0.5) -> pd.DataFrame:\n    model.eval()\n    pred_rows = []\n    for i, row in test_df.iterrows():\n        img = cv2.imread(row['img_path'], cv2.IMREAD_UNCHANGED)\n        if img.ndim == 2:\n            img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)\n        else:\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        probs = predict_mask(img)\n        mask = postprocess(probs, min_area=min_area, thr=thr)\n        annotation = rle_encode(mask)\n        pred_rows.append({'case_id': int(row['case_id']), 'annotation': annotation})\n        if (i+1) % 200 == 0:\n            print(f'Processed {i+1}/{len(test_df)}')\n    sub = pd.DataFrame(pred_rows).sort_values('case_id')\n    sub.to_csv(output_csv, index=False)\n    print('Wrote submission ->', output_csv)\n    return sub\n\n# You can call generate_submission() later or let the training cell trigger it on best epochs\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T06:52:40.383398Z","iopub.execute_input":"2025-11-08T06:52:40.383754Z","iopub.status.idle":"2025-11-08T06:52:40.398446Z","shell.execute_reply.started":"2025-11-08T06:52:40.383723Z","shell.execute_reply":"2025-11-08T06:52:40.397647Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize a few samples\nrows = 2\ncols = 3\nfig, axes = plt.subplots(rows, cols, figsize=(12, 8))\nfor ax in axes.ravel():\n    idx = random.randint(0, len(train_ds)-1)\n    img, mask = train_ds[idx]\n    img_np = (img.permute(1,2,0).numpy() * np.array(STD) + np.array(MEAN))\n    img_np = np.clip(img_np, 0, 1)\n    ax.imshow(img_np)\n    ax.imshow(mask.squeeze().numpy(), alpha=0.4, cmap='Reds')\n    ax.axis('off')\nplt.tight_layout()\nplt.show()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T06:52:49.352696Z","iopub.execute_input":"2025-11-08T06:52:49.353356Z","iopub.status.idle":"2025-11-08T06:52:51.416839Z","shell.execute_reply.started":"2025-11-08T06:52:49.353335Z","shell.execute_reply":"2025-11-08T06:52:51.416068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inference-only: load best_model.pt and create /kaggle/working/submission.csv\n\n# 1) Installs (quiet)\nimport sys, subprocess, importlib, os\ndef pip_install(pkg):\n    try:\n        importlib.import_module(pkg.split('==')[0].split('[')[0])\n    except Exception:\n        subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', pkg])\nfor pkg in [\n    'albumentations==1.4.7',\n    'segmentation-models-pytorch==0.3.3',\n    'timm==1.0.9',\n    'opencv-python-headless==4.10.0.84',\n]:\n    pip_install(pkg)\n\n# 2) Imports\nfrom pathlib import Path\nimport numpy as np, pandas as pd, cv2, torch\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport segmentation_models_pytorch as smp\nfrom tqdm.auto import tqdm\n\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nIMG_SIZE = 768\nMEAN = (0.485, 0.456, 0.406)\nSTD = (0.229, 0.224, 0.225)\nENCODER = 'timm-efficientnet-b0'   # must match training\nprint('Device:', DEVICE)\n\n# 3) Locate dataset root (expects test_images/)\ndef find_dataset_root() -> Path:\n    base = Path('/kaggle/input')\n    cands = []\n    if base.exists():\n        for root, dirs, files in os.walk(base):\n            p = Path(root)\n            if (p / 'test_images').exists():\n                cands.append(p)\n    if not cands:\n        raise FileNotFoundError('Could not find dataset directory with test_images under /kaggle/input')\n    # pick the largest by size\n    cands.sort(key=lambda p: sum(f.stat().st_size for f in p.rglob('*') if f.is_file()), reverse=True)\n    return cands[0]\n\nDATA_DIR = find_dataset_root()\nTEST_IMG_DIR = DATA_DIR / 'test_images'\nprint('DATA_DIR =', DATA_DIR)\n\n# 4) Find best_model.pt (upload it as a Dataset/Input or keep in working)\ndef find_best_model_path() -> Path:\n    # search under /kaggle/input for best_model.pt\n    base = Path('/kaggle/input')\n    hits = []\n    if base.exists():\n        for root, dirs, files in os.walk(base):\n            if 'best_model.pt' in files:\n                hits.append(Path(root) / 'best_model.pt')\n    if hits:\n        # pick the most recent by mtime\n        hits.sort(key=lambda p: p.stat().st_mtime, reverse=True)\n        return hits[0]\n    # fallback if user attached it to the notebook or re-uploaded to working\n    fallback = Path('/kaggle/working/best_model.pt')\n    if fallback.exists():\n        return fallback\n    raise FileNotFoundError('best_model.pt not found. Add it as a Kaggle Dataset/Input, or place in /kaggle/working.')\n\nBEST_PATH = find_best_model_path()\nprint('Using checkpoint:', BEST_PATH)\n\n# 5) Augs and helpers\ndef make_valid_aug(size: int):\n    return A.Compose([\n        A.LongestMaxSize(max_size=size),\n        A.PadIfNeeded(min_height=size, min_width=size, border_mode=cv2.BORDER_CONSTANT, value=(0, 0, 0), mask_value=0),\n        A.Normalize(mean=MEAN, std=STD),\n        ToTensorV2(),\n    ])\n\ndef rle_encode(mask: np.ndarray) -> str:\n    mask = (mask > 0).astype(np.uint8)\n    pixels = mask.flatten(order='C')\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] = runs[1::2] - runs[::2]\n    if len(runs) == 0:\n        return 'authentic'\n    return '[' + ' '.join(map(str, runs.tolist())) + ']'\n\n@torch.no_grad()\ndef predict_mask(model, img: np.ndarray) -> np.ndarray:\n    aug = make_valid_aug(IMG_SIZE)\n    out = aug(image=img)\n    tensor = out['image'].unsqueeze(0).to(DEVICE)\n    logits = model(tensor)\n    probs = torch.sigmoid(logits)[0, 0].cpu().numpy()\n    return probs\n\ndef postprocess(probs: np.ndarray, min_area: int = 100, thr: float = 0.5) -> np.ndarray:\n    mask = (probs >= thr).astype(np.uint8)\n    num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(mask, connectivity=8)\n    cleaned = np.zeros_like(mask)\n    for i in range(1, num_labels):\n        if stats[i, cv2.CC_STAT_AREA] >= min_area:\n            cleaned[labels == i] = 1\n    return cleaned\n\ndef list_test_images(test_img_dir: Path) -> pd.DataFrame:\n    rows = [{'case_id': p.stem, 'img_path': str(p)} for p in sorted(test_img_dir.glob('*.png'))]\n    return pd.DataFrame(rows)\n\n# 6) Build model and load checkpoint\nmodel = smp.Unet(encoder_name=ENCODER, encoder_weights=None, in_channels=3, classes=1, activation=None)\nckpt = torch.load(BEST_PATH, map_location=DEVICE)\nmodel.load_state_dict(ckpt['state_dict'])\nmodel.to(DEVICE).eval()\nprint('Loaded epoch:', ckpt.get('epoch'))\n\n# 7) Generate submission\ntest_df = list_test_images(TEST_IMG_DIR)\npred_rows = []\nfor i, row in tqdm(test_df.iterrows(), total=len(test_df), desc='Infer test'):\n    img = cv2.imread(row['img_path'], cv2.IMREAD_UNCHANGED)\n    if img.ndim == 2:\n        img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)\n    else:\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    probs = predict_mask(model, img)\n    mask = postprocess(probs, min_area=100, thr=0.5)\n    annotation = rle_encode(mask)\n    pred_rows.append({'case_id': int(row['case_id']), 'annotation': annotation})\n\nsub = pd.DataFrame(pred_rows).sort_values('case_id')\nout_path = '/kaggle/working/submission.csv'\nsub.to_csv(out_path, index=False)\nprint('Wrote', out_path)\ndisplay(sub.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T06:56:01.52846Z","iopub.execute_input":"2025-11-08T06:56:01.529265Z","iopub.status.idle":"2025-11-08T06:56:11.408311Z","shell.execute_reply.started":"2025-11-08T06:56:01.529238Z","shell.execute_reply":"2025-11-08T06:56:11.407679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# FINAL: Build submission.csv from saved best_model.pt using competition-style JSON RLEs\n\nimport os, json, cv2, torch\nimport numpy as np, pandas as pd\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport segmentation_models_pytorch as smp\nimport numba\nfrom numba import njit\n\n# -------- config (adjust if needed) ----------\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nIMG_SIZE = 768\nMEAN, STD = (0.485, 0.456, 0.406), (0.229, 0.224, 0.225)\nENCODER = 'timm-efficientnet-b0'     # must match training\nBEST_MODEL_PATH = Path('/kaggle/working/best_model.pt')  # your saved checkpoint\n\nTHRESH = 0.50            # probability threshold\nMIN_MASK_PIXELS = 100    # min connected-component size\nMIN_COVERAGE = 0.0       # optional coverage filter lower bound\nMAX_COVERAGE = 0.95      # optional coverage filter upper bound\n# --------------------------------------------\n\ndef find_competition_root():\n    base = Path('/kaggle/input')\n    best, best_rows = None, -1\n    for root, dirs, files in os.walk(base):\n        p = Path(root)\n        if (p/'test_images').exists() and (p/'sample_submission.csv').exists():\n            try:\n                nrows = sum(1 for _ in open(p/'sample_submission.csv')) - 1\n            except Exception:\n                nrows = len(pd.read_csv(p/'sample_submission.csv'))\n            if nrows > best_rows:\n                best_rows, best = nrows, p\n    if best is None:\n        raise FileNotFoundError('Attach the competition data (test_images + sample_submission.csv).')\n    print(f'Using DATA_DIR={best} | sample rows={best_rows}')\n    return best\n\nDATA_DIR = find_competition_root()\nTEST_DIR = DATA_DIR / 'test_images'\nSAMPLE_SUB = DATA_DIR / 'sample_submission.csv'\n\n# ---------------- RLE per metric (JSON arrays, ';'-separated) ----------------\n@numba.jit(nopython=True)\ndef _rle_encode_jit(x: np.ndarray, fg_val: int = 1) -> list:\n    dots = np.where(x.T.flatten() == fg_val)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if b > prev + 1:\n            run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\ndef rle_encode_instances(masks: list[np.ndarray], fg_val: int = 1) -> str:\n    if len(masks) == 0:\n        return 'authentic'\n    return ';'.join([json.dumps(_rle_encode_jit(m.astype(np.uint8), fg_val)) for m in masks])\n# ---------------------------------------------------------------------------\n\n# aug\ndef valid_aug(size):\n    return A.Compose([\n        A.LongestMaxSize(max_size=size),\n        A.PadIfNeeded(min_height=size, min_width=size, border_mode=cv2.BORDER_CONSTANT),\n        A.Normalize(mean=MEAN, std=STD),\n        ToTensorV2(),\n    ])\n\naug = valid_aug(IMG_SIZE)\n\n# model\nassert BEST_MODEL_PATH.exists(), f'Missing checkpoint: {BEST_MODEL_PATH}'\nmodel = smp.Unet(encoder_name=ENCODER, encoder_weights=None, in_channels=3, classes=1, activation=None).to(DEVICE)\nckpt = torch.load(BEST_MODEL_PATH, map_location=DEVICE)\nmodel.load_state_dict(ckpt['state_dict'])\nmodel.eval()\nprint('Loaded epoch:', ckpt.get('epoch'))\n\n@torch.no_grad()\ndef predict_probs(img: np.ndarray) -> np.ndarray:\n    out = aug(image=img)\n    x = out['image'].unsqueeze(0).to(DEVICE)\n    y = model(x)\n    return torch.sigmoid(y)[0, 0].cpu().numpy()\n\ndef binarize_and_instances(probs: np.ndarray, thr: float, min_pixels: int) -> list[np.ndarray]:\n    m = (probs >= thr).astype(np.uint8)\n    num, labels, stats, _ = cv2.connectedComponentsWithStats(m, connectivity=8)\n    instances = []\n    for i in range(1, num):\n        if stats[i, cv2.CC_STAT_AREA] >= min_pixels:\n            inst = (labels == i).astype(np.uint8)\n            cov = inst.sum() / inst.size\n            if MIN_COVERAGE <= cov <= MAX_COVERAGE:\n                instances.append(inst)\n    return instances\n\n# sample order ensures correct rows\nsample = pd.read_csv(SAMPLE_SUB)\nsample['case_id'] = sample['case_id'].astype(str)\n\n# we assume .png naming per competition; fallback to common exts if .png missing\nEXTS = ['png','jpg','jpeg','tif','tiff','bmp','gif','webp']\ndef find_image_by_id(cid: str) -> Path | None:\n    p = TEST_DIR / f'{cid}.png'\n    if p.exists(): return p\n    for ext in EXTS:\n        q = TEST_DIR / f'{cid}.{ext}'\n        if q.exists(): return q\n    # recursive search as last resort\n    for ext in EXTS:\n        for q in TEST_DIR.rglob(f'{cid}.{ext}'):\n            return q\n    return None\n\nrows = []\nfor _, r in tqdm(sample.iterrows(), total=len(sample), desc='Predict'):\n    cid = str(r['case_id'])\n    p = find_image_by_id(cid)\n    if p is None:\n        rows.append({'case_id': int(cid), 'annotation': 'authentic'})\n        continue\n    img = cv2.imread(str(p), cv2.IMREAD_UNCHANGED)\n    if img is None:\n        rows.append({'case_id': int(cid), 'annotation': 'authentic'})\n        continue\n    if img.ndim == 2:\n        img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)\n    else:\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n    probs = predict_probs(img)\n    instances = binarize_and_instances(probs, thr=THRESH, min_pixels=MIN_MASK_PIXELS)\n    annotation = rle_encode_instances(instances)\n    rows.append({'case_id': int(cid), 'annotation': annotation})\n\nsub = pd.DataFrame(rows)\nout_path = '/kaggle/working/submission.csv'\nsub.to_csv(out_path, index=False)\nprint('Wrote', out_path, '| rows =', len(sub))\ndisplay(sub.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T07:09:45.584883Z","iopub.execute_input":"2025-11-08T07:09:45.585848Z","iopub.status.idle":"2025-11-08T07:09:51.773395Z","shell.execute_reply.started":"2025-11-08T07:09:45.585808Z","shell.execute_reply":"2025-11-08T07:09:51.772622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}