{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":113558,"databundleVersionId":14174843,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-25T05:48:32.072241Z","iopub.execute_input":"2025-10-25T05:48:32.072513Z","iopub.status.idle":"2025-10-25T05:48:32.329791Z","shell.execute_reply.started":"2025-10-25T05:48:32.072492Z","shell.execute_reply":"2025-10-25T05:48:32.328990Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ====================================================\n# Recod.ai/LUC - Scientific Image Forgery Detection\n# Multi-GPU (2× T4) Training Notebook\n# ====================================================\n\nimport os, gc, random\nimport numpy as np\nfrom glob import glob\nimport matplotlib.pyplot as plt\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.distributed as dist\nimport torch.multiprocessing as mp\nfrom torch.nn.parallel import DistributedDataParallel as DDP\nfrom torch.cuda.amp import autocast, GradScaler\nimport albumentations as A\nimport cv2\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)\nwarnings.filterwarnings(\"ignore\", category=UserWarning)\n\n\n# ====================================================\n# 1. Paths and Globals\n# ====================================================\nBASE_PATH = \"/kaggle/input/recodai-luc-scientific-image-forgery-detection\"\nTRAIN_IMG_PATH = os.path.join(BASE_PATH, \"train_images\")\nTRAIN_MASK_PATH = os.path.join(BASE_PATH, \"train_masks\")\nos.environ[\"TORCH_DISTRIBUTED_DEBUG\"] = \"OFF\"  # set \"DETAIL\" for debug\n\n# ====================================================\n# 2. Helper functions\n# ====================================================\ndef is_valid_image(img_path):\n    try:\n        img = plt.imread(img_path)\n        return img is not None and img.size > 0 and min(img.shape[:2]) > 8\n    except Exception:\n        return False\n\ndef is_valid_mask(mask_path):\n    try:\n        mask = np.load(mask_path, allow_pickle=True)\n        return mask is not None and mask.size > 0 and len(mask.shape) == 2\n    except Exception:\n        return False\n\n# ====================================================\n# 3. Dataset\n# ====================================================\nclass ForgeryDataset(Dataset):\n    def __init__(self, image_paths, mask_paths, size=512, augment=False):\n        self.image_paths = image_paths\n        self.mask_paths = mask_paths\n        self.size = size\n        self.augment = augment\n\n        self.tf_train = A.Compose([\n            A.Resize(self.size, self.size),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.RandomBrightnessContrast(p=0.2),\n            A.Affine(scale=(0.9, 1.1), translate_percent=(0.05, 0.05), rotate=(-10, 10), p=0.4)\n        ], is_check_shapes=False)\n        self.tf_val = A.Compose([A.Resize(self.size, self.size)], is_check_shapes=False)\n\n    def __len__(self): return len(self.image_paths)\n\n    def safe_img(self, p):\n        try: img = plt.imread(p)\n        except: img = np.zeros((256,256,3), np.uint8)\n        if img.ndim == 2: img = np.stack([img]*3, -1)\n        if img.shape[2] == 4: img = img[:,:,:3]\n        if img.dtype != np.uint8:\n            img = (img*255).astype(np.uint8) if img.max()<=1 else img.astype(np.uint8)\n        return img\n\n    def safe_mask(self, p, shape):\n        if not p or not os.path.exists(p): return np.zeros(shape[:2], np.uint8)\n        try:\n            m = np.load(p); \n            if m.ndim!=2: return np.zeros(shape[:2], np.uint8)\n            m = (m>0).astype(np.uint8)\n            if m.shape!=shape[:2]:\n                m = cv2.resize(m, (shape[1], shape[0]), interpolation=cv2.INTER_NEAREST)\n            return m\n        except: return np.zeros(shape[:2], np.uint8)\n\n    def __getitem__(self, i):\n        imgp = self.image_paths[i]\n        name = os.path.basename(imgp).replace(\".png\",\"\")\n        img = self.safe_img(imgp)\n        mask = self.safe_mask(mask_paths.get(name), img.shape)\n        tfm = self.tf_train if self.augment else self.tf_val\n        aug = tfm(image=img, mask=mask)\n        img, mask = aug[\"image\"], aug[\"mask\"]\n        img = torch.tensor(img.transpose(2,0,1), dtype=torch.float32)/255.\n        mask = torch.tensor(mask, dtype=torch.float32).unsqueeze(0)\n        return img, mask\n\n# ====================================================\n# 4. U-Net Model\n# ====================================================\nclass ConvBlock(nn.Module):\n    def __init__(self, c1, c2):\n        super().__init__()\n        self.seq = nn.Sequential(\n            nn.Conv2d(c1, c2, 3, padding=1), nn.BatchNorm2d(c2), nn.ReLU(True),\n            nn.Conv2d(c2, c2, 3, padding=1), nn.BatchNorm2d(c2), nn.ReLU(True)\n        )\n    def forward(self, x): return self.seq(x)\n\nclass UNet(nn.Module):\n    def __init__(self, in_ch=3, out_ch=1, base=32):\n        super().__init__()\n        self.enc1 = ConvBlock(in_ch, base)\n        self.enc2 = ConvBlock(base, base*2)\n        self.enc3 = ConvBlock(base*2, base*4)\n        self.enc4 = ConvBlock(base*4, base*8)\n        self.pool = nn.MaxPool2d(2)\n        self.bottleneck = ConvBlock(base*8, base*16)\n        self.up4 = nn.ConvTranspose2d(base*16, base*8, 2, stride=2)\n        self.dec4 = ConvBlock(base*16, base*8)\n        self.up3 = nn.ConvTranspose2d(base*8, base*4, 2, stride=2)\n        self.dec3 = ConvBlock(base*8, base*4)\n        self.up2 = nn.ConvTranspose2d(base*4, base*2, 2, stride=2)\n        self.dec2 = ConvBlock(base*4, base*2)\n        self.up1 = nn.ConvTranspose2d(base*2, base, 2, stride=2)\n        self.dec1 = ConvBlock(base*2, base)\n        self.out_conv = nn.Conv2d(base, out_ch, 1)\n    def forward(self, x):\n        e1 = self.enc1(x)\n        e2 = self.enc2(self.pool(e1))\n        e3 = self.enc3(self.pool(e2))\n        e4 = self.enc4(self.pool(e3))\n        b  = self.bottleneck(self.pool(e4))\n        d4 = self.dec4(torch.cat([self.up4(b), e4], 1))\n        d3 = self.dec3(torch.cat([self.up3(d4), e3], 1))\n        d2 = self.dec2(torch.cat([self.up2(d3), e2], 1))\n        d1 = self.dec1(torch.cat([self.up1(d2), e1], 1))\n        return self.out_conv(d1)\n\n# ====================================================\n# 5. Loss, optimizer, AMP, etc.\n# ====================================================\ndef dice_loss(pred, target, eps=1e-6):\n    pred = torch.sigmoid(pred)\n    inter = (pred*target).sum((1,2,3))\n    union = pred.sum((1,2,3)) + target.sum((1,2,3))\n    return 1 - ((2*inter+eps)/(union+eps)).mean()\n\ndef make_loss(device, pos_weight=8.0):\n    bce = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([pos_weight], device=device))\n    def loss_fn(p, t):\n        return 0.7*bce(p,t) + 0.3*dice_loss(p,t)\n    return loss_fn\n\n# ====================================================\n# 6. DDP Setup\n# ====================================================\ndef setup(rank, world_size):\n    dist.init_process_group(\"nccl\", rank=rank, world_size=world_size)\n    torch.cuda.set_device(rank)\n\ndef cleanup():\n    dist.destroy_process_group()\n\n# ====================================================\n# 7. Training function (per GPU process)\n# ====================================================\ndef train_ddp(rank, world_size):\n    setup(rank, world_size)\n    device = torch.device(f\"cuda:{rank}\")\n    print(f\"Rank {rank} ready.\")\n\n    # Data\n    forged = glob(os.path.join(TRAIN_IMG_PATH, \"forged\", \"*.png\"))\n    auth   = glob(os.path.join(TRAIN_IMG_PATH, \"authentic\", \"*.png\"))\n    mask_files = glob(os.path.join(TRAIN_MASK_PATH, \"*.npy\"))\n    global mask_paths\n    mask_paths = {os.path.basename(p).replace(\".npy\",\"\"):p for p in mask_files if is_valid_mask(p)}\n\n    forged_valid = [p for p in forged if is_valid_image(p)]\n    random.shuffle(forged_valid)\n    forged_train, forged_val = train_test_split(forged_valid, test_size=0.15, random_state=42)\n\n    # Datasets & Samplers\n    size_stages = [512, 768]\n    BATCH = 8\n    EPOCHS = 6\n    EARLY_STOP = 3\n\n    model = UNet(base=32).to(device)\n    model = DDP(model, device_ids=[rank], output_device=rank, find_unused_parameters=False)\n\n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n    scaler = GradScaler()\n    loss_fn = make_loss(device, pos_weight=8.0)\n\n    best_val = 1e9\n\n    for size in size_stages:\n        if rank == 0:\n            print(f\"\\n=== Stage {size} ===\")\n        train_ds = ForgeryDataset(forged_train, mask_paths, size=size, augment=True)\n        val_ds   = ForgeryDataset(forged_val, mask_paths, size=size, augment=False)\n\n        train_sampler = torch.utils.data.distributed.DistributedSampler(train_ds, num_replicas=world_size, rank=rank, shuffle=True)\n        val_sampler   = torch.utils.data.distributed.DistributedSampler(val_ds,   num_replicas=world_size, rank=rank, shuffle=False)\n        train_dl = DataLoader(train_ds, batch_size=BATCH, sampler=train_sampler, num_workers=2, pin_memory=True)\n        val_dl   = DataLoader(val_ds,   batch_size=BATCH, sampler=val_sampler,   num_workers=2, pin_memory=True)\n\n        early = 0\n        for epoch in range(EPOCHS):\n            train_sampler.set_epoch(epoch)\n            model.train()\n            total_train = 0\n            for imgs, masks in tqdm(train_dl, disable=(rank!=0)):\n                imgs, masks = imgs.to(device, non_blocking=True), masks.to(device, non_blocking=True)\n                optimizer.zero_grad()\n                with autocast():\n                    preds = model(imgs)\n                    loss = loss_fn(preds, masks)\n                scaler.scale(loss).backward()\n                scaler.step(optimizer)\n                scaler.update()\n                total_train += loss.item()\n            avg_train = total_train / len(train_dl)\n\n            # Validation\n            model.eval()\n            total_val = 0\n            with torch.no_grad():\n                for imgs, masks in val_dl:\n                    imgs, masks = imgs.to(device, non_blocking=True), masks.to(device, non_blocking=True)\n                    with autocast():\n                        preds = model(imgs)\n                        vloss = loss_fn(preds, masks)\n                    total_val += vloss.item()\n            avg_val = total_val / len(val_dl)\n\n            if rank == 0:\n                print(f\"[{size}] Epoch {epoch+1}/{EPOCHS} - Train={avg_train:.4f}, Val={avg_val:.4f}\")\n            if avg_val < best_val:\n                best_val = avg_val\n                if rank == 0:\n                    torch.save(model.module.state_dict(), \"model_final_ddp.pth\")\n                    print(\"✅ Saved checkpoint.\")\n                early = 0\n            else:\n                early += 1\n            if early >= EARLY_STOP:\n                if rank == 0: print(\"⏹️ Early stop triggered.\")\n                break\n        torch.cuda.empty_cache()\n        gc.collect()\n\n    cleanup()\n    if rank == 0:\n        print(\"\\n🎯 Training complete. model_final_ddp.pth saved.\")\n\n# ====================================================\n# 8. Launch DataParallel (simpler for notebooks)\n# ====================================================\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device(s): {torch.cuda.device_count()} GPU(s) detected\")\n\nmodel = UNet(base=32).to(DEVICE)\nif torch.cuda.device_count() > 1:\n    print(\"✅ Using nn.DataParallel across GPUs\")\n    model = nn.DataParallel(model)\n\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\nloss_fn = make_loss(DEVICE, pos_weight=8.0)\nscaler = torch.amp.GradScaler(\"cuda\")\n\n# Your same training loop from train_ddp, but now in single-process:\nstages = [512, 768]\nEPOCHS = 6\nBATCH_SIZE = 16\nEARLY_STOP = 3\n\n# ====================================================\n# Prepare training/validation splits (forged-first)\n# ====================================================\n\n# Locate all image and mask files again (if not already loaded)\nforged_images = glob(os.path.join(TRAIN_IMG_PATH, \"forged\", \"*.png\"))\nauth_images   = glob(os.path.join(TRAIN_IMG_PATH, \"authentic\", \"*.png\"))\nmask_files    = glob(os.path.join(TRAIN_MASK_PATH, \"*.npy\"))\n\n# Filter valid mask files\nmask_paths = {\n    os.path.basename(p).replace(\".npy\", \"\"): p\n    for p in mask_files\n    if os.path.exists(p)\n}\n\n# Verify images are valid\nforged_valid = [p for p in forged_images if is_valid_image(p)]\nprint(f\"✅ Found {len(forged_valid)} valid forged images\")\n\n# Split into train/val (forged-only first phase)\nfrom sklearn.model_selection import train_test_split\nforged_train, forged_val = train_test_split(forged_valid, test_size=0.15, random_state=42)\n\nprint(f\"Train forged images: {len(forged_train)} | Val forged images: {len(forged_val)}\")\n\n\n# Reuse your forged_train, forged_val definitions\nfor size in stages:\n    print(f\"\\n=== Stage {size} ===\")\n    train_ds = ForgeryDataset(forged_train, mask_paths, size=size, augment=True)\n    val_ds   = ForgeryDataset(forged_val,   mask_paths, size=size, augment=False)\n    train_dl = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, pin_memory=True)\n    val_dl   = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, pin_memory=True)\n\n    best_val = np.inf\n    early = 0\n\n    for epoch in range(EPOCHS):\n        model.train()\n        total_train = 0\n        for imgs, masks in tqdm(train_dl):\n            imgs, masks = imgs.to(DEVICE), masks.to(DEVICE)\n            optimizer.zero_grad()\n            with torch.amp.autocast(\"cuda\"):\n                preds = model(imgs)\n                loss = loss_fn(preds, masks)\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            total_train += loss.item()\n        train_loss = total_train / len(train_dl)\n\n        # Validation\n        model.eval()\n        total_val = 0\n        with torch.no_grad():\n            for imgs, masks in val_dl:\n                imgs, masks = imgs.to(DEVICE), masks.to(DEVICE)\n                with torch.amp.autocast(\"cuda\"):\n                    preds = model(imgs)\n                    val_loss = loss_fn(preds, masks)\n                total_val += val_loss.item()\n        val_loss = total_val / len(val_dl)\n        print(f\"[{size}] Epoch {epoch+1}/{EPOCHS} | Train={train_loss:.4f} | Val={val_loss:.4f}\")\n\n        if val_loss < best_val:\n            best_val = val_loss\n            torch.save(model.state_dict(), \"model_final.pth\")\n            print(\"✅ Model improved and saved.\")\n            early = 0\n        else:\n            early += 1\n        if early >= EARLY_STOP:\n            print(\"⏹️ Early stopping triggered.\")\n            break\n\nprint(\"\\n🎯 Training complete. model_final.pth saved.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T05:59:44.738156Z","iopub.execute_input":"2025-10-25T05:59:44.738909Z","iopub.status.idle":"2025-10-25T06:27:32.646745Z","shell.execute_reply.started":"2025-10-25T05:59:44.738884Z","shell.execute_reply":"2025-10-25T06:27:32.645573Z"}},"outputs":[],"execution_count":null}]}