{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":113558,"databundleVersionId":14174843,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport cv2\nimport json\nimport torch\nimport numpy as np\nimport pandas as pd\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nimport time\nimport warnings\n\n# 忽略不必要的警告\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-09T07:15:12.305671Z","iopub.execute_input":"2025-11-09T07:15:12.306138Z","iopub.status.idle":"2025-11-09T07:15:12.312625Z","shell.execute_reply.started":"2025-11-09T07:15:12.306110Z","shell.execute_reply":"2025-11-09T07:15:12.311319Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"All Config.","metadata":{}},{"cell_type":"code","source":"# cell 2\nclass CFG:\n    # 1. 路径\n    BASE_PATH = '/kaggle/input/recodai-luc-scientific-image-forgery-detection'\n    TRAIN_AUTH_PATH = f'{BASE_PATH}/train_images/authentic'\n    TRAIN_FORGED_PATH = f'{BASE_PATH}/train_images/forged'\n    TRAIN_MASKS_PATH = f'{BASE_PATH}/train_masks'\n    TEST_PATH = f'{BASE_PATH}/test_images'\n    SAMPLE_SUB_PATH = f'{BASE_PATH}/sample_submission.csv'\n    \n    # 2. 设备\n    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    # 3. 训练参数 (平衡性能和时间)\n    IMG_SIZE = 256        # <-- 关键修改：降回 128x128，节省 4 倍时间\n    BATCH_SIZE = 16       # 如果 GPU 内存溢出 (OOM)，请降到 8\n    NUM_EPOCHS = 80       # 80个！\n    LR = 0.001            # 学习率\n    VAL_SPLIT = 0.2       # 20% 的数据用于验证\n    SEED = 42             # 保证可复现性\n    \n    # 4. 数据加载器优化 (!!! 关键速度提升 !!!)\n    NUM_WORKERS = 2       # 使用 2 个 worker 来预加载数据，而不是 0\n    PIN_MEMORY = True     # 锁定内存，加快 GPU 数据传输\n    \n    # 5. 推理参数\n    INFERENCE_THRESHOLD = 0.5  # 预测掩码的阈值\n    MIN_MASK_SIZE = 100        # 小于此像素数的掩码将被忽略\n\n# 打印配置\nprint(f\"Using device: {CFG.DEVICE}\")\nprint(f\"Image size: {CFG.IMG_SIZE}x{CFG.IMG_SIZE}\")\nprint(f\"Batch size: {CFG.BATCH_SIZE}\")\nprint(f\"Epochs: {CFG.NUM_EPOCHS}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T07:15:42.137198Z","iopub.execute_input":"2025-11-09T07:15:42.137589Z","iopub.status.idle":"2025-11-09T07:15:42.146151Z","shell.execute_reply.started":"2025-11-09T07:15:42.137557Z","shell.execute_reply":"2025-11-09T07:15:42.144790Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"RLE function","metadata":{}},{"cell_type":"code","source":"#cell 3\ndef rle_encode(mask):\n    \"\"\"\n    Fast RLE encoding (column-major)\n    \"\"\"\n    pixels = mask.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    \n    # 转换为 JSON 格式\n    if len(runs) == 0:\n        return json.dumps([])\n    else:\n        return json.dumps([int(x) for x in runs])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T07:16:12.703432Z","iopub.execute_input":"2025-11-09T07:16:12.703824Z","iopub.status.idle":"2025-11-09T07:16:12.710060Z","shell.execute_reply.started":"2025-11-09T07:16:12.703793Z","shell.execute_reply":"2025-11-09T07:16:12.708911Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"dataset","metadata":{}},{"cell_type":"code","source":"#cell 4\nclass ForgeryDataset(Dataset):\n    def __init__(self, samples, masks_path, img_size, is_train=True):\n        self.samples = samples\n        self.masks_path = masks_path\n        self.img_size = img_size\n        self.is_train = is_train\n\n    def __len__(self):\n        return len(self.samples)\n\n    def __getitem__(self, idx):\n        img_path, is_forged = self.samples[idx]\n        \n        # 1. 加载图像并调整大小\n        # 使用 OpenCV 加载更快\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (self.img_size, self.img_size))\n        \n        # 2. 加载或创建掩码\n        mask_path = os.path.join(self.masks_path, f\"{os.path.basename(img_path).split('.')[0]}.npy\")\n        \n        if is_forged and os.path.exists(mask_path):\n            try:\n                mask = np.load(mask_path)\n                # 处理多通道掩码\n                if mask.ndim == 3:\n                    mask = mask.max(axis=0) if mask.shape[0] <= 10 else mask.max(axis=-1)\n                mask = cv2.resize(mask.astype(np.uint8), (self.img_size, self.img_size))\n                mask = (mask > 0).astype(np.float32)\n            except Exception as e:\n                # print(f\"Warning: Error loading mask {mask_path}. Using empty mask. Error: {e}\")\n                mask = np.zeros((self.img_size, self.img_size), dtype=np.float32)\n        else:\n            # *** 这是关键：为真实图像创建空掩码 ***\n            mask = np.zeros((self.img_size, self.img_size), dtype=np.float32)\n            \n        # 3. 归一化和转换\n        img = img.astype(np.float32) / 255.0\n        img = torch.from_numpy(img).permute(2, 0, 1) # HWC -> CWH\n        mask = torch.from_numpy(mask).unsqueeze(0)   # HW -> 1HW\n        \n        return img, mask","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T07:16:30.588512Z","iopub.execute_input":"2025-11-09T07:16:30.589418Z","iopub.status.idle":"2025-11-09T07:16:30.598571Z","shell.execute_reply.started":"2025-11-09T07:16:30.589362Z","shell.execute_reply":"2025-11-09T07:16:30.597649Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"fastunet","metadata":{}},{"cell_type":"code","source":"#cell 5\n# --- 替换 单元 5 的全部内容 ---\n\nclass FastUNet(nn.Module):\n    \"\"\"\n    Extremely lightweight U-Net for fast training\n    (来自原版 ultra-fast notebook)\n    \"\"\"\n    \n    def __init__(self, in_channels=3, out_channels=1):\n        super().__init__()\n        \n        # Encoder (downsampling)\n        self.enc1 = self.conv_block(in_channels, 32)\n        self.enc2 = self.conv_block(32, 64)\n        self.enc3 = self.conv_block(64, 128)\n        \n        # Bottleneck\n        self.bottleneck = self.conv_block(128, 256)\n        \n        # Decoder (upsampling)\n        self.up3 = nn.ConvTranspose2d(256, 128, 2, 2)\n        self.dec3 = self.conv_block(256, 128) # 128 (up) + 128 (e3) = 256\n        \n        self.up2 = nn.ConvTranspose2d(128, 64, 2, 2)\n        self.dec2 = self.conv_block(128, 64)  # 64 (up) + 64 (e2) = 128\n        \n        self.up1 = nn.ConvTranspose2d(64, 32, 2, 2)\n        self.dec1 = self.conv_block(64, 32)   # 32 (up) + 32 (e1) = 64\n        \n        # Output\n        self.out = nn.Conv2d(32, out_channels, 1)\n        \n        self.pool = nn.MaxPool2d(2, 2)\n    \n    def conv_block(self, in_ch, out_ch):\n        return nn.Sequential(\n            nn.Conv2d(in_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_ch, out_ch, 3, padding=1),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True)\n        )\n    \n    def forward(self, x):\n        # Encoder\n        e1 = self.enc1(x)\n        e2 = self.enc2(self.pool(e1))\n        e3 = self.enc3(self.pool(e2))\n        \n        # Bottleneck\n        b = self.bottleneck(self.pool(e3))\n        \n        # Decoder\n        d3 = self.up3(b)\n        d3 = torch.cat([d3, e3], dim=1)\n        d3 = self.dec3(d3)\n        \n        d2 = self.up2(d3)\n        d2 = torch.cat([d2, e2], dim=1)\n        d2 = self.dec2(d2)\n        \n        d1 = self.up1(d2)\n        d1 = torch.cat([d1, e1], dim=1)\n        d1 = self.dec1(d1)\n        \n        return torch.sigmoid(self.out(d1))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"HybridLoss","metadata":{}},{"cell_type":"code","source":"#cell 6\nclass DiceLoss(nn.Module):\n    def __init__(self, smooth=1e-6):\n        super(DiceLoss, self).__init__()\n        self.smooth = smooth\n\n    def forward(self, inputs, targets):\n        inputs = inputs.view(-1)\n        targets = targets.view(-1)\n        intersection = (inputs * targets).sum()\n        dice = (2. * intersection + self.smooth) / (inputs.sum() + targets.sum() + self.smooth)\n        return 1 - dice\n\nclass HybridLoss(nn.Module):\n    def __init__(self, bce_weight=0.5, dice_weight=0.5):\n        super(HybridLoss, self).__init__()\n        self.bce = nn.BCELoss()\n        self.dice = DiceLoss()\n        self.bce_weight = bce_weight\n        self.dice_weight = dice_weight\n\n    def forward(self, inputs, targets):\n        bce_loss = self.bce(inputs, targets)\n        dice_loss = self.dice(inputs, targets)\n        return self.bce_weight * bce_loss + self.dice_weight * dice_loss","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"training and validaiton","metadata":{}},{"cell_type":"code","source":"#cell 7\ndef train_epoch(model, loader, optimizer, criterion, device):\n    model.train()\n    total_loss = 0\n    \n    for imgs, masks in tqdm(loader, desc=\"Training\"):\n        imgs = imgs.to(device, non_blocking=True)\n        masks = masks.to(device, non_blocking=True)\n        \n        optimizer.zero_grad()\n        outputs = model(imgs)\n        loss = criterion(outputs, masks)\n        loss.backward()\n        optimizer.step()\n        \n        total_loss += loss.item()\n    \n    return total_loss / len(loader)\n\ndef validate_epoch(model, loader, criterion, device):\n    model.eval()\n    total_loss = 0\n    \n    with torch.no_grad():\n        for imgs, masks in tqdm(loader, desc=\"Validating\"):\n            imgs = imgs.to(device, non_blocking=True)\n            masks = masks.to(device, non_blocking=True)\n            \n            outputs = model(imgs)\n            loss = criterion(outputs, masks)\n            total_loss += loss.item()\n    \n    return total_loss / len(loader)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"main","metadata":{}},{"cell_type":"code","source":"#cell8\n# 设置随机种子\nnp.random.seed(CFG.SEED)\ntorch.manual_seed(CFG.SEED)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed(CFG.SEED)\n\nprint(\"Starting data preparation...\")\n\n# 1. 收集所有样本\nall_samples = []\nfor file in os.listdir(CFG.TRAIN_AUTH_PATH):\n    all_samples.append((os.path.join(CFG.TRAIN_AUTH_PATH, file), 0))\n\nfor file in os.listdir(CFG.TRAIN_FORGED_PATH):\n    all_samples.append((os.path.join(CFG.TRAIN_FORGED_PATH, file), 1))\n\nprint(f\"Total samples collected: {len(all_samples)}\")\n\n# 2. 分割训练集和验证集\ntrain_samples, val_samples = train_test_split(all_samples, \n                                              test_size=CFG.VAL_SPLIT, \n                                              random_state=CFG.SEED,\n                                              # 确保伪造/真实的比例在两组中相似\n                                              stratify=[s[1] for s in all_samples])\n\nprint(f\"Training samples: {len(train_samples)}\")\nprint(f\"Validation samples: {len(val_samples)}\")\n\n# 3. 创建 Datasets\ntrain_dataset = ForgeryDataset(train_samples, CFG.TRAIN_MASKS_PATH, CFG.IMG_SIZE, is_train=True)\nval_dataset = ForgeryDataset(val_samples, CFG.TRAIN_MASKS_PATH, CFG.IMG_SIZE, is_train=False)\n\n# 4. 创建 DataLoaders (!!! 关键速度优化 !!!)\ntrain_loader = DataLoader(\n    train_dataset,\n    batch_size=CFG.BATCH_SIZE,\n    shuffle=True,\n    num_workers=CFG.NUM_WORKERS,  # 使用多进程加载数据\n    pin_memory=CFG.PIN_MEMORY     # 锁定内存\n)\n\nval_loader = DataLoader(\n    val_dataset,\n    batch_size=CFG.BATCH_SIZE,\n    shuffle=False,\n    num_workers=CFG.NUM_WORKERS,\n    pin_memory=CFG.PIN_MEMORY\n)\n\nprint(\"\\nStarting model training...\")\nstart_time = time.time()\n\n# 5. 初始化模型、损失和优化器\nmodel = FastUNet().to(CFG.DEVICE)\ncriterion = HybridLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=CFG.LR)\n# 修正后的代码：每 8 个 epoch 降低学习率，以获得更长时间的收敛\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=8, gamma=0.1)\n\nbest_val_loss = float('inf')\nbest_model_path = 'best_model.pth'\n\n# 6. 训练循环\nfor epoch in range(CFG.NUM_EPOCHS):\n    print(f\"\\n--- Epoch {epoch+1}/{CFG.NUM_EPOCHS} ---\")\n    \n    train_loss = train_epoch(model, train_loader, optimizer, criterion, CFG.DEVICE)\n    val_loss = validate_epoch(model, val_loader, criterion, CFG.DEVICE)\n    scheduler.step()\n    \n    print(f\"Epoch {epoch+1}: Train Loss = {train_loss:.4f}, Val Loss = {val_loss:.4f}\")\n    \n    # 保存最佳模型\n    if val_loss < best_val_loss:\n        print(f\"Validation loss improved ({best_val_loss:.4f} -> {val_loss:.4f}). Saving model...\")\n        best_val_loss = val_loss\n        torch.save(model.state_dict(), best_model_path)\n\nend_time = time.time()\nprint(f\"\\nTraining finished in {(end_time - start_time) / 60:.2f} minutes.\")\nprint(f\"Best model saved to {best_model_path} with Val Loss: {best_val_loss:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"inference and submit","metadata":{}},{"cell_type":"code","source":"#cell 9\nprint(\"Starting inference...\")\n\n# 1. 加载我们保存的最佳模型\nmodel = FastUNet().to(CFG.DEVICE)\nmodel.load_state_dict(torch.load(best_model_path))\nmodel.eval()\n\npredictions = {}\ntest_files = sorted(os.listdir(CFG.TEST_PATH))\n\nwith torch.no_grad():\n    for file in tqdm(test_files, desc=\"Generating Predictions\"):\n        case_id = file.split('.')[0]\n        \n        # 1. 加载和预处理测试图像\n        img_path = os.path.join(CFG.TEST_PATH, file)\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        original_size = img.shape[:2] # (H, W)\n        \n        # 缩放到模型输入尺寸\n        img_resized = cv2.resize(img, (CFG.IMG_SIZE, CFG.IMG_SIZE))\n        img_tensor = (img_resized.astype(np.float32) / 255.0)\n        img_tensor = torch.from_numpy(img_tensor).permute(2, 0, 1).unsqueeze(0).to(CFG.DEVICE)\n        \n        # 2. 模型预测\n        mask_pred = model(img_tensor)[0, 0].cpu().numpy()\n        \n        # 3. 后处理\n        # 缩放回原始尺寸\n        mask_pred = cv2.resize(mask_pred, (original_size[1], original_size[0]), \n                               interpolation=cv2.INTER_NEAREST)\n        \n        # 应用阈值\n        mask_binary = (mask_pred > CFG.INFERENCE_THRESHOLD).astype(np.uint8)\n        \n        # (可选) 形态学后处理，清理小噪点\n        # kernel = np.ones((3, 3), np.uint8)\n        # mask_binary = cv2.morphologyEx(mask_binary, cv2.MORPH_OPEN, kernel, iterations=1)\n        \n        # 4. RLE 编码\n        if mask_binary.sum() < CFG.MIN_MASK_SIZE:\n            predictions[case_id] = \"authentic\"\n        else:\n            predictions[case_id] = rle_encode(mask_binary)\n\nprint(\"Inference complete. Creating submission file...\")\n\n# 5. 创建提交文件\nsample_df = pd.read_csv(CFG.SAMPLE_SUB_PATH)\nsubmission_data = []\n\nfor case_id in sample_df['case_id']:\n    annotation = predictions.get(str(case_id), \"authentic\")\n    submission_data.append({'case_id': case_id, 'annotation': annotation})\n\nsubmission_df = pd.DataFrame(submission_data)\nsubmission_df.to_csv('submission.csv', index=False)\n\nprint(\"submission.csv created successfully!\")\nprint(submission_df.head())","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}