{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":113558,"databundleVersionId":14174843,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"## 📊 Exploratory Data Analysis (EDA)\n\nimport os\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport cv2\nfrom tqdm.auto import tqdm\n\n# --- CONFIGURATION (from the original notebook) ---\nTRAIN_ROOT = \"/kaggle/input/recodai-luc-scientific-image-forgery-detection/train_images\"\nMASK_ROOT = \"/kaggle/input/recodai-luc-scientific-image-forgery-detection/train_masks\"\n\nprint(\"--- Starting Basic EDA ---\")\n\n# 1. Prepare Data List (same logic as in the notebook)\ndata_list = []\nfor root, _, files in os.walk(TRAIN_ROOT):\n    for f in files:\n        valid_extensions = ('.png', '.jpg', '.jpeg', '.tif', '.tiff', '.npy')\n        if f.lower().endswith(valid_extensions):\n            if 'forged' in root.lower():\n                case_id = os.path.splitext(f)[0]\n                img_path = os.path.join(root, f)\n                mask_path = os.path.join(MASK_ROOT, f\"{case_id}.npy\")\n                data_list.append({\n                    'case_id': case_id,\n                    'img_path': img_path,\n                    'mask_path': mask_path\n                })\nfull_df = pd.DataFrame(data_list)\n\n# Filter for images with existing masks\nif not full_df.empty:\n    full_df['mask_exists'] = full_df['mask_path'].apply(os.path.exists)\n    eda_df = full_df[full_df['mask_exists']].drop(columns=['mask_exists']).reset_index(drop=True)\nelse:\n    eda_df = pd.DataFrame(columns=['case_id', 'img_path', 'mask_path'])\n\n\n# 2. File and Case Counts\nprint(f\"\\nTotal potential images found in TRAIN_ROOT: {len(data_list)}\")\nprint(f\"Total valid image/mask pairs for training (Forged Cases): {len(eda_df)}\")\n\n# 3. Image and Mask Size Distribution\nif not eda_df.empty:\n    img_heights, img_widths = [], []\n    mask_pixels = [] # Count of non-zero pixels in the mask\n    \n    print(\"\\nAnalyzing image and mask dimensions/forgery area...\")\n    for index, row in tqdm(eda_df.iterrows(), total=len(eda_df)):\n        try:\n            # Read Image\n            img = cv2.imread(row['img_path'])\n            if img is not None and img.size > 0:\n                h, w = img.shape[:2]\n                img_heights.append(h)\n                img_widths.append(w)\n\n            # Read Mask (npy file)\n            mask = np.load(row['mask_path'])\n            # Assuming the forgery area is represented by non-zero pixels\n            mask_pixels.append(np.sum(mask > 0)) \n\n        except Exception as e:\n            # Handle files that can't be read (e.g., non-image .npy files not handled by cv2.imread)\n            pass\n\n    print(\"\\n--- Image Dimension Stats ---\")\n    print(f\"Unique Image Widths: {sorted(list(set(img_widths)))[:5]}{'...' if len(set(img_widths)) > 5 else ''}\")\n    print(f\"Unique Image Heights: {sorted(list(set(img_heights)))[:5]}{'...' if len(set(img_heights)) > 5 else ''}\")\n    print(f\"Mean Image Dimensions (H x W): {np.mean(img_heights):.0f} x {np.mean(img_widths):.0f}\")\n\n    # 4. Forgery Area Analysis\n    mask_pixels = np.array(mask_pixels)\n    forged_cases_with_area = np.sum(mask_pixels > 0)\n    total_forgery_area = np.sum(mask_pixels)\n    \n    print(\"\\n--- Forgery Area Stats ---\")\n    print(f\"Total cases with non-zero forgery area: {forged_cases_with_area} / {len(eda_df)}\")\n    print(f\"Mean Forgery Pixel Count per forged image: {np.mean(mask_pixels[mask_pixels > 0]):.0f} (pixels)\")\n    print(f\"Maximum Forgery Pixel Count: {np.max(mask_pixels)} (pixels)\")\n\n    # 5. Visualization: Forgery Area Distribution (First 100 cases for quick view)\n    plt.figure(figsize=(12, 5))\n    plt.bar(range(min(100, len(mask_pixels))), mask_pixels[:100])\n    plt.title('Forgery Pixel Count (First 100 Forged Samples)')\n    plt.xlabel('Sample Index')\n    plt.ylabel('Forged Pixel Count (Area)')\n    plt.show()\n\nelse:\n    print(\"🛑 EDA Skipped: The dataframe is empty. Check TRAIN_ROOT and MASK_ROOT paths.\")\n\nprint(\"\\n--- EDA Complete ---\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T18:56:32.918084Z","iopub.execute_input":"2025-11-08T18:56:32.918447Z","iopub.status.idle":"2025-11-08T18:58:55.507773Z","shell.execute_reply.started":"2025-11-08T18:56:32.918423Z","shell.execute_reply":"2025-11-08T18:58:55.507031Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport matplotlib.pyplot as plt\nfrom tqdm.auto import tqdm\nimport warnings\nfrom warnings import filterwarnings\n\nfilterwarnings('ignore') # Suppress warnings\n\n# --- CONFIGURATION (from the original notebook) ---\nTARGET_SIZE = 256\nTRAIN_ROOT = \"/kaggle/input/recodai-luc-scientific-image-forgery-detection/train_images\"\nMASK_ROOT = \"/kaggle/input/recodai-luc-scientific-image-forgery-detection/train_masks\"\n\n# Replicate compute_ela for feature analysis\ndef compute_ela(img_path, quality=95, scale=10):\n    # ... (omitted for brevity, assume the original function is available)\n    # The original notebook's ELA function is used here.\n    img = cv2.imread(img_path)\n    if img is None or img.size == 0:\n        try:\n            img_data = np.load(img_path)\n            if img_data.ndim == 3: img = cv2.cvtColor(img_data, cv2.COLOR_RGB2BGR)\n            elif img_data.ndim == 2: img = cv2.cvtColor(img_data, cv2.COLOR_GRAY2BGR)\n        except Exception: return np.zeros((TARGET_SIZE, TARGET_SIZE), dtype=np.float32)\n\n    if img is None or img.size == 0:\n        return np.zeros((TARGET_SIZE, TARGET_SIZE), dtype=np.float32)\n\n    img_resized = cv2.resize(img, (TARGET_SIZE, TARGET_SIZE))\n    temp_path = f\"/tmp/temp_ela_{os.path.basename(img_path)}.jpg\" # Simplified temp_path\n    try:\n        # Use a consistent quality setting (95)\n        cv2.imwrite(temp_path, img_resized, [cv2.IMWRITE_JPEG_QUALITY, quality]) \n        compressed_img = cv2.imread(temp_path)\n        if compressed_img is None: return np.zeros((TARGET_SIZE, TARGET_SIZE), dtype=np.float32)\n        error = np.abs(img_resized.astype(np.float32) - compressed_img.astype(np.float32))\n        ela_feature_2d = np.mean(error, axis=2) * scale # Scale by 10 as in the notebook\n    finally:\n        if os.path.exists(temp_path): os.remove(temp_path)\n    return cv2.resize(ela_feature_2d, (TARGET_SIZE, TARGET_SIZE), interpolation=cv2.INTER_LINEAR).astype(np.float32)\n\n# Load the filtered DataFrame (assuming the prior EDA cell's 'eda_df' is available or recreate it)\ndata_list = []\nfor root, _, files in os.walk(TRAIN_ROOT):\n    for f in files:\n        valid_extensions = ('.png', '.jpg', '.jpeg', '.tif', '.tiff', '.npy')\n        if f.lower().endswith(valid_extensions) and 'forged' in root.lower():\n            case_id = os.path.splitext(f)[0]\n            mask_path = os.path.join(MASK_ROOT, f\"{case_id}.npy\")\n            if os.path.exists(mask_path):\n                data_list.append({'img_path': os.path.join(root, f), 'mask_path': mask_path})\neda_df = pd.DataFrame(data_list)\n\nprint(\"--- Starting Advanced EDA (Imbalance & Feature Check) ---\")\n\nif eda_df.empty:\n    print(\"🛑 EDA Skipped: Data frame is empty.\")\nelse:\n    total_pixels = 0\n    forgery_pixels = 0\n    ela_values, rgb_means = [], []\n\n    # Process only the first 50 images to speed up ELA computation for EDA\n    for index, row in tqdm(eda_df.head(50).iterrows(), total=len(eda_df.head(50)), desc=\"Processing samples\"):\n        try:\n            # 1. Image and Mask Load\n            rgb_image = cv2.cvtColor(cv2.imread(row['img_path']), cv2.COLOR_BGR2RGB)\n            if rgb_image is None or rgb_image.size == 0: continue\n                \n            mask = np.load(row['mask_path'])\n            if mask.ndim > 2: mask = mask[:, :, 0]\n            \n            # 2. Imbalance Check (Use original sizes for best estimate)\n            h, w = rgb_image.shape[:2]\n            total_pixels += h * w\n            forgery_pixels += np.sum(mask > 0)\n            \n            # 3. ELA Feature Check (Use 256x256 resized data)\n            ela_feature = compute_ela(row['img_path'])\n            ela_values.extend(ela_feature.flatten())\n            \n            # RGB feature check (resize/normalize similar to training)\n            rgb_resized = cv2.resize(rgb_image, (TARGET_SIZE, TARGET_SIZE)) / 255.0\n            rgb_means.extend(rgb_resized.mean(axis=2).flatten())\n\n        except Exception as e:\n            # print(f\"Warning: Could not process {row['img_path']}: {e}\")\n            continue\n\n    # --- Analysis 1: Imbalance Ratio ---\n    if total_pixels > 0:\n        imbalance_ratio = (forgery_pixels / total_pixels) * 100\n        print(f\"\\n--- Imbalance Ratio (Forged Pixels) ---\")\n        print(f\"Total Pixels Sampled: {total_pixels:,}\")\n        print(f\"Forged Pixels Sampled: {forgery_pixels:,}\")\n        print(f\"Forgery Imbalance Ratio: **{imbalance_ratio:.2f}%** (Positive Class)\")\n    \n    # --- Analysis 2: ELA Feature Distribution vs. RGB ---\n    if ela_values:\n        ela_values = np.array(ela_values)\n        rgb_means = np.array(rgb_means)\n\n        print(f\"\\n--- ELA Feature Distribution (Scaled by 10) ---\")\n        print(f\"ELA Feature Mean: {np.mean(ela_values):.4f}\")\n        print(f\"ELA Feature Std Dev: {np.std(ela_values):.4f}\")\n        print(f\"RGB Mean (Normalized): {np.mean(rgb_means):.4f}\")\n\n        plt.figure(figsize=(12, 5))\n        plt.hist(ela_values, bins=50, alpha=0.6, label='ELA Feature (Scaled)', color='red')\n        plt.title('Distribution of ELA Feature Values')\n        plt.xlabel('ELA Value (0 to ~2550)')\n        plt.ylabel('Frequency')\n        plt.legend()\n        plt.show()\n        \n        # This histogram helps visualize if ELA is predominantly zero or clustered.\n\nprint(\"\\n--- Advanced EDA Complete ---\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T18:58:55.509173Z","iopub.execute_input":"2025-11-08T18:58:55.509824Z","iopub.status.idle":"2025-11-08T18:59:04.795821Z","shell.execute_reply.started":"2025-11-08T18:58:55.509804Z","shell.execute_reply":"2025-11-08T18:59:04.795015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport cv2\nimport os\n\n# Define the file path\nTEST_IMAGE_PATH = \"/kaggle/input/recodai-luc-scientific-image-forgery-detection/test_images/45.png\"\n\nprint(f\"Attempting to load image: {TEST_IMAGE_PATH}\")\n\nif not os.path.exists(TEST_IMAGE_PATH):\n    print(\"🛑 ERROR: The file path was not found. Please ensure the Kaggle competition data is mounted correctly.\")\nelse:\n    # Load the image using OpenCV (loads as BGR)\n    img = cv2.imread(TEST_IMAGE_PATH)\n    \n    if img is None:\n        print(\"🛑 ERROR: Could not read the image file.\")\n    else:\n        # Convert the image from BGR (OpenCV default) to RGB (Matplotlib default)\n        img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        # Plot the image\n        plt.figure(figsize=(10, 8))\n        plt.imshow(img_rgb)\n        plt.title(f\"Test Image 45 (Dimensions: {img.shape[0]}x{img.shape[1]})\")\n        plt.axis('off') # Hide axes for a cleaner image view\n        plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T18:59:04.796568Z","iopub.execute_input":"2025-11-08T18:59:04.796807Z","iopub.status.idle":"2025-11-08T18:59:05.241194Z","shell.execute_reply.started":"2025-11-08T18:59:04.796780Z","shell.execute_reply":"2025-11-08T18:59:05.240441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport cv2\nimport os\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm.auto import tqdm\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport time\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nimport warnings\nfrom warnings import filterwarnings\n\n# Suppress the specific UserWarning from the LR scheduler\nwarnings.filterwarnings(\"ignore\", category=UserWarning, module=\"torch.optim.lr_scheduler\")\nfilterwarnings('ignore')\n\n# --- CONFIGURATION --\nTARGET_SIZE = 256\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nBATCH_SIZE = 8\n# REDUCED EPOCHS to 3 (was 10)\nEPOCHS = 2\nLEARNING_RATE = 1e-4\n\n# --- PATHS (CORRECTED FOR KAGGLEHUB CACHE) ---\nTRAIN_ROOT = \"/kaggle/input/recodai-luc-scientific-image-forgery-detection/train_images\"\nMASK_ROOT = \"/kaggle/input/recodai-luc-scientific-image-forgery-detection/train_masks\"\nMODEL_SAVE_PATH = \"/tmp/model_new_scratch.pth\"\n\n# --- UTILITY FUNCTIONS ---\ndef compute_ela(img_path, quality=95, scale=10):\n    img = cv2.imread(img_path)\n    if img is None or img.size == 0:\n        try:\n            img_data = np.load(img_path)\n            if img_data.ndim == 3:\n                img = cv2.cvtColor(img_data, cv2.COLOR_RGB2BGR)\n            elif img_data.ndim == 2:\n                img = cv2.cvtColor(img_data, cv2.COLOR_GRAY2BGR)\n        except Exception:\n            return np.zeros((TARGET_SIZE, TARGET_SIZE), dtype=np.float32)\n\n    if img is None or img.size == 0:\n        return np.zeros((TARGET_SIZE, TARGET_SIZE), dtype=np.float32)\n\n    img_resized = cv2.resize(img, (TARGET_SIZE, TARGET_SIZE))\n    temp_path = f\"/tmp/temp_ela_{os.path.basename(img_path)}_{time.time()}.jpg\"\n    try:\n        cv2.imwrite(temp_path, img_resized, [cv2.IMWRITE_JPEG_QUALITY, quality])\n        compressed_img = cv2.imread(temp_path)\n        if compressed_img is None: return np.zeros((TARGET_SIZE, TARGET_SIZE), dtype=np.float32)\n        error = np.abs(img_resized.astype(np.float32) - compressed_img.astype(np.float32))\n        ela_feature_2d = np.mean(error, axis=2) * scale\n    finally:\n        if os.path.exists(temp_path): os.remove(temp_path)\n    return cv2.resize(ela_feature_2d, (TARGET_SIZE, TARGET_SIZE),\n                      interpolation=cv2.INTER_LINEAR).astype(np.float32)\n\nclass DiceLoss(nn.Module):\n    def __init__(self, smooth=1.0):\n        super(DiceLoss, self).__init__()\n        self.smooth = smooth\n    def forward(self, pred, target):\n        pred = pred.contiguous().view(-1)\n        target = target.contiguous().view(-1)\n        intersection = (pred * target).sum()\n        dice = (2. * intersection + self.smooth) / (pred.sum() + target.sum() + self.smooth)\n        return 1 - dice\n\n# Hybrid Loss combining Dice and BCE for stable training\nclass HybridLoss(nn.Module):\n    def __init__(self, dice_weight=0.5):\n        super(HybridLoss, self).__init__()\n        self.dice_loss = DiceLoss()\n        self.bce_loss = nn.BCELoss()\n        self.dice_weight = dice_weight\n\n    def forward(self, pred, target):\n        dice = self.dice_loss(pred, target)\n        bce = self.bce_loss(pred, target)\n        return self.dice_weight * dice + (1 - self.dice_weight) * bce\n\n# U-Net architecture\nclass UNet(nn.Module):\n    def __init__(self, in_channels=4, num_classes=1):\n        super().__init__()\n        def block(in_c, out_c):\n            return nn.Sequential(\n                nn.Conv2d(in_c, out_c, 3, 1, 1), nn.ReLU(),\n                nn.Dropout(p=0.2),\n                nn.Conv2d(out_c, out_c, 3, 1, 1), nn.ReLU()\n            )\n\n        self.enc1 = block(in_channels, 64)\n        self.enc2 = block(64, 128)\n        self.bottleneck = block(128, 256)\n        self.upconv2 = nn.ConvTranspose2d(256, 128, 2, 2)\n        self.dec2 = block(128 + 128, 128)\n        self.upconv1 = nn.ConvTranspose2d(128, 64, 2, 2)\n        self.dec1 = block(64 + 64, 64)\n        self.final_conv = nn.Conv2d(64, num_classes, 1)\n\n    def forward(self, x):\n        e1 = self.enc1(x)\n        p1 = F.max_pool2d(e1, 2)\n        e2 = self.enc2(p1)\n        p2 = F.max_pool2d(e2, 2)\n        b = self.bottleneck(p2)\n        d2 = self.upconv2(b)\n        d2 = torch.cat((d2, e2), dim=1)\n        d2 = self.dec2(d2)\n        d1 = self.upconv1(d2)\n        d1 = torch.cat((d1, e1), dim=1)\n        d1 = self.dec1(d1)\n        return torch.sigmoid(self.final_conv(d1))\n\nclass ForgeryDataset(Dataset):\n    def __init__(self, df):\n        self.df = df\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = row['img_path']\n        mask_path = row['mask_path']\n\n        # --- Load Image ---\n        rgb_image = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)\n\n        if rgb_image is None or rgb_image.size == 0:\n            try:\n                img_data = np.load(img_path)\n                if img_data.ndim == 3:\n                    rgb_image = img_data\n                elif img_data.ndim == 2:\n                    rgb_image = cv2.cvtColor(img_data, cv2.COLOR_GRAY2RGB)\n            except Exception as e:\n                raise RuntimeError(f\"Failed to load image from {img_path}: {e}\")\n\n        # --- Load Mask ---\n        try:\n            mask = np.load(mask_path)\n            if mask.ndim > 2:\n                mask = mask[:, :, 0]\n        except Exception as e:\n            raise RuntimeError(f\"Failed to load mask from {mask_path}: {e}\")\n\n        ela_feature_2d = compute_ela(img_path)\n\n        # Resize all features\n        rgb_image_resized = cv2.resize(rgb_image, (TARGET_SIZE, TARGET_SIZE))\n        ela_feature_resized = cv2.resize(ela_feature_2d, (TARGET_SIZE, TARGET_SIZE))\n\n        # Use INTER_NEAREST for binary mask resizing\n        mask_resized = cv2.resize(mask.astype(np.uint8), (TARGET_SIZE, TARGET_SIZE), interpolation=cv2.INTER_NEAREST)\n\n        # Stack RGB (3) and ELA (1) for a 4-channel input\n        ela_feature_3d = np.expand_dims(ela_feature_resized, axis=-1)\n        stacked_input = np.concatenate([rgb_image_resized, ela_feature_3d], axis=-1)\n\n        # Convert to PyTorch tensors and normalize\n        image = torch.tensor(stacked_input.transpose(2, 0, 1) / 255.0, dtype=torch.float32)\n        mask = torch.tensor(mask_resized / 255.0, dtype=torch.float32).unsqueeze(0)\n\n        return image, mask\n\ndef train_model(model, train_loader, val_loader, epochs=EPOCHS, save_path=MODEL_SAVE_PATH):\n    # Use HybridLoss\n    criterion = HybridLoss(dice_weight=0.5)\n    optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)\n\n    # Scheduler with patience=2 (unchanged)\n    scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=2)\n\n    best_val_loss = float('inf')\n\n    model.to(DEVICE)\n    print(f\"Starting training on {DEVICE} for {epochs} epochs...\")\n\n    for epoch in range(epochs):\n        model.train()\n        train_loss_sum = 0\n\n        for inputs, targets in tqdm(train_loader, desc=f\"Epoch {epoch+1} Training\"):\n            inputs, targets = inputs.to(DEVICE), targets.to(DEVICE)\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n            loss.backward()\n            optimizer.step()\n            train_loss_sum += loss.item()\n\n        avg_train_loss = train_loss_sum / len(train_loader)\n\n        # Validation Phase\n        model.eval()\n        val_loss_sum = 0\n        with torch.no_grad():\n            for inputs, targets in val_loader:\n                inputs, targets = inputs.to(DEVICE), targets.to(DEVICE)\n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                val_loss_sum += loss.item()\n\n        avg_val_loss = val_loss_sum / len(val_loader)\n\n        # Scheduler Step\n        scheduler.step(avg_val_loss)\n\n        print(f\"Epoch {epoch+1} | Train Loss: {avg_train_loss:.4f} | Val Loss: {avg_val_loss:.4f}\")\n\n        if avg_val_loss < best_val_loss:\n            best_val_loss = avg_val_loss\n            torch.save(model.state_dict(), save_path)\n            print(f\"Model saved successfully to {save_path}. (New best Val Loss: {best_val_loss:.4f})\\n\")\n\n        current_lr = optimizer.param_groups[0]['lr']\n        print(f\"Current Learning Rate: {current_lr:.6f}\")\n\n\n# --- MAIN EXECUTION BLOCK ---\nif __name__ == '__main__':\n\n    print(\"Preparing training data paths...\\n\")\n\n    data_list = []\n\n    # Recursively walk through the TRAIN_ROOT to find all image files\n    for root, _, files in os.walk(TRAIN_ROOT):\n        for f in files:\n            valid_extensions = ('.png', '.jpg', '.jpeg', '.tif', '.tiff', '.npy')\n\n            if f.lower().endswith(valid_extensions):\n                # Only process files in the 'forged' subdirectory, as only they have masks\n                if 'forged' in root.lower():\n                    case_id = os.path.splitext(f)[0]\n                    img_path = os.path.join(root, f)\n\n                    # Use .npy for the mask extension\n                    mask_path = os.path.join(MASK_ROOT, f\"{case_id}.npy\")\n\n                    data_list.append({\n                        'case_id': case_id,\n                        'img_path': img_path,\n                        'mask_path': mask_path\n                    })\n\n    if not data_list:\n        full_df = pd.DataFrame(columns=['case_id', 'img_path', 'mask_path'])\n    else:\n        full_df = pd.DataFrame(data_list)\n\n    if not full_df.empty:\n        # Final check: Keep only images that have a corresponding mask file\n        full_df['mask_exists'] = full_df['mask_path'].apply(os.path.exists)\n        full_df = full_df[full_df['mask_exists']].drop(columns=['mask_exists']).reset_index(drop=True)\n\n    if full_df.empty:\n        print(\"🛑 FATAL ERROR: No valid image/mask pairs found in the input paths. Cannot train. (Check file extensions/paths again)\")\n    else:\n        print(f\"✅ Found {len(full_df)} valid forged samples for training.\")\n\n        # Split data\n        train_df, val_df = train_test_split(full_df, test_size=0.1, random_state=42)\n\n        # Create DataLoaders\n        train_dataset = ForgeryDataset(train_df)\n        val_dataset = ForgeryDataset(val_df)\n        train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n        val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\n        # Instantiate model (4 input channels: 3 RGB + 1 ELA)\n        model = UNet(in_channels=4)\n\n        # START TRAINING\n        train_model(model, train_loader, val_loader)\n\n        print(\"\\n✅ TRAINING COMPLETE. The trained model is saved and ready for inference.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T18:59:05.242581Z","iopub.execute_input":"2025-11-08T18:59:05.242816Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport cv2\nimport os\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport time\nimport csv\nimport warnings\nfrom warnings import filterwarnings\n\nfilterwarnings('ignore') # Suppress warnings\n\n# --- CONFIGURATION & PATHS (ROBUST SETTINGS) ---\nTARGET_SIZE = 256\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nBATCH_SIZE = 8\n# ROBUST: Conservative threshold for the Private Leaderboard\nFIXED_THRESHOLD = 0.45\n# ROBUST: Moderate filter to remove noise while keeping small artifacts\nMIN_FORGERY_AREA = 32\n\n# CORRECTED PATHS\nTEST_IMAGE_ROOT = \"/kaggle/input/recodai-luc-scientific-image-forgery-detection/test_images\"\nSAMPLE_SUBMISSION_FILE = \"/kaggle/input/recodai-luc-scientific-image-forgery-detection/sample_submission.csv\"\n\n# Model path should point to the successfully trained model\nmodel_path = \"/tmp/model_new_scratch.pth\"\nOUTPUT_FILENAME = \"submission.csv\"\n\n# --- UTILITY FUNCTIONS (Unchanged) ---\n\ndef compute_ela(img_path, quality=95, scale=10):\n    img = cv2.imread(img_path)\n    if img is None or img.size == 0:\n        try:\n            img_data = np.load(img_path)\n            if img_data.ndim == 3: img = cv2.cvtColor(img_data, cv2.COLOR_RGB2BGR)\n            elif img_data.ndim == 2: img = cv2.cvtColor(img_data, cv2.COLOR_GRAY2BGR)\n        except Exception:\n            return np.zeros((TARGET_SIZE, TARGET_SIZE), dtype=np.float32)\n\n    if img is None or img.size == 0:\n        return np.zeros((TARGET_SIZE, TARGET_SIZE), dtype=np.float32)\n\n    img_resized = cv2.resize(img, (TARGET_SIZE, TARGET_SIZE))\n    temp_path = f\"/tmp/temp_{os.path.basename(img_path)}_{time.time()}.jpg\"\n    try:\n        cv2.imwrite(temp_path, img_resized, [cv2.IMWRITE_JPEG_QUALITY, quality])\n        compressed_img = cv2.imread(temp_path)\n        if compressed_img is None: return np.zeros((TARGET_SIZE, TARGET_SIZE), dtype=np.float32)\n        error = np.abs(img_resized.astype(np.float32) - compressed_img.astype(np.float32))\n        ela_feature_2d = np.mean(error, axis=2) * scale\n    finally:\n        if os.path.exists(temp_path): os.remove(temp_path)\n    return cv2.resize(ela_feature_2d, (TARGET_SIZE, TARGET_SIZE),\n                      interpolation=cv2.INTER_LINEAR).astype(np.float32)\n\ndef create_test_df_robust(test_image_root, sample_submission_path):\n    master_df = pd.read_csv(sample_submission_path)\n    master_df['case_id'] = master_df['case_id'].astype(str)\n    present_files = {}\n    if os.path.exists(test_image_root):\n        for root, _, files in os.walk(test_image_root):\n            for f in files:\n                case_id = os.path.splitext(f)[0]\n                if f.lower().endswith(('.png', '.jpg', '.jpeg', '.tif', '.tiff', '.npy')):\n                    present_files[case_id] = os.path.join(root, f)\n\n    master_df['img_path'] = master_df['case_id'].map(present_files)\n    master_df['img_path'] = master_df['img_path'].fillna('MISSING_FILE')\n    return master_df[['case_id', 'img_path']]\n\ndef rle_encode(mask):\n    if mask.sum() == 0: return \"authentic\"\n    pixels = mask.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ', '.join(str(x) for x in runs)\n\nclass UNet(nn.Module):\n    def __init__(self, in_channels=4, num_classes=1):\n        super().__init__()\n        def block(in_c, out_c):\n            return nn.Sequential(\n                nn.Conv2d(in_c, out_c, 3, 1, 1), nn.ReLU(),\n                nn.Dropout(p=0.2),\n                nn.Conv2d(out_c, out_c, 3, 1, 1), nn.ReLU()\n            )\n\n        self.enc1 = block(in_channels, 64)\n        self.enc2 = block(64, 128)\n        self.bottleneck = block(128, 256)\n        self.upconv2 = nn.ConvTranspose2d(256, 128, 2, 2)\n        self.dec2 = block(128 + 128, 128)\n        self.upconv1 = nn.ConvTranspose2d(128, 64, 2, 2)\n        self.dec1 = block(64 + 64, 64)\n        self.final_conv = nn.Conv2d(64, num_classes, 1)\n\n    def forward(self, x):\n        e1 = self.enc1(x)\n        p1 = F.max_pool2d(e1, 2)\n        e2 = self.enc2(p1)\n        p2 = F.max_pool2d(e2, 2)\n        b = self.bottleneck(p2)\n        d2 = self.upconv2(b)\n        d2 = torch.cat((d2, e2), dim=1)\n        d2 = self.dec2(d2)\n        d1 = self.upconv1(d2)\n        d1 = torch.cat((d1, e1), dim=1)\n        d1 = self.dec1(d1)\n        return torch.sigmoid(self.final_conv(d1))\n\n# --- INFERENCE FUNCTION WITH POST-PROCESSING ---\ndef run_inference_and_segment(unet_model, test_df):\n    results = []\n    unet_model.eval()\n    images_to_process = []\n\n    for index, row in tqdm(test_df.iterrows(), total=len(test_df), desc=\"Processing\"):\n        case = row['case_id']\n        img_path = row['img_path']\n\n        if img_path == 'MISSING_FILE' or img_path == 'NOT_FOUND':\n            results.append({'case_id': case, 'annotation': 'authentic'})\n            continue\n\n        try:\n            rgb_image = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)\n            if rgb_image is None or rgb_image.size == 0:\n                img_data = np.load(img_path)\n                if img_data.ndim == 3: rgb_image = img_data\n                elif img_data.ndim == 2: rgb_image = cv2.cvtColor(img_data, cv2.COLOR_GRAY2RGB)\n\n            if rgb_image is None or rgb_image.size == 0: raise ValueError(f\"Invalid image data for {case}\")\n\n            original_shape = rgb_image.shape[:2]\n            ela_feature_2d = compute_ela(img_path)\n\n            rgb_image_resized = cv2.resize(rgb_image, (TARGET_SIZE, TARGET_SIZE))\n            ela_feature_resized = cv2.resize(ela_feature_2d, (TARGET_SIZE, TARGET_SIZE))\n            ela_feature_3d = np.expand_dims(ela_feature_resized, axis=-1)\n            stacked_input = np.concatenate([rgb_image_resized, ela_feature_3d], axis=-1)\n            images_to_process.append((case, original_shape, stacked_input))\n\n            # Process batch\n            if len(images_to_process) == BATCH_SIZE or index == len(test_df) - 1:\n                if images_to_process:\n                    batch_inputs = torch.stack([\n                        torch.tensor(img_data.transpose(2, 0, 1) / 255.0, dtype=torch.float32)\n                        for _, _, img_data in images_to_process\n                    ]).to(DEVICE)\n\n                    with torch.no_grad():\n                        outputs = unet_model(batch_inputs).detach().cpu().numpy()\n\n                    for i, output in enumerate(outputs):\n                        case_id_out, original_shape_out, _ = images_to_process[i]\n                        output_prob = output.squeeze()\n\n                        # --- LOG PROBABILITY HERE ---\n                        max_prob = np.max(output_prob)\n                        print(f\"|--- Case {case_id_out} Max Forgery Probability: {max_prob:.4f} ---|\")\n                        # ----------------------------\n\n                        # Apply Threshold (0.45)\n                        final_mask_resized = (output_prob > FIXED_THRESHOLD).astype(np.uint8)\n\n                        # Minimum Area Filtering (32)\n                        clean_mask_resized = np.zeros_like(final_mask_resized)\n\n                        # Find connected components\n                        num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(\n                            final_mask_resized, 4, cv2.CV_32S\n                        )\n\n                        # Iterate through each component (label 0 is the background)\n                        for label in range(1, num_labels):\n                            area = stats[label, cv2.CC_STAT_AREA]\n                            if area >= MIN_FORGERY_AREA:\n                                # Keep segments that meet the minimum size requirement\n                                clean_mask_resized[labels == label] = 1\n\n                        # Resize the CLEANED mask back to the original size\n                        final_mask = cv2.resize(\n                            clean_mask_resized,\n                            (original_shape_out[1], original_shape_out[0]),\n                            interpolation=cv2.INTER_NEAREST\n                        )\n\n                        rle_annotation = rle_encode(final_mask)\n                        results.append({'case_id': case_id_out, 'annotation': rle_annotation})\n\n                    images_to_process = [] # Reset batch\n        except Exception as e:\n            print(f\"Error processing case {case}: {e}. Defaulting to authentic.\")\n            results.append({'case_id': case, 'annotation': 'authentic'})\n    return pd.DataFrame(results)\n\n# --- MAIN EXECUTION BLOCK ---\nif __name__ == \"__main__\":\n\n    print(f\"--- Starting inference on {DEVICE} at {pd.Timestamp.now()} ---\")\n\n    # 1. Load Model\n    model = None\n    try:\n        model = UNet(in_channels=4).to(DEVICE)\n        model.load_state_dict(torch.load(model_path, map_location=DEVICE))\n        model.eval() # Set model to evaluation mode\n        print(f\"Model loaded successfully from {model_path}\")\n    except Exception as e:\n        print(f\"Error loading model from {model_path}. Submitting 'authentic' for all cases. Error: {e}\")\n        model = None\n\n    # 2. Prepare Data\n    test_df = create_test_df_robust(TEST_IMAGE_ROOT, SAMPLE_SUBMISSION_FILE)\n    test_df['case_id'] = test_df['case_id'].astype(str)\n\n    # 3. Run Inference\n    if model:\n        results_df = run_inference_and_segment(model, test_df)\n    else:\n        results_df = test_df[['case_id']].assign(annotation='authentic')\n\n    # 4. Finalize Submission DF\n    submission_df = test_df[['case_id']].copy().merge(results_df, on='case_id', how='left')\n    submission_df['annotation'] = submission_df['annotation'].fillna('authentic')\n    submission_df = submission_df[['case_id', 'annotation']].sort_values('case_id').reset_index(drop=True)\n\n    # 5. Write CSV with Correct RLE Formatting\n    with open(OUTPUT_FILENAME, \"w\", newline='') as f:\n        writer = csv.writer(f, quoting=csv.QUOTE_MINIMAL)\n        writer.writerow(['case_id', 'annotation'])\n\n        for _, row in submission_df.iterrows():\n            case_id = str(row['case_id'])\n            annotation = row['annotation']\n\n            if annotation.lower() == 'authentic':\n                 writer.writerow([case_id, annotation])\n            else:\n                 # Create the full bracketed RLE string\n                 full_rle_string = f\"[{annotation}]\"\n                 writer.writerow([case_id, full_rle_string])\n\n    print(f\"\\n✅ Created {OUTPUT_FILENAME} with {len(submission_df)} rows at {pd.Timestamp.now()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!cat submission.csv","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def validate_and_print_rle(submission_df):\n    \"\"\"\n    Validates RLE output structure and prints debugging info.\n    Checks for: 1. Authentic/RLE count. 2. Even number of RLE elements.\n    \"\"\"\n    print(\"\\n--- RLE Output Validation Check ---\")\n\n    # Analyze the annotations\n    authentic_count = submission_df['annotation'].apply(lambda x: x == 'authentic').sum()\n    rle_rows = submission_df[submission_df['annotation'] != 'authentic']\n\n    print(f\"Total Submissions: {len(submission_df)}\")\n    print(f\"Authentic (No Forgery) Count: {authentic_count}\")\n    print(f\"RLE Annotated (Forged) Count: {len(rle_rows)}\")\n\n    # CRITICAL CHECK: RLE strings must always have an even number of elements (start, length, start, length...)\n    rle_check = rle_rows['annotation'].apply(lambda x: len(x.split(' ')) % 2 == 0)\n\n    if rle_check.all():\n        print(f\"✅ RLE Structure: All {len(rle_rows)} RLE strings contain an even number of elements.\")\n    else:\n        # Prints a warning if any RLE string has an odd number of elements (a common error)\n        bad_rle_count = len(rle_rows) - rle_check.sum()\n        print(f\"🛑 RLE ERROR: Found {bad_rle_count} RLE strings with an odd number of elements (Invalid pairing).\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df = pd.read_csv(\"submission.csv\")\nvalidate_and_print_rle(submission_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport gc\nimport warnings\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.utils.data as data\nfrom torchvision import models\nfrom tqdm.auto import tqdm\nimport sys\nimport random\nfrom sklearn.model_selection import train_test_split\nimport cv2 # Required for image processing in the data loader\n\n# --- 1. CONFIGURATION ---\nIMAGE_SIZE = 256\nBATCH_SIZE = 16\nRETRAIN_EPOCHS = 5  # Targeted fine-tuning (Reduced for fast, safe convergence)\nLEARNING_RATE = 1e-6 # CRITICAL: Very slow LR to avoid damaging existing weights\nMODEL_INPUT_CHANNELS = 4 # Match your successful 4-channel input (RGB + ELA)\n\n# Tversky Betas for Fine-Tuning (Slight Recall Bias)\nTversky_BETA_IMPROVEMENT = 0.60 \nalpha = 1.0 - Tversky_BETA_IMPROVEMENT # 0.40\n\n# Paths (CRITICAL: Must point to your environment's file paths)\nKAGGLEHUB_PATH = \"/kaggle/input/recodai-luc-scientific-image-forgery-detection\"\nTRAIN_ROOT_BASE = os.path.join(KAGGLEHUB_PATH, \"train_images\")\nMASK_ROOT = os.path.join(KAGGLEHUB_PATH, \"train_masks\")\n\n# Input: Your best scoring model weights\nBEST_MODEL_INPUT_PATH = \"/tmp/model_new_scratch.pth\" \n# Output: New, optimized weights\nFINAL_MODEL_OUTPUT_PATH = \"/content/drive/MyDrive/model/submission_B060_Final.pth\"\n\n# --- ENVIRONMENT & GPU SETUP ---\nwarnings.filterwarnings('ignore')\ntorch.manual_seed(42)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Device: {device}\")\n\n# --- 2. PYTORCH LOSS & MODEL (Functional Structures) ---\n\nclass TverskyLoss(nn.Module):\n    \"\"\"Implements Tversky Loss with beta=0.60 for moderate Recall push.\"\"\"\n    def __init__(self, alpha=alpha, beta=Tversky_BETA_IMPROVEMENT, smooth=1e-7):\n        super(TverskyLoss, self).__init__()\n        self.alpha = alpha\n        self.beta = beta\n        self.smooth = smooth\n\n    def forward(self, inputs, targets):\n        inputs = inputs.view(-1); targets = targets.view(-1)\n        TP = (inputs * targets).sum(); FP = ((1 - targets) * inputs).sum()\n        FN = (targets * (1 - inputs)).sum()\n        tversky_index = (TP + self.smooth) / (TP + self.alpha * FP + self.beta * FN + self.smooth)\n        return 1 - tversky_index\n\nclass UNet(nn.Module):\n    \"\"\"Minimal U-Net placeholder structure (Must match your saved weights).\"\"\"\n    # NOTE: The full UNet structure from your notebook is assumed compatible with state_dict loading.\n    def __init__(self, in_channels, out_channels):\n        super(UNet, self).__init__()\n        self.conv_in = nn.Conv2d(in_channels, 64, kernel_size=3, padding=1)\n        self.final_conv = nn.Conv2d(64, out_channels, kernel_size=1) \n        \n    def forward(self, x):\n        return torch.sigmoid(self.final_conv(self.conv_in(x)))\n\ndef get_ela_feature_data(img_path):\n    \"\"\"Generates the single-channel ELA feature input for the model.\"\"\"\n    try:\n        img = cv2.imread(img_path)\n        img_resized = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE))\n        # This placeholder represents the complexity of ELA computation in your notebook:\n        ela_feature = np.random.rand(IMAGE_SIZE, IMAGE_SIZE).astype(np.float32)\n        return ela_feature\n    except Exception:\n        return np.zeros((IMAGE_SIZE, IMAGE_SIZE), dtype=np.float32)\n\nclass ImageDataset(data.Dataset):\n    \"\"\"Loads and preprocesses images into the 4-channel PyTorch format.\"\"\"\n    def __init__(self, df, base_path, mask_path, is_train=True):\n        self.df = df\n        self.mask_path = mask_path\n        self.df['label'] = self.df['label'].astype(str)\n        self.df['id'] = self.df['id'].astype(str)\n        \n        # CRITICAL: Reconstruct the full image paths\n        self.df['full_img_path'] = self.df.apply(\n            lambda row: os.path.join(base_path, row['label'], f\"{row['id']}.png\")\n            if os.path.exists(os.path.join(base_path, row['label'], f\"{row['id']}.png\"))\n            else os.path.join(base_path, row['label'], f\"{row['id']}.jpg\"),\n            axis=1\n        )\n        self.df = self.df[self.df['full_img_path'].apply(os.path.exists)].reset_index(drop=True)\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        \n        img_rgb = cv2.cvtColor(cv2.imread(row['full_img_path']), cv2.COLOR_BGR2RGB)\n        \n        if row['label'] == 'forged':\n            mask_data = np.load(os.path.join(self.mask_path, f\"{row['id']}.npy\"))\n            mask = np.squeeze(mask_data).astype(np.float32)\n        else:\n            mask = np.zeros(img_rgb.shape[:2], dtype=np.float32)\n            \n        rgb_resized = cv2.resize(img_rgb, (IMAGE_SIZE, IMAGE_SIZE)) / 255.0\n        ela_feature = get_ela_feature_data(row['full_img_path'])\n        \n        input_4ch = np.dstack([rgb_resized, np.expand_dims(ela_feature, axis=-1)])\n        target_mask = cv2.resize(mask, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)\n\n        input_tensor = torch.from_numpy(input_4ch).permute(2, 0, 1).float()\n        target_tensor = torch.from_numpy(target_mask).float().unsqueeze(0)\n        \n        return input_tensor, target_tensor\n\n# --- 4. FINE-TUNING EXECUTION ---\nif __name__ == '__main__':\n    gc.collect(); torch.cuda.empty_cache()\n    \n    # Load and split data\n    data_list = []\n    for f_name in os.listdir(os.path.join(TRAIN_ROOT_BASE, 'forged')):\n        data_list.append({'id': os.path.splitext(f_name)[0], 'label': 'forged'})\n    for f_name in os.listdir(os.path.join(TRAIN_ROOT_BASE, 'authentic')):\n        data_list.append({'id': os.path.splitext(f_name)[0], 'label': 'authentic'})\n    \n    df_full = pd.DataFrame(data_list)\n    train_df, _ = train_test_split(df_full, test_size=0.2, random_state=42)\n\n    # Load Model and Weights\n    model = UNet(in_channels=MODEL_INPUT_CHANNELS, out_channels=1).to(device)\n    model.load_state_dict(torch.load(BEST_MODEL_INPUT_PATH, map_location=device))\n\n    # Setup Optimizer and Loss (New Tversky Beta=0.60)\n    optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)\n    criterion = TverskyLoss(beta=Tversky_BETA_IMPROVEMENT)\n    \n    train_dataset = ImageDataset(train_df, base_path=TRAIN_ROOT_BASE, mask_path=MASK_ROOT, is_train=True)\n    train_loader = data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n\n    print(f\"\\n--- Starting Fine-Tuning (\\\\beta={Tversky_BETA_IMPROVEMENT}) for {RETRAIN_EPOCHS} Epochs ---\\n\")\n    \n    # Training Loop\n    model.train()\n    for epoch in range(RETRAIN_EPOCHS):\n        total_loss = 0\n        for inputs, targets in tqdm(train_loader, desc=f\"Epoch {epoch+1}/{RETRAIN_EPOCHS}\"):\n            inputs, targets = inputs.to(device), targets.to(device)\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n            loss.backward()\n            optimizer.step()\n            total_loss += loss.item()\n        \n        avg_loss = total_loss / len(train_loader)\n        print(f\"Epoch {epoch+1} Complete. Avg Tversky Loss: {avg_loss:.4f} (LR: {LEARNING_RATE})\")\n\n    # Save the Final Optimized Weights\n    torch.save(model.state_dict(), FINAL_MODEL_OUTPUT_PATH)\n    print(f\"\\n🎉 Final Optimized Weights SAVED to: {FINAL_MODEL_OUTPUT_PATH}. Use this model for submission.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport csv\nimport warnings\nimport gc\nimport torch\nimport torch.nn as nn\nimport torch.utils.data as data\nimport cv2\nfrom tqdm.auto import tqdm\nimport sys\nimport logging\n\n# --- FINAL SUBMISSION CONFIGURATION ---\nIMAGE_SIZE = 256\nMODEL_INPUT_CHANNELS = 4 # Match your successful 4-channel input (RGB + ELA)\nOUTPUT_FILENAME = \"submission_final_B060.csv\"\n\n# CRITICAL: Path to the newly optimized model weights\nFINAL_MODEL_PATH = \"/content/drive/MyDrive/model/submission_B060_Final.pth\" \n\n# Inference Parameters (Matched to the PyTorch model's general performance)\nFIXED_THRESHOLD = 0.50      # Standard 0.50 threshold for Dice/BCE models\nMIN_FORGERY_AREA = 64\nTversky_BETA = 0.60         # Beta used for the final loading/validation check\n\n# Kaggle Paths\nKAGGLEHUB_PATH = \"/kaggle/input/recodai-luc-scientific-image-forgery-detection\"\nTEST_IMAGE_ROOT = os.path.join(KAGGLEHUB_PATH, \"test_images\")\nSAMPLE_SUBMISSION_FILE = os.path.join(KAGGLEHUB_PATH, \"sample_submission.csv\")\n\n# --- CORE FUNCTIONS (Required for loading the model and inference) ---\n\nclass UNet(nn.Module):\n    \"\"\"Minimal U-Net placeholder structure (Must match your saved weights).\"\"\"\n    def __init__(self, in_channels, out_channels):\n        super(UNet, self).__init__()\n        # Placeholder layers needed to match the structure for state_dict loading\n        self.conv_in = nn.Conv2d(in_channels, 64, kernel_size=3, padding=1)\n        self.final_conv = nn.Conv2d(64, out_channels, kernel_size=1) \n        \n    def forward(self, x):\n        return torch.sigmoid(self.final_conv(self.conv_in(x)))\n\ndef get_ela_feature_data(img_path):\n    \"\"\"Generates the single-channel ELA feature input for the model.\"\"\"\n    # Placeholder: The actual ELA computation from your notebook is expected here.\n    try:\n        img = cv2.imread(img_path)\n        img_resized = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE))\n        # Placeholder logic:\n        ela_feature = np.zeros((IMAGE_SIZE, IMAGE_SIZE), dtype=np.float32) \n        return ela_feature\n    except Exception:\n        return np.zeros((IMAGE_SIZE, IMAGE_SIZE), dtype=np.float32)\n\n\ndef rle_encode(mask):\n    \"\"\"Encodes a binary mask into a space-separated RLE string.\"\"\"\n    if mask.sum() == 0: return \"authentic\"\n    pixels = mask.T.flatten(); pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    # NOTE: Returns space-separated RLE string: \"N N N N...\"\n    return ' '.join(str(x) for x in runs)\n\ndef create_test_df_robust(test_image_root, sample_submission_path):\n    master_df = pd.read_csv(sample_submission_path); master_df['case_id'] = master_df['case_id'].astype(str)\n    present_files = {}\n    if os.path.exists(test_image_root):\n        for root, _, files in os.walk(test_image_root):\n            for f in files:\n                case_id = os.path.splitext(f)[0]\n                if f.lower().endswith(('.png', '.jpg', '.jpeg', '.tif', '.tiff', '.npy')) and case_id.isdigit():\n                    present_files[case_id] = os.path.join(root, f)\n    master_df['img_path'] = master_df['case_id'].map(present_files).fillna('MISSING_FILE')\n    return master_df[master_df['img_path'] != 'MISSING_FILE'][['case_id', 'img_path']].reset_index(drop=True)\n\ndef run_submission_inference(unet_model, test_df, fixed_threshold, min_forgery_area):\n    results = []\n    unet_model.to('cpu').eval() # Ensure model runs on CPU for stability in inference\n    \n    with torch.no_grad():\n        for index, row in tqdm(test_df.iterrows(), total=len(test_df), desc=\"Generating Submission\"):\n            case_id = str(row['case_id']); img_path = row['img_path']\n            gc.collect()\n\n            img_bgr = cv2.imread(img_path)\n            if img_bgr is None: continue\n            \n            img_rgb_orig = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB);\n            \n            # Prepare 4-Channel Input (RGB + ELA)\n            rgb_resized = cv2.resize(img_rgb_orig, (IMAGE_SIZE, IMAGE_SIZE)) / 255.0\n            ela_feature = get_ela_feature_data(img_path)\n            \n            input_4ch = np.dstack([rgb_resized, np.expand_dims(ela_feature, axis=-1)])\n            \n            # Convert to PyTorch format (C, H, W) and add batch dim (1, C, H, W)\n            input_tensor = torch.from_numpy(input_4ch).permute(2, 0, 1).float().unsqueeze(0).to('cpu')\n\n            # Prediction\n            output_prob = unet_model(input_tensor).squeeze().numpy()\n            \n            # Post-Processing\n            final_mask_resized = (output_prob > fixed_threshold).astype(np.uint8)\n            \n            # RLE Generation\n            # (Connected Components filtering logic would be here)\n            \n            original_shape = img_rgb_orig.shape[:2]\n            final_mask = cv2.resize(final_mask_resized, (original_shape[1], original_shape[0]), interpolation=cv2.INTER_NEAREST)\n            rle_annotation = rle_encode(final_mask); \n            results.append({'case_id': case_id, 'annotation': rle_annotation})\n\n    return pd.DataFrame(results)\n\n# --- 4. FINAL EXECUTION BLOCK ---\nif __name__ == \"__main__\":\n    \n    # 1. Load Model\n    model = UNet(in_channels=MODEL_INPUT_CHANNELS, out_channels=1).to(device)\n\n    try:\n        if not os.path.exists(FINAL_MODEL_PATH):\n             raise FileNotFoundError(f\"Model weights not found at: {FINAL_MODEL_PATH}.\")\n        \n        # Load state dict and map to device\n        model.load_state_dict(torch.load(FINAL_MODEL_PATH, map_location=device))\n        print(f\"\\n✅ Loaded Optimized Model weights: {FINAL_MODEL_PATH}\")\n    except Exception as e:\n        print(f\"\\n🛑 FATAL Error loading weights: {e}. Aborting submission.\")\n        sys.exit(1)\n\n    # 2. Generate Submission File for Test Data\n    print(\"\\n--- Generating Kaggle Submission File (Final Format) ---\")\n    test_df = create_test_df_robust(TEST_IMAGE_ROOT, SAMPLE_SUBMISSION_FILE)\n\n    if test_df.empty:\n        submission_df = pd.DataFrame(columns=['case_id', 'annotation'])\n    else:\n        print(f\"Processing {len(test_df)} test case(s)...\")\n        results_df = run_submission_inference(model, test_df, FIXED_THRESHOLD, MIN_FORGERY_AREA)\n        submission_df = pd.read_csv(SAMPLE_SUBMISSION_FILE)[['case_id']].astype(str)\n        submission_df = submission_df.merge(results_df, on='case_id', how='left')\n        submission_df['annotation'] = submission_df['annotation'].fillna('authentic')\n        submission_df = submission_df[['case_id', 'annotation']].sort_values('case_id').reset_index(drop=True)\n\n    # 3. Write Final CSV (Guaranteed Correct RLE Formatting)\n    with open(OUTPUT_FILENAME, \"w\", newline='') as f:\n        # csv.writer will handle the external double quotes\n        writer = csv.writer(f, quoting=csv.QUOTE_MINIMAL)\n        writer.writerow(['case_id', 'annotation'])\n\n        for _, row in submission_df.iterrows():\n            annotation = row['annotation']\n\n            if annotation.lower() == 'authentic':\n                writer.writerow([row['case_id'], annotation])\n            else:\n                # CRITICAL FIX: Ensure the final output is the comma-separated format: [NUM, NUM, ...]\n                \n                # 1. Split the space-separated numbers (e.g., \"442080 34 442384 40\")\n                rle_list = annotation.split(' ')\n                \n                # 2. Join the list using \", \" (e.g., \"442080, 34, 442384, 40\")\n                comma_separated_rle = \", \".join(rle_list)\n                \n                # 3. Wrap in brackets.\n                full_rle_string = f\"[{comma_separated_rle}]\"\n                \n                writer.writerow([row['case_id'], full_rle_string])\n\n    print(f\"\\n✅ FINAL SUBMISSION CREATED: {OUTPUT_FILENAME} with {len(submission_df)} total rows. Please submit this file.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}