{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":113558,"databundleVersionId":14174843,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nimport subprocess\n\nprint(\"--- Python Environment Details ---\")\nprint(f\"Python version: {sys.version}\")\nprint(\"-\" * 30)\n\n# Function to get installed package version\ndef get_package_version(package_name):\n    try:\n        # Using importlib.metadata for Python 3.8+\n        from importlib.metadata import version\n        return version(package_name)\n    except (ImportError, ModuleNotFoundError):\n        # Fallback for older Python or if package not found easily\n        try:\n            return subprocess.check_output([sys.executable, \"-m\", \"pip\", \"show\", package_name]).decode().split('\\n')[1].split(': ')[1]\n        except Exception:\n            return \"Not Installed\"\n\nprint(f\"numpy version: {get_package_version('numpy')}\")\nprint(f\"scipy version: {get_package_version('scipy')}\")\nprint(f\"matplotlib version: {get_package_version('matplotlib')}\")\nprint(f\"pandas version: {get_package_version('pandas')}\")\nprint(f\"torch version: {get_package_version('torch')}\")\nprint(f\"torchvision version: {get_package_version('torchvision')}\")\nprint(f\"segmentation_models_pytorch version: {get_package_version('segmentation_models_pytorch')}\")\nprint(f\"scikit-learn version: {get_package_version('scikit-learn')}\")\nprint(\"-\" * 30)\n\n\nprint(\"\\n--- Attempting minimal installations ---\")\n\n# Uninstall segmentation_models_pytorch if it was partially installed and conflicting\n# We'll install it clean later.\n# We are NOT uninstalling numpy, scipy, matplotlib, pandas here. We want to keep Kaggle's defaults.\n!pip uninstall -y segmentation-models-pytorch --quiet\n\n# Only install segmentation_models_pytorch and scikit-learn.\n# We trust Kaggle's numpy/scipy/matplotlib/pandas to be mostly compatible with each other.\n# The challenge is adding new libraries without breaking the core.\n!pip install segmentation_models_pytorch scikit-learn --quiet\n\nprint(\"\\n--- Verification of critical libraries after minimal install ---\")\nprint(f\"segmentation_models_pytorch version: {get_package_version('segmentation_models_pytorch')}\")\nprint(f\"scikit-learn version: {get_package_version('scikit-learn')}\")\nprint(\"-\" * 30)\n\n# --- Standard Imports ---\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom PIL import Image # For handling image files\nimport matplotlib.pyplot as plt # For plotting and visualization\nimport glob # For finding file paths\n\n# PyTorch imports\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms # Use torchvision transforms for augmentation\n\n# Segmentation model library\nimport segmentation_models_pytorch as smp\n\n# Scikit-learn for data splitting\nfrom sklearn.model_selection import train_test_split\n\nprint(\"\\nAll standard imports attempted.\")\nprint(\"If no errors above, imports were successful!\")\n\n\n# Optional: Print current directory contents to confirm data structure\nprint(\"\\n--- Listing input directory contents (first 10 files) ---\")\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        if \"mask\" in filename or \"train_images\" in dirname or \"test_images\" in dirname or \"submission\" in filename:\n            print(os.path.join(dirname, filename))\n            # Limit output to prevent overwhelming the console\n            if len(filenames) > 100: # Heuristic\n                print(\"... (truncated list for brevity)\")\n                break\nprint(\"--- End of input directory listing ---\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T06:12:54.247874Z","iopub.execute_input":"2025-10-30T06:12:54.248213Z","iopub.status.idle":"2025-10-30T06:13:07.255707Z","shell.execute_reply.started":"2025-10-30T06:12:54.248187Z","shell.execute_reply":"2025-10-30T06:13:07.254908Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Official RLE functions from the Kaggle metric notebook\n# https://www.kaggle.com/code/metric/recodai-f1?scriptVersionId=270092713&cellId=1\n\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]]) # Add sentinels\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef rle_decode(mask_rle, shape=(256, 256)): # Default shape, adjust based on your image size\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return\n    Returns numpy array, 1 - mask, 0 - background\n    '''\n    s = mask_rle.split()\n    if not s: # Handle empty string for 'authentic' case or missing mask\n        return np.zeros(shape[0]*shape[1], dtype=np.uint8).reshape(shape)\n    \n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1 # RLE is 1-indexed, Python arrays are 0-indexed\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape)\n\nprint(\"RLE functions defined.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T06:13:07.257403Z","iopub.execute_input":"2025-10-30T06:13:07.257990Z","iopub.status.idle":"2025-10-30T06:13:07.265236Z","shell.execute_reply.started":"2025-10-30T06:13:07.257970Z","shell.execute_reply":"2025-10-30T06:13:07.264571Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Configuration ---\nDATA_DIR = '/kaggle/input/recodai-luc-scientific-image-forgery-detection' \nIMG_SIZE = 256 # Standardize image size for training\nBATCH_SIZE = 16\nNUM_EPOCHS = 15 # Increased epochs slightly, you may need more\nLEARNING_RATE = 1e-4\n\n# --- Paths ---\ntrain_image_forged_dir = os.path.join(DATA_DIR, 'train_images', 'forged')\ntrain_image_authentic_dir = os.path.join(DATA_DIR, 'train_images', 'authentic') \ntrain_mask_dir = os.path.join(DATA_DIR, 'train_masks') \n\ntest_image_dir = os.path.join(DATA_DIR, 'test_images') \n\n# Get all training image paths\ntrain_forged_image_paths = sorted(glob.glob(os.path.join(train_image_forged_dir, '*.png')))\ntrain_authentic_image_paths = sorted(glob.glob(os.path.join(train_image_authentic_dir, '*.png')))\ntrain_image_paths = train_forged_image_paths + train_authentic_image_paths\n\n# Get all mask paths\nall_mask_paths = sorted(glob.glob(os.path.join(train_mask_dir, '*.png')))\n\nimage_id_to_mask_paths = {}\nfor mask_path in all_mask_paths:\n    mask_basename = os.path.basename(mask_path)\n    image_id_part = mask_basename.split('_')[0].split('.')[0] \n    \n    if image_id_part not in image_id_to_mask_paths:\n        image_id_to_mask_paths[image_id_part] = []\n    image_id_to_mask_paths[image_id_part].append(mask_path)\n\n\ntrain_data_list = []\nfor img_path in train_image_paths:\n    img_basename = os.path.basename(img_path)\n    img_id = img_basename.split('.')[0] \n    \n    mask_info = image_id_to_mask_paths.get(img_id, []) \n    train_data_list.append((img_path, mask_info))\n\nprint(f\"Found {len(train_forged_image_paths)} forged training images.\")\nprint(f\"Found {len(train_authentic_image_paths)} authentic training images.\")\nprint(f\"Total training images: {len(train_data_list)}\")\nprint(f\"Found {len(all_mask_paths)} total mask files.\") \n\ntest_image_paths = sorted(glob.glob(os.path.join(test_image_dir, '*.png')))\nprint(f\"Found {len(test_image_paths)} test images.\")\n\nprint(\"\\nFirst 5 training data entries (image path, mask paths):\")\nfor i in range(min(5, len(train_data_list))):\n    img_path, mask_paths = train_data_list[i]\n    print(f\"Image: {os.path.basename(img_path)}, Masks: {[os.path.basename(p) for p in mask_paths] if mask_paths else 'authentic'}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T06:13:07.265865Z","iopub.execute_input":"2025-10-30T06:13:07.266040Z","iopub.status.idle":"2025-10-30T06:13:07.308625Z","shell.execute_reply.started":"2025-10-30T06:13:07.266026Z","shell.execute_reply":"2025-10-30T06:13:07.307876Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ForgeryDataset(Dataset):\n    def __init__(self, data_list, img_size, transform=None, is_test=False):\n        self.data_list = data_list \n        self.img_size = img_size\n        self.transform = transform\n        self.is_test = is_test\n\n    def __len__(self):\n        return len(self.data_list)\n\n    def __getitem__(self, idx):\n        if self.is_test:\n            img_path = self.data_list[idx]\n            image = Image.open(img_path).convert(\"RGB\") # Use PIL.Image\n            \n            original_w, original_h = image.size # PIL gives (width, height)\n\n            if self.transform:\n                image = self.transform(image)\n            \n            return image, os.path.basename(img_path).split('.')[0], (original_h, original_w)\n            \n        else: # Training/Validation\n            img_path, mask_paths = self.data_list[idx]\n            \n            image = Image.open(img_path).convert(\"RGB\") # Use PIL.Image\n\n            # Initialize an empty mask (all zeros) as a PIL Image\n            combined_mask = Image.fromarray(np.zeros(image.size[::-1], dtype=np.uint8)) # size[::-1] for (H,W)\n\n            # If there are mask paths, combine them\n            if mask_paths:\n                mask_arrays = []\n                for m_path in mask_paths:\n                    mask_img = Image.open(m_path).convert(\"L\") # Open as grayscale\n                    mask_array = np.array(mask_img)\n                    mask_arrays.append((mask_array > 0).astype(np.uint8)) # Binary mask\n                \n                if mask_arrays: # Combine multiple masks if they exist\n                    combined_mask_array = np.logical_or.reduce(mask_arrays).astype(np.uint8)\n                    combined_mask = Image.fromarray(combined_mask_array)\n\n            if self.transform:\n                # Apply transforms to both image and mask\n                # Note: torchvision.transforms are usually applied sequentially\n                # We need a custom transform for paired image-mask augmentation\n                # For simplicity, we'll resize here and apply basic random augmentations.\n                # For more advanced paired transforms, one might need a custom composed transform.\n                \n                # Resize first\n                image = transforms.Resize((self.img_size, self.img_size))(image)\n                mask = transforms.Resize((self.img_size, self.img_size), interpolation=transforms.InterpolationMode.NEAREST)(combined_mask)\n                \n                # Apply random augmentations (e.g., flip) to both\n                if np.random.rand() < 0.5: # Random horizontal flip\n                    image = transforms.functional.hflip(image)\n                    mask = transforms.functional.hflip(mask)\n                if np.random.rand() < 0.5: # Random vertical flip\n                    image = transforms.functional.vflip(image)\n                    mask = transforms.functional.vflip(mask)\n\n                # Convert to Tensor and Normalize\n                image = transforms.ToTensor()(image)\n                mask = transforms.ToTensor()(mask)\n                \n                # Normalize image\n                image = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])(image)\n\n            else:\n                # Default resize and ToTensor\n                image = transforms.Compose([\n                    transforms.Resize((self.img_size, self.img_size)),\n                    transforms.ToTensor(),\n                    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n                ])(image)\n                mask = transforms.Compose([\n                    transforms.Resize((self.img_size, self.img_size), interpolation=transforms.InterpolationMode.NEAREST),\n                    transforms.ToTensor(),\n                ])(combined_mask)\n            \n            return image, mask\n\nprint(\"ForgeryDataset class defined (using PIL and torchvision).\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T06:13:07.310441Z","iopub.execute_input":"2025-10-30T06:13:07.310661Z","iopub.status.idle":"2025-10-30T06:13:07.321201Z","shell.execute_reply.started":"2025-10-30T06:13:07.310646Z","shell.execute_reply":"2025-10-30T06:13:07.320470Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Augmentations (Simplified torchvision style) ---\n# Basic transforms including normalization and conversion to tensor\ndef get_train_transforms(img_size):\n    return transforms.Compose([\n        transforms.Resize((img_size, img_size)),\n        # Random augmentations like flips are handled in Dataset __getitem__ for paired transforms\n        transforms.ToTensor(), # Converts image to CxHxW and scales pixels to [0, 1]\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), # ImageNet stats\n    ])\n\ndef get_val_test_transforms(img_size):\n    return transforms.Compose([\n        transforms.Resize((img_size, img_size)),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ])\n\nprint(\"Augmentation transforms defined (using torchvision).\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T06:13:07.321975Z","iopub.execute_input":"2025-10-30T06:13:07.322253Z","iopub.status.idle":"2025-10-30T06:13:07.341291Z","shell.execute_reply.started":"2025-10-30T06:13:07.322232Z","shell.execute_reply":"2025-10-30T06:13:07.340659Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Data Split ---\ntrain_data, val_data = train_test_split(train_data_list, test_size=0.2, random_state=42)\n\n# Pass transform functions to the Dataset\ntrain_dataset = ForgeryDataset(train_data, IMG_SIZE, get_train_transforms(IMG_SIZE), is_test=False)\nval_dataset = ForgeryDataset(val_data, IMG_SIZE, get_val_test_transforms(IMG_SIZE), is_test=False)\ntest_dataset = ForgeryDataset(test_image_paths, IMG_SIZE, get_val_test_transforms(IMG_SIZE), is_test=True)\n\n\ntrain_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\nval_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\nprint(f\"Train samples: {len(train_dataset)}, Val samples: {len(val_dataset)}, Test samples: {len(test_dataset)}\")\nprint(\"DataLoaders created.\")\n\n# --- Model ---\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {DEVICE}\")\n\nmodel = smp.Unet(\n    encoder_name=\"resnet34\",        \n    encoder_weights=\"imagenet\",     \n    in_channels=3,                  \n    classes=1,                      \n)\nmodel.to(DEVICE)\n\n# --- Loss Function ---\nclass CombinedLoss(nn.Module):\n    def __init__(self, dice_weight=0.5, bce_weight=0.5):\n        super().__init__()\n        self.dice_loss = smp.losses.DiceLoss(mode='binary', from_logits=True)\n        self.bce_loss = smp.losses.SoftBCEWithLogitsLoss()\n        self.dice_weight = dice_weight\n        self.bce_weight = bce_weight\n\n    def forward(self, pred, target):\n        dice = self.dice_loss(pred, target)\n        bce = self.bce_loss(pred, target)\n        return self.dice_weight * dice + self.bce_weight * bce\n\nloss_fn = CombinedLoss(dice_weight=0.5, bce_weight=0.5)\n\n# --- Optimizer ---\noptimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)\n\n# --- Learning Rate Scheduler ---\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5, verbose=True)\n\nprint(\"Model, Loss, Optimizer, and Scheduler initialized.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T06:13:07.342099Z","iopub.execute_input":"2025-10-30T06:13:07.342846Z","iopub.status.idle":"2025-10-30T06:13:07.955635Z","shell.execute_reply.started":"2025-10-30T06:13:07.342820Z","shell.execute_reply":"2025-10-30T06:13:07.954821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Training Loop ---\nbest_val_loss = float('inf')\nMODEL_SAVE_PATH = 'best_model.pth' \n\nprint(f\"Starting training for {NUM_EPOCHS} epochs...\")\nfor epoch in range(NUM_EPOCHS):\n    model.train()\n    running_loss = 0.0\n    for batch_idx, (images, masks) in enumerate(train_loader):\n        images, masks = images.to(DEVICE), masks.to(DEVICE)\n\n        optimizer.zero_grad()\n        outputs = model(images)\n        loss = loss_fn(outputs, masks)\n        loss.backward()\n        optimizer.step()\n\n        running_loss += loss.item()\n\n        if (batch_idx + 1) % 50 == 0: \n            print(f\"Epoch {epoch+1}/{NUM_EPOCHS}, Batch {batch_idx+1}/{len(train_loader)}, Train Loss: {running_loss / (batch_idx+1):.4f}\")\n\n    avg_train_loss = running_loss / len(train_loader)\n\n    # --- Validation Loop ---\n    model.eval()\n    val_loss = 0.0\n    with torch.no_grad():\n        for images, masks in val_loader:\n            images, masks = images.to(DEVICE), masks.to(DEVICE)\n            outputs = model(images)\n            loss = loss_fn(outputs, masks)\n            val_loss += loss.item()\n    \n    avg_val_loss = val_loss / len(val_loader)\n    print(f\"Epoch {epoch+1}/{NUM_EPOCHS}, Avg Train Loss: {avg_train_loss:.4f}, Avg Val Loss: {avg_val_loss:.4f}\")\n\n    scheduler.step(avg_val_loss)\n\n    if avg_val_loss < best_val_loss:\n        best_val_loss = avg_val_loss\n        torch.save(model.state_dict(), MODEL_SAVE_PATH)\n        print(f\"Saved best model with validation loss: {best_val_loss:.4f}\")\n\nprint(\"\\nTraining complete!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T06:13:07.956515Z","iopub.execute_input":"2025-10-30T06:13:07.957369Z","iopub.status.idle":"2025-10-30T06:43:57.769972Z","shell.execute_reply.started":"2025-10-30T06:13:07.957349Z","shell.execute_reply":"2025-10-30T06:43:57.769153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Load Best Model for Prediction ---\nmodel.load_state_dict(torch.load(MODEL_SAVE_PATH))\nmodel.eval()\nprint(f\"Loaded best model from {MODEL_SAVE_PATH} for prediction.\")\n\n# --- Prediction and Submission ---\nsubmission_data = []\nPREDICTION_THRESHOLD = 0.5 \n\nprint(\"Generating predictions for test images...\")\nwith torch.no_grad():\n    for images, image_ids, original_sizes in test_loader:\n        images = images.to(DEVICE)\n        outputs = model(images)\n        probabilities = torch.sigmoid(outputs).cpu().numpy() \n\n        for i in range(probabilities.shape[0]):\n            img_id = image_ids[i]\n            # original_sizes is a tuple of (H_tensor, W_tensor) for the batch\n            original_h, original_w = original_sizes[0][i].item(), original_sizes[1][i].item()\n\n            pred_mask = probabilities[i, 0] # Get the single channel mask (HxW)\n            \n            # Resize the predicted mask back to the original image size\n            # Use interpolation=cv2.INTER_LINEAR here is more suitable for probabilities\n            pred_mask_resized = transforms.ToPILImage()(torch.from_numpy(pred_mask)).resize((original_w, original_h), Image.BILINEAR)\n            pred_mask_resized = np.array(pred_mask_resized) # Convert back to numpy array\n\n            binary_mask = (pred_mask_resized > PREDICTION_THRESHOLD).astype(np.uint8)\n            \n            if np.sum(binary_mask) == 0: \n                submission_data.append({'case_id': img_id, 'annotation': 'authentic'})\n            else:\n                rle_encoded_mask = rle_encode(binary_mask)\n                submission_data.append({'case_id': img_id, 'annotation': rle_encoded_mask})\n\nsubmission_df = pd.DataFrame(submission_data)\n\nSUBMISSION_FILE_PATH = 'submission.csv'\nsubmission_df.to_csv(SUBMISSION_FILE_PATH, index=False)\n\nprint(f\"Submission file created successfully at {SUBMISSION_FILE_PATH}\")\nprint(\"\\nFirst 5 rows of submission.csv:\")\nprint(submission_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T06:43:57.771004Z","iopub.execute_input":"2025-10-30T06:43:57.771296Z","iopub.status.idle":"2025-10-30T06:43:58.150259Z","shell.execute_reply.started":"2025-10-30T06:43:57.771274Z","shell.execute_reply":"2025-10-30T06:43:58.149209Z"}},"outputs":[],"execution_count":null}]}