{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":113558,"databundleVersionId":14174843,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\nimport sys\nimport glob\nimport albumentations as A\n# --- Phase 1: Setup and Data Loading ---\n# !!! IMPORTANT: VERIFY THIS PATH IN YOUR KAGGLE DATA SIDEBAR !!!\nKAGGLE_DIR = '../input/recodai-luc-scientific-image-forgery-detection/'\nTRAIN_IMG_BASE_DIR = os.path.join(KAGGLE_DIR, 'train_images')\nTRAIN_MASK_BASE_DIR = os.path.join(KAGGLE_DIR, 'train_masks')\nTEST_IMG_DIR = os.path.join(KAGGLE_DIR, 'test_images')\n# Configuration\nIMG_SIZE = 256\nBATCH_SIZE = 32\nEPOCHS = 20 # Increased epochs for better convergence\n# --- Helper Functions (RLE & Metric) ---\ndef rle_encode(img):\n    \"\"\" Encodes a binary image mask into RLE format \"\"\"\n    pixels = img.flatten()\n    pixels = np.concatenate([np.array(), pixels,])\n    runs = np.where(pixels[1:] != pixels[:-1]) + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\ndef combined_loss(y_true, y_pred, smooth=1e-6):\n    \"\"\" Combined Dice Loss and Binary Cross-Entropy (BCE) Loss \"\"\"\n    y_true_f = tf.cast(tf.flatten(y_true), tf.float32)\n    y_pred_f = tf.cast(tf.flatten(y_pred), tf.float32)\n    intersection = tf.reduce_sum(y_true_f * y_pred_f)\n    den = tf.reduce_sum(y_true_f + y_pred_f)\n    dice_loss_val = 1.0 - (2. * intersection + smooth) / (den + smooth)\n    bce_loss_val = tf.keras.losses.binary_crossentropy(y_true, y_pred)\n    return bce_loss_val + dice_loss_val\n# --- Data Augmentation Pipeline (Albumentations) ---\n# This pipeline applies random flips, rotations, and minor distortions during training.\ntrain_augment = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.RandomRotate90(p=0.5),\n    A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=10, p=0.5),\n    A.ElasticTransform(p=0.2, alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03),\n])\n# --- Data Loading Function (Handles subdirectories and augmentation) ---\ndef load_data(img_base_dir, mask_base_dir=None, augment=False):\n    images = []\n    masks = []\n    img_ids = []\n    if not os.path.exists(img_base_dir):\n        print(f\"Error: Base directory not found at {img_base_dir}\", file=sys.stderr)\n        return np.array(images), np.array(masks), img_ids\n    img_paths = sorted(glob.glob(os.path.join(img_base_dir, '**', '*.png'), recursive=True))\n    print(f\"Found {len(img_paths)} image files across subdirectories.\")\n    for img_path in tqdm(img_paths):\n        img = cv2.imread(img_path, cv2.IMREAD_COLOR)\n        if img is None:\n            print(f\"Warning: Could not read image {img_path}. Skipping.\", file=sys.stderr)\n            continue\n        # Load mask if available\n        mask = None\n        if mask_base_dir:\n            relative_path = os.path.relpath(img_path, img_base_dir)\n            mask_path = os.path.join(mask_base_dir, relative_path)\n            mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE)\n            if mask is None:\n                print(f\"Warning: Could not read mask {mask_path}. Skipping associated image.\", file=sys.stderr)\n                continue\n        # Apply augmentation only during training phases\n        if augment and mask is not None:\n            augmented = train_augment(image=img, mask=mask)\n            img = augmented['image']\n            mask = augmented['mask']\n        # Resize and normalize\n        img = cv2.resize(img, (IMG_SIZE, IMG_SIZE)) / 255.0\n        images.append(img)\n        img_name = os.path.basename(img_path)\n        img_ids.append(os.path.splitext(img_name))\n        if mask is not None:\n            mask = cv2.resize(mask, (IMG_SIZE, IMG_SIZE))\n            mask = np.expand_dims(mask, axis=-1) / 255.0\n            masks.append(mask)\n        elif mask_base_dir: # Handle cases where we skip due to missing mask\n             images.pop()\n             img_ids.pop()\n    return np.array(images), np.array(masks), img_ids\n# --- U-Net Model Definition ---\ndef build_unet(input_shape):\n    inputs = tf.keras.layers.Input(input_shape)\n    # ... (U-Net layers as defined previously) ...\n    c1 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(inputs)\n    c1 = tf.keras.layers.Dropout(0.1)(c1)\n    c1 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c1)\n    p1 = tf.keras.layers.MaxPooling2D((2, 2))(c1)\n    c2 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p1)\n    c2 = tf.keras.layers.Dropout(0.1)(c2)\n    c2 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c2)\n    p2 = tf.keras.layers.MaxPooling2D((2, 2))(c2)\n    c3 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p2)\n    c3 = tf.keras.layers.Dropout(0.2)(c3)\n    c3 = tf.keras.layers.Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c3)\n    u4 = tf.keras.layers.Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c3)\n    u4 = tf.keras.layers.concatenate([u4, c2])\n    c4 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u4)\n    c4 = tf.keras.layers.Dropout(0.1)(c4)\n    c4 = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c4)\n    u5 = tf.keras.layers.Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c4)\n    u5 = tf.keras.layers.concatenate([u5, c1])\n    c5 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u5)\n    c5 = tf.keras.layers.Dropout(0.1)(c5)\n    c5 = tf.keras.layers.Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c5)\n    outputs = tf.keras.layers.Conv2D(1, (1, 1), activation='sigmoid')(c5)\n    model = tf.keras.Model(inputs=[inputs], outputs=[outputs])\n    model.compile(optimizer='adam', loss=combined_loss, metrics=['accuracy']) # Using combined loss now\n    return model\n# --- Main execution block ---\nif __name__ == \"__main__\":\n    # --- 1. Load Data ---\n    print(\"Loading training data from subdirectories with augmentation...\")\n    # Use augment=True for training data\n    train_images, train_masks, train_ids = load_data(TRAIN_IMG_BASE_DIR, TRAIN_MASK_BASE_DIR, augment=True)\n    if len(train_images) == 0:\n        print(f\"FATAL ERROR: No training images loaded. Check KAGGLE_DIR or file structure.\")\n        sys.exit(1)\n    print(f\"Loaded {len(train_images)} training images successfully.\")\n    X_train, X_val, y_train, y_val = train_test_split(train_images, train_masks, test_size=0.2, random_state=42)\n    # --- 2. Build and Train Model ---\n    input_shape = (IMG_SIZE, IMG_SIZE, 3)\n    model = build_unet(input_shape)\n    print(\"Starting U-Net model training...\")\n    model.fit(\n        X_train, y_train,\n        validation_data=(X_val, y_val),\n        batch_size=BATCH_SIZE,\n        epochs=EPOCHS,\n        verbose=1\n    )\n    print(\"Training complete.\")\n    # --- 3. Prediction and Submission ---\n    print(\"Loading test data for prediction (no augmentation)...\")\n    # Use augment=False for test data\n    test_images, _, test_ids = load_data(TEST_IMG_DIR, augment=False)\n    if len(test_images) == 0:\n        print(\"Error: No test images loaded. Generating empty submission.\")\n        pd.DataFrame({'case_id': [], 'annotation': []}).to_csv('submission.csv', index=False)\n        sys.exit(1)\n    print(f\"Generating predictions on {len(test_images)} test images...\")\n    predictions = model.predict(test_images, batch_size=BATCH_SIZE)\n    # Post-process predictions to binary masks (optimize this threshold later)\n    THRESHOLD = 0.5\n    predicted_masks = (predictions > THRESHOLD).astype(np.uint8)\n    submission_df = pd.DataFrame(columns=['case_id', 'annotation'])\n    annotations = []\n    for i, case_id in enumerate(tqdm(test_ids)):\n        mask = predicted_masks[i].squeeze()\n        rle_annotation = rle_encode(mask)\n        if np.sum(mask) == 0 or rle_annotation == ' ':\n             annotations.append('authentic')\n        else:\n            annotations.append(rle_annotation)\n    submission_df['case_id'] = test_ids\n    submission_df['annotation'] = annotations\n    submission_file_path = 'submission.csv'\n    submission_df.to_csv(submission_file_path, index=False)\n    print(f\"Submission file '{submission_file_path}' created successfully.\")\n","metadata":{"_uuid":"e58ac2df-18ce-4c89-875f-e1e1038e0378","_cell_guid":"f007254d-ff7d-42fb-b371-b77454f80593","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-11-07T07:36:27.557964Z","iopub.execute_input":"2025-11-07T07:36:27.5585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}