{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":117682,"databundleVersionId":15062069,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-27T16:16:47.669336Z","iopub.execute_input":"2026-01-27T16:16:47.669676Z","iopub.status.idle":"2026-01-27T16:16:49.841216Z","shell.execute_reply.started":"2026-01-27T16:16:47.669648Z","shell.execute_reply":"2026-01-27T16:16:49.840549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport cv2\nimport os\nfrom pathlib import Path\nimport warnings\nwarnings.filterwarnings('ignore')\n!pip install segmentation-models-pytorch\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.optim as optim\nfrom torchvision import transforms\nimport segmentation_models_pytorch as smp","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T16:16:49.842519Z","iopub.execute_input":"2026-01-27T16:16:49.842866Z","iopub.status.idle":"2026-01-27T16:17:06.571977Z","shell.execute_reply.started":"2026-01-27T16:16:49.842841Z","shell.execute_reply":"2026-01-27T16:17:06.571339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed=42):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nset_seed(42)\n\n# Check GPU availability\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T16:17:06.572931Z","iopub.execute_input":"2026-01-27T16:17:06.573217Z","iopub.status.idle":"2026-01-27T16:17:06.654129Z","shell.execute_reply.started":"2026-01-27T16:17:06.573193Z","shell.execute_reply":"2026-01-27T16:17:06.653555Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nDATA_PATH = '/kaggle/input/vesuvius-challenge-surface-detection'\nTRAIN_LABELS = f'{DATA_PATH}/train_labels'\nTRAIN_IMAGES = f'{DATA_PATH}/train_images'\nTEST_IMAGES = f'{DATA_PATH}/test_images'\n\n\n# Load CSV files\ntrain_df = pd.read_csv(f'{DATA_PATH}/train.csv')\ntest_df = pd.read_csv(f'{DATA_PATH}/test.csv')\n\nprint(\"Data loaded successfully!\")\nprint(f\"\\nTraining samples: {len(train_df)}\")\nprint(f\"Test samples: {len(test_df)}\")\nprint(f\"\\nTrain CSV columns: {train_df.columns.tolist()}\")\nprint(f\"\\nFirst few rows of train.csv:\")\nprint(train_df.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T16:17:06.655002Z","iopub.execute_input":"2026-01-27T16:17:06.655237Z","iopub.status.idle":"2026-01-27T16:17:06.690958Z","shell.execute_reply.started":"2026-01-27T16:17:06.655217Z","shell.execute_reply":"2026-01-27T16:17:06.690366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef explore_data(idx=0):\n    \"\"\"\n    Visualize training images and their masks - with file existence check\n    \"\"\"\n    row = train_df.iloc[idx]\n    image_id = str(row['id'])  # Convert to string\n    \n    # Check if file exists\n    img_path = f'{TRAIN_IMAGES}/{image_id}.tif'\n    if not os.path.exists(img_path):\n        print(f\"File not found for ID {image_id}, trying next sample...\")\n        # Try next index\n        if idx + 1 < len(train_df):\n            return explore_data(idx + 1)\n        else:\n            print(\"No valid samples found\")\n            return\n    \n    # Load TIFF image\n    img = cv2.imread(img_path, cv2.IMREAD_UNCHANGED)\n    \n    if img is None:\n        print(f\" Could not load: {img_path}\")\n        return explore_data(idx + 1) if idx + 1 < len(train_df) else None\n    \n    # Convert to RGB\n    if len(img.shape) == 2:  # Grayscale\n        img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)\n    elif img.shape[2] == 4:  # RGBA\n        img = cv2.cvtColor(img, cv2.COLOR_BGRA2RGB)\n    else:  # BGR\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    # Load mask\n    mask_path = f'{TRAIN_LABELS}/{image_id}.tif'\n    mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE)\n    \n    if mask is None:\n        print(f\"❌ Could not load mask for {image_id}\")\n        return explore_data(idx + 1) if idx + 1 < len(train_df) else None\n    \n    # Print info\n    print(f\"✅ Image ID: {image_id}\")\n    print(f\"Image shape: {img.shape}\")\n    print(f\"Mask shape: {mask.shape}\")\n    print(f\"Mask unique values: {np.unique(mask)}\")\n    print(f\"Surface coverage: {(mask > 0).sum() / mask.size * 100:.2f}%\")\n    \n    # Visualize using plt directly\n    from matplotlib import pyplot as plt\n    fig = plt.figure(figsize=(15, 5))\n    \n    ax1 = fig.add_subplot(1, 3, 1)\n    ax1.imshow(img)\n    ax1.set_title(f'Image: {image_id}')\n    ax1.axis('off')\n    \n    ax2 = fig.add_subplot(1, 3, 2)\n    ax2.imshow(mask, cmap='gray')\n    ax2.set_title('Surface Mask')\n    ax2.axis('off')\n    \n    ax3 = fig.add_subplot(1, 3, 3)\n    ax3.imshow(img)\n    ax3.imshow(mask, cmap='Reds', alpha=0.4)\n    ax3.set_title('Overlay')\n    ax3.axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n\n\nexplore_data(idx=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T16:17:06.692558Z","iopub.execute_input":"2026-01-27T16:17:06.692793Z","iopub.status.idle":"2026-01-27T16:17:08.116322Z","shell.execute_reply.started":"2026-01-27T16:17:06.692773Z","shell.execute_reply":"2026-01-27T16:17:08.115402Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\nimport cv2\nimport numpy as np\n\nclass VesuviusDataset(Dataset):\n    \"\"\"\n    Dataset for Vesuvius surface detection\n    \"\"\"\n    def __init__(self, df, mode='train', transform=None):\n        self.df = df.reset_index(drop=True)\n        self.mode = mode\n        self.transform = transform\n        self.images_dir = TRAIN_IMAGES if mode == 'train' else TEST_IMAGES\n        self.labels_dir = TRAIN_LABELS if mode == 'train' else None\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        image_id = str(row['id'])  # Convert to string\n        \n        # Load TIFF image\n        img_path = f'{self.images_dir}/{image_id}.tif'\n        image = cv2.imread(img_path, cv2.IMREAD_UNCHANGED)\n        \n        if image is None:\n            raise ValueError(f\"Could not load image: {img_path}\")\n        \n        # Convert to RGB\n        if len(image.shape) == 2:\n            image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)\n        elif image.shape[2] == 4:\n            image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGB)\n        else:\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        # Load mask if training\n        if self.mode == 'train':\n            mask_path = f'{self.labels_dir}/{image_id}.tif'\n            mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE)\n            if mask is None:\n                raise ValueError(f\"Could not load mask: {mask_path}\")\n            mask = (mask > 0).astype(np.float32)\n        else:\n            mask = np.zeros((image.shape[0], image.shape[1]), dtype=np.float32)\n        \n        # Apply transformations\n        if self.transform:\n            augmented = self.transform(image=image, mask=mask)\n            image = augmented['image']\n            mask = augmented['mask']\n        \n        # Convert to tensor\n        image = torch.from_numpy(image).permute(2, 0, 1).float() / 255.0\n        mask = torch.from_numpy(mask).unsqueeze(0).float()\n        \n        return image, mask, image_id","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T16:17:08.117348Z","iopub.execute_input":"2026-01-27T16:17:08.117609Z","iopub.status.idle":"2026-01-27T16:17:08.126514Z","shell.execute_reply.started":"2026-01-27T16:17:08.117587Z","shell.execute_reply":"2026-01-27T16:17:08.125902Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"try:\n    import albumentations as A\n    \n    # Training augmentations\n    train_transform = A.Compose([\n        A.Resize(512, 512),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.RandomRotate90(p=0.5),\n        A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.15, rotate_limit=45, p=0.5),\n        A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.3),\n        A.GaussNoise(p=0.2),\n    ])\n    \n    # Validation/Test augmentations\n    val_transform = A.Compose([\n        A.Resize(512, 512),\n    ])\n    \n    print(\"Albumentations loaded successfully\")\n    \nexcept ImportError:\n    print(\"Albumentations not found. Installing...\")\n    print(\"Run: !pip install albumentations\")\n    train_transform = None\n    val_transform = None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T16:17:08.127411Z","iopub.execute_input":"2026-01-27T16:17:08.127748Z","iopub.status.idle":"2026-01-27T16:17:08.877520Z","shell.execute_reply.started":"2026-01-27T16:17:08.127717Z","shell.execute_reply":"2026-01-27T16:17:08.876771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nclass VesuviusModel(nn.Module):\n    \"\"\"\n    U-Net segmentation model with pretrained encoder\n    \"\"\"\n    def __init__(self, encoder_name='resnet34', encoder_weights='imagenet'):\n        super().__init__()\n        \n        self.model = smp.Unet(\n            encoder_name=encoder_name,\n            encoder_weights=encoder_weights,\n            in_channels=3,  # RGB images\n            classes=1,      # Binary segmentation\n            activation=None\n        )\n    \n    def forward(self, x):\n        return torch.sigmoid(self.model(x))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T16:17:08.878600Z","iopub.execute_input":"2026-01-27T16:17:08.879057Z","iopub.status.idle":"2026-01-27T16:17:08.883905Z","shell.execute_reply.started":"2026-01-27T16:17:08.879034Z","shell.execute_reply":"2026-01-27T16:17:08.883164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class DiceLoss(nn.Module):\n    def __init__(self, smooth=1.0):\n        super().__init__()\n        self.smooth = smooth\n    \n    def forward(self, pred, target):\n        pred = pred.view(-1)\n        target = target.view(-1)\n        intersection = (pred * target).sum()\n        dice = (2. * intersection + self.smooth) / (pred.sum() + target.sum() + self.smooth)\n        return 1 - dice\n\nclass CombinedLoss(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.bce = nn.BCELoss()\n        self.dice = DiceLoss()\n    \n    def forward(self, pred, target):\n        return 0.5 * self.bce(pred, target) + 0.5 * self.dice(pred, target)\n\ndef dice_coefficient(pred, target, threshold=0.5):\n    pred = (pred > threshold).float()\n    target = (target > threshold).float()\n    intersection = (pred * target).sum()\n    dice = (2. * intersection) / (pred.sum() + target.sum() + 1e-8)\n    return dice.item()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T16:17:08.884961Z","iopub.execute_input":"2026-01-27T16:17:08.885272Z","iopub.status.idle":"2026-01-27T16:17:08.905266Z","shell.execute_reply.started":"2026-01-27T16:17:08.885246Z","shell.execute_reply":"2026-01-27T16:17:08.904572Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_epoch(model, dataloader, optimizer, criterion, device):\n    model.train()\n    total_loss = 0\n    total_dice = 0\n    \n    for images, masks, _ in dataloader:\n        images = images.to(device)\n        masks = masks.to(device)\n        \n        optimizer.zero_grad()\n        outputs = model(images)\n        loss = criterion(outputs, masks)\n        loss.backward()\n        optimizer.step()\n        \n        total_loss += loss.item()\n        total_dice += dice_coefficient(outputs, masks)\n    \n    return total_loss / len(dataloader), total_dice / len(dataloader)\n\ndef validate(model, dataloader, criterion, device):\n    model.eval()\n    total_loss = 0\n    total_dice = 0\n    \n    with torch.no_grad():\n        for images, masks, _ in dataloader:\n            images = images.to(device)\n            masks = masks.to(device)\n            outputs = model(images)\n            loss = criterion(outputs, masks)\n            total_loss += loss.item()\n            total_dice += dice_coefficient(outputs, masks)\n    \n    return total_loss / len(dataloader), total_dice / len(dataloader)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T16:17:08.906195Z","iopub.execute_input":"2026-01-27T16:17:08.906471Z","iopub.status.idle":"2026-01-27T16:17:08.917556Z","shell.execute_reply.started":"2026-01-27T16:17:08.906444Z","shell.execute_reply":"2026-01-27T16:17:08.916885Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_model(num_epochs=15, batch_size=8, learning_rate=1e-4):\n    \"\"\"\n    Main training function\n    \"\"\"\n    # Split data (80-20 split)\n    from sklearn.model_selection import train_test_split\n    train_data, val_data = train_test_split(train_df, test_size=0.2, random_state=42)\n    \n    print(f\"Training samples: {len(train_data)}\")\n    print(f\"Validation samples: {len(val_data)}\")\n    \n    # Create datasets\n    train_dataset = VesuviusDataset(train_data, mode='train', transform=train_transform)\n    val_dataset = VesuviusDataset(val_data, mode='train', transform=val_transform)\n    \n    # Create dataloaders\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=2)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=2)\n    \n    # Initialize model\n    model = VesuviusModel(encoder_name='resnet34').to(device)\n    criterion = CombinedLoss()\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=3, factor=0.5)\n    \n    # Training loop\n    best_dice = 0\n    history = {'train_loss': [], 'train_dice': [], 'val_loss': [], 'val_dice': []}\n    \n    for epoch in range(num_epochs):\n        print(f\"\\nEpoch {epoch+1}/{num_epochs}\")\n        print(\"-\" * 60)\n        \n        train_loss, train_dice = train_epoch(model, train_loader, optimizer, criterion, device)\n        val_loss, val_dice = validate(model, val_loader, criterion, device)\n        scheduler.step(val_loss)\n        \n        history['train_loss'].append(train_loss)\n        history['train_dice'].append(train_dice)\n        history['val_loss'].append(val_loss)\n        history['val_dice'].append(val_dice)\n        \n        print(f\"Train - Loss: {train_loss:.4f}, Dice: {train_dice:.4f}\")\n        print(f\"Val   - Loss: {val_loss:.4f}, Dice: {val_dice:.4f}\")\n        \n        if val_dice > best_dice:\n            best_dice = val_dice\n            torch.save(model.state_dict(), 'best_model.pth')\n            print(f\"✅ New best model! Dice: {best_dice:.4f}\")\n    \n    # Plot history\n    fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n    axes[0].plot(history['train_loss'], label='Train')\n    axes[0].plot(history['val_loss'], label='Val')\n    axes[0].set_xlabel('Epoch')\n    axes[0].set_ylabel('Loss')\n    axes[0].legend()\n    axes[0].set_title('Loss')\n    \n    axes[1].plot(history['train_dice'], label='Train')\n    axes[1].plot(history['val_dice'], label='Val')\n    axes[1].set_xlabel('Epoch')\n    axes[1].set_ylabel('Dice Score')\n    axes[1].legend()\n    axes[1].set_title('Dice Score')\n    plt.tight_layout()\n    plt.savefig('training_history.png', dpi=100, bbox_inches='tight')\n    plt.show()\n    \n    return model, history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T16:17:08.918515Z","iopub.execute_input":"2026-01-27T16:17:08.918833Z","iopub.status.idle":"2026-01-27T16:17:08.933812Z","shell.execute_reply.started":"2026-01-27T16:17:08.918803Z","shell.execute_reply":"2026-01-27T16:17:08.933177Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Filter out missing files\nprint(\"Filtering for existing files...\")\ndef file_exists(row):\n    return os.path.exists(f'{TRAIN_IMAGES}/{row[\"id\"]}.tif')\n\ntrain_df = train_df[train_df.apply(file_exists, axis=1)].reset_index(drop=True)\nprint(f\"Samples with existing files: {len(train_df)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T16:17:08.934684Z","iopub.execute_input":"2026-01-27T16:17:08.934948Z","iopub.status.idle":"2026-01-27T16:17:09.515677Z","shell.execute_reply.started":"2026-01-27T16:17:08.934916Z","shell.execute_reply":"2026-01-27T16:17:09.514906Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nmodel, history = train_model(num_epochs=5, batch_size=4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T16:17:09.516610Z","iopub.execute_input":"2026-01-27T16:17:09.516896Z","iopub.status.idle":"2026-01-27T16:27:20.687052Z","shell.execute_reply.started":"2026-01-27T16:17:09.516866Z","shell.execute_reply":"2026-01-27T16:27:20.686202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader\n\n# 1.RLE Encoder \ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formatted\n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\n# 2. Define the submission function\ndef create_submission(model, output_path='/kaggle/working/submission.csv'):\n    print(f\"Creating submission at: {output_path}\")\n    model.eval()\n    predictions = []\n    \n    # Ensure test_df exists\n    if 'test_df' not in globals():\n        print(\"Error: test_df is not defined.\")\n        return None\n\n    test_dataset = VesuviusDataset(test_df, mode='test', transform=val_transform)\n    test_loader = DataLoader(test_dataset, batch_size=4, shuffle=False, num_workers=2)\n    \n    print(f\"Generating predictions for {len(test_dataset)} items...\")\n    \n    with torch.no_grad():\n        for images, _, image_ids in test_loader:\n            images = images.to(device)\n            outputs = model(images)\n            \n            for i, image_id in enumerate(image_ids):\n                # Get prediction mask\n                pred_mask = outputs[i, 0].cpu().numpy()\n                pred_binary = (pred_mask > 0.5).astype(np.uint8)\n                \n                # ENCODE TO RLE (Fixes the missing encoding)\n                rle_str = rle_encode(pred_binary)\n                \n                predictions.append({\n                    'Id': image_id,      \n                    'Predicted': rle_str \n                })\n    \n    # Create submission DataFrame\n    if len(predictions) > 0:\n        submission_df = pd.DataFrame(predictions)\n        submission_df.to_csv(output_path, index=False)\n        print(f\"✅ Submission saved to {output_path}\")\n        print(submission_df.head()) # Print first few rows to verify\n        return submission_df\n    else:\n        print(\"Error: No predictions were generated.\")\n        return None\n\n# main function\nif __name__ == \"__main__\":\n    try:\n        submission = create_submission(model, output_path='submission.csv')\n    except Exception as e:\n        print(f\"An error occurred: {e}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-27T16:27:20.690017Z","iopub.execute_input":"2026-01-27T16:27:20.690519Z","iopub.status.idle":"2026-01-27T16:27:21.635307Z","shell.execute_reply.started":"2026-01-27T16:27:20.690488Z","shell.execute_reply":"2026-01-27T16:27:21.634528Z"}},"outputs":[],"execution_count":null}]}