{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":117682,"databundleVersionId":14443416,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:32:05.028315Z","iopub.execute_input":"2025-11-14T14:32:05.028783Z","iopub.status.idle":"2025-11-14T14:32:06.088305Z","shell.execute_reply.started":"2025-11-14T14:32:05.028751Z","shell.execute_reply":"2025-11-14T14:32:06.087105Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Import libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image, ImageSequence\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport zipfile\nimport warnings \nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:32:06.090013Z","iopub.execute_input":"2025-11-14T14:32:06.090389Z","iopub.status.idle":"2025-11-14T14:32:06.096092Z","shell.execute_reply.started":"2025-11-14T14:32:06.090366Z","shell.execute_reply":"2025-11-14T14:32:06.094968Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Loading and Preprocessing","metadata":{}},{"cell_type":"code","source":"CONFIG = {\n    'IMG_HEIGHT': 128,  # Adjust based on your memory constraints\n    'IMG_WIDTH': 128,\n    'IMG_DEPTH': 32,    # Number of slices to use\n    'BATCH_SIZE': 4,\n    'EPOCHS': 3,\n    'LEARNING_RATE': 1e-4,\n    'DATA_PATH': '/kaggle/input/vesuvius-challenge-surface-detection',\n    'TRAIN_IMAGES': '/kaggle/input/vesuvius-challenge-surface-detection/train_images',\n    'TRAIN_LABELS': '/kaggle/input/vesuvius-challenge-surface-detection/train_labels',\n    'TEST_IMAGES': '/kaggle/input/vesuvius-challenge-surface-detection/test_images',\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:32:06.097242Z","iopub.execute_input":"2025-11-14T14:32:06.097651Z","iopub.status.idle":"2025-11-14T14:32:06.128262Z","shell.execute_reply.started":"2025-11-14T14:32:06.097625Z","shell.execute_reply":"2025-11-14T14:32:06.127083Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_volume(path):\n    \"\"\"Load a 3D volume from a tif file.\"\"\"\n    im = Image.open(path)\n    slices = []\n    for page in ImageSequence.Iterator(im):\n        slice_array = np.array(page)\n        slices.append(slice_array)\n    volume = np.stack(slices, axis=0)\n    return volume\n\ndef normalize_volume(volume):\n    \"\"\"Normalize volume to [0, 1] range.\"\"\"\n    volume = volume.astype(np.float32)\n    vmin, vmax = volume.min(), volume.max()\n    if vmax > vmin:\n        volume = (volume - vmin) / (vmax - vmin)\n    return volume\n\ndef load_data_paths():\n    \"\"\"Load paths to training data.\"\"\"\n    train_df = pd.read_csv(f\"{CONFIG['DATA_PATH']}/train.csv\")\n    \n    data = []\n    for idx, row in train_df.iterrows():\n        img_id = row['id']\n        img_path = f\"{CONFIG['TRAIN_IMAGES']}/{img_id}.tif\"\n        label_path = f\"{CONFIG['TRAIN_LABELS']}/{img_id}.tif\"\n        \n        if os.path.exists(img_path) and os.path.exists(label_path):\n            data.append({\n                'id': img_id,\n                'image_path': img_path,\n                'label_path': label_path\n            })\n    \n    return pd.DataFrame(data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:32:06.129625Z","iopub.execute_input":"2025-11-14T14:32:06.129904Z","iopub.status.idle":"2025-11-14T14:32:06.151207Z","shell.execute_reply.started":"2025-11-14T14:32:06.129872Z","shell.execute_reply":"2025-11-14T14:32:06.150028Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prepare_patch(volume, target_depth, target_height, target_width):\n    \"\"\"Extract or resize a patch from volume.\"\"\"\n    d, h, w = volume.shape\n    \n    # Simple center crop or pad\n    if d > target_depth:\n        start_d = (d - target_depth) // 2\n        volume = volume[start_d:start_d + target_depth]\n    elif d < target_depth:\n        pad_d = target_depth - d\n        volume = np.pad(volume, ((0, pad_d), (0, 0), (0, 0)), mode='constant')\n    \n    if h > target_height or w > target_width:\n        start_h = max(0, (h - target_height) // 2)\n        start_w = max(0, (w - target_width) // 2)\n        volume = volume[:, start_h:start_h + target_height, start_w:start_w + target_width]\n    \n    # Pad if needed\n    d, h, w = volume.shape\n    if h < target_height or w < target_width:\n        pad_h = max(0, target_height - h)\n        pad_w = max(0, target_width - w)\n        volume = np.pad(volume, ((0, 0), (0, pad_h), (0, pad_w)), mode='constant')\n    \n    return volume","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:32:06.153992Z","iopub.execute_input":"2025-11-14T14:32:06.154287Z","iopub.status.idle":"2025-11-14T14:32:06.177218Z","shell.execute_reply.started":"2025-11-14T14:32:06.154266Z","shell.execute_reply":"2025-11-14T14:32:06.176116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def data_generator(df, batch_size, is_training=True):\n    \"\"\"Generator for training data.\"\"\"\n    indices = np.arange(len(df))\n    \n    while True:\n        if is_training:\n            np.random.shuffle(indices)\n        \n        for start_idx in range(0, len(df), batch_size):\n            batch_indices = indices[start_idx:start_idx + batch_size]\n            batch_images = []\n            batch_labels = []\n            \n            for idx in batch_indices:\n                row = df.iloc[idx]\n                \n                # Load volumes\n                img_vol = load_volume(row['image_path'])\n                label_vol = load_volume(row['label_path'])\n                \n                # Prepare patches\n                img_vol = prepare_patch(img_vol, CONFIG['IMG_DEPTH'], \n                                       CONFIG['IMG_HEIGHT'], CONFIG['IMG_WIDTH'])\n                label_vol = prepare_patch(label_vol, CONFIG['IMG_DEPTH'],\n                                         CONFIG['IMG_HEIGHT'], CONFIG['IMG_WIDTH'])\n                \n                # Normalize\n                img_vol = normalize_volume(img_vol)\n                \n                # Convert labels (0=background, 1=ink, 2=ignore) to binary\n                label_vol = (label_vol == 1).astype(np.float32)\n                \n                # Add channel dimension\n                img_vol = img_vol[..., np.newaxis]\n                label_vol = label_vol[..., np.newaxis]\n                \n                batch_images.append(img_vol)\n                batch_labels.append(label_vol)\n            \n            yield np.array(batch_images), np.array(batch_labels)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:32:06.178137Z","iopub.execute_input":"2025-11-14T14:32:06.178474Z","iopub.status.idle":"2025-11-14T14:32:06.209638Z","shell.execute_reply.started":"2025-11-14T14:32:06.178447Z","shell.execute_reply":"2025-11-14T14:32:06.208368Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Plots","metadata":{}},{"cell_type":"code","source":"df = load_data_paths()\nprint(f\"Found {len(df)} training volumes\\n\")\n\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:32:06.210642Z","iopub.execute_input":"2025-11-14T14:32:06.210994Z","iopub.status.idle":"2025-11-14T14:32:06.3232Z","shell.execute_reply.started":"2025-11-14T14:32:06.210959Z","shell.execute_reply":"2025-11-14T14:32:06.322141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"VISUALIZING FIRST SAMPLE\")\nprint(\"=\"*80)\n\nrow = df.iloc[0]\nimg_vol = load_volume(row['image_path'])\nlabel_vol = load_volume(row['label_path'])\nimg_vol_norm = normalize_volume(img_vol)\n\nprint(f\"Loaded id: {row['id']}\")\nprint(f\"  Image shape: {img_vol.shape}\")\nprint(f\"  Label shape: {label_vol.shape}\")\nprint(f\"  Image dtype: {img_vol.dtype}, range: [{img_vol.min()}, {img_vol.max()}]\")\nprint(f\"  Label unique values: {np.unique(label_vol)}\")\n\n# Quick statistics\nink_pixels = np.sum(label_vol == 1)\ntotal_pixels = label_vol.size\nink_percentage = 100 * ink_pixels / total_pixels\nprint(f\"  Ink pixels: {ink_pixels:,} ({ink_percentage:.3f}%)\")\nprint()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:32:06.324293Z","iopub.execute_input":"2025-11-14T14:32:06.324542Z","iopub.status.idle":"2025-11-14T14:32:09.702851Z","shell.execute_reply.started":"2025-11-14T14:32:06.324521Z","shell.execute_reply":"2025-11-14T14:32:09.701626Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def visualize_sample_slices(volume, label=None, num_slices=8, title=\"Volume Slices\", cmap='gray'):\n    \"\"\"\n    Visualize evenly spaced slices from a 3D volume.\n    \"\"\"\n    depth = volume.shape[0]\n    indices = np.linspace(0, depth-1, num_slices, dtype=int)\n    \n    if label is not None:\n        fig, axes = plt.subplots(2, num_slices, figsize=(20, 5))\n        for i, idx in enumerate(indices):\n            axes[0, i].imshow(volume[idx], cmap=cmap)\n            axes[0, i].set_title(f'Slice {idx}')\n            axes[0, i].axis('off')\n            \n            axes[1, i].imshow(label[idx], cmap='jet', vmin=0, vmax=2)\n            axes[1, i].set_title(f'Label {idx}')\n            axes[1, i].axis('off')\n        \n        axes[0, 0].set_ylabel('Image', fontsize=12, rotation=0, labelpad=40)\n        axes[1, 0].set_ylabel('Mask', fontsize=12, rotation=0, labelpad=40)\n    else:\n        fig, axes = plt.subplots(1, num_slices, figsize=(20, 3))\n        for i, idx in enumerate(indices):\n            axes[i].imshow(volume[idx], cmap=cmap)\n            axes[i].set_title(f'Slice {idx}')\n            axes[i].axis('off')\n    \n    plt.suptitle(title, fontsize=16, y=1.02)\n    plt.tight_layout()\n    plt.show()\n    plt.close()\n\n\n # Slice views\nprint(\"\\nSlice views (8 evenly spaced slices)...\")\nvisualize_sample_slices(img_vol_norm, label_vol, num_slices=8, \n                       title=f\"Sample: {row['id']} - Slice Views\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:32:09.704188Z","iopub.execute_input":"2025-11-14T14:32:09.704688Z","iopub.status.idle":"2025-11-14T14:32:11.489828Z","shell.execute_reply.started":"2025-11-14T14:32:09.704658Z","shell.execute_reply":"2025-11-14T14:32:11.488363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def visualize_orthogonal_views(volume, label=None, slice_indices=None):\n    \"\"\"\n    Show orthogonal views (axial, sagittal, coronal) of a 3D volume.\n    \"\"\"\n    if slice_indices is None:\n        slice_indices = {\n            'z': volume.shape[0] // 2,\n            'y': volume.shape[1] // 2,\n            'x': volume.shape[2] // 2\n        }\n    \n    if label is not None:\n        fig, axes = plt.subplots(2, 3, figsize=(15, 10))\n        \n        # Image views\n        axes[0, 0].imshow(volume[slice_indices['z']], cmap='gray')\n        axes[0, 0].set_title(f\"Axial (Z={slice_indices['z']})\")\n        axes[0, 0].set_xlabel('X'); axes[0, 0].set_ylabel('Y')\n        \n        axes[0, 1].imshow(volume[:, slice_indices['y'], :], cmap='gray', aspect='auto')\n        axes[0, 1].set_title(f\"Coronal (Y={slice_indices['y']})\")\n        axes[0, 1].set_xlabel('X'); axes[0, 1].set_ylabel('Z')\n        \n        axes[0, 2].imshow(volume[:, :, slice_indices['x']], cmap='gray', aspect='auto')\n        axes[0, 2].set_title(f\"Sagittal (X={slice_indices['x']})\")\n        axes[0, 2].set_xlabel('Y'); axes[0, 2].set_ylabel('Z')\n        \n        # Label views\n        axes[1, 0].imshow(label[slice_indices['z']], cmap='jet', vmin=0, vmax=2)\n        axes[1, 0].set_title(\"Mask - Axial\")\n        axes[1, 0].set_xlabel('X'); axes[1, 0].set_ylabel('Y')\n        \n        axes[1, 1].imshow(label[:, slice_indices['y'], :], cmap='jet', vmin=0, vmax=2, aspect='auto')\n        axes[1, 1].set_title(\"Mask - Coronal\")\n        axes[1, 1].set_xlabel('X'); axes[1, 1].set_ylabel('Z')\n        \n        axes[1, 2].imshow(label[:, :, slice_indices['x']], cmap='jet', vmin=0, vmax=2, aspect='auto')\n        axes[1, 2].set_title(\"Mask - Sagittal\")\n        axes[1, 2].set_xlabel('Y'); axes[1, 2].set_ylabel('Z')\n    else:\n        fig, axes = plt.subplots(1, 3, figsize=(15, 5))\n        \n        axes[0].imshow(volume[slice_indices['z']], cmap='gray')\n        axes[0].set_title(f\"Axial (Z={slice_indices['z']})\")\n        \n        axes[1].imshow(volume[:, slice_indices['y'], :], cmap='gray', aspect='auto')\n        axes[1].set_title(f\"Coronal (Y={slice_indices['y']})\")\n        \n        axes[2].imshow(volume[:, :, slice_indices['x']], cmap='gray', aspect='auto')\n        axes[2].set_title(f\"Sagittal (X={slice_indices['x']})\")\n    \n    plt.tight_layout()\n    plt.show()\n    plt.close()\n\n\n# Orthogonal views\nprint(\"\\nOrthogonal views (3 planes)...\")\nvisualize_orthogonal_views(img_vol_norm, label_vol)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:32:11.490785Z","iopub.execute_input":"2025-11-14T14:32:11.491091Z","iopub.status.idle":"2025-11-14T14:32:12.976843Z","shell.execute_reply.started":"2025-11-14T14:32:11.491068Z","shell.execute_reply":"2025-11-14T14:32:12.975961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def visualize_overlay(volume, label, slice_idx=None, alpha=0.5):\n    \"\"\"\n    Visualize image with mask overlay.\n    \"\"\"\n    if slice_idx is None:\n        slice_idx = volume.shape[0] // 2\n    \n    fig, axes = plt.subplots(1, 3, figsize=(18, 6))\n    \n    # Original image\n    axes[0].imshow(volume[slice_idx], cmap='gray')\n    axes[0].set_title(f'Image (Slice {slice_idx})')\n    axes[0].axis('off')\n    \n    # Mask only\n    mask_display = np.ma.masked_where(label[slice_idx] == 0, label[slice_idx])\n    axes[1].imshow(volume[slice_idx], cmap='gray')\n    axes[1].imshow(mask_display, cmap='Reds', alpha=0.7, vmin=0, vmax=2)\n    axes[1].set_title('Mask Only')\n    axes[1].axis('off')\n    \n    # Overlay\n    axes[2].imshow(volume[slice_idx], cmap='gray')\n    axes[2].imshow(mask_display, cmap='jet', alpha=alpha, vmin=0, vmax=2)\n    axes[2].set_title(f'Overlay (alpha={alpha})')\n    axes[2].axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n    plt.close()\n\n\n# Overlay visualization\nprint(\"\\n Image-Mask overlay...\")\nvisualize_overlay(img_vol_norm, label_vol)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:32:12.978174Z","iopub.execute_input":"2025-11-14T14:32:12.97863Z","iopub.status.idle":"2025-11-14T14:32:13.748657Z","shell.execute_reply.started":"2025-11-14T14:32:12.978593Z","shell.execute_reply":"2025-11-14T14:32:13.747436Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def visualize_multiple_samples(data_df, num_samples=3, slices_per_sample=4):\n    \"\"\"\n    Visualize multiple samples from the dataset.\n    \"\"\"\n    num_samples = min(num_samples, len(data_df))\n    \n    fig, axes = plt.subplots(num_samples * 2, slices_per_sample, \n                             figsize=(4*slices_per_sample, 4*num_samples))\n    \n    if num_samples == 1:\n        axes = axes.reshape(2, -1)\n    \n    for sample_idx in range(num_samples):\n        row = data_df.iloc[sample_idx]\n        \n        # Load volumes\n        img_vol = load_volume(row['image_path'])\n        label_vol = load_volume(row['label_path'])\n        img_vol = normalize_volume(img_vol)\n        \n        # Select slices\n        depth = img_vol.shape[0]\n        slice_indices = np.linspace(0, depth-1, slices_per_sample, dtype=int)\n        \n        for i, slice_idx in enumerate(slice_indices):\n            # Image\n            axes[sample_idx*2, i].imshow(img_vol[slice_idx], cmap='gray')\n            axes[sample_idx*2, i].set_title(f'Sample {sample_idx+1}, Slice {slice_idx}')\n            axes[sample_idx*2, i].axis('off')\n            \n            # Label\n            axes[sample_idx*2+1, i].imshow(label_vol[slice_idx], cmap='jet', vmin=0, vmax=2)\n            axes[sample_idx*2+1, i].axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n    plt.close()\n\nif len(df) >= 3:\n    num_samples = min(3, len(df))\n    visualize_multiple_samples(df, num_samples=num_samples, slices_per_sample=4)\nelse:\n    print(f\"Only {len(df)} sample(s) available.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:32:13.7498Z","iopub.execute_input":"2025-11-14T14:32:13.750441Z","iopub.status.idle":"2025-11-14T14:32:23.004664Z","shell.execute_reply.started":"2025-11-14T14:32:13.7504Z","shell.execute_reply":"2025-11-14T14:32:23.003333Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model (3D U-Net)\n","metadata":{}},{"cell_type":"code","source":"def conv_block(inputs, filters, kernel_size=3):\n    \"\"\"Convolutional block with two conv layers.\"\"\"\n    x = layers.Conv3D(filters, kernel_size, padding='same')(inputs)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation('relu')(x)\n    \n    x = layers.Conv3D(filters, kernel_size, padding='same')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation('relu')(x)\n    \n    return x\n\ndef encoder_block(inputs, filters):\n    \"\"\"Encoder block with conv and pooling.\"\"\"\n    x = conv_block(inputs, filters)\n    p = layers.MaxPooling3D((2, 2, 2))(x)\n    return x, p\n\ndef decoder_block(inputs, skip, filters):\n    \"\"\"Decoder block with upsampling and concatenation.\"\"\"\n    x = layers.Conv3DTranspose(filters, 2, strides=2, padding='same')(inputs)\n    x = layers.Concatenate()([x, skip])\n    x = conv_block(x, filters)\n    return x\n\ndef build_unet_3d(input_shape, num_classes=1):\n    \"\"\"Build 3D U-Net model.\"\"\"\n    inputs = keras.Input(shape=input_shape)\n    \n    # Encoder\n    s1, p1 = encoder_block(inputs, 32)\n    s2, p2 = encoder_block(p1, 64)\n    s3, p3 = encoder_block(p2, 128)\n    s4, p4 = encoder_block(p3, 256)\n    \n    # Bridge\n    b = conv_block(p4, 512)\n    \n    # Decoder\n    d1 = decoder_block(b, s4, 256)\n    d2 = decoder_block(d1, s3, 128)\n    d3 = decoder_block(d2, s2, 64)\n    d4 = decoder_block(d3, s1, 32)\n    \n    # Output\n    outputs = layers.Conv3D(num_classes, 1, padding='same', activation='sigmoid')(d4)\n    \n    model = keras.Model(inputs=inputs, outputs=outputs, name='3D-UNet')\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:47:03.318846Z","iopub.execute_input":"2025-11-14T14:47:03.319811Z","iopub.status.idle":"2025-11-14T14:47:03.330524Z","shell.execute_reply.started":"2025-11-14T14:47:03.319763Z","shell.execute_reply":"2025-11-14T14:47:03.329483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def dice_coefficient(y_true, y_pred, smooth=1e-6):\n    \"\"\"Dice coefficient metric.\"\"\"\n    y_true_f = tf.reshape(y_true, [-1])\n    y_pred_f = tf.reshape(y_pred, [-1])\n    intersection = tf.reduce_sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (tf.reduce_sum(y_true_f) + tf.reduce_sum(y_pred_f) + smooth)\n\ndef dice_loss(y_true, y_pred):\n    \"\"\"Dice loss function.\"\"\"\n    return 1 - dice_coefficient(y_true, y_pred)\n\ndef combined_loss(y_true, y_pred):\n    \"\"\"Combined BCE and Dice loss.\"\"\"\n    bce = tf.keras.losses.binary_crossentropy(y_true, y_pred)\n    dice = dice_loss(y_true, y_pred)\n    return bce + dice\n\ndef train_model():\n    print(\"Loading data paths...\")\n    df = load_data_paths()\n    print(f\"Found {len(df)} training volumes\")\n    \n    if len(df) == 0:\n        print(\"No training data found! Check your data paths.\")\n        return None\n    \n    # Split data\n    train_df, val_df = train_test_split(df, test_size=0.2, random_state=42)\n    print(f\"Training samples: {len(train_df)}, Validation samples: {len(val_df)}\")\n    \n    # Create model\n    print(\"Building model...\")\n    input_shape = (CONFIG['IMG_DEPTH'], CONFIG['IMG_HEIGHT'], CONFIG['IMG_WIDTH'], 1)\n    model = build_unet_3d(input_shape)\n    \n    # Compile\n    model.compile(\n        optimizer=keras.optimizers.Adam(CONFIG['LEARNING_RATE']),\n        loss=combined_loss,\n        metrics=[dice_coefficient, 'binary_accuracy']\n    )\n    \n    print(model.summary())\n    \n    # Callbacks\n    callbacks = [\n        keras.callbacks.ModelCheckpoint(\n            'best_model.h5',\n            save_best_only=True,\n            monitor='val_dice_coefficient',\n            mode='max'\n        ),\n        keras.callbacks.ReduceLROnPlateau(\n            monitor='val_loss',\n            factor=0.5,\n            patience=3,\n            min_lr=1e-7\n        ),\n        keras.callbacks.EarlyStopping(\n            monitor='val_dice_coefficient',\n            patience=5,\n            mode='max',\n            restore_best_weights=True\n        )\n    ]\n    \n    # Train\n    steps_per_epoch = len(train_df) // CONFIG['BATCH_SIZE']\n    validation_steps = max(1, len(val_df) // CONFIG['BATCH_SIZE'])\n    \n    print(\"Starting training...\")\n    history = model.fit(\n        data_generator(train_df, CONFIG['BATCH_SIZE'], is_training=True),\n        steps_per_epoch=steps_per_epoch,\n        epochs=CONFIG['EPOCHS'],\n        validation_data=data_generator(val_df, CONFIG['BATCH_SIZE'], is_training=False),\n        validation_steps=validation_steps,\n        callbacks=callbacks\n    )\n    \n    return model, history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:47:09.790626Z","iopub.execute_input":"2025-11-14T14:47:09.791072Z","iopub.status.idle":"2025-11-14T14:47:09.807062Z","shell.execute_reply.started":"2025-11-14T14:47:09.791037Z","shell.execute_reply":"2025-11-14T14:47:09.805843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_submission(model):\n    \"\"\"Create submission file.\"\"\"\n    print(\"Creating submission...\")\n    \n    test_df = pd.read_csv(f\"{CONFIG['DATA_PATH']}/test.csv\")\n    os.makedirs('submission', exist_ok=True)\n    \n    for idx, row in tqdm(test_df.iterrows(), total=len(test_df)):\n        img_id = row['id']\n        img_path = f\"{CONFIG['TEST_IMAGES']}/{img_id}.tif\"\n        \n        if not os.path.exists(img_path):\n            print(f\"Warning: {img_path} not found\")\n            continue\n        \n        # Load and predict\n        volume = load_volume(img_path)\n        original_shape = volume.shape\n        \n        # Prepare for model\n        volume_proc = prepare_patch(volume, CONFIG['IMG_DEPTH'],\n                                   CONFIG['IMG_HEIGHT'], CONFIG['IMG_WIDTH'])\n        volume_proc = normalize_volume(volume_proc)\n        volume_proc = volume_proc[..., np.newaxis]\n        volume_proc = np.expand_dims(volume_proc, 0)\n        \n        # Predict\n        pred = model.predict(volume_proc, verbose=0)[0, ..., 0]\n        \n        # Threshold\n        pred_binary = (pred > 0.5).astype(np.uint8)\n        \n        # Resize back to original shape\n        pred_resized = np.zeros(original_shape, dtype=np.uint8)\n        d_min = min(pred_binary.shape[0], original_shape[0])\n        h_min = min(pred_binary.shape[1], original_shape[1])\n        w_min = min(pred_binary.shape[2], original_shape[2])\n        pred_resized[:d_min, :h_min, :w_min] = pred_binary[:d_min, :h_min, :w_min]\n        \n        # Save as tif\n        images = [Image.fromarray(pred_resized[i]) for i in range(pred_resized.shape[0])]\n        images[0].save(\n            f'submission/{img_id}.tif',\n            save_all=True,\n            append_images=images[1:]\n        )\n    \n    # Create zip\n    with zipfile.ZipFile('submission.zip', 'w') as zipf:\n        for file in os.listdir('submission'):\n            zipf.write(os.path.join('submission', file), file)\n    \n    print(\"Submission created: submission.zip\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:47:15.608218Z","iopub.execute_input":"2025-11-14T14:47:15.608517Z","iopub.status.idle":"2025-11-14T14:47:15.620597Z","shell.execute_reply.started":"2025-11-14T14:47:15.608499Z","shell.execute_reply":"2025-11-14T14:47:15.619516Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model, history = train_model()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T14:47:20.838512Z","iopub.execute_input":"2025-11-14T14:47:20.838824Z","execution_failed":"2025-11-14T16:23:06.477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if model is not None:\n    plt.figure(figsize=(12, 4))\n    \n    plt.subplot(1, 2, 1)\n    plt.plot(history.history['loss'], label='Train Loss')\n    plt.plot(history.history['val_loss'], label='Val Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend()\n    plt.title('Loss')\n    \n    plt.subplot(1, 2, 2)\n    plt.plot(history.history['dice_coefficient'], label='Train Dice')\n    plt.plot(history.history['val_dice_coefficient'], label='Val Dice')\n    plt.xlabel('Epoch')\n    plt.ylabel('Dice Coefficient')\n    plt.legend()\n    plt.title('Dice Coefficient')\n    \n    plt.tight_layout()\n    plt.savefig('training_history.png')\n    plt.show()\n    \n    # Create submission\n    create_submission(model)\n    \n    print(\"\\nDone\")\nelse:\n    print(\"\\nTraining failed\")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-14T16:23:06.477Z"}},"outputs":[],"execution_count":null}]}