{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[],"gpuType":"T4"},"accelerator":"GPU","kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":117682,"databundleVersionId":14443416,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q imagecodecs","metadata":{"id":"PArzWOl22cmc","trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:36:17.632624Z","iopub.execute_input":"2025-12-06T12:36:17.633053Z","iopub.status.idle":"2025-12-06T12:36:22.793607Z","shell.execute_reply.started":"2025-12-06T12:36:17.633019Z","shell.execute_reply":"2025-12-06T12:36:22.792888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PreprocessConfig:\n    # UBAH INI:\n    # DATA_DIR = \"/content/vesuvius_data\"  # ❌ Colab path\n    DATA_DIR = \"/kaggle/input/vesuvius-challenge-surface-detection\"  # ✅ Kaggle path\n    \n    # UBAH INI:\n    # OUTPUT_DIR = \"/content/preprocessed_data\"  # ❌ Colab path\n    OUTPUT_DIR = \"/kaggle/working/preprocessed_data\"  # ✅ Kaggle path","metadata":{"id":"ylMZSVBTcHY8","outputId":"25d7a3e1-0580-44f4-82bc-6d3887576583","trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:36:22.795369Z","iopub.execute_input":"2025-12-06T12:36:22.795630Z","iopub.status.idle":"2025-12-06T12:36:22.799735Z","shell.execute_reply.started":"2025-12-06T12:36:22.795605Z","shell.execute_reply":"2025-12-06T12:36:22.799006Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls /kaggle/input/vesuvius-challenge-surface-detection","metadata":{"id":"ljX3RARrZ22N","outputId":"28e7b26f-495f-47ee-a7cf-71b62b681e58","trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:36:22.801356Z","iopub.execute_input":"2025-12-06T12:36:22.801721Z","iopub.status.idle":"2025-12-06T12:36:22.925217Z","shell.execute_reply.started":"2025-12-06T12:36:22.801701Z","shell.execute_reply":"2025-12-06T12:36:22.924550Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"MULAI DARI AWAL LAGI","metadata":{"id":"qEO-JUaftZi3"}},{"cell_type":"code","source":"\"\"\"\nSTEP 0: INSTALL DEPENDENCIES\nJalankan cell ini DULU, lalu RESTART RUNTIME sebelum jalankan code lain!\n\"\"\"\n\nimport subprocess\nimport sys\n\nprint(\"=\"*70)\nprint(\"INSTALLING REQUIRED PACKAGES\")\nprint(\"=\"*70)\n\n# Install imagecodecs\nprint(\"\\n[1/1] Installing imagecodecs (for LZW compression support)...\")\nsubprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', 'imagecodecs'])\n\nprint(\"\\n\" + \"=\"*70)\nprint(\"✅ INSTALLATION COMPLETE!\")\nprint(\"=\"*70)\nprint(\"\\n⚠️  IMPORTANT: Please RESTART RUNTIME now!\")\nprint(\"   (Runtime → Restart runtime)\")\nprint(\"\\nAfter restart, you can run the exploration and preprocessing scripts.\")\nprint(\"=\"*70)","metadata":{"id":"UFWHukXNsiZP","outputId":"15342039-9e12-40c8-eae4-11607b318510","trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:36:22.926889Z","iopub.execute_input":"2025-12-06T12:36:22.927129Z","iopub.status.idle":"2025-12-06T12:36:25.923769Z","shell.execute_reply.started":"2025-12-06T12:36:22.927105Z","shell.execute_reply":"2025-12-06T12:36:25.922969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Install imagecodecs\nimport subprocess\nimport sys\nsubprocess.check_call([sys.executable, '-m', 'pip', 'install', 'imagecodecs'])\n\nprint(\"=\"*70)\nprint(\"✅ imagecodecs installed!\")\nprint(\"⚠️  RESTART KERNEL NOW!\")\nprint(\"=\"*70)\nprint(\"\\nSteps:\")\nprint(\"1. Click 'Session' → 'Restart Session'\")\nprint(\"2. Run the exploration script again\")","metadata":{"id":"W-DcgDWItc5t","outputId":"3b4460c8-3df8-4001-f88f-6c9553e7de8c","trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:36:25.924824Z","iopub.execute_input":"2025-12-06T12:36:25.925193Z","iopub.status.idle":"2025-12-06T12:36:28.892234Z","shell.execute_reply.started":"2025-12-06T12:36:25.925161Z","shell.execute_reply":"2025-12-06T12:36:28.891410Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# SCRIPT EKSPLORASI DATASET VESUVIUS - KAGGLE VERSION (NO IMAGECODECS)\n# Dataset ini berisi 3D CT scan volumes (320x320x320)\n# Uses PIL as alternative to avoid imagecodecs dependency\n# ============================================================================\n\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ============================================================================\n# PATH CONFIGURATION - KAGGLE\n# ============================================================================\nDATA_DIR = \"/kaggle/input/vesuvius-challenge-surface-detection\"\nOUTPUT_DIR = \"/kaggle/working\"\n\nprint(\"=\"*70)\nprint(\"EKSPLORASI DATASET VESUVIUS - 3D CT SCAN (KAGGLE)\")\nprint(\"=\"*70)\n\n# ============================================================================\n# HELPER FUNCTION: Load Multi-page TIFF dengan PIL\n# ============================================================================\ndef load_multipage_tiff(path, max_frames=None):\n    \"\"\"\n    Load multi-page TIFF file menggunakan PIL\n    Slower than tifffile but works without imagecodecs\n    \"\"\"\n    frames = []\n    try:\n        with Image.open(path) as img:\n            n_frames = 0\n            # Count total frames\n            try:\n                while True:\n                    img.seek(n_frames)\n                    n_frames += 1\n            except EOFError:\n                pass\n            \n            # Load frames\n            if max_frames:\n                n_frames = min(n_frames, max_frames)\n            \n            for i in range(n_frames):\n                img.seek(i)\n                frames.append(np.array(img))\n            \n        return np.stack(frames, axis=0)\n    except Exception as e:\n        print(f\"Error loading {path.name}: {e}\")\n        return None\n\n# ============================================================================\n# 1. CEK STRUKTUR FOLDER\n# ============================================================================\nprint(\"\\n[1] STRUKTUR FOLDER:\")\nprint(\"-\"*70)\n\ndata_path = Path(DATA_DIR)\n\nif not data_path.exists():\n    print(f\"❌ ERROR: Data path tidak ditemukan: {DATA_DIR}\")\n    print(\"\\n⚠️  Pastikan Anda sudah Add Data di Kaggle:\")\n    print(\"   1. Klik 'Add data' button\")\n    print(\"   2. Search: 'vesuvius-challenge-surface-detection'\")\n    print(\"   3. Click 'Add'\")\nelse:\n    print(f\"✅ Data path found: {DATA_DIR}\\n\")\n    for item in sorted(data_path.iterdir()):\n        if item.is_dir():\n            count = len(list(item.iterdir()))\n            print(f\"📁 {item.name:25s} → {count} files\")\n        else:\n            size_mb = item.stat().st_size / (1024*1024)\n            print(f\"📄 {item.name:25s} → {size_mb:.2f} MB\")\n\n    # ============================================================================\n    # 2. ANALISIS train.csv DAN test.csv\n    # ============================================================================\n    print(\"\\n[2] ANALISIS CSV FILES:\")\n    print(\"-\"*70)\n\n    train_csv = pd.read_csv(data_path / \"train.csv\")\n    test_csv = pd.read_csv(data_path / \"test.csv\")\n\n    print(f\"train.csv shape: {train_csv.shape}\")\n    print(f\"train.csv columns: {train_csv.columns.tolist()}\")\n    print(f\"\\nSample train.csv:\")\n    print(train_csv.head(10))\n\n    print(f\"\\ntest.csv shape: {test_csv.shape}\")\n    print(f\"test.csv columns: {test_csv.columns.tolist()}\")\n    print(f\"\\nSample test.csv:\")\n    print(test_csv.head())\n\n    # Analisis unique values\n    print(f\"\\n📊 Unique scroll_ids in train: {train_csv['scroll_id'].nunique()}\")\n    print(f\"Scroll_id distribution:\")\n    print(train_csv['scroll_id'].value_counts())\n\n    # ============================================================================\n    # 3. ANALISIS STRUKTUR 3D VOLUMES\n    # ============================================================================\n    print(\"\\n[3] ANALISIS 3D VOLUMES:\")\n    print(\"-\"*70)\n\n    train_images_dir = data_path / \"train_images\"\n    train_labels_dir = data_path / \"train_labels\"\n\n    image_files = sorted(list(train_images_dir.glob(\"*.tif\")))\n    label_files = sorted(list(train_labels_dir.glob(\"*.tif\")))\n\n    print(f\"Training volumes: {len(image_files)}\")\n    print(f\"Label volumes: {len(label_files)}\")\n\n    # Load sample volume\n    if len(image_files) > 0:\n        sample_img_path = image_files[0]\n        sample_label_path = train_labels_dir / sample_img_path.name\n\n        print(f\"\\n📦 Loading sample: {sample_img_path.name}\")\n        print(\"   (Using PIL - this may take 30-60 seconds...)\")\n\n        try:\n            # Load 3D volume using PIL (load only first 100 slices for speed)\n            img_volume = load_multipage_tiff(sample_img_path, max_frames=100)\n            \n            if img_volume is not None:\n                print(f\"✓ Image volume loaded!\")\n                print(f\"  Shape: {img_volume.shape} (D x H x W)\")\n                print(f\"  Note: Loaded first 100 slices only for speed\")\n                print(f\"  Dtype: {img_volume.dtype}\")\n                print(f\"  Size: {img_volume.nbytes / (1024**2):.2f} MB\")\n                print(f\"  Min: {img_volume.min()}, Max: {img_volume.max()}\")\n                print(f\"  Mean: {img_volume.mean():.2f}, Std: {img_volume.std():.2f}\")\n\n                # Load label volume jika ada\n                if sample_label_path.exists():\n                    print(f\"\\n📦 Loading label volume...\")\n                    label_volume = load_multipage_tiff(sample_label_path, max_frames=100)\n                    \n                    if label_volume is not None:\n                        print(f\"✓ Label volume loaded!\")\n                        print(f\"  Shape: {label_volume.shape}\")\n                        print(f\"  Dtype: {label_volume.dtype}\")\n                        print(f\"  Unique values: {np.unique(label_volume)}\")\n                        print(f\"  Surface pixels: {(label_volume > 0).sum()}\")\n                        print(f\"  Coverage: {(label_volume > 0).sum() / label_volume.size * 100:.4f}%\")\n\n                        # ================================================================\n                        # VISUALISASI MULTIPLE SLICES\n                        # ================================================================\n                        print(\"\\n[4] VISUALISASI SLICES:\")\n                        print(\"-\"*70)\n\n                        # Ambil beberapa slices dari tengah volume\n                        depth = img_volume.shape[0]\n                        slice_indices = [depth//4, depth//2, 3*depth//4]\n\n                        fig, axes = plt.subplots(3, 3, figsize=(15, 15))\n\n                        for i, slice_idx in enumerate(slice_indices):\n                            # Image slice\n                            img_slice = img_volume[slice_idx]\n                            label_slice = label_volume[slice_idx]\n\n                            # Normalisasi untuk visualisasi\n                            img_normalized = (img_slice - img_slice.min()) / (img_slice.max() - img_slice.min() + 1e-8)\n\n                            # Column 1: CT scan slice\n                            axes[i, 0].imshow(img_slice, cmap='gray')\n                            axes[i, 0].set_title(f'CT Slice {slice_idx}/{depth}\\nDepth: {slice_idx}')\n                            axes[i, 0].axis('off')\n\n                            # Column 2: Label slice\n                            axes[i, 1].imshow(label_slice, cmap='hot')\n                            axes[i, 1].set_title(f'Surface Label\\nPixels: {(label_slice > 0).sum()}')\n                            axes[i, 1].axis('off')\n\n                            # Column 3: Overlay\n                            overlay = np.stack([img_normalized]*3, axis=-1)\n                            overlay[:,:,0] = np.where(label_slice > 0, 1.0, overlay[:,:,0])\n                            axes[i, 2].imshow(overlay)\n                            axes[i, 2].set_title('Overlay (Red = Surface)')\n                            axes[i, 2].axis('off')\n\n                        plt.tight_layout()\n                        \n                        # Save to /kaggle/working\n                        save_path = f'{OUTPUT_DIR}/volume_slices_visualization.png'\n                        plt.savefig(save_path, dpi=150, bbox_inches='tight')\n                        print(f\"✓ Saved: {save_path}\")\n                        plt.close()\n\n                        # ================================================================\n                        # STATISTIK PER SLICE\n                        # ================================================================\n                        print(\"\\n[5] STATISTIK PER SLICE:\")\n                        print(\"-\"*70)\n\n                        surface_per_slice = [(label_volume[i] > 0).sum() for i in range(depth)]\n\n                        plt.figure(figsize=(12, 4))\n\n                        plt.subplot(1, 2, 1)\n                        plt.plot(surface_per_slice, linewidth=2)\n                        plt.xlabel('Slice Index')\n                        plt.ylabel('Surface Pixels')\n                        plt.title('Surface Pixels per Slice (First 100)')\n                        plt.grid(True, alpha=0.3)\n\n                        plt.subplot(1, 2, 2)\n                        plt.hist(surface_per_slice, bins=30, edgecolor='black')\n                        plt.xlabel('Surface Pixels')\n                        plt.ylabel('Frequency')\n                        plt.title('Distribution of Surface Pixels')\n                        plt.grid(True, alpha=0.3)\n\n                        plt.tight_layout()\n                        \n                        # Save to /kaggle/working\n                        save_path = f'{OUTPUT_DIR}/surface_distribution.png'\n                        plt.savefig(save_path, dpi=150, bbox_inches='tight')\n                        print(f\"✓ Saved: {save_path}\")\n                        plt.close()\n\n                        # Slice dengan surface terbanyak\n                        max_surface_idx = np.argmax(surface_per_slice)\n                        print(f\"\\nSlice dengan surface terbanyak: {max_surface_idx}\")\n                        print(f\"Surface pixels: {surface_per_slice[max_surface_idx]}\")\n\n                        # Statistik keseluruhan\n                        non_zero_slices = sum(1 for x in surface_per_slice if x > 0)\n                        print(f\"\\nSlices dengan surface: {non_zero_slices}/{depth} ({non_zero_slices/depth*100:.1f}%)\")\n                        print(f\"Rata-rata surface per slice: {np.mean(surface_per_slice):.1f}\")\n                        print(f\"Median surface per slice: {np.median(surface_per_slice):.1f}\")\n                    else:\n                        print(f\"⚠️ Failed to load label volume\")\n                else:\n                    print(f\"⚠️ Label tidak ditemukan untuk {sample_img_path.name}\")\n            else:\n                print(f\"⚠️ Failed to load image volume\")\n\n        except Exception as e:\n            print(f\"❌ Error: {e}\")\n            import traceback\n            traceback.print_exc()\n\n    # ============================================================================\n    # 6. RINGKASAN DATASET\n    # ============================================================================\n    print(\"\\n\" + \"=\"*70)\n    print(\"RINGKASAN DATASET\")\n    print(\"=\"*70)\n\n    print(f\"\"\"\n📊 Dataset Information:\n   • Type: 3D CT Scan Volumes for Surface Detection\n   • Task: Segmentation (detect papyrus surface in 3D volume)\n   • Training volumes: {len(image_files)}\n   • Label volumes: {len(label_files)}\n   • Volume shape: 320 x 320 x 320 (D x H x W)\n   • Data type: uint8 (0-255)\n   • Loading method: PIL (slower but no imagecodecs needed)\n\n🎯 Preprocessing Strategy:\n   1. Normalisasi per volume atau per slice\n   2. Denoising 3D atau per slice 2D\n   3. Extract 2D slices untuk training (lebih efficient)\n   4. Patch extraction dari setiap slice\n   5. Oversample slices yang memiliki surface\n\n⚡ Rekomendasi:\n   • Process slice-by-slice (2D approach) untuk efisiensi\n   • Atau gunakan 3D patches jika GPU kuat\n   • Skip slices tanpa surface untuk training efficiency\n   • Gunakan weighted loss karena extreme class imbalance\n\n💾 Output Location:\n   • Visualizations saved to: {OUTPUT_DIR}/\n   • Accessible in Kaggle notebook output\n\n⚠️  Note on PIL vs tifffile:\n   • PIL is slower but works without imagecodecs\n   • For preprocessing, consider using imagecodecs for speed:\n     1. !pip install imagecodecs\n     2. Restart kernel\n     3. Use tifffile.imread() instead\n\"\"\")\n\n    print(\"=\"*70)\n    print(\"✅ EKSPLORASI SELESAI!\")\n    print(\"=\"*70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:36:28.893356Z","iopub.execute_input":"2025-12-06T12:36:28.893698Z","iopub.status.idle":"2025-12-06T12:36:33.500323Z","shell.execute_reply.started":"2025-12-06T12:36:28.893677Z","shell.execute_reply":"2025-12-06T12:36:33.499628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# VESUVIUS 3D CT SCAN PREPROCESSING - KAGGLE OPTIMIZED (FIXED)\n# Memory-efficient preprocessing with immediate save and garbage collection\n# ============================================================================\n\nimport numpy as np\nimport cv2\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nfrom scipy.ndimage import gaussian_filter, median_filter\nfrom skimage.morphology import binary_opening, binary_closing, disk\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport json\nimport pandas as pd\nimport tifffile\nimport gc\nimport shutil\n\n# ============================================================================\n# KONFIGURASI PREPROCESSING - KAGGLE PATHS\n# ============================================================================\n\nclass PreprocessConfig:\n    \"\"\"Konfigurasi untuk preprocessing 3D CT scan Vesuvius\"\"\"\n\n    # Path - KAGGLE SPECIFIC\n    DATA_DIR = \"/kaggle/input/vesuvius-challenge-surface-detection\"\n    OUTPUT_DIR = \"/kaggle/working/preprocessed_data\"\n\n    # Processing mode\n    PROCESS_AS_2D = True\n\n    # Memory optimization - ENHANCED\n    SAVE_PATCHES_IMMEDIATELY = True\n    CLEAR_MEMORY_AFTER_VOLUME = True\n    BATCH_SIZE_VOLUMES = 5  # Reduced for better memory management\n    DELETE_TEMP_DURING_COPY = True  # Delete temp files while copying\n\n    # Slice filtering - AGGRESSIVE\n    SKIP_EMPTY_SLICES = True  # Skip slices without surface\n    MIN_SURFACE_PIXELS = 500  # Higher threshold\n    SAMPLE_SLICES = True      # Sample only some slices per volume\n    SLICE_SAMPLE_RATE = 0.5   # Use only 50% of slices\n\n    # Normalisasi\n    NORM_METHOD = \"minmax\"\n    CLIP_PERCENTILE = (1, 99)\n    NORMALIZE_PER_VOLUME = False\n\n    # Denoising - DISABLE untuk speed dan size\n    USE_DENOISING = False  # Disabled for speed/space\n    DENOISE_METHOD = \"gaussian\"\n    GAUSSIAN_SIGMA = 0.5\n    MEDIAN_SIZE = 3\n\n    # Patch extraction - SUPER AGGRESSIVE\n    PATCH_SIZE = 256\n    STRIDE = 256  # NO OVERLAP! Reduces patches by 75%\n    MIN_MASK_RATIO = 0.03  # Much higher threshold (3% vs 1%)\n    MAX_PATCHES_PER_VOLUME = 300  # Hard limit per volume\n\n    # Mask cleaning\n    USE_MASK_CLEANING = False  # Disabled for speed\n    MORPHOLOGY_KERNEL_SIZE = 2\n\n    # Class imbalance handling\n    OVERSAMPLE_FACTOR = 1  # NO OVERSAMPLE to save space\n\n    # Data split\n    TEST_SIZE = 0.2\n    RANDOM_SEED = 42\n    \n    # Compression - CRITICAL FOR KAGGLE\n    USE_COMPRESSION = True  # Save as compressed uint8\n    SAVE_AS_UINT8 = True    # Convert float32 to uint8 (4x smaller)\n\n    # Visualisasi\n    SAVE_PREVIEW = True\n    NUM_PREVIEW = 5\n\n    # Limit untuk testing (None = process all)\n    MAX_VOLUMES = 100  # Change to 50 for testing\n    \n    # Skip patches without surface to save space\n    SKIP_NO_SURFACE_PATCHES = True\n\n# ============================================================================\n# UTILITY FUNCTIONS\n# ============================================================================\n\ndef create_directories(output_dir):\n    \"\"\"Buat struktur folder output\"\"\"\n    dirs = [\n        f\"{output_dir}/train/images\",\n        f\"{output_dir}/train/masks\",\n        f\"{output_dir}/val/images\",\n        f\"{output_dir}/val/masks\",\n        f\"{output_dir}/preview\",\n        f\"{output_dir}/stats\",\n        f\"{output_dir}/temp/images\",\n        f\"{output_dir}/temp/masks\"\n    ]\n    for d in dirs:\n        Path(d).mkdir(parents=True, exist_ok=True)\n    print(\"✅ Directories created\")\n\ndef load_volume(path):\n    \"\"\"Load 3D volume dengan error handling\"\"\"\n    try:\n        volume = tifffile.imread(str(path))\n        return volume\n    except Exception as e:\n        print(f\"  ❌ Error loading {path.name}: {e}\")\n        return None\n\ndef clear_memory():\n    \"\"\"Force garbage collection\"\"\"\n    gc.collect()\n\ndef get_memory_usage():\n    \"\"\"Get current memory usage (if available)\"\"\"\n    try:\n        import psutil\n        process = psutil.Process()\n        mem = process.memory_info().rss / 1024 / 1024  # MB\n        return f\"{mem:.1f} MB\"\n    except:\n        return \"N/A\"\n\n# ============================================================================\n# NORMALISASI\n# ============================================================================\n\ndef normalize_slice(slice_2d, method=\"minmax\", clip_percentile=(1, 99)):\n    \"\"\"Normalisasi single 2D slice\"\"\"\n    if clip_percentile:\n        low, high = np.percentile(slice_2d, clip_percentile)\n        slice_2d = np.clip(slice_2d, low, high)\n\n    if method == \"minmax\":\n        img_min = slice_2d.min()\n        img_max = slice_2d.max()\n        if img_max - img_min > 0:\n            normalized = (slice_2d - img_min) / (img_max - img_min)\n        else:\n            normalized = slice_2d - img_min\n    elif method == \"standardize\":\n        mean = slice_2d.mean()\n        std = slice_2d.std()\n        if std > 0:\n            normalized = (slice_2d - mean) / std\n        else:\n            normalized = slice_2d - mean\n    else:\n        normalized = slice_2d\n\n    return normalized.astype(np.float32)\n\ndef normalize_volume(volume, method=\"minmax\", clip_percentile=(1, 99)):\n    \"\"\"Normalisasi entire 3D volume\"\"\"\n    if clip_percentile:\n        low, high = np.percentile(volume, clip_percentile)\n        volume = np.clip(volume, low, high)\n\n    if method == \"minmax\":\n        vol_min = volume.min()\n        vol_max = volume.max()\n        if vol_max - vol_min > 0:\n            normalized = (volume - vol_min) / (vol_max - vol_min)\n        else:\n            normalized = volume - vol_min\n    elif method == \"standardize\":\n        mean = volume.mean()\n        std = volume.std()\n        if std > 0:\n            normalized = (volume - mean) / std\n        else:\n            normalized = volume - mean\n    else:\n        normalized = volume\n\n    return normalized.astype(np.float32)\n\n# ============================================================================\n# DENOISING\n# ============================================================================\n\ndef denoise_slice(slice_2d, method=\"gaussian\", sigma=0.5, kernel_size=3):\n    \"\"\"Denoise 2D slice\"\"\"\n    if method == \"gaussian\":\n        denoised = gaussian_filter(slice_2d, sigma=sigma)\n    elif method == \"median\":\n        denoised = median_filter(slice_2d, size=kernel_size)\n    elif method == \"bilateral\":\n        img_uint8 = (slice_2d * 255).astype(np.uint8)\n        denoised = cv2.bilateralFilter(img_uint8, kernel_size, 75, 75)\n        denoised = denoised.astype(np.float32) / 255.0\n    else:\n        denoised = slice_2d\n\n    return denoised\n\n# ============================================================================\n# MASK CLEANING\n# ============================================================================\n\ndef clean_mask(mask, kernel_size=2):\n    \"\"\"Clean mask dengan morphological operations\"\"\"\n    mask_binary = (mask > 0).astype(np.uint8)\n    kernel = disk(kernel_size)\n    mask_opened = binary_opening(mask_binary, kernel)\n    mask_cleaned = binary_closing(mask_opened, kernel)\n    return mask_cleaned.astype(np.uint8)\n\n# ============================================================================\n# PATCH EXTRACTION\n# ============================================================================\n\ndef extract_patches_2d(image, mask, patch_size=256, stride=192, min_mask_ratio=0.001):\n    \"\"\"Extract patches dari 2D slice dengan sliding window overlap\"\"\"\n    h, w = image.shape\n    patches = []\n\n    y_steps = []\n    x_steps = []\n\n    # Generate Y positions\n    y = 0\n    while y + patch_size <= h:\n        y_steps.append(y)\n        y += stride\n    if len(y_steps) > 0 and y_steps[-1] + patch_size < h:\n        y_steps.append(h - patch_size)\n\n    # Generate X positions\n    x = 0\n    while x + patch_size <= w:\n        x_steps.append(x)\n        x += stride\n    if len(x_steps) > 0 and x_steps[-1] + patch_size < w:\n        x_steps.append(w - patch_size)\n\n    # Extract patches\n    for y in y_steps:\n        for x in x_steps:\n            img_patch = image[y:y+patch_size, x:x+patch_size]\n            mask_patch = mask[y:y+patch_size, x:x+patch_size]\n\n            if img_patch.shape != (patch_size, patch_size):\n                continue\n\n            mask_ratio = mask_patch.sum() / (patch_size * patch_size)\n            has_mask = mask_ratio >= min_mask_ratio\n\n            patches.append({\n                'image': img_patch,\n                'mask': mask_patch,\n                'has_mask': has_mask,\n                'mask_ratio': mask_ratio,\n                'position': (y, x)\n            })\n\n    return patches\n\n# ============================================================================\n# PROCESS SINGLE VOLUME\n# ============================================================================\n\ndef process_volume_as_slices(volume_path, label_path, config):\n    \"\"\"Process 3D volume dengan extract 2D slices - SUPER OPTIMIZED\"\"\"\n    \n    # Load volumes\n    img_volume = load_volume(volume_path)\n    label_volume = load_volume(label_path)\n\n    if img_volume is None or label_volume is None:\n        return []\n\n    volume_id = volume_path.stem\n    depth, height, width = img_volume.shape\n\n    print(f\"  📦 Shape: {img_volume.shape} | Mem: {get_memory_usage()}\")\n\n    # Normalisasi per volume jika diminta\n    if config.NORMALIZE_PER_VOLUME:\n        img_volume = normalize_volume(img_volume, config.NORM_METHOD,\n                                      config.CLIP_PERCENTILE)\n\n    # OPTIMIZATION: Sample only some slices if enabled\n    if config.SAMPLE_SLICES:\n        slice_indices = np.random.choice(\n            depth, \n            size=int(depth * config.SLICE_SAMPLE_RATE), \n            replace=False\n        )\n        slice_indices = sorted(slice_indices)\n    else:\n        slice_indices = range(depth)\n\n    # Process setiap slice\n    all_patches_metadata = []\n    surface_count = 0\n\n    for slice_idx in slice_indices:\n        img_slice = img_volume[slice_idx]\n        label_slice = label_volume[slice_idx]\n\n        surface_pixels = (label_slice > 0).sum()\n\n        # SKIP: Empty slices\n        if config.SKIP_EMPTY_SLICES and surface_pixels < config.MIN_SURFACE_PIXELS:\n            continue\n\n        if not config.NORMALIZE_PER_VOLUME:\n            img_slice = normalize_slice(img_slice, config.NORM_METHOD,\n                                       config.CLIP_PERCENTILE)\n\n        if config.USE_DENOISING:\n            img_slice = denoise_slice(img_slice, config.DENOISE_METHOD,\n                                     config.GAUSSIAN_SIGMA, config.MEDIAN_SIZE)\n\n        if config.USE_MASK_CLEANING:\n            label_slice = clean_mask(label_slice, config.MORPHOLOGY_KERNEL_SIZE)\n\n        patches = extract_patches_2d(img_slice, label_slice,\n                                     config.PATCH_SIZE, config.STRIDE,\n                                     config.MIN_MASK_RATIO)\n\n        # Filter: Skip patches without surface if enabled\n        if config.SKIP_NO_SURFACE_PATCHES:\n            patches = [p for p in patches if p['has_mask']]\n\n        for patch in patches:\n            patch['volume_id'] = volume_id\n            patch['slice_idx'] = slice_idx\n            patch['has_surface'] = surface_pixels > 0\n\n        all_patches_metadata.extend(patches)\n\n        if surface_pixels > 0:\n            surface_count += 1\n        \n        # HARD LIMIT: Stop if too many patches\n        if len(all_patches_metadata) >= config.MAX_PATCHES_PER_VOLUME:\n            print(f\"  ⚠️  Hit limit: {config.MAX_PATCHES_PER_VOLUME} patches\")\n            all_patches_metadata = all_patches_metadata[:config.MAX_PATCHES_PER_VOLUME]\n            break\n\n    print(f\"  ✅ {len(all_patches_metadata)} patches from {surface_count} slices\")\n\n    # Clear memory aggressively\n    del img_volume, label_volume\n    clear_memory()\n\n    return all_patches_metadata\n\n# ============================================================================\n# SAVE PATCHES\n# ============================================================================\n\ndef save_patch_immediate(patch, split_name, idx, config):\n    \"\"\"Save single patch immediately ke disk - COMPRESSED\"\"\"\n    img_path = f\"{config.OUTPUT_DIR}/{split_name}/images/patch_{idx:06d}.npy\"\n    mask_path = f\"{config.OUTPUT_DIR}/{split_name}/masks/patch_{idx:06d}.npy\"\n\n    # Convert to uint8 to save 4x space (float32 → uint8)\n    if config.SAVE_AS_UINT8:\n        img_uint8 = (patch['image'] * 255).astype(np.uint8)\n        mask_uint8 = patch['mask'].astype(np.uint8)\n        \n        # Save compressed\n        if config.USE_COMPRESSION:\n            np.savez_compressed(img_path.replace('.npy', '.npz'), data=img_uint8)\n            np.savez_compressed(mask_path.replace('.npy', '.npz'), data=mask_uint8)\n        else:\n            np.save(img_path, img_uint8)\n            np.save(mask_path, mask_uint8)\n    else:\n        np.save(img_path, patch['image'])\n        np.save(mask_path, patch['mask'])\n\n    return {\n        'has_mask': patch['has_mask'],\n        'mask_ratio': patch['mask_ratio'],\n        'volume_id': patch.get('volume_id', 'unknown'),\n        'slice_idx': patch.get('slice_idx', -1)\n    }\n\n# ============================================================================\n# VISUALISASI\n# ============================================================================\n\ndef save_preview_samples(output_dir, num_samples=5):\n    \"\"\"Save preview samples dari hasil preprocessing\"\"\"\n    config = PreprocessConfig()\n    \n    # Load beberapa random patches untuk preview\n    train_img_dir = Path(f\"{output_dir}/train/images\")\n    train_mask_dir = Path(f\"{output_dir}/train/masks\")\n    \n    # Support both .npy and .npz files\n    patch_files = list(train_img_dir.glob(\"*.npy\")) + list(train_img_dir.glob(\"*.npz\"))\n    patch_files = sorted(patch_files)\n    \n    if len(patch_files) == 0:\n        print(\"  ⚠️  No patches for preview\")\n        return\n    \n    # Random sample\n    sample_indices = np.random.choice(len(patch_files), \n                                     min(num_samples, len(patch_files)), \n                                     replace=False)\n    \n    fig, axes = plt.subplots(num_samples, 3, figsize=(12, 4*num_samples))\n    if num_samples == 1:\n        axes = axes.reshape(1, -1)\n    \n    for i, idx in enumerate(sample_indices):\n        patch_file = patch_files[idx]\n        \n        # Load image\n        if patch_file.suffix == '.npz':\n            img = np.load(patch_file)['data']\n            mask_file = train_mask_dir / patch_file.name\n            mask = np.load(mask_file)['data']\n        else:\n            img = np.load(patch_file)\n            mask_file = train_mask_dir / patch_file.name\n            mask = np.load(mask_file)\n        \n        # Normalize for display if uint8\n        if img.dtype == np.uint8:\n            img_display = img.astype(np.float32) / 255.0\n        else:\n            img_display = img\n        \n        # Image\n        axes[i, 0].imshow(img_display, cmap='gray')\n        axes[i, 0].set_title(f'Image: {patch_file.stem}')\n        axes[i, 0].axis('off')\n        \n        # Mask\n        axes[i, 1].imshow(mask, cmap='hot')\n        axes[i, 1].set_title(f'Mask (ratio: {mask.sum()/(256*256):.4f})')\n        axes[i, 1].axis('off')\n        \n        # Overlay\n        overlay = np.stack([img_display]*3, axis=-1)\n        overlay[:,:,0] = np.where(mask > 0, 1.0, overlay[:,:,0])\n        axes[i, 2].imshow(overlay)\n        axes[i, 2].set_title('Overlay (Red = Surface)')\n        axes[i, 2].axis('off')\n    \n    plt.tight_layout()\n    plt.savefig(f\"{output_dir}/preview/samples.png\", dpi=150, bbox_inches='tight')\n    plt.close()\n    print(\"  ✅ Preview saved\")\n\n# ============================================================================\n# MAIN PREPROCESSING PIPELINE - OPTIMIZED\n# ============================================================================\n\ndef preprocess_vesuvius_dataset():\n    \"\"\"Main preprocessing pipeline - HIGHLY OPTIMIZED FOR KAGGLE\"\"\"\n    config = PreprocessConfig()\n\n    print(\"=\"*70)\n    print(\"VESUVIUS PREPROCESSING - KAGGLE OPTIMIZED\")\n    print(\"=\"*70)\n    \n    # Get volume paths FIRST - FIXED ORDER\n    train_images_dir = Path(config.DATA_DIR) / \"train_images\"\n    train_labels_dir = Path(config.DATA_DIR) / \"train_labels\"\n\n    volume_files = sorted(list(train_images_dir.glob(\"*.tif\")))\n\n    if config.MAX_VOLUMES:\n        volume_files = volume_files[:config.MAX_VOLUMES]\n\n    print(f\"\\n📦 Found {len(volume_files)} volumes\")\n\n    # NOW print configuration\n    print(f\"\\n⚙️  Configuration:\")\n    print(f\"   • Mode: 2D slice processing\")\n    print(f\"   • Patch size: {config.PATCH_SIZE}x{config.PATCH_SIZE}\")\n    print(f\"   • Stride: {config.STRIDE} (overlap: {config.PATCH_SIZE - config.STRIDE})\")\n    print(f\"   • Min mask ratio: {config.MIN_MASK_RATIO} (higher = fewer patches)\")\n    print(f\"   • Max patches/volume: {config.MAX_PATCHES_PER_VOLUME}\")\n    print(f\"   • Slice sampling: {config.SLICE_SAMPLE_RATE*100:.0f}% of slices\")\n    print(f\"   • Skip empty slices: {config.SKIP_EMPTY_SLICES}\")\n    print(f\"   • Oversample: {config.OVERSAMPLE_FACTOR}x\")\n    print(f\"   • Compression: uint8 + npz = 16x smaller!\")\n    print(f\"   • Skip no-surface patches: {config.SKIP_NO_SURFACE_PATCHES}\")\n    print(f\"   • Memory: GC every {config.BATCH_SIZE_VOLUMES} volumes\")\n    if config.MAX_VOLUMES:\n        print(f\"   • ⚠️  TEST MODE: {config.MAX_VOLUMES} volumes only\")\n    \n    # Estimate output size\n    estimated_patches = len(volume_files) * config.MAX_PATCHES_PER_VOLUME\n    estimated_size_gb = (estimated_patches * 256 * 256 * 2) / (1024**3)  # uint8 img+mask\n    estimated_size_gb = estimated_size_gb * 0.5  # compression factor\n    print(f\"\\n📊 Estimated output: ~{estimated_size_gb:.1f} GB\")\n    if estimated_size_gb > 18:\n        print(f\"   ⚠️  WARNING: May exceed Kaggle disk limit (20 GB)!\")\n        print(f\"   💡 Consider: Reduce MAX_PATCHES_PER_VOLUME or MAX_VOLUMES\")\n\n    create_directories(config.OUTPUT_DIR)\n\n    # Statistics\n    stats = {\n        'volumes_processed': 0,\n        'volumes_failed': 0,\n        'total_patches': 0\n    }\n\n    # =======================================================================\n    # PHASE 1: PROCESS & SAVE TO TEMP\n    # =======================================================================\n    print(f\"\\n{'='*70}\")\n    print(\"PHASE 1: PROCESSING & SAVING\")\n    print(f\"{'='*70}\\n\")\n\n    patch_idx = 0\n    all_patch_metadata = []\n\n    for i, vol_path in enumerate(volume_files, 1):\n        label_path = train_labels_dir / vol_path.name\n\n        print(f\"[{i}/{len(volume_files)}] {vol_path.name}\")\n\n        if not label_path.exists():\n            print(f\"  ⚠️  Label not found\")\n            stats['volumes_failed'] += 1\n            continue\n\n        try:\n            # Process volume\n            patches = process_volume_as_slices(vol_path, label_path, config)\n\n            if len(patches) == 0:\n                stats['volumes_failed'] += 1\n                continue\n\n            # Save immediately\n            for patch in tqdm(patches, desc=\"  💾 Saving\", leave=False):\n                metadata = save_patch_immediate(patch, \"temp\", patch_idx, config)\n                all_patch_metadata.append(metadata)\n                patch_idx += 1\n\n            stats['volumes_processed'] += 1\n            stats['total_patches'] += len(patches)\n\n            # Memory management\n            if config.CLEAR_MEMORY_AFTER_VOLUME and i % config.BATCH_SIZE_VOLUMES == 0:\n                print(f\"  🧹 Clearing memory... (Total: {stats['total_patches']} patches)\")\n                clear_memory()\n\n        except Exception as e:\n            print(f\"  ❌ Error: {e}\")\n            stats['volumes_failed'] += 1\n            continue\n\n    print(f\"\\n{'='*70}\")\n    print(\"PHASE 1 COMPLETE\")\n    print(f\"{'='*70}\")\n    print(f\"✅ Processed: {stats['volumes_processed']}/{len(volume_files)}\")\n    print(f\"❌ Failed: {stats['volumes_failed']}\")\n    print(f\"📦 Total patches: {stats['total_patches']}\")\n\n    if stats['total_patches'] == 0:\n        print(\"\\n❌ ERROR: No patches extracted!\")\n        return None\n\n    # =======================================================================\n    # PHASE 2: OVERSAMPLE & SPLIT\n    # =======================================================================\n    print(f\"\\n{'='*70}\")\n    print(\"PHASE 2: OVERSAMPLING & SPLIT\")\n    print(f\"{'='*70}\\n\")\n\n    with_surface = [i for i, m in enumerate(all_patch_metadata) if m['has_mask']]\n    without_surface = [i for i, m in enumerate(all_patch_metadata) if not m['has_mask']]\n\n    print(f\"With surface: {len(with_surface)}\")\n    print(f\"Without surface: {len(without_surface)}\")\n\n    # Oversample\n    oversampled_indices = with_surface * config.OVERSAMPLE_FACTOR + without_surface\n    np.random.shuffle(oversampled_indices)\n\n    print(f\"After oversample: {len(oversampled_indices)}\")\n\n    # Split\n    train_indices, val_indices = train_test_split(\n        oversampled_indices,\n        test_size=config.TEST_SIZE,\n        random_state=config.RANDOM_SEED\n    )\n\n    print(f\"\\n📊 Train: {len(train_indices)} | Val: {len(val_indices)}\")\n\n    # =======================================================================\n    # PHASE 3: ORGANIZE FILES (MEMORY EFFICIENT)\n    # =======================================================================\n    print(f\"\\n{'='*70}\")\n    print(\"PHASE 3: ORGANIZING FILES\")\n    print(f\"{'='*70}\\n\")\n\n    # Determine file extension based on compression setting\n    file_ext = \".npz\" if config.USE_COMPRESSION else \".npy\"\n\n    # Copy train files\n    print(\"📂 Copying train files...\")\n    for new_idx, old_idx in enumerate(tqdm(train_indices)):\n        old_img = f\"{config.OUTPUT_DIR}/temp/images/patch_{old_idx:06d}{file_ext}\"\n        old_mask = f\"{config.OUTPUT_DIR}/temp/masks/patch_{old_idx:06d}{file_ext}\"\n        new_img = f\"{config.OUTPUT_DIR}/train/images/patch_{new_idx:06d}{file_ext}\"\n        new_mask = f\"{config.OUTPUT_DIR}/train/masks/patch_{new_idx:06d}{file_ext}\"\n\n        shutil.copy2(old_img, new_img)\n        shutil.copy2(old_mask, new_mask)\n\n    # Copy val files\n    print(\"📂 Copying val files...\")\n    for new_idx, old_idx in enumerate(tqdm(val_indices)):\n        old_img = f\"{config.OUTPUT_DIR}/temp/images/patch_{old_idx:06d}{file_ext}\"\n        old_mask = f\"{config.OUTPUT_DIR}/temp/masks/patch_{old_idx:06d}{file_ext}\"\n        new_img = f\"{config.OUTPUT_DIR}/val/images/patch_{new_idx:06d}{file_ext}\"\n        new_mask = f\"{config.OUTPUT_DIR}/val/masks/patch_{new_idx:06d}{file_ext}\"\n\n        shutil.copy2(old_img, new_img)\n        shutil.copy2(old_mask, new_mask)\n\n    # Clean temp\n    print(\"\\n🧹 Removing temp files...\")\n    shutil.rmtree(Path(config.OUTPUT_DIR) / \"temp\")\n    clear_memory()\n\n    # =======================================================================\n    # SAVE STATS & PREVIEW\n    # =======================================================================\n    print(f\"\\n{'='*70}\")\n    print(\"SAVING STATS & PREVIEW\")\n    print(f\"{'='*70}\\n\")\n\n    # Stats\n    stats['train_patches'] = len(train_indices)\n    stats['val_patches'] = len(val_indices)\n    stats['config'] = {\n        'patch_size': config.PATCH_SIZE,\n        'stride': config.STRIDE,\n        'oversample_factor': config.OVERSAMPLE_FACTOR\n    }\n\n    stats_path = f\"{config.OUTPUT_DIR}/stats/preprocessing_stats.json\"\n    with open(stats_path, 'w') as f:\n        json.dump(stats, f, indent=2)\n    print(f\"✅ Stats saved: {stats_path}\")\n\n    # Preview\n    if config.SAVE_PREVIEW:\n        print(\"\\n🎨 Generating preview...\")\n        save_preview_samples(config.OUTPUT_DIR, config.NUM_PREVIEW)\n\n    # =======================================================================\n    # FINAL SUMMARY\n    # =======================================================================\n    print(f\"\\n{'='*70}\")\n    print(\"✅ PREPROCESSING COMPLETE!\")\n    print(f\"{'='*70}\")\n    print(f\"\\n📁 Output: {config.OUTPUT_DIR}\")\n    print(f\"📊 Train: {len(train_indices)} patches\")\n    print(f\"📊 Val: {len(val_indices)} patches\")\n    print(f\"\\n💾 Files ready for training!\")\n    print(f\"{'='*70}\\n\")\n\n    return stats\n\n# ============================================================================\n# RUN\n# ============================================================================\n\nif __name__ == \"__main__\":\n    np.random.seed(42)\n    stats = preprocess_vesuvius_dataset()\n    \n    if stats:\n        print(\"\\n📈 Final Statistics:\")\n        print(json.dumps(stats, indent=2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:36:33.501383Z","iopub.execute_input":"2025-12-06T12:36:33.501625Z","iopub.status.idle":"2025-12-06T12:40:23.430211Z","shell.execute_reply.started":"2025-12-06T12:36:33.501606Z","shell.execute_reply":"2025-12-06T12:40:23.429401Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install -q segmentation-models-pytorch albumentations","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:40:23.431127Z","iopub.execute_input":"2025-12-06T12:40:23.431567Z","iopub.status.idle":"2025-12-06T12:41:40.230886Z","shell.execute_reply.started":"2025-12-06T12:40:23.431538Z","shell.execute_reply":"2025-12-06T12:41:40.230191Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install -q \"numpy<2.0\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:41:40.231943Z","iopub.execute_input":"2025-12-06T12:41:40.232202Z","iopub.status.idle":"2025-12-06T12:41:49.429962Z","shell.execute_reply.started":"2025-12-06T12:41:40.232168Z","shell.execute_reply":"2025-12-06T12:41:49.429232Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nprint(f\"NumPy version: {np.__version__}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:41:49.432602Z","iopub.execute_input":"2025-12-06T12:41:49.432862Z","iopub.status.idle":"2025-12-06T12:41:49.437135Z","shell.execute_reply.started":"2025-12-06T12:41:49.432838Z","shell.execute_reply":"2025-12-06T12:41:49.436417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nimport json\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(f\"✅ NumPy: {np.__version__}\")\nprint(f\"✅ PyTorch: {torch.__version__}\")\nprint(f\"✅ CUDA: {torch.cuda.is_available()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:41:49.438043Z","iopub.execute_input":"2025-12-06T12:41:49.438392Z","iopub.status.idle":"2025-12-06T12:41:52.254277Z","shell.execute_reply.started":"2025-12-06T12:41:49.438369Z","shell.execute_reply":"2025-12-06T12:41:52.253450Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# CONFIGURATION\n# ============================================================================\n\nclass TrainingConfig:\n    \"\"\"Training configuration\"\"\"\n    \n    # Paths - KAGGLE\n    DATA_DIR = \"/kaggle/working/preprocessed_data\"\n    OUTPUT_DIR = \"/kaggle/working/training_output\"\n    \n    # Model\n    MODEL_NAME = \"UnetPlusPlus\"\n    IN_CHANNELS = 1\n    NUM_CLASSES = 1\n    BASE_FILTERS = 32  # Starting filters (32, 64, 128, 256, 512)\n    \n    # Training\n    EPOCHS = 20\n    BATCH_SIZE = 32\n    LEARNING_RATE = 2e-4\n    WEIGHT_DECAY = 1e-5\n    \n    # Loss weights\n    DICE_WEIGHT = 0.5\n    BCE_WEIGHT = 0.5\n    \n    # Optimizer & Scheduler\n    SCHEDULER = \"CosineAnnealingLR\"\n    T_MAX = 30\n    \n    # Training settings\n    NUM_WORKERS = 4\n    PIN_MEMORY = True\n    MIXED_PRECISION = True\n    PREFETCH_FACTOR = 2\n    PERSISTENT_WORKERS = True\n    \n    # Checkpointing\n    SAVE_BEST_ONLY = True\n    SAVE_EVERY_N_EPOCHS = 5\n    EARLY_STOPPING_PATIENCE = 5\n    \n    # Augmentation\n    USE_AUGMENTATION = True\n    \n    # Random seed\n    SEED = 42\n    \n    # Device\n    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:55:06.525137Z","iopub.execute_input":"2025-12-06T12:55:06.525429Z","iopub.status.idle":"2025-12-06T12:55:06.531678Z","shell.execute_reply.started":"2025-12-06T12:55:06.525403Z","shell.execute_reply":"2025-12-06T12:55:06.530970Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# U-NET++ MODEL ARCHITECTURE (PURE PYTORCH)\n# ============================================================================\n\nclass ConvBlock(nn.Module):\n    \"\"\"Double Convolution Block\"\"\"\n    def __init__(self, in_ch, out_ch):\n        super().__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(in_ch, out_ch, 3, padding=1, bias=False),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_ch, out_ch, 3, padding=1, bias=False),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True)\n        )\n    \n    def forward(self, x):\n        return self.conv(x)\n\nclass UNetPlusPlus(nn.Module):\n    \"\"\"\n    U-Net++ (Nested U-Net) Architecture\n    Paper: https://arxiv.org/abs/1807.10165\n    \"\"\"\n    def __init__(self, in_channels=1, num_classes=1, base_filters=32, deep_supervision=False):\n        super().__init__()\n        \n        self.deep_supervision = deep_supervision\n        filters = [base_filters, base_filters*2, base_filters*4, base_filters*8, base_filters*16]\n        \n        # Encoder (Downsampling path)\n        self.conv0_0 = ConvBlock(in_channels, filters[0])\n        self.conv1_0 = ConvBlock(filters[0], filters[1])\n        self.conv2_0 = ConvBlock(filters[1], filters[2])\n        self.conv3_0 = ConvBlock(filters[2], filters[3])\n        self.conv4_0 = ConvBlock(filters[3], filters[4])\n        \n        # Nested skip pathways - Level 0\n        self.conv0_1 = ConvBlock(filters[0] + filters[1], filters[0])\n        self.conv0_2 = ConvBlock(filters[0]*2 + filters[1], filters[0])\n        self.conv0_3 = ConvBlock(filters[0]*3 + filters[1], filters[0])\n        self.conv0_4 = ConvBlock(filters[0]*4 + filters[1], filters[0])\n        \n        # Nested skip pathways - Level 1\n        self.conv1_1 = ConvBlock(filters[1] + filters[2], filters[1])\n        self.conv1_2 = ConvBlock(filters[1]*2 + filters[2], filters[1])\n        self.conv1_3 = ConvBlock(filters[1]*3 + filters[2], filters[1])\n        \n        # Nested skip pathways - Level 2\n        self.conv2_1 = ConvBlock(filters[2] + filters[3], filters[2])\n        self.conv2_2 = ConvBlock(filters[2]*2 + filters[3], filters[2])\n        \n        # Nested skip pathways - Level 3\n        self.conv3_1 = ConvBlock(filters[3] + filters[4], filters[3])\n        \n        # Pooling\n        self.pool = nn.MaxPool2d(2)\n        \n        # Upsampling\n        self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        \n        # Final output\n        if deep_supervision:\n            self.final1 = nn.Conv2d(filters[0], num_classes, 1)\n            self.final2 = nn.Conv2d(filters[0], num_classes, 1)\n            self.final3 = nn.Conv2d(filters[0], num_classes, 1)\n            self.final4 = nn.Conv2d(filters[0], num_classes, 1)\n        else:\n            self.final = nn.Conv2d(filters[0], num_classes, 1)\n    \n    def forward(self, x):\n        # Encoder path\n        x0_0 = self.conv0_0(x)\n        x1_0 = self.conv1_0(self.pool(x0_0))\n        x2_0 = self.conv2_0(self.pool(x1_0))\n        x3_0 = self.conv3_0(self.pool(x2_0))\n        x4_0 = self.conv4_0(self.pool(x3_0))\n        \n        # Nested connections - Column 1\n        x0_1 = self.conv0_1(torch.cat([x0_0, self.up(x1_0)], 1))\n        x1_1 = self.conv1_1(torch.cat([x1_0, self.up(x2_0)], 1))\n        x2_1 = self.conv2_1(torch.cat([x2_0, self.up(x3_0)], 1))\n        x3_1 = self.conv3_1(torch.cat([x3_0, self.up(x4_0)], 1))\n        \n        # Nested connections - Column 2\n        x0_2 = self.conv0_2(torch.cat([x0_0, x0_1, self.up(x1_1)], 1))\n        x1_2 = self.conv1_2(torch.cat([x1_0, x1_1, self.up(x2_1)], 1))\n        x2_2 = self.conv2_2(torch.cat([x2_0, x2_1, self.up(x3_1)], 1))\n        \n        # Nested connections - Column 3\n        x0_3 = self.conv0_3(torch.cat([x0_0, x0_1, x0_2, self.up(x1_2)], 1))\n        x1_3 = self.conv1_3(torch.cat([x1_0, x1_1, x1_2, self.up(x2_2)], 1))\n        \n        # Nested connections - Column 4\n        x0_4 = self.conv0_4(torch.cat([x0_0, x0_1, x0_2, x0_3, self.up(x1_3)], 1))\n        \n        # Output\n        if self.deep_supervision:\n            output1 = self.final1(x0_1)\n            output2 = self.final2(x0_2)\n            output3 = self.final3(x0_3)\n            output4 = self.final4(x0_4)\n            return [output1, output2, output3, output4]\n        else:\n            return self.final(x0_4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:55:08.876428Z","iopub.execute_input":"2025-12-06T12:55:08.876710Z","iopub.status.idle":"2025-12-06T12:55:08.891708Z","shell.execute_reply.started":"2025-12-06T12:55:08.876688Z","shell.execute_reply":"2025-12-06T12:55:08.891062Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# DATASET\n# ============================================================================\n\nclass VesuviusDataset(Dataset):\n    \"\"\"Dataset for Vesuvius Challenge\"\"\"\n    \n    def __init__(self, data_dir, split='train', augment=False):\n        self.data_dir = Path(data_dir)\n        self.split = split\n        self.augment = augment\n        \n        self.img_dir = self.data_dir / split / \"images\"\n        self.mask_dir = self.data_dir / split / \"masks\"\n        \n        self.image_files = sorted(list(self.img_dir.glob(\"*.npz\"))) or \\\n                          sorted(list(self.img_dir.glob(\"*.npy\")))\n        \n        print(f\"[{split.upper()}] Found {len(self.image_files)} samples\")\n    \n    def __len__(self):\n        return len(self.image_files)\n    \n    def __getitem__(self, idx):\n        img_path = self.image_files[idx]\n        \n        # Load data\n        if img_path.suffix == '.npz':\n            image = np.load(img_path)['data']\n            mask_path = self.mask_dir / img_path.name\n            mask = np.load(mask_path)['data']\n        else:\n            image = np.load(img_path)\n            mask_path = self.mask_dir / img_path.name\n            mask = np.load(mask_path)\n        \n        # Convert to float32 [0, 1]\n        if image.dtype == np.uint8:\n            image = image.astype(np.float32) / 255.0\n        \n        mask = (mask > 0).astype(np.float32)\n        \n        # Simple augmentation\n        if self.augment:\n            # Random horizontal flip\n            if np.random.random() > 0.5:\n                image = np.fliplr(image).copy()\n                mask = np.fliplr(mask).copy()\n            \n            # Random vertical flip\n            if np.random.random() > 0.5:\n                image = np.flipud(image).copy()\n                mask = np.flipud(mask).copy()\n            \n            # Random rotation (90, 180, 270)\n            k = np.random.randint(0, 4)\n            if k > 0:\n                image = np.rot90(image, k).copy()\n                mask = np.rot90(mask, k).copy()\n        \n        # Convert to torch tensors (C, H, W)\n        image = torch.from_numpy(image).unsqueeze(0)  # (1, H, W)\n        mask = torch.from_numpy(mask).unsqueeze(0)    # (1, H, W)\n        \n        return image, mask","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:55:09.057669Z","iopub.execute_input":"2025-12-06T12:55:09.057884Z","iopub.status.idle":"2025-12-06T12:55:09.065811Z","shell.execute_reply.started":"2025-12-06T12:55:09.057866Z","shell.execute_reply":"2025-12-06T12:55:09.065215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# LOSS FUNCTIONS\n# ============================================================================\n\nclass DiceLoss(nn.Module):\n    \"\"\"Dice Loss for binary segmentation\"\"\"\n    def __init__(self, smooth=1e-6):\n        super().__init__()\n        self.smooth = smooth\n    \n    def forward(self, pred, target):\n        pred = torch.sigmoid(pred)\n        pred = pred.view(-1)\n        target = target.view(-1)\n        \n        intersection = (pred * target).sum()\n        dice = (2. * intersection + self.smooth) / (pred.sum() + target.sum() + self.smooth)\n        \n        return 1 - dice\n\nclass CombinedLoss(nn.Module):\n    \"\"\"Dice + BCE Loss\"\"\"\n    def __init__(self, dice_weight=0.5, bce_weight=0.5):\n        super().__init__()\n        self.dice_weight = dice_weight\n        self.bce_weight = bce_weight\n        self.dice_loss = DiceLoss()\n        self.bce_loss = nn.BCEWithLogitsLoss()\n    \n    def forward(self, pred, target):\n        dice = self.dice_loss(pred, target)\n        bce = self.bce_loss(pred, target)\n        return self.dice_weight * dice + self.bce_weight * bce\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:55:09.211293Z","iopub.execute_input":"2025-12-06T12:55:09.211705Z","iopub.status.idle":"2025-12-06T12:55:09.217438Z","shell.execute_reply.started":"2025-12-06T12:55:09.211685Z","shell.execute_reply":"2025-12-06T12:55:09.216781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# METRICS\n# ============================================================================\n\ndef dice_coefficient(pred, target, smooth=1e-6):\n    \"\"\"Calculate Dice coefficient\"\"\"\n    pred = pred.view(-1)\n    target = target.view(-1)\n    intersection = (pred * target).sum()\n    dice = (2. * intersection + smooth) / (pred.sum() + target.sum() + smooth)\n    return dice\n\ndef iou_score(pred, target, smooth=1e-6):\n    \"\"\"Calculate IoU score\"\"\"\n    pred = pred.view(-1)\n    target = target.view(-1)\n    intersection = (pred * target).sum()\n    union = pred.sum() + target.sum() - intersection\n    iou = (intersection + smooth) / (union + smooth)\n    return iou\n\nclass MetricTracker:\n    \"\"\"Track metrics\"\"\"\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.values = []\n    \n    def update(self, value):\n        self.values.append(value)\n    \n    def avg(self):\n        return np.mean(self.values) if self.values else 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:55:09.379741Z","iopub.execute_input":"2025-12-06T12:55:09.379961Z","iopub.status.idle":"2025-12-06T12:55:09.386142Z","shell.execute_reply.started":"2025-12-06T12:55:09.379943Z","shell.execute_reply":"2025-12-06T12:55:09.385437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# TRAINING & VALIDATION\n# ============================================================================\n\ndef train_epoch(model, loader, criterion, optimizer, device, scaler=None):\n    \"\"\"Train one epoch\"\"\"\n    model.train()\n    losses, dices, ious = MetricTracker(), MetricTracker(), MetricTracker()\n    \n    pbar = tqdm(loader, desc=\"Training\")\n    for images, masks in pbar:\n        images, masks = images.to(device), masks.to(device)\n        \n        optimizer.zero_grad()\n        \n        if scaler:\n            with torch.cuda.amp.autocast():\n                outputs = model(images)\n                loss = criterion(outputs, masks)\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n        else:\n            outputs = model(images)\n            loss = criterion(outputs, masks)\n            loss.backward()\n            optimizer.step()\n        \n        with torch.no_grad():\n            pred = (torch.sigmoid(outputs) > 0.5).float()\n            dice = dice_coefficient(pred, masks)\n            iou = iou_score(pred, masks)\n        \n        losses.update(loss.item())\n        dices.update(dice.item())\n        ious.update(iou.item())\n        \n        pbar.set_postfix({\n            'loss': f'{losses.avg():.4f}',\n            'dice': f'{dices.avg():.4f}',\n            'iou': f'{ious.avg():.4f}'\n        })\n    \n    return {'loss': losses.avg(), 'dice': dices.avg(), 'iou': ious.avg()}\n\n@torch.no_grad()\ndef validate_epoch(model, loader, criterion, device):\n    \"\"\"Validate one epoch\"\"\"\n    model.eval()\n    losses, dices, ious = MetricTracker(), MetricTracker(), MetricTracker()\n    \n    pbar = tqdm(loader, desc=\"Validation\")\n    for images, masks in pbar:\n        images, masks = images.to(device), masks.to(device)\n        \n        outputs = model(images)\n        loss = criterion(outputs, masks)\n        \n        pred = (torch.sigmoid(outputs) > 0.5).float()\n        dice = dice_coefficient(pred, masks)\n        iou = iou_score(pred, masks)\n        \n        losses.update(loss.item())\n        dices.update(dice.item())\n        ious.update(iou.item())\n        \n        pbar.set_postfix({\n            'loss': f'{losses.avg():.4f}',\n            'dice': f'{dices.avg():.4f}',\n            'iou': f'{ious.avg():.4f}'\n        })\n    \n    return {'loss': losses.avg(), 'dice': dices.avg(), 'iou': ious.avg()}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:55:09.534744Z","iopub.execute_input":"2025-12-06T12:55:09.534992Z","iopub.status.idle":"2025-12-06T12:55:09.544614Z","shell.execute_reply.started":"2025-12-06T12:55:09.534975Z","shell.execute_reply":"2025-12-06T12:55:09.543947Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# VISUALIZATION\n# ============================================================================\n\ndef visualize_predictions(model, dataset, device, num_samples=5, save_path=None):\n    \"\"\"Visualize predictions\"\"\"\n    model.eval()\n    fig, axes = plt.subplots(num_samples, 3, figsize=(12, 4*num_samples))\n    if num_samples == 1:\n        axes = axes.reshape(1, -1)\n    \n    indices = np.random.choice(len(dataset), num_samples, replace=False)\n    \n    with torch.no_grad():\n        for i, idx in enumerate(indices):\n            image, mask = dataset[idx]\n            img_input = image.unsqueeze(0).to(device)\n            \n            output = model(img_input)\n            pred = torch.sigmoid(output).cpu().numpy()[0, 0]\n            pred_binary = (pred > 0.5).astype(np.float32)\n            \n            img_np = image.cpu().numpy()[0]\n            mask_np = mask.cpu().numpy()[0]\n            \n            dice = dice_coefficient(\n                torch.from_numpy(pred_binary),\n                torch.from_numpy(mask_np)\n            ).item()\n            \n            axes[i, 0].imshow(img_np, cmap='gray')\n            axes[i, 0].set_title('Input')\n            axes[i, 0].axis('off')\n            \n            axes[i, 1].imshow(mask_np, cmap='hot')\n            axes[i, 1].set_title('Ground Truth')\n            axes[i, 1].axis('off')\n            \n            axes[i, 2].imshow(pred_binary, cmap='hot')\n            axes[i, 2].set_title(f'Prediction (Dice: {dice:.4f})')\n            axes[i, 2].axis('off')\n    \n    plt.tight_layout()\n    if save_path:\n        plt.savefig(save_path, dpi=150, bbox_inches='tight')\n        print(f\"✅ Saved: {save_path}\")\n    plt.close()\n\ndef plot_history(history, save_path=None):\n    \"\"\"Plot training curves\"\"\"\n    fig, axes = plt.subplots(1, 3, figsize=(15, 4))\n    epochs = range(1, len(history['train_loss']) + 1)\n    \n    # Loss\n    axes[0].plot(epochs, history['train_loss'], 'o-', label='Train')\n    axes[0].plot(epochs, history['val_loss'], 's-', label='Val')\n    axes[0].set_xlabel('Epoch')\n    axes[0].set_ylabel('Loss')\n    axes[0].set_title('Loss')\n    axes[0].legend()\n    axes[0].grid(True, alpha=0.3)\n    \n    # Dice\n    axes[1].plot(epochs, history['train_dice'], 'o-', label='Train')\n    axes[1].plot(epochs, history['val_dice'], 's-', label='Val')\n    axes[1].set_xlabel('Epoch')\n    axes[1].set_ylabel('Dice Score')\n    axes[1].set_title('Dice Score')\n    axes[1].legend()\n    axes[1].grid(True, alpha=0.3)\n    \n    # IoU\n    axes[2].plot(epochs, history['train_iou'], 'o-', label='Train')\n    axes[2].plot(epochs, history['val_iou'], 's-', label='Val')\n    axes[2].set_xlabel('Epoch')\n    axes[2].set_ylabel('IoU Score')\n    axes[2].set_title('IoU Score')\n    axes[2].legend()\n    axes[2].grid(True, alpha=0.3)\n    \n    plt.tight_layout()\n    if save_path:\n        plt.savefig(save_path, dpi=150, bbox_inches='tight')\n        print(f\"✅ Saved: {save_path}\")\n    plt.close()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:55:09.691175Z","iopub.execute_input":"2025-12-06T12:55:09.691455Z","iopub.status.idle":"2025-12-06T12:55:09.704404Z","shell.execute_reply.started":"2025-12-06T12:55:09.691433Z","shell.execute_reply":"2025-12-06T12:55:09.703742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# MAIN TRAINING\n# ============================================================================\n\ndef train_model(config=None):\n    \"\"\"Main training function\"\"\"\n    if config is None:\n        config = TrainingConfig()\n    \n    print(\"=\"*70)\n    print(\"VESUVIUS U-NET++ TRAINING (PURE PYTORCH)\")\n    print(\"=\"*70)\n    \n    torch.manual_seed(config.SEED)\n    np.random.seed(config.SEED)\n    \n    Path(config.OUTPUT_DIR).mkdir(parents=True, exist_ok=True)\n    \n    print(f\"\\n⚙️  Configuration:\")\n    print(f\"   • Model: {config.MODEL_NAME}\")\n    print(f\"   • Base filters: {config.BASE_FILTERS}\")\n    print(f\"   • Batch size: {config.BATCH_SIZE}\")\n    print(f\"   • Epochs: {config.EPOCHS}\")\n    print(f\"   • Learning rate: {config.LEARNING_RATE}\")\n    print(f\"   • Device: {config.DEVICE}\")\n    \n    # Datasets\n    print(f\"\\n📦 Loading datasets...\")\n    train_dataset = VesuviusDataset(\n        config.DATA_DIR, 'train', augment=config.USE_AUGMENTATION\n    )\n    val_dataset = VesuviusDataset(config.DATA_DIR, 'val', augment=False)\n    \n    train_loader = DataLoader(\n    train_dataset, \n    batch_size=config.BATCH_SIZE,\n    num_workers=config.NUM_WORKERS,\n    prefetch_factor=2,\n    persistent_workers=True,\n    pin_memory=True,\n    shuffle=True\n    )\n    val_loader = DataLoader(\n        val_dataset, batch_size=config.BATCH_SIZE, shuffle=False,\n        num_workers=config.NUM_WORKERS, pin_memory=config.PIN_MEMORY\n    )\n    \n    # Model\n    print(f\"\\n🏗️  Building model...\")\n    model = UNetPlusPlus(\n        in_channels=config.IN_CHANNELS,\n        num_classes=config.NUM_CLASSES,\n        base_filters=config.BASE_FILTERS\n    ).to(config.DEVICE)\n    \n    total_params = sum(p.numel() for p in model.parameters())\n    print(f\"   • Parameters: {total_params:,}\")\n    \n    # Loss, optimizer, scheduler\n    criterion = CombinedLoss(config.DICE_WEIGHT, config.BCE_WEIGHT)\n    optimizer = torch.optim.AdamW(\n        model.parameters(), lr=config.LEARNING_RATE, weight_decay=config.WEIGHT_DECAY\n    )\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=config.T_MAX)\n    scaler = torch.cuda.amp.GradScaler() if config.MIXED_PRECISION else None\n    \n    # Training loop\n    history = {\n        'train_loss': [], 'train_dice': [], 'train_iou': [],\n        'val_loss': [], 'val_dice': [], 'val_iou': []\n    }\n    \n    best_dice = 0.0\n    epochs_no_improve = 0\n    \n    print(f\"\\n{'='*70}\")\n    print(\"STARTING TRAINING\")\n    print(f\"{'='*70}\\n\")\n    \n    for epoch in range(1, config.EPOCHS + 1):\n        print(f\"\\nEpoch {epoch}/{config.EPOCHS}\")\n        print(\"-\" * 70)\n        \n        train_metrics = train_epoch(model, train_loader, criterion, optimizer, config.DEVICE, scaler)\n        val_metrics = validate_epoch(model, val_loader, criterion, config.DEVICE)\n        \n        history['train_loss'].append(train_metrics['loss'])\n        history['train_dice'].append(train_metrics['dice'])\n        history['train_iou'].append(train_metrics['iou'])\n        history['val_loss'].append(val_metrics['loss'])\n        history['val_dice'].append(val_metrics['dice'])\n        history['val_iou'].append(val_metrics['iou'])\n        \n        print(f\"\\n📊 Epoch {epoch} Summary:\")\n        print(f\"   Train - Loss: {train_metrics['loss']:.4f} | Dice: {train_metrics['dice']:.4f} | IoU: {train_metrics['iou']:.4f}\")\n        print(f\"   Val   - Loss: {val_metrics['loss']:.4f} | Dice: {val_metrics['dice']:.4f} | IoU: {val_metrics['iou']:.4f}\")\n        print(f\"   LR: {optimizer.param_groups[0]['lr']:.6f}\")\n        \n        scheduler.step()\n        \n        is_best = val_metrics['dice'] > best_dice\n        if is_best:\n            best_dice = val_metrics['dice']\n            epochs_no_improve = 0\n            torch.save({\n                'epoch': epoch,\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'best_dice': best_dice\n            }, f\"{config.OUTPUT_DIR}/best_model.pth\")\n            print(f\"   ✅ Best model saved! (Dice: {best_dice:.4f})\")\n        else:\n            epochs_no_improve += 1\n        \n        if epoch % config.SAVE_EVERY_N_EPOCHS == 0:\n            torch.save(model.state_dict(), f\"{config.OUTPUT_DIR}/checkpoint_epoch_{epoch}.pth\")\n        \n        if epochs_no_improve >= config.EARLY_STOPPING_PATIENCE:\n            print(f\"\\n⚠️  Early stopping at epoch {epoch}\")\n            break\n    \n    print(f\"\\n{'='*70}\")\n    print(\"TRAINING COMPLETE!\")\n    print(f\"{'='*70}\")\n    print(f\"✅ Best Dice: {best_dice:.4f}\")\n    \n    # Save history\n    with open(f\"{config.OUTPUT_DIR}/history.json\", 'w') as f:\n        json.dump(history, f, indent=2)\n    \n    # Plots\n    plot_history(history, f\"{config.OUTPUT_DIR}/training_curves.png\")\n    visualize_predictions(model, val_dataset, config.DEVICE, 5, f\"{config.OUTPUT_DIR}/predictions.png\")\n    \n    print(f\"\\n📁 Outputs saved to: {config.OUTPUT_DIR}\\n\")\n    \n    return model, history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:55:09.849837Z","iopub.execute_input":"2025-12-06T12:55:09.850422Z","iopub.status.idle":"2025-12-06T12:55:09.863052Z","shell.execute_reply.started":"2025-12-06T12:55:09.850403Z","shell.execute_reply":"2025-12-06T12:55:09.862240Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# RUN\n# ============================================================================\n\nif __name__ == \"__main__\":\n    config = TrainingConfig()\n    model, history = train_model(config)\n    \n    print(\"\\n🎉 Training complete!\")\n    print(f\"📊 Best Dice: {max(history['val_dice']):.4f}\")\n    print(f\"📊 Best IoU: {max(history['val_iou']):.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T12:55:22.471601Z","iopub.execute_input":"2025-12-06T12:55:22.472078Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# SIMPAN MODEL\n# ============================================================================\n\nimport shutil\nfrom pathlib import Path\n\n# Buat folder penyimpanan\nsave_dir = \"/kaggle/working/saved_model\"\nPath(save_dir).mkdir(exist_ok=True)\n\n# Copy model file\nshutil.copy(\n    \"/kaggle/working/training_output/best_model.pth\",\n    f\"{save_dir}/best_model.pth\"\n)\n\nprint(\"✅ Model saved!\")\nprint(f\"📁 Location: {save_dir}/best_model.pth\")\nprint(\"\\n📌 Download dari Output tab\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}