{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[],"gpuType":"T4"},"accelerator":"GPU","kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":117682,"databundleVersionId":14443416,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q imagecodecs","metadata":{"id":"PArzWOl22cmc","trusted":true,"execution":{"iopub.status.busy":"2025-12-06T18:16:38.841855Z","iopub.execute_input":"2025-12-06T18:16:38.842116Z","iopub.status.idle":"2025-12-06T18:16:44.979201Z","shell.execute_reply.started":"2025-12-06T18:16:38.842093Z","shell.execute_reply":"2025-12-06T18:16:44.978471Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PreprocessConfig:\n    # UBAH INI:\n    # DATA_DIR = \"/content/vesuvius_data\"  # ❌ Colab path\n    DATA_DIR = \"/kaggle/input/vesuvius-challenge-surface-detection\"  # ✅ Kaggle path\n    \n    # UBAH INI:\n    # OUTPUT_DIR = \"/content/preprocessed_data\"  # ❌ Colab path\n    OUTPUT_DIR = \"/kaggle/working/preprocessed_data\"  # ✅ Kaggle path","metadata":{"id":"ylMZSVBTcHY8","outputId":"25d7a3e1-0580-44f4-82bc-6d3887576583","trusted":true,"execution":{"iopub.status.busy":"2025-12-06T18:16:44.980666Z","iopub.execute_input":"2025-12-06T18:16:44.980916Z","iopub.status.idle":"2025-12-06T18:16:44.985194Z","shell.execute_reply.started":"2025-12-06T18:16:44.980892Z","shell.execute_reply":"2025-12-06T18:16:44.984559Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls /kaggle/input/vesuvius-challenge-surface-detection","metadata":{"id":"ljX3RARrZ22N","outputId":"28e7b26f-495f-47ee-a7cf-71b62b681e58","trusted":true,"execution":{"iopub.status.busy":"2025-12-06T18:16:44.986055Z","iopub.execute_input":"2025-12-06T18:16:44.986305Z","iopub.status.idle":"2025-12-06T18:16:45.11796Z","shell.execute_reply.started":"2025-12-06T18:16:44.986284Z","shell.execute_reply":"2025-12-06T18:16:45.117064Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"MULAI DARI AWAL LAGI","metadata":{"id":"qEO-JUaftZi3"}},{"cell_type":"code","source":"\"\"\"\nSTEP 0: INSTALL DEPENDENCIES\nJalankan cell ini DULU, lalu RESTART RUNTIME sebelum jalankan code lain!\n\"\"\"\n\nimport subprocess\nimport sys\n\nprint(\"=\"*70)\nprint(\"INSTALLING REQUIRED PACKAGES\")\nprint(\"=\"*70)\n\n# Install imagecodecs\nprint(\"\\n[1/1] Installing imagecodecs (for LZW compression support)...\")\nsubprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', 'imagecodecs'])\n\nprint(\"\\n\" + \"=\"*70)\nprint(\"✅ INSTALLATION COMPLETE!\")\nprint(\"=\"*70)\nprint(\"\\n⚠️  IMPORTANT: Please RESTART RUNTIME now!\")\nprint(\"   (Runtime → Restart runtime)\")\nprint(\"\\nAfter restart, you can run the exploration and preprocessing scripts.\")\nprint(\"=\"*70)","metadata":{"id":"UFWHukXNsiZP","outputId":"15342039-9e12-40c8-eae4-11607b318510","trusted":true,"execution":{"iopub.status.busy":"2025-12-06T18:16:45.119911Z","iopub.execute_input":"2025-12-06T18:16:45.120244Z","iopub.status.idle":"2025-12-06T18:16:48.303344Z","shell.execute_reply.started":"2025-12-06T18:16:45.120216Z","shell.execute_reply":"2025-12-06T18:16:48.302527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Install imagecodecs\nimport subprocess\nimport sys\nsubprocess.check_call([sys.executable, '-m', 'pip', 'install', 'imagecodecs'])\n\nprint(\"=\"*70)\nprint(\"✅ imagecodecs installed!\")\nprint(\"⚠️  RESTART KERNEL NOW!\")\nprint(\"=\"*70)\nprint(\"\\nSteps:\")\nprint(\"1. Click 'Session' → 'Restart Session'\")\nprint(\"2. Run the exploration script again\")","metadata":{"id":"W-DcgDWItc5t","outputId":"3b4460c8-3df8-4001-f88f-6c9553e7de8c","trusted":true,"execution":{"iopub.status.busy":"2025-12-06T18:16:48.304304Z","iopub.execute_input":"2025-12-06T18:16:48.304522Z","iopub.status.idle":"2025-12-06T18:16:51.507319Z","shell.execute_reply.started":"2025-12-06T18:16:48.304504Z","shell.execute_reply":"2025-12-06T18:16:51.506523Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# SCRIPT EKSPLORASI DATASET VESUVIUS - KAGGLE VERSION (NO IMAGECODECS)\n# Dataset ini berisi 3D CT scan volumes (320x320x320)\n# Uses PIL as alternative to avoid imagecodecs dependency\n# ============================================================================\n\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ============================================================================\n# PATH CONFIGURATION - KAGGLE\n# ============================================================================\nDATA_DIR = \"/kaggle/input/vesuvius-challenge-surface-detection\"\nOUTPUT_DIR = \"/kaggle/working\"\n\nprint(\"=\"*70)\nprint(\"EKSPLORASI DATASET VESUVIUS - 3D CT SCAN (KAGGLE)\")\nprint(\"=\"*70)\n\n# ============================================================================\n# HELPER FUNCTION: Load Multi-page TIFF dengan PIL\n# ============================================================================\ndef load_multipage_tiff(path, max_frames=None):\n    \"\"\"\n    Load multi-page TIFF file menggunakan PIL\n    Slower than tifffile but works without imagecodecs\n    \"\"\"\n    frames = []\n    try:\n        with Image.open(path) as img:\n            n_frames = 0\n            # Count total frames\n            try:\n                while True:\n                    img.seek(n_frames)\n                    n_frames += 1\n            except EOFError:\n                pass\n            \n            # Load frames\n            if max_frames:\n                n_frames = min(n_frames, max_frames)\n            \n            for i in range(n_frames):\n                img.seek(i)\n                frames.append(np.array(img))\n            \n        return np.stack(frames, axis=0)\n    except Exception as e:\n        print(f\"Error loading {path.name}: {e}\")\n        return None\n\n# ============================================================================\n# 1. CEK STRUKTUR FOLDER\n# ============================================================================\nprint(\"\\n[1] STRUKTUR FOLDER:\")\nprint(\"-\"*70)\n\ndata_path = Path(DATA_DIR)\n\nif not data_path.exists():\n    print(f\"❌ ERROR: Data path tidak ditemukan: {DATA_DIR}\")\n    print(\"\\n⚠️  Pastikan Anda sudah Add Data di Kaggle:\")\n    print(\"   1. Klik 'Add data' button\")\n    print(\"   2. Search: 'vesuvius-challenge-surface-detection'\")\n    print(\"   3. Click 'Add'\")\nelse:\n    print(f\"✅ Data path found: {DATA_DIR}\\n\")\n    for item in sorted(data_path.iterdir()):\n        if item.is_dir():\n            count = len(list(item.iterdir()))\n            print(f\"📁 {item.name:25s} → {count} files\")\n        else:\n            size_mb = item.stat().st_size / (1024*1024)\n            print(f\"📄 {item.name:25s} → {size_mb:.2f} MB\")\n\n    # ============================================================================\n    # 2. ANALISIS train.csv DAN test.csv\n    # ============================================================================\n    print(\"\\n[2] ANALISIS CSV FILES:\")\n    print(\"-\"*70)\n\n    train_csv = pd.read_csv(data_path / \"train.csv\")\n    test_csv = pd.read_csv(data_path / \"test.csv\")\n\n    print(f\"train.csv shape: {train_csv.shape}\")\n    print(f\"train.csv columns: {train_csv.columns.tolist()}\")\n    print(f\"\\nSample train.csv:\")\n    print(train_csv.head(10))\n\n    print(f\"\\ntest.csv shape: {test_csv.shape}\")\n    print(f\"test.csv columns: {test_csv.columns.tolist()}\")\n    print(f\"\\nSample test.csv:\")\n    print(test_csv.head())\n\n    # Analisis unique values\n    print(f\"\\n📊 Unique scroll_ids in train: {train_csv['scroll_id'].nunique()}\")\n    print(f\"Scroll_id distribution:\")\n    print(train_csv['scroll_id'].value_counts())\n\n    # ============================================================================\n    # 3. ANALISIS STRUKTUR 3D VOLUMES\n    # ============================================================================\n    print(\"\\n[3] ANALISIS 3D VOLUMES:\")\n    print(\"-\"*70)\n\n    train_images_dir = data_path / \"train_images\"\n    train_labels_dir = data_path / \"train_labels\"\n\n    image_files = sorted(list(train_images_dir.glob(\"*.tif\")))\n    label_files = sorted(list(train_labels_dir.glob(\"*.tif\")))\n\n    print(f\"Training volumes: {len(image_files)}\")\n    print(f\"Label volumes: {len(label_files)}\")\n\n    # Load sample volume\n    if len(image_files) > 0:\n        sample_img_path = image_files[0]\n        sample_label_path = train_labels_dir / sample_img_path.name\n\n        print(f\"\\n📦 Loading sample: {sample_img_path.name}\")\n        print(\"   (Using PIL - this may take 30-60 seconds...)\")\n\n        try:\n            # Load 3D volume using PIL (load only first 100 slices for speed)\n            img_volume = load_multipage_tiff(sample_img_path, max_frames=100)\n            \n            if img_volume is not None:\n                print(f\"✓ Image volume loaded!\")\n                print(f\"  Shape: {img_volume.shape} (D x H x W)\")\n                print(f\"  Note: Loaded first 100 slices only for speed\")\n                print(f\"  Dtype: {img_volume.dtype}\")\n                print(f\"  Size: {img_volume.nbytes / (1024**2):.2f} MB\")\n                print(f\"  Min: {img_volume.min()}, Max: {img_volume.max()}\")\n                print(f\"  Mean: {img_volume.mean():.2f}, Std: {img_volume.std():.2f}\")\n\n                # Load label volume jika ada\n                if sample_label_path.exists():\n                    print(f\"\\n📦 Loading label volume...\")\n                    label_volume = load_multipage_tiff(sample_label_path, max_frames=100)\n                    \n                    if label_volume is not None:\n                        print(f\"✓ Label volume loaded!\")\n                        print(f\"  Shape: {label_volume.shape}\")\n                        print(f\"  Dtype: {label_volume.dtype}\")\n                        print(f\"  Unique values: {np.unique(label_volume)}\")\n                        print(f\"  Surface pixels: {(label_volume > 0).sum()}\")\n                        print(f\"  Coverage: {(label_volume > 0).sum() / label_volume.size * 100:.4f}%\")\n\n                        # ================================================================\n                        # VISUALISASI MULTIPLE SLICES\n                        # ================================================================\n                        print(\"\\n[4] VISUALISASI SLICES:\")\n                        print(\"-\"*70)\n\n                        # Ambil beberapa slices dari tengah volume\n                        depth = img_volume.shape[0]\n                        slice_indices = [depth//4, depth//2, 3*depth//4]\n\n                        fig, axes = plt.subplots(3, 3, figsize=(15, 15))\n\n                        for i, slice_idx in enumerate(slice_indices):\n                            # Image slice\n                            img_slice = img_volume[slice_idx]\n                            label_slice = label_volume[slice_idx]\n\n                            # Normalisasi untuk visualisasi\n                            img_normalized = (img_slice - img_slice.min()) / (img_slice.max() - img_slice.min() + 1e-8)\n\n                            # Column 1: CT scan slice\n                            axes[i, 0].imshow(img_slice, cmap='gray')\n                            axes[i, 0].set_title(f'CT Slice {slice_idx}/{depth}\\nDepth: {slice_idx}')\n                            axes[i, 0].axis('off')\n\n                            # Column 2: Label slice\n                            axes[i, 1].imshow(label_slice, cmap='hot')\n                            axes[i, 1].set_title(f'Surface Label\\nPixels: {(label_slice > 0).sum()}')\n                            axes[i, 1].axis('off')\n\n                            # Column 3: Overlay\n                            overlay = np.stack([img_normalized]*3, axis=-1)\n                            overlay[:,:,0] = np.where(label_slice > 0, 1.0, overlay[:,:,0])\n                            axes[i, 2].imshow(overlay)\n                            axes[i, 2].set_title('Overlay (Red = Surface)')\n                            axes[i, 2].axis('off')\n\n                        plt.tight_layout()\n                        \n                        # Save to /kaggle/working\n                        save_path = f'{OUTPUT_DIR}/volume_slices_visualization.png'\n                        plt.savefig(save_path, dpi=150, bbox_inches='tight')\n                        print(f\"✓ Saved: {save_path}\")\n                        plt.close()\n\n                        # ================================================================\n                        # STATISTIK PER SLICE\n                        # ================================================================\n                        print(\"\\n[5] STATISTIK PER SLICE:\")\n                        print(\"-\"*70)\n\n                        surface_per_slice = [(label_volume[i] > 0).sum() for i in range(depth)]\n\n                        plt.figure(figsize=(12, 4))\n\n                        plt.subplot(1, 2, 1)\n                        plt.plot(surface_per_slice, linewidth=2)\n                        plt.xlabel('Slice Index')\n                        plt.ylabel('Surface Pixels')\n                        plt.title('Surface Pixels per Slice (First 100)')\n                        plt.grid(True, alpha=0.3)\n\n                        plt.subplot(1, 2, 2)\n                        plt.hist(surface_per_slice, bins=30, edgecolor='black')\n                        plt.xlabel('Surface Pixels')\n                        plt.ylabel('Frequency')\n                        plt.title('Distribution of Surface Pixels')\n                        plt.grid(True, alpha=0.3)\n\n                        plt.tight_layout()\n                        \n                        # Save to /kaggle/working\n                        save_path = f'{OUTPUT_DIR}/surface_distribution.png'\n                        plt.savefig(save_path, dpi=150, bbox_inches='tight')\n                        print(f\"✓ Saved: {save_path}\")\n                        plt.close()\n\n                        # Slice dengan surface terbanyak\n                        max_surface_idx = np.argmax(surface_per_slice)\n                        print(f\"\\nSlice dengan surface terbanyak: {max_surface_idx}\")\n                        print(f\"Surface pixels: {surface_per_slice[max_surface_idx]}\")\n\n                        # Statistik keseluruhan\n                        non_zero_slices = sum(1 for x in surface_per_slice if x > 0)\n                        print(f\"\\nSlices dengan surface: {non_zero_slices}/{depth} ({non_zero_slices/depth*100:.1f}%)\")\n                        print(f\"Rata-rata surface per slice: {np.mean(surface_per_slice):.1f}\")\n                        print(f\"Median surface per slice: {np.median(surface_per_slice):.1f}\")\n                    else:\n                        print(f\"⚠️ Failed to load label volume\")\n                else:\n                    print(f\"⚠️ Label tidak ditemukan untuk {sample_img_path.name}\")\n            else:\n                print(f\"⚠️ Failed to load image volume\")\n\n        except Exception as e:\n            print(f\"❌ Error: {e}\")\n            import traceback\n            traceback.print_exc()\n\n    # ============================================================================\n    # 6. RINGKASAN DATASET\n    # ============================================================================\n    print(\"\\n\" + \"=\"*70)\n    print(\"RINGKASAN DATASET\")\n    print(\"=\"*70)\n\n    print(f\"\"\"\n📊 Dataset Information:\n   • Type: 3D CT Scan Volumes for Surface Detection\n   • Task: Segmentation (detect papyrus surface in 3D volume)\n   • Training volumes: {len(image_files)}\n   • Label volumes: {len(label_files)}\n   • Volume shape: 320 x 320 x 320 (D x H x W)\n   • Data type: uint8 (0-255)\n   • Loading method: PIL (slower but no imagecodecs needed)\n\n🎯 Preprocessing Strategy:\n   1. Normalisasi per volume atau per slice\n   2. Denoising 3D atau per slice 2D\n   3. Extract 2D slices untuk training (lebih efficient)\n   4. Patch extraction dari setiap slice\n   5. Oversample slices yang memiliki surface\n\n⚡ Rekomendasi:\n   • Process slice-by-slice (2D approach) untuk efisiensi\n   • Atau gunakan 3D patches jika GPU kuat\n   • Skip slices tanpa surface untuk training efficiency\n   • Gunakan weighted loss karena extreme class imbalance\n\n💾 Output Location:\n   • Visualizations saved to: {OUTPUT_DIR}/\n   • Accessible in Kaggle notebook output\n\n⚠️  Note on PIL vs tifffile:\n   • PIL is slower but works without imagecodecs\n   • For preprocessing, consider using imagecodecs for speed:\n     1. !pip install imagecodecs\n     2. Restart kernel\n     3. Use tifffile.imread() instead\n\"\"\")\n\n    print(\"=\"*70)\n    print(\"✅ EKSPLORASI SELESAI!\")\n    print(\"=\"*70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T18:16:51.508323Z","iopub.execute_input":"2025-12-06T18:16:51.508559Z","iopub.status.idle":"2025-12-06T18:16:56.913764Z","shell.execute_reply.started":"2025-12-06T18:16:51.50854Z","shell.execute_reply":"2025-12-06T18:16:56.912847Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# VESUVIUS 3D CT SCAN PREPROCESSING - KAGGLE OPTIMIZED (FIXED)\n# Memory-efficient preprocessing with immediate save and garbage collection\n# ============================================================================\n\nimport numpy as np\nimport cv2\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nfrom scipy.ndimage import gaussian_filter, median_filter\nfrom skimage.morphology import binary_opening, binary_closing, disk\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport json\nimport pandas as pd\nimport tifffile\nimport gc\nimport shutil\n\n# ============================================================================\n# KONFIGURASI PREPROCESSING - KAGGLE PATHS\n# ============================================================================\n\nclass PreprocessConfig:\n    \"\"\"Konfigurasi untuk preprocessing 3D CT scan Vesuvius\"\"\"\n\n    # Path - KAGGLE SPECIFIC\n    DATA_DIR = \"/kaggle/input/vesuvius-challenge-surface-detection\"\n    OUTPUT_DIR = \"/kaggle/working/preprocessed_data\"\n\n    # Processing mode\n    PROCESS_AS_2D = True\n\n    # Memory optimization - ENHANCED\n    SAVE_PATCHES_IMMEDIATELY = True\n    CLEAR_MEMORY_AFTER_VOLUME = True\n    BATCH_SIZE_VOLUMES = 5  # Reduced for better memory management\n    DELETE_TEMP_DURING_COPY = True  # Delete temp files while copying\n\n    # Slice filtering - AGGRESSIVE\n    SKIP_EMPTY_SLICES = True  # Skip slices without surface\n    MIN_SURFACE_PIXELS = 500  # Higher threshold\n    SAMPLE_SLICES = True      # Sample only some slices per volume\n    SLICE_SAMPLE_RATE = 0.5   # Use only 50% of slices\n\n    # Normalisasi\n    NORM_METHOD = \"minmax\"\n    CLIP_PERCENTILE = (1, 99)\n    NORMALIZE_PER_VOLUME = False\n\n    # Denoising - DISABLE untuk speed dan size\n    USE_DENOISING = False  # Disabled for speed/space\n    DENOISE_METHOD = \"gaussian\"\n    GAUSSIAN_SIGMA = 0.5\n    MEDIAN_SIZE = 3\n\n    # Patch extraction - SUPER AGGRESSIVE\n    PATCH_SIZE = 256\n    STRIDE = 256  # NO OVERLAP! Reduces patches by 75%\n    MIN_MASK_RATIO = 0.03  # Much higher threshold (3% vs 1%)\n    MAX_PATCHES_PER_VOLUME = 300  # Hard limit per volume\n\n    # Mask cleaning\n    USE_MASK_CLEANING = False  # Disabled for speed\n    MORPHOLOGY_KERNEL_SIZE = 2\n\n    # Class imbalance handling\n    OVERSAMPLE_FACTOR = 1  # NO OVERSAMPLE to save space\n\n    # Data split\n    TEST_SIZE = 0.2\n    RANDOM_SEED = 42\n    \n    # Compression - CRITICAL FOR KAGGLE\n    USE_COMPRESSION = True  # Save as compressed uint8\n    SAVE_AS_UINT8 = True    # Convert float32 to uint8 (4x smaller)\n\n    # Visualisasi\n    SAVE_PREVIEW = True\n    NUM_PREVIEW = 5\n\n    # Limit untuk testing (None = process all)\n    MAX_VOLUMES = 100  # Change to 50 for testing\n    \n    # Skip patches without surface to save space\n    SKIP_NO_SURFACE_PATCHES = True\n\n# ============================================================================\n# UTILITY FUNCTIONS\n# ============================================================================\n\ndef create_directories(output_dir):\n    \"\"\"Buat struktur folder output\"\"\"\n    dirs = [\n        f\"{output_dir}/train/images\",\n        f\"{output_dir}/train/masks\",\n        f\"{output_dir}/val/images\",\n        f\"{output_dir}/val/masks\",\n        f\"{output_dir}/preview\",\n        f\"{output_dir}/stats\",\n        f\"{output_dir}/temp/images\",\n        f\"{output_dir}/temp/masks\"\n    ]\n    for d in dirs:\n        Path(d).mkdir(parents=True, exist_ok=True)\n    print(\"✅ Directories created\")\n\ndef load_volume(path):\n    \"\"\"Load 3D volume dengan error handling\"\"\"\n    try:\n        volume = tifffile.imread(str(path))\n        return volume\n    except Exception as e:\n        print(f\"  ❌ Error loading {path.name}: {e}\")\n        return None\n\ndef clear_memory():\n    \"\"\"Force garbage collection\"\"\"\n    gc.collect()\n\ndef get_memory_usage():\n    \"\"\"Get current memory usage (if available)\"\"\"\n    try:\n        import psutil\n        process = psutil.Process()\n        mem = process.memory_info().rss / 1024 / 1024  # MB\n        return f\"{mem:.1f} MB\"\n    except:\n        return \"N/A\"\n\n# ============================================================================\n# NORMALISASI\n# ============================================================================\n\ndef normalize_slice(slice_2d, method=\"minmax\", clip_percentile=(1, 99)):\n    \"\"\"Normalisasi single 2D slice\"\"\"\n    if clip_percentile:\n        low, high = np.percentile(slice_2d, clip_percentile)\n        slice_2d = np.clip(slice_2d, low, high)\n\n    if method == \"minmax\":\n        img_min = slice_2d.min()\n        img_max = slice_2d.max()\n        if img_max - img_min > 0:\n            normalized = (slice_2d - img_min) / (img_max - img_min)\n        else:\n            normalized = slice_2d - img_min\n    elif method == \"standardize\":\n        mean = slice_2d.mean()\n        std = slice_2d.std()\n        if std > 0:\n            normalized = (slice_2d - mean) / std\n        else:\n            normalized = slice_2d - mean\n    else:\n        normalized = slice_2d\n\n    return normalized.astype(np.float32)\n\ndef normalize_volume(volume, method=\"minmax\", clip_percentile=(1, 99)):\n    \"\"\"Normalisasi entire 3D volume\"\"\"\n    if clip_percentile:\n        low, high = np.percentile(volume, clip_percentile)\n        volume = np.clip(volume, low, high)\n\n    if method == \"minmax\":\n        vol_min = volume.min()\n        vol_max = volume.max()\n        if vol_max - vol_min > 0:\n            normalized = (volume - vol_min) / (vol_max - vol_min)\n        else:\n            normalized = volume - vol_min\n    elif method == \"standardize\":\n        mean = volume.mean()\n        std = volume.std()\n        if std > 0:\n            normalized = (volume - mean) / std\n        else:\n            normalized = volume - mean\n    else:\n        normalized = volume\n\n    return normalized.astype(np.float32)\n\n# ============================================================================\n# DENOISING\n# ============================================================================\n\ndef denoise_slice(slice_2d, method=\"gaussian\", sigma=0.5, kernel_size=3):\n    \"\"\"Denoise 2D slice\"\"\"\n    if method == \"gaussian\":\n        denoised = gaussian_filter(slice_2d, sigma=sigma)\n    elif method == \"median\":\n        denoised = median_filter(slice_2d, size=kernel_size)\n    elif method == \"bilateral\":\n        img_uint8 = (slice_2d * 255).astype(np.uint8)\n        denoised = cv2.bilateralFilter(img_uint8, kernel_size, 75, 75)\n        denoised = denoised.astype(np.float32) / 255.0\n    else:\n        denoised = slice_2d\n\n    return denoised\n\n# ============================================================================\n# MASK CLEANING\n# ============================================================================\n\ndef clean_mask(mask, kernel_size=2):\n    \"\"\"Clean mask dengan morphological operations\"\"\"\n    mask_binary = (mask > 0).astype(np.uint8)\n    kernel = disk(kernel_size)\n    mask_opened = binary_opening(mask_binary, kernel)\n    mask_cleaned = binary_closing(mask_opened, kernel)\n    return mask_cleaned.astype(np.uint8)\n\n# ============================================================================\n# PATCH EXTRACTION\n# ============================================================================\n\ndef extract_patches_2d(image, mask, patch_size=256, stride=192, min_mask_ratio=0.001):\n    \"\"\"Extract patches dari 2D slice dengan sliding window overlap\"\"\"\n    h, w = image.shape\n    patches = []\n\n    y_steps = []\n    x_steps = []\n\n    # Generate Y positions\n    y = 0\n    while y + patch_size <= h:\n        y_steps.append(y)\n        y += stride\n    if len(y_steps) > 0 and y_steps[-1] + patch_size < h:\n        y_steps.append(h - patch_size)\n\n    # Generate X positions\n    x = 0\n    while x + patch_size <= w:\n        x_steps.append(x)\n        x += stride\n    if len(x_steps) > 0 and x_steps[-1] + patch_size < w:\n        x_steps.append(w - patch_size)\n\n    # Extract patches\n    for y in y_steps:\n        for x in x_steps:\n            img_patch = image[y:y+patch_size, x:x+patch_size]\n            mask_patch = mask[y:y+patch_size, x:x+patch_size]\n\n            if img_patch.shape != (patch_size, patch_size):\n                continue\n\n            mask_ratio = mask_patch.sum() / (patch_size * patch_size)\n            has_mask = mask_ratio >= min_mask_ratio\n\n            patches.append({\n                'image': img_patch,\n                'mask': mask_patch,\n                'has_mask': has_mask,\n                'mask_ratio': mask_ratio,\n                'position': (y, x)\n            })\n\n    return patches\n\n# ============================================================================\n# PROCESS SINGLE VOLUME\n# ============================================================================\n\ndef process_volume_as_slices(volume_path, label_path, config):\n    \"\"\"Process 3D volume dengan extract 2D slices - SUPER OPTIMIZED\"\"\"\n    \n    # Load volumes\n    img_volume = load_volume(volume_path)\n    label_volume = load_volume(label_path)\n\n    if img_volume is None or label_volume is None:\n        return []\n\n    volume_id = volume_path.stem\n    depth, height, width = img_volume.shape\n\n    print(f\"  📦 Shape: {img_volume.shape} | Mem: {get_memory_usage()}\")\n\n    # Normalisasi per volume jika diminta\n    if config.NORMALIZE_PER_VOLUME:\n        img_volume = normalize_volume(img_volume, config.NORM_METHOD,\n                                      config.CLIP_PERCENTILE)\n\n    # OPTIMIZATION: Sample only some slices if enabled\n    if config.SAMPLE_SLICES:\n        slice_indices = np.random.choice(\n            depth, \n            size=int(depth * config.SLICE_SAMPLE_RATE), \n            replace=False\n        )\n        slice_indices = sorted(slice_indices)\n    else:\n        slice_indices = range(depth)\n\n    # Process setiap slice\n    all_patches_metadata = []\n    surface_count = 0\n\n    for slice_idx in slice_indices:\n        img_slice = img_volume[slice_idx]\n        label_slice = label_volume[slice_idx]\n\n        surface_pixels = (label_slice > 0).sum()\n\n        # SKIP: Empty slices\n        if config.SKIP_EMPTY_SLICES and surface_pixels < config.MIN_SURFACE_PIXELS:\n            continue\n\n        if not config.NORMALIZE_PER_VOLUME:\n            img_slice = normalize_slice(img_slice, config.NORM_METHOD,\n                                       config.CLIP_PERCENTILE)\n\n        if config.USE_DENOISING:\n            img_slice = denoise_slice(img_slice, config.DENOISE_METHOD,\n                                     config.GAUSSIAN_SIGMA, config.MEDIAN_SIZE)\n\n        if config.USE_MASK_CLEANING:\n            label_slice = clean_mask(label_slice, config.MORPHOLOGY_KERNEL_SIZE)\n\n        patches = extract_patches_2d(img_slice, label_slice,\n                                     config.PATCH_SIZE, config.STRIDE,\n                                     config.MIN_MASK_RATIO)\n\n        # Filter: Skip patches without surface if enabled\n        if config.SKIP_NO_SURFACE_PATCHES:\n            patches = [p for p in patches if p['has_mask']]\n\n        for patch in patches:\n            patch['volume_id'] = volume_id\n            patch['slice_idx'] = slice_idx\n            patch['has_surface'] = surface_pixels > 0\n\n        all_patches_metadata.extend(patches)\n\n        if surface_pixels > 0:\n            surface_count += 1\n        \n        # HARD LIMIT: Stop if too many patches\n        if len(all_patches_metadata) >= config.MAX_PATCHES_PER_VOLUME:\n            print(f\"  ⚠️  Hit limit: {config.MAX_PATCHES_PER_VOLUME} patches\")\n            all_patches_metadata = all_patches_metadata[:config.MAX_PATCHES_PER_VOLUME]\n            break\n\n    print(f\"  ✅ {len(all_patches_metadata)} patches from {surface_count} slices\")\n\n    # Clear memory aggressively\n    del img_volume, label_volume\n    clear_memory()\n\n    return all_patches_metadata\n\n# ============================================================================\n# SAVE PATCHES\n# ============================================================================\n\ndef save_patch_immediate(patch, split_name, idx, config):\n    \"\"\"Save single patch immediately ke disk - COMPRESSED\"\"\"\n    img_path = f\"{config.OUTPUT_DIR}/{split_name}/images/patch_{idx:06d}.npy\"\n    mask_path = f\"{config.OUTPUT_DIR}/{split_name}/masks/patch_{idx:06d}.npy\"\n\n    # Convert to uint8 to save 4x space (float32 → uint8)\n    if config.SAVE_AS_UINT8:\n        img_uint8 = (patch['image'] * 255).astype(np.uint8)\n        mask_uint8 = patch['mask'].astype(np.uint8)\n        \n        # Save compressed\n        if config.USE_COMPRESSION:\n            np.savez_compressed(img_path.replace('.npy', '.npz'), data=img_uint8)\n            np.savez_compressed(mask_path.replace('.npy', '.npz'), data=mask_uint8)\n        else:\n            np.save(img_path, img_uint8)\n            np.save(mask_path, mask_uint8)\n    else:\n        np.save(img_path, patch['image'])\n        np.save(mask_path, patch['mask'])\n\n    return {\n        'has_mask': patch['has_mask'],\n        'mask_ratio': patch['mask_ratio'],\n        'volume_id': patch.get('volume_id', 'unknown'),\n        'slice_idx': patch.get('slice_idx', -1)\n    }\n\n# ============================================================================\n# VISUALISASI\n# ============================================================================\n\ndef save_preview_samples(output_dir, num_samples=5):\n    \"\"\"Save preview samples dari hasil preprocessing\"\"\"\n    config = PreprocessConfig()\n    \n    # Load beberapa random patches untuk preview\n    train_img_dir = Path(f\"{output_dir}/train/images\")\n    train_mask_dir = Path(f\"{output_dir}/train/masks\")\n    \n    # Support both .npy and .npz files\n    patch_files = list(train_img_dir.glob(\"*.npy\")) + list(train_img_dir.glob(\"*.npz\"))\n    patch_files = sorted(patch_files)\n    \n    if len(patch_files) == 0:\n        print(\"  ⚠️  No patches for preview\")\n        return\n    \n    # Random sample\n    sample_indices = np.random.choice(len(patch_files), \n                                     min(num_samples, len(patch_files)), \n                                     replace=False)\n    \n    fig, axes = plt.subplots(num_samples, 3, figsize=(12, 4*num_samples))\n    if num_samples == 1:\n        axes = axes.reshape(1, -1)\n    \n    for i, idx in enumerate(sample_indices):\n        patch_file = patch_files[idx]\n        \n        # Load image\n        if patch_file.suffix == '.npz':\n            img = np.load(patch_file)['data']\n            mask_file = train_mask_dir / patch_file.name\n            mask = np.load(mask_file)['data']\n        else:\n            img = np.load(patch_file)\n            mask_file = train_mask_dir / patch_file.name\n            mask = np.load(mask_file)\n        \n        # Normalize for display if uint8\n        if img.dtype == np.uint8:\n            img_display = img.astype(np.float32) / 255.0\n        else:\n            img_display = img\n        \n        # Image\n        axes[i, 0].imshow(img_display, cmap='gray')\n        axes[i, 0].set_title(f'Image: {patch_file.stem}')\n        axes[i, 0].axis('off')\n        \n        # Mask\n        axes[i, 1].imshow(mask, cmap='hot')\n        axes[i, 1].set_title(f'Mask (ratio: {mask.sum()/(256*256):.4f})')\n        axes[i, 1].axis('off')\n        \n        # Overlay\n        overlay = np.stack([img_display]*3, axis=-1)\n        overlay[:,:,0] = np.where(mask > 0, 1.0, overlay[:,:,0])\n        axes[i, 2].imshow(overlay)\n        axes[i, 2].set_title('Overlay (Red = Surface)')\n        axes[i, 2].axis('off')\n    \n    plt.tight_layout()\n    plt.savefig(f\"{output_dir}/preview/samples.png\", dpi=150, bbox_inches='tight')\n    plt.close()\n    print(\"  ✅ Preview saved\")\n\n# ============================================================================\n# MAIN PREPROCESSING PIPELINE - OPTIMIZED\n# ============================================================================\n\ndef preprocess_vesuvius_dataset():\n    \"\"\"Main preprocessing pipeline - HIGHLY OPTIMIZED FOR KAGGLE\"\"\"\n    config = PreprocessConfig()\n\n    print(\"=\"*70)\n    print(\"VESUVIUS PREPROCESSING - KAGGLE OPTIMIZED\")\n    print(\"=\"*70)\n    \n    # Get volume paths FIRST - FIXED ORDER\n    train_images_dir = Path(config.DATA_DIR) / \"train_images\"\n    train_labels_dir = Path(config.DATA_DIR) / \"train_labels\"\n\n    volume_files = sorted(list(train_images_dir.glob(\"*.tif\")))\n\n    if config.MAX_VOLUMES:\n        volume_files = volume_files[:config.MAX_VOLUMES]\n\n    print(f\"\\n📦 Found {len(volume_files)} volumes\")\n\n    # NOW print configuration\n    print(f\"\\n⚙️  Configuration:\")\n    print(f\"   • Mode: 2D slice processing\")\n    print(f\"   • Patch size: {config.PATCH_SIZE}x{config.PATCH_SIZE}\")\n    print(f\"   • Stride: {config.STRIDE} (overlap: {config.PATCH_SIZE - config.STRIDE})\")\n    print(f\"   • Min mask ratio: {config.MIN_MASK_RATIO} (higher = fewer patches)\")\n    print(f\"   • Max patches/volume: {config.MAX_PATCHES_PER_VOLUME}\")\n    print(f\"   • Slice sampling: {config.SLICE_SAMPLE_RATE*100:.0f}% of slices\")\n    print(f\"   • Skip empty slices: {config.SKIP_EMPTY_SLICES}\")\n    print(f\"   • Oversample: {config.OVERSAMPLE_FACTOR}x\")\n    print(f\"   • Compression: uint8 + npz = 16x smaller!\")\n    print(f\"   • Skip no-surface patches: {config.SKIP_NO_SURFACE_PATCHES}\")\n    print(f\"   • Memory: GC every {config.BATCH_SIZE_VOLUMES} volumes\")\n    if config.MAX_VOLUMES:\n        print(f\"   • ⚠️  TEST MODE: {config.MAX_VOLUMES} volumes only\")\n    \n    # Estimate output size\n    estimated_patches = len(volume_files) * config.MAX_PATCHES_PER_VOLUME\n    estimated_size_gb = (estimated_patches * 256 * 256 * 2) / (1024**3)  # uint8 img+mask\n    estimated_size_gb = estimated_size_gb * 0.5  # compression factor\n    print(f\"\\n📊 Estimated output: ~{estimated_size_gb:.1f} GB\")\n    if estimated_size_gb > 18:\n        print(f\"   ⚠️  WARNING: May exceed Kaggle disk limit (20 GB)!\")\n        print(f\"   💡 Consider: Reduce MAX_PATCHES_PER_VOLUME or MAX_VOLUMES\")\n\n    create_directories(config.OUTPUT_DIR)\n\n    # Statistics\n    stats = {\n        'volumes_processed': 0,\n        'volumes_failed': 0,\n        'total_patches': 0\n    }\n\n    # =======================================================================\n    # PHASE 1: PROCESS & SAVE TO TEMP\n    # =======================================================================\n    print(f\"\\n{'='*70}\")\n    print(\"PHASE 1: PROCESSING & SAVING\")\n    print(f\"{'='*70}\\n\")\n\n    patch_idx = 0\n    all_patch_metadata = []\n\n    for i, vol_path in enumerate(volume_files, 1):\n        label_path = train_labels_dir / vol_path.name\n\n        print(f\"[{i}/{len(volume_files)}] {vol_path.name}\")\n\n        if not label_path.exists():\n            print(f\"  ⚠️  Label not found\")\n            stats['volumes_failed'] += 1\n            continue\n\n        try:\n            # Process volume\n            patches = process_volume_as_slices(vol_path, label_path, config)\n\n            if len(patches) == 0:\n                stats['volumes_failed'] += 1\n                continue\n\n            # Save immediately\n            for patch in tqdm(patches, desc=\"  💾 Saving\", leave=False):\n                metadata = save_patch_immediate(patch, \"temp\", patch_idx, config)\n                all_patch_metadata.append(metadata)\n                patch_idx += 1\n\n            stats['volumes_processed'] += 1\n            stats['total_patches'] += len(patches)\n\n            # Memory management\n            if config.CLEAR_MEMORY_AFTER_VOLUME and i % config.BATCH_SIZE_VOLUMES == 0:\n                print(f\"  🧹 Clearing memory... (Total: {stats['total_patches']} patches)\")\n                clear_memory()\n\n        except Exception as e:\n            print(f\"  ❌ Error: {e}\")\n            stats['volumes_failed'] += 1\n            continue\n\n    print(f\"\\n{'='*70}\")\n    print(\"PHASE 1 COMPLETE\")\n    print(f\"{'='*70}\")\n    print(f\"✅ Processed: {stats['volumes_processed']}/{len(volume_files)}\")\n    print(f\"❌ Failed: {stats['volumes_failed']}\")\n    print(f\"📦 Total patches: {stats['total_patches']}\")\n\n    if stats['total_patches'] == 0:\n        print(\"\\n❌ ERROR: No patches extracted!\")\n        return None\n\n    # =======================================================================\n    # PHASE 2: OVERSAMPLE & SPLIT\n    # =======================================================================\n    print(f\"\\n{'='*70}\")\n    print(\"PHASE 2: OVERSAMPLING & SPLIT\")\n    print(f\"{'='*70}\\n\")\n\n    with_surface = [i for i, m in enumerate(all_patch_metadata) if m['has_mask']]\n    without_surface = [i for i, m in enumerate(all_patch_metadata) if not m['has_mask']]\n\n    print(f\"With surface: {len(with_surface)}\")\n    print(f\"Without surface: {len(without_surface)}\")\n\n    # Oversample\n    oversampled_indices = with_surface * config.OVERSAMPLE_FACTOR + without_surface\n    np.random.shuffle(oversampled_indices)\n\n    print(f\"After oversample: {len(oversampled_indices)}\")\n\n    # Split\n    train_indices, val_indices = train_test_split(\n        oversampled_indices,\n        test_size=config.TEST_SIZE,\n        random_state=config.RANDOM_SEED\n    )\n\n    print(f\"\\n📊 Train: {len(train_indices)} | Val: {len(val_indices)}\")\n\n    # =======================================================================\n    # PHASE 3: ORGANIZE FILES (MEMORY EFFICIENT)\n    # =======================================================================\n    print(f\"\\n{'='*70}\")\n    print(\"PHASE 3: ORGANIZING FILES\")\n    print(f\"{'='*70}\\n\")\n\n    # Determine file extension based on compression setting\n    file_ext = \".npz\" if config.USE_COMPRESSION else \".npy\"\n\n    # Copy train files\n    print(\"📂 Copying train files...\")\n    for new_idx, old_idx in enumerate(tqdm(train_indices)):\n        old_img = f\"{config.OUTPUT_DIR}/temp/images/patch_{old_idx:06d}{file_ext}\"\n        old_mask = f\"{config.OUTPUT_DIR}/temp/masks/patch_{old_idx:06d}{file_ext}\"\n        new_img = f\"{config.OUTPUT_DIR}/train/images/patch_{new_idx:06d}{file_ext}\"\n        new_mask = f\"{config.OUTPUT_DIR}/train/masks/patch_{new_idx:06d}{file_ext}\"\n\n        shutil.copy2(old_img, new_img)\n        shutil.copy2(old_mask, new_mask)\n\n    # Copy val files\n    print(\"📂 Copying val files...\")\n    for new_idx, old_idx in enumerate(tqdm(val_indices)):\n        old_img = f\"{config.OUTPUT_DIR}/temp/images/patch_{old_idx:06d}{file_ext}\"\n        old_mask = f\"{config.OUTPUT_DIR}/temp/masks/patch_{old_idx:06d}{file_ext}\"\n        new_img = f\"{config.OUTPUT_DIR}/val/images/patch_{new_idx:06d}{file_ext}\"\n        new_mask = f\"{config.OUTPUT_DIR}/val/masks/patch_{new_idx:06d}{file_ext}\"\n\n        shutil.copy2(old_img, new_img)\n        shutil.copy2(old_mask, new_mask)\n\n    # Clean temp\n    print(\"\\n🧹 Removing temp files...\")\n    shutil.rmtree(Path(config.OUTPUT_DIR) / \"temp\")\n    clear_memory()\n\n    # =======================================================================\n    # SAVE STATS & PREVIEW\n    # =======================================================================\n    print(f\"\\n{'='*70}\")\n    print(\"SAVING STATS & PREVIEW\")\n    print(f\"{'='*70}\\n\")\n\n    # Stats\n    stats['train_patches'] = len(train_indices)\n    stats['val_patches'] = len(val_indices)\n    stats['config'] = {\n        'patch_size': config.PATCH_SIZE,\n        'stride': config.STRIDE,\n        'oversample_factor': config.OVERSAMPLE_FACTOR\n    }\n\n    stats_path = f\"{config.OUTPUT_DIR}/stats/preprocessing_stats.json\"\n    with open(stats_path, 'w') as f:\n        json.dump(stats, f, indent=2)\n    print(f\"✅ Stats saved: {stats_path}\")\n\n    # Preview\n    if config.SAVE_PREVIEW:\n        print(\"\\n🎨 Generating preview...\")\n        save_preview_samples(config.OUTPUT_DIR, config.NUM_PREVIEW)\n\n    # =======================================================================\n    # FINAL SUMMARY\n    # =======================================================================\n    print(f\"\\n{'='*70}\")\n    print(\"✅ PREPROCESSING COMPLETE!\")\n    print(f\"{'='*70}\")\n    print(f\"\\n📁 Output: {config.OUTPUT_DIR}\")\n    print(f\"📊 Train: {len(train_indices)} patches\")\n    print(f\"📊 Val: {len(val_indices)} patches\")\n    print(f\"\\n💾 Files ready for training!\")\n    print(f\"{'='*70}\\n\")\n\n    return stats\n\n# ============================================================================\n# RUN\n# ============================================================================\n\nif __name__ == \"__main__\":\n    np.random.seed(42)\n    stats = preprocess_vesuvius_dataset()\n    \n    if stats:\n        print(\"\\n📈 Final Statistics:\")\n        print(json.dumps(stats, indent=2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T18:16:56.915037Z","iopub.execute_input":"2025-12-06T18:16:56.915361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install -q segmentation-models-pytorch albumentations","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install -q \"numpy<2.0\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nprint(f\"NumPy version: {np.__version__}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # ============================================================================\n# # OPTIMIZED TRAINING CONFIGURATION - TARGET: 5 MIN/EPOCH\n# # ============================================================================\n\n# import torch\n# import torch.nn as nn\n# import torch.nn.functional as F\n# from torch.utils.data import Dataset, DataLoader\n# from pathlib import Path\n# from tqdm.auto import tqdm\n# import matplotlib.pyplot as plt\n# import json\n# import warnings\n# import numpy as np\n# from functools import lru_cache\n# warnings.filterwarnings('ignore')\n\n# # ============================================================================\n# # OPTIMIZED CONFIGURATION\n# # ============================================================================\n\n# class OptimizedTrainingConfig:\n#     \"\"\"Optimized training configuration for faster training\"\"\"\n    \n#     # Paths - KAGGLE\n#     DATA_DIR = \"/kaggle/working/preprocessed_data\"\n#     OUTPUT_DIR = \"/kaggle/working/training_output\"\n    \n#     # Model - REDUCED SIZE\n#     MODEL_NAME = \"UnetPlusPlus\"\n#     IN_CHANNELS = 1\n#     NUM_CLASSES = 1\n#     BASE_FILTERS = 16  # 32 → 16 (4x fewer params, much faster)\n    \n#     # Training - OPTIMIZED\n#     EPOCHS = 20\n#     BATCH_SIZE = 96  # 64 → 96 (even faster!)\n#     LEARNING_RATE = 4e-4  # Increased for faster convergence\n#     WEIGHT_DECAY = 1e-5\n    \n#     # CRITICAL: Sample subset for faster training\n#     TRAIN_SUBSET_RATIO = 0.25  # Use only 25% of data = ~5.9k samples\n#     # This gives: 5896/96 = 61 iterations/epoch (12x speedup!)\n    \n#     # Loss weights\n#     DICE_WEIGHT = 0.5\n#     BCE_WEIGHT = 0.5\n    \n#     # Optimizer & Scheduler\n#     SCHEDULER = \"CosineAnnealingLR\"\n#     T_MAX = 20\n    \n#     # Training settings - OPTIMIZED FOR SPEED\n#     NUM_WORKERS = 2  # 4 → 2 (less overhead on Kaggle)\n#     PIN_MEMORY = True\n#     MIXED_PRECISION = True\n#     PREFETCH_FACTOR = 2  # Reduced from 4\n#     PERSISTENT_WORKERS = True\n    \n#     # Gradient accumulation (if needed for smaller batch)\n#     ACCUMULATION_STEPS = 1\n    \n#     # Checkpointing\n#     SAVE_BEST_ONLY = True\n#     SAVE_EVERY_N_EPOCHS = 10  # Less frequent saving\n#     EARLY_STOPPING_PATIENCE = 7  # More patience\n    \n#     # Augmentation - DISABLED for speed\n#     USE_AUGMENTATION = False  # Augmentation is slow!\n    \n#     # Validation frequency\n#     VALIDATE_EVERY_N_EPOCHS = 2  # Validate every 2 epochs instead of every epoch!\n    \n#     # Random seed\n#     SEED = 42\n    \n#     # Device\n#     DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# # ============================================================================\n# # OPTIMIZED DATASET WITH CACHING\n# # ============================================================================\n\n# class OptimizedVesuviusDataset(Dataset):\n#     \"\"\"Optimized dataset with memory caching and faster loading\"\"\"\n    \n#     def __init__(self, data_dir, split='train', augment=False, subset_ratio=1.0):\n#         self.data_dir = Path(data_dir)\n#         self.split = split\n#         self.augment = augment\n        \n#         self.img_dir = self.data_dir / split / \"images\"\n#         self.mask_dir = self.data_dir / split / \"masks\"\n        \n#         image_files = sorted(list(self.img_dir.glob(\"*.npz\"))) or \\\n#                      sorted(list(self.img_dir.glob(\"*.npy\")))\n        \n#         # CRITICAL: Sample subset for faster training\n#         if subset_ratio < 1.0:\n#             n_samples = int(len(image_files) * subset_ratio)\n#             indices = np.random.RandomState(42).choice(\n#                 len(image_files), n_samples, replace=False\n#             )\n#             self.image_files = [image_files[i] for i in sorted(indices)]\n#             print(f\"[{split.upper()}] Sampled {len(self.image_files)}/{len(image_files)} samples ({subset_ratio*100:.0f}%)\")\n#         else:\n#             self.image_files = image_files\n#             print(f\"[{split.upper()}] Found {len(self.image_files)} samples\")\n        \n#         # Pre-cache file paths for faster access\n#         self.mask_files = [self.mask_dir / f.name for f in self.image_files]\n        \n#         # Check file format\n#         self.use_npz = self.image_files[0].suffix == '.npz'\n    \n#     def __len__(self):\n#         return len(self.image_files)\n    \n#     def __getitem__(self, idx):\n#         # Optimized loading\n#         if self.use_npz:\n#             image = np.load(self.image_files[idx])['data']\n#             mask = np.load(self.mask_files[idx])['data']\n#         else:\n#             image = np.load(self.image_files[idx])\n#             mask = np.load(self.mask_files[idx])\n        \n#         # Fast conversion\n#         if image.dtype == np.uint8:\n#             image = image.astype(np.float32) * (1.0 / 255.0)\n        \n#         mask = (mask > 0).astype(np.float32)\n        \n#         # Augmentation disabled by default for speed\n#         if self.augment:\n#             # Minimal augmentation\n#             if np.random.random() > 0.5:\n#                 image = np.fliplr(image).copy()\n#                 mask = np.fliplr(mask).copy()\n        \n#         # Convert to tensors\n#         image = torch.from_numpy(image).unsqueeze(0)\n#         mask = torch.from_numpy(mask).unsqueeze(0)\n        \n#         return image, mask\n\n# # ============================================================================\n# # LIGHTWEIGHT U-NET++ MODEL\n# # ============================================================================\n\n# class ConvBlock(nn.Module):\n#     \"\"\"Optimized double convolution block\"\"\"\n#     def __init__(self, in_ch, out_ch):\n#         super().__init__()\n#         self.conv = nn.Sequential(\n#             nn.Conv2d(in_ch, out_ch, 3, padding=1, bias=False),\n#             nn.BatchNorm2d(out_ch),\n#             nn.ReLU(inplace=True),\n#             nn.Conv2d(out_ch, out_ch, 3, padding=1, bias=False),\n#             nn.BatchNorm2d(out_ch),\n#             nn.ReLU(inplace=True)\n#         )\n    \n#     def forward(self, x):\n#         return self.conv(x)\n\n# class LightweightUNetPlusPlus(nn.Module):\n#     \"\"\"Lightweight U-Net++ for faster training\"\"\"\n#     def __init__(self, in_channels=1, num_classes=1, base_filters=16):\n#         super().__init__()\n        \n#         filters = [base_filters, base_filters*2, base_filters*4, base_filters*8, base_filters*16]\n        \n#         # Encoder\n#         self.conv0_0 = ConvBlock(in_channels, filters[0])\n#         self.conv1_0 = ConvBlock(filters[0], filters[1])\n#         self.conv2_0 = ConvBlock(filters[1], filters[2])\n#         self.conv3_0 = ConvBlock(filters[2], filters[3])\n#         self.conv4_0 = ConvBlock(filters[3], filters[4])\n        \n#         # Nested skip pathways\n#         self.conv0_1 = ConvBlock(filters[0] + filters[1], filters[0])\n#         self.conv0_2 = ConvBlock(filters[0]*2 + filters[1], filters[0])\n#         self.conv0_3 = ConvBlock(filters[0]*3 + filters[1], filters[0])\n#         self.conv0_4 = ConvBlock(filters[0]*4 + filters[1], filters[0])\n        \n#         self.conv1_1 = ConvBlock(filters[1] + filters[2], filters[1])\n#         self.conv1_2 = ConvBlock(filters[1]*2 + filters[2], filters[1])\n#         self.conv1_3 = ConvBlock(filters[1]*3 + filters[2], filters[1])\n        \n#         self.conv2_1 = ConvBlock(filters[2] + filters[3], filters[2])\n#         self.conv2_2 = ConvBlock(filters[2]*2 + filters[3], filters[2])\n        \n#         self.conv3_1 = ConvBlock(filters[3] + filters[4], filters[3])\n        \n#         # Pooling & Upsampling\n#         self.pool = nn.MaxPool2d(2)\n#         self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        \n#         # Output\n#         self.final = nn.Conv2d(filters[0], num_classes, 1)\n    \n#     def forward(self, x):\n#         # Encoder\n#         x0_0 = self.conv0_0(x)\n#         x1_0 = self.conv1_0(self.pool(x0_0))\n#         x2_0 = self.conv2_0(self.pool(x1_0))\n#         x3_0 = self.conv3_0(self.pool(x2_0))\n#         x4_0 = self.conv4_0(self.pool(x3_0))\n        \n#         # Nested connections\n#         x0_1 = self.conv0_1(torch.cat([x0_0, self.up(x1_0)], 1))\n#         x1_1 = self.conv1_1(torch.cat([x1_0, self.up(x2_0)], 1))\n#         x2_1 = self.conv2_1(torch.cat([x2_0, self.up(x3_0)], 1))\n#         x3_1 = self.conv3_1(torch.cat([x3_0, self.up(x4_0)], 1))\n        \n#         x0_2 = self.conv0_2(torch.cat([x0_0, x0_1, self.up(x1_1)], 1))\n#         x1_2 = self.conv1_2(torch.cat([x1_0, x1_1, self.up(x2_1)], 1))\n#         x2_2 = self.conv2_2(torch.cat([x2_0, x2_1, self.up(x3_1)], 1))\n        \n#         x0_3 = self.conv0_3(torch.cat([x0_0, x0_1, x0_2, self.up(x1_2)], 1))\n#         x1_3 = self.conv1_3(torch.cat([x1_0, x1_1, x1_2, self.up(x2_2)], 1))\n        \n#         x0_4 = self.conv0_4(torch.cat([x0_0, x0_1, x0_2, x0_3, self.up(x1_3)], 1))\n        \n#         return self.final(x0_4)\n\n# # ============================================================================\n# # LOSS FUNCTIONS (SAME)\n# # ============================================================================\n\n# class DiceLoss(nn.Module):\n#     def __init__(self, smooth=1e-6):\n#         super().__init__()\n#         self.smooth = smooth\n    \n#     def forward(self, pred, target):\n#         pred = torch.sigmoid(pred)\n#         pred = pred.view(-1)\n#         target = target.view(-1)\n        \n#         intersection = (pred * target).sum()\n#         dice = (2. * intersection + self.smooth) / (pred.sum() + target.sum() + self.smooth)\n        \n#         return 1 - dice\n\n# class CombinedLoss(nn.Module):\n#     def __init__(self, dice_weight=0.5, bce_weight=0.5):\n#         super().__init__()\n#         self.dice_weight = dice_weight\n#         self.bce_weight = bce_weight\n#         self.dice_loss = DiceLoss()\n#         self.bce_loss = nn.BCEWithLogitsLoss()\n    \n#     def forward(self, pred, target):\n#         dice = self.dice_loss(pred, target)\n#         bce = self.bce_loss(pred, target)\n#         return self.dice_weight * dice + self.bce_weight * bce\n\n# # ============================================================================\n# # METRICS (SAME)\n# # ============================================================================\n\n# def dice_coefficient(pred, target, smooth=1e-6):\n#     pred = pred.view(-1)\n#     target = target.view(-1)\n#     intersection = (pred * target).sum()\n#     dice = (2. * intersection + smooth) / (pred.sum() + target.sum() + smooth)\n#     return dice\n\n# def iou_score(pred, target, smooth=1e-6):\n#     pred = pred.view(-1)\n#     target = target.view(-1)\n#     intersection = (pred * target).sum()\n#     union = pred.sum() + target.sum() - intersection\n#     iou = (intersection + smooth) / (union + smooth)\n#     return iou\n\n# class MetricTracker:\n#     def __init__(self):\n#         self.reset()\n    \n#     def reset(self):\n#         self.values = []\n    \n#     def update(self, value):\n#         self.values.append(value)\n    \n#     def avg(self):\n#         return np.mean(self.values) if self.values else 0\n\n# # ============================================================================\n# # OPTIMIZED TRAINING & VALIDATION\n# # ============================================================================\n\n# def train_epoch(model, loader, criterion, optimizer, device, scaler=None):\n#     \"\"\"Ultra-optimized training loop\"\"\"\n#     model.train()\n#     total_loss = 0.0\n#     total_dice = 0.0\n#     num_batches = 0\n    \n#     # Removed tqdm for speed - overhead is significant!\n#     for images, masks in loader:\n#         images, masks = images.to(device, non_blocking=True), masks.to(device, non_blocking=True)\n        \n#         optimizer.zero_grad(set_to_none=True)\n        \n#         if scaler:\n#             with torch.cuda.amp.autocast():\n#                 outputs = model(images)\n#                 loss = criterion(outputs, masks)\n#             scaler.scale(loss).backward()\n#             scaler.step(optimizer)\n#             scaler.update()\n#         else:\n#             outputs = model(images)\n#             loss = criterion(outputs, masks)\n#             loss.backward()\n#             optimizer.step()\n        \n#         # Only calculate metrics every 10 batches for speed\n#         if num_batches % 10 == 0:\n#             with torch.no_grad():\n#                 pred = (torch.sigmoid(outputs) > 0.5).float()\n#                 dice = dice_coefficient(pred, masks)\n#                 total_dice += dice.item()\n        \n#         total_loss += loss.item()\n#         num_batches += 1\n    \n#     avg_loss = total_loss / num_batches\n#     avg_dice = total_dice / (num_batches // 10 + 1)\n    \n#     return {'loss': avg_loss, 'dice': avg_dice, 'iou': avg_dice * 0.9}  # Approximate IoU\n\n# @torch.no_grad()\n# def validate_epoch(model, loader, criterion, device):\n#     \"\"\"Ultra-fast validation - calculate on subset only\"\"\"\n#     model.eval()\n#     total_loss = 0.0\n#     total_dice = 0.0\n#     num_batches = 0\n    \n#     # Only validate on first 20 batches for speed!\n#     max_batches = min(20, len(loader))\n    \n#     for i, (images, masks) in enumerate(loader):\n#         if i >= max_batches:\n#             break\n            \n#         images, masks = images.to(device, non_blocking=True), masks.to(device, non_blocking=True)\n        \n#         outputs = model(images)\n#         loss = criterion(outputs, masks)\n        \n#         pred = (torch.sigmoid(outputs) > 0.5).float()\n#         dice = dice_coefficient(pred, masks)\n        \n#         total_loss += loss.item()\n#         total_dice += dice.item()\n#         num_batches += 1\n    \n#     avg_loss = total_loss / num_batches\n#     avg_dice = total_dice / num_batches\n    \n#     return {'loss': avg_loss, 'dice': avg_dice, 'iou': avg_dice * 0.9}\n\n# # ============================================================================\n# # VISUALIZATION (SAME)\n# # ============================================================================\n\n# def visualize_predictions(model, dataset, device, num_samples=5, save_path=None):\n#     model.eval()\n#     fig, axes = plt.subplots(num_samples, 3, figsize=(12, 4*num_samples))\n#     if num_samples == 1:\n#         axes = axes.reshape(1, -1)\n    \n#     indices = np.random.choice(len(dataset), min(num_samples, len(dataset)), replace=False)\n    \n#     with torch.no_grad():\n#         for i, idx in enumerate(indices):\n#             image, mask = dataset[idx]\n#             img_input = image.unsqueeze(0).to(device)\n            \n#             output = model(img_input)\n#             pred = torch.sigmoid(output).cpu().numpy()[0, 0]\n#             pred_binary = (pred > 0.5).astype(np.float32)\n            \n#             img_np = image.cpu().numpy()[0]\n#             mask_np = mask.cpu().numpy()[0]\n            \n#             dice = dice_coefficient(\n#                 torch.from_numpy(pred_binary),\n#                 torch.from_numpy(mask_np)\n#             ).item()\n            \n#             axes[i, 0].imshow(img_np, cmap='gray')\n#             axes[i, 0].set_title('Input')\n#             axes[i, 0].axis('off')\n            \n#             axes[i, 1].imshow(mask_np, cmap='hot')\n#             axes[i, 1].set_title('Ground Truth')\n#             axes[i, 1].axis('off')\n            \n#             axes[i, 2].imshow(pred_binary, cmap='hot')\n#             axes[i, 2].set_title(f'Prediction (Dice: {dice:.4f})')\n#             axes[i, 2].axis('off')\n    \n#     plt.tight_layout()\n#     if save_path:\n#         plt.savefig(save_path, dpi=150, bbox_inches='tight')\n#         print(f\"✅ Saved: {save_path}\")\n#     plt.close()\n\n# def plot_history(history, save_path=None):\n#     fig, axes = plt.subplots(1, 3, figsize=(15, 4))\n#     epochs = range(1, len(history['train_loss']) + 1)\n    \n#     axes[0].plot(epochs, history['train_loss'], 'o-', label='Train')\n#     axes[0].plot(epochs, history['val_loss'], 's-', label='Val')\n#     axes[0].set_xlabel('Epoch')\n#     axes[0].set_ylabel('Loss')\n#     axes[0].set_title('Loss')\n#     axes[0].legend()\n#     axes[0].grid(True, alpha=0.3)\n    \n#     axes[1].plot(epochs, history['train_dice'], 'o-', label='Train')\n#     axes[1].plot(epochs, history['val_dice'], 's-', label='Val')\n#     axes[1].set_xlabel('Epoch')\n#     axes[1].set_ylabel('Dice Score')\n#     axes[1].set_title('Dice Score')\n#     axes[1].legend()\n#     axes[1].grid(True, alpha=0.3)\n    \n#     axes[2].plot(epochs, history['train_iou'], 'o-', label='Train')\n#     axes[2].plot(epochs, history['val_iou'], 's-', label='Val')\n#     axes[2].set_xlabel('Epoch')\n#     axes[2].set_ylabel('IoU Score')\n#     axes[2].set_title('IoU Score')\n#     axes[2].legend()\n#     axes[2].grid(True, alpha=0.3)\n    \n#     plt.tight_layout()\n#     if save_path:\n#         plt.savefig(save_path, dpi=150, bbox_inches='tight')\n#         print(f\"✅ Saved: {save_path}\")\n#     plt.close()\n\n# # ============================================================================\n# # OPTIMIZED MAIN TRAINING\n# # ============================================================================\n\n# def train_model_optimized(config=None):\n#     \"\"\"Optimized training - TARGET: 5 min/epoch\"\"\"\n#     if config is None:\n#         config = OptimizedTrainingConfig()\n    \n#     print(\"=\"*70)\n#     print(\"VESUVIUS OPTIMIZED TRAINING - TARGET: 5 MIN/EPOCH\")\n#     print(\"=\"*70)\n    \n#     torch.manual_seed(config.SEED)\n#     np.random.seed(config.SEED)\n    \n#     # Enable cuDNN benchmarking for faster training\n#     torch.backends.cudnn.benchmark = True\n    \n#     Path(config.OUTPUT_DIR).mkdir(parents=True, exist_ok=True)\n    \n#     print(f\"\\n⚙️  Configuration:\")\n#     print(f\"   • Model: {config.MODEL_NAME} (Lightweight)\")\n#     print(f\"   • Base filters: {config.BASE_FILTERS}\")\n#     print(f\"   • Batch size: {config.BATCH_SIZE}\")\n#     print(f\"   • Epochs: {config.EPOCHS}\")\n#     print(f\"   • Learning rate: {config.LEARNING_RATE}\")\n#     print(f\"   • Subset ratio: {config.TRAIN_SUBSET_RATIO*100:.0f}%\")\n#     print(f\"   • Device: {config.DEVICE}\")\n    \n#     # Datasets with subset sampling\n#     print(f\"\\n📦 Loading datasets...\")\n#     train_dataset = OptimizedVesuviusDataset(\n#         config.DATA_DIR, 'train', \n#         augment=config.USE_AUGMENTATION,\n#         subset_ratio=config.TRAIN_SUBSET_RATIO\n#     )\n#     val_dataset = OptimizedVesuviusDataset(\n#         config.DATA_DIR, 'val', \n#         augment=False,\n#         subset_ratio=0.3  # Use 30% of val for faster validation\n#     )\n    \n#     train_loader = DataLoader(\n#         train_dataset, \n#         batch_size=config.BATCH_SIZE,\n#         num_workers=config.NUM_WORKERS,\n#         prefetch_factor=config.PREFETCH_FACTOR,\n#         persistent_workers=config.PERSISTENT_WORKERS,\n#         pin_memory=config.PIN_MEMORY,\n#         shuffle=True\n#     )\n#     val_loader = DataLoader(\n#         val_dataset, \n#         batch_size=config.BATCH_SIZE*2,  # Larger batch for validation\n#         num_workers=config.NUM_WORKERS,\n#         prefetch_factor=config.PREFETCH_FACTOR,\n#         persistent_workers=config.PERSISTENT_WORKERS,\n#         pin_memory=config.PIN_MEMORY,\n#         shuffle=False\n#     )\n    \n#     print(f\"   • Train batches: {len(train_loader)}\")\n#     print(f\"   • Val batches: {len(val_loader)} (using {min(20, len(val_loader))} for speed)\")\n#     print(f\"   • Estimated time/epoch: ~{len(train_loader) * 2.0 / 60:.1f} min\")\n    \n#     # Lightweight model\n#     print(f\"\\n🏗️  Building lightweight model...\")\n#     model = LightweightUNetPlusPlus(\n#         in_channels=config.IN_CHANNELS,\n#         num_classes=config.NUM_CLASSES,\n#         base_filters=config.BASE_FILTERS\n#     ).to(config.DEVICE)\n    \n#     total_params = sum(p.numel() for p in model.parameters())\n#     print(f\"   • Parameters: {total_params:,}\")\n    \n#     # Loss, optimizer, scheduler\n#     criterion = CombinedLoss(config.DICE_WEIGHT, config.BCE_WEIGHT)\n#     optimizer = torch.optim.AdamW(\n#         model.parameters(), \n#         lr=config.LEARNING_RATE, \n#         weight_decay=config.WEIGHT_DECAY\n#     )\n#     scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n#         optimizer, T_max=config.T_MAX\n#     )\n#     scaler = torch.cuda.amp.GradScaler() if config.MIXED_PRECISION else None\n    \n#     # Training loop\n#     history = {\n#         'train_loss': [], 'train_dice': [], 'train_iou': [],\n#         'val_loss': [], 'val_dice': [], 'val_iou': []\n#     }\n    \n#     best_dice = 0.0\n#     epochs_no_improve = 0\n    \n#     print(f\"\\n{'='*70}\")\n#     print(\"STARTING OPTIMIZED TRAINING\")\n#     print(f\"{'='*70}\\n\")\n    \n#     import time\n#     total_training_time = 0\n    \n#     for epoch in range(1, config.EPOCHS + 1):\n#         print(f\"\\nEpoch {epoch}/{config.EPOCHS}\", end=\" \")\n        \n#         epoch_start = time.time()\n        \n#         train_metrics = train_epoch(\n#             model, train_loader, criterion, optimizer, config.DEVICE, scaler\n#         )\n        \n#         # Validate only every N epochs for speed!\n#         if epoch % config.VALIDATE_EVERY_N_EPOCHS == 0 or epoch == 1:\n#             val_metrics = validate_epoch(\n#                 model, val_loader, criterion, config.DEVICE\n#             )\n#             print(f\"| Train Loss: {train_metrics['loss']:.4f}, Dice: {train_metrics['dice']:.4f} | Val Loss: {val_metrics['loss']:.4f}, Dice: {val_metrics['dice']:.4f}\", end=\"\")\n#         else:\n#             val_metrics = history['val_loss'][-1] if history['val_loss'] else train_metrics  # Use last val metrics\n#             print(f\"| Train Loss: {train_metrics['loss']:.4f}, Dice: {train_metrics['dice']:.4f} | (validation skipped)\", end=\"\")\n        \n#         epoch_time = time.time() - epoch_start\n#         total_training_time += epoch_time\n        \n#         print(f\" | Time: {epoch_time/60:.2f} min\")\n        \n#         history['train_loss'].append(train_metrics['loss'])\n#         history['train_dice'].append(train_metrics['dice'])\n#         history['train_iou'].append(train_metrics['iou'])\n#         history['val_loss'].append(val_metrics['loss'])\n#         history['val_dice'].append(val_metrics['dice'])\n#         history['val_iou'].append(val_metrics['iou'])\n        \n#         scheduler.step()\n        \n#         # Only save if validation was performed\n#         if epoch % config.VALIDATE_EVERY_N_EPOCHS == 0:\n#             is_best = val_metrics['dice'] > best_dice\n#             if is_best:\n#                 best_dice = val_metrics['dice']\n#                 epochs_no_improve = 0\n#                 torch.save({\n#                     'epoch': epoch,\n#                     'model_state_dict': model.state_dict(),\n#                     'optimizer_state_dict': optimizer.state_dict(),\n#                     'best_dice': best_dice\n#                 }, f\"{config.OUTPUT_DIR}/best_model.pth\")\n#                 print(f\"   ✅ Best model saved! (Dice: {best_dice:.4f})\")\n#             else:\n#                 epochs_no_improve += 1\n        \n#         if epoch % config.SAVE_EVERY_N_EPOCHS == 0:\n#             torch.save(model.state_dict(), f\"{config.OUTPUT_DIR}/checkpoint_epoch_{epoch}.pth\")\n        \n#         if epochs_no_improve >= config.EARLY_STOPPING_PATIENCE:\n#             print(f\"\\n⚠️  Early stopping at epoch {epoch}\")\n#             break\n    \n#     print(f\"\\n{'='*70}\")\n#     print(\"TRAINING COMPLETE!\")\n#     print(f\"{'='*70}\")\n#     print(f\"✅ Best Dice: {best_dice:.4f}\")\n#     print(f\"⏱️  Total time: {total_training_time/60:.1f} minutes\")\n#     print(f\"⏱️  Avg time/epoch: {total_training_time/epoch/60:.2f} minutes\")\n    \n#     # Save history\n#     with open(f\"{config.OUTPUT_DIR}/history.json\", 'w') as f:\n#         json.dump(history, f, indent=2)\n    \n#     # Plots\n#     plot_history(history, f\"{config.OUTPUT_DIR}/training_curves.png\")\n#     visualize_predictions(model, val_dataset, config.DEVICE, 5, f\"{config.OUTPUT_DIR}/predictions.png\")\n    \n#     print(f\"\\n📁 Outputs saved to: {config.OUTPUT_DIR}\\n\")\n    \n#     return model, history\n\n# # ============================================================================\n# # RUN OPTIMIZED TRAINING\n# # ============================================================================\n\n# if __name__ == \"__main__\":\n#     config = OptimizedTrainingConfig()\n#     model, history = train_model_optimized(config)\n    \n#     print(\"\\n🎉 Training complete!\")\n#     print(f\"📊 Best Dice: {max(history['val_dice']):.4f}\")\n#     print(f\"📊 Best IoU: {max(history['val_iou']):.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# VERIFY GPU MEMORY - JALANKAN SETELAH RESTART KERNEL\n# ============================================================================\n\nimport torch\nimport gc\n\n# Force garbage collection\ngc.collect()\n\n# Clear CUDA cache\nif torch.cuda.is_available():\n    torch.cuda.empty_cache()\n    torch.cuda.synchronize()\n    \n    # Print memory status\n    total_mem = torch.cuda.get_device_properties(0).total_memory / 1024**3\n    reserved_mem = torch.cuda.memory_reserved(0) / 1024**3\n    allocated_mem = torch.cuda.memory_allocated(0) / 1024**3\n    \n    print(\"=\"*70)\n    print(\"GPU MEMORY STATUS\")\n    print(\"=\"*70)\n    print(f\"Total:     {total_mem:.2f} GB\")\n    print(f\"Reserved:  {reserved_mem:.2f} GB\")\n    print(f\"Allocated: {allocated_mem:.2f} GB\")\n    print(f\"Free:      {total_mem - reserved_mem:.2f} GB\")\n    print(\"=\"*70)\n    \n    if reserved_mem > 1.0:\n        print(\"\\n⚠️  WARNING: GPU memory still occupied!\")\n        print(\"   Please RESTART KERNEL again\")\n    else:\n        print(\"\\n✅ GPU memory cleared! Ready to train.\")\n        print(f\"   Available: {total_mem - reserved_mem:.2f} GB\")\nelse:\n    print(\"❌ CUDA not available!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# OPTIMIZED TRAINING CONFIGURATION - TARGET: 5 MIN/EPOCH\n# ============================================================================\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nimport json\nimport warnings\nimport numpy as np\nfrom functools import lru_cache\nwarnings.filterwarnings('ignore')\n\n# ============================================================================\n# OPTIMIZED CONFIGURATION\n# ============================================================================\n\nclass OptimizedTrainingConfig:\n    \"\"\"Optimized training configuration for faster training\"\"\"\n    \n    # Paths - KAGGLE\n    DATA_DIR = \"/kaggle/working/preprocessed_data\"\n    OUTPUT_DIR = \"/kaggle/working/training_output\"\n    \n    # Model - REDUCED SIZE\n    MODEL_NAME = \"UnetPlusPlus\"\n    IN_CHANNELS = 1\n    NUM_CLASSES = 1\n    BASE_FILTERS = 16  # 32 → 16 (4x fewer params, much faster)\n    \n    # Training - OPTIMIZED FOR GPU MEMORY\n    EPOCHS = 20\n    BATCH_SIZE = 48  # Safe for T4 GPU (15GB)\n    LEARNING_RATE = 3e-4\n    WEIGHT_DECAY = 1e-5\n    \n    # CRITICAL: Sample subset for faster training\n    TRAIN_SUBSET_RATIO = 0.25  # Use only 25% of data = ~5.9k samples\n    # This gives: 5896/48 = 123 iterations/epoch (6x speedup!)\n    \n    # Gradient accumulation to simulate larger batch\n    ACCUMULATION_STEPS = 2  # Effective batch = 48*2 = 96\n    \n    # Loss weights\n    DICE_WEIGHT = 0.5\n    BCE_WEIGHT = 0.5\n    \n    # Optimizer & Scheduler\n    SCHEDULER = \"CosineAnnealingLR\"\n    T_MAX = 20\n    \n    # Training settings - OPTIMIZED FOR SPEED & MEMORY\n    NUM_WORKERS = 2\n    PIN_MEMORY = True\n    MIXED_PRECISION = True  # Critical for memory savings\n    PREFETCH_FACTOR = 2\n    PERSISTENT_WORKERS = True\n    \n    # Memory management\n    EMPTY_CACHE_EVERY_N_BATCHES = 50  # Clear cache periodically\n    \n    # Checkpointing\n    SAVE_BEST_ONLY = True\n    SAVE_EVERY_N_EPOCHS = 10  # Less frequent saving\n    EARLY_STOPPING_PATIENCE = 7  # More patience\n    \n    # Augmentation - DISABLED for speed\n    USE_AUGMENTATION = False  # Augmentation is slow!\n    \n    # Validation frequency\n    VALIDATE_EVERY_N_EPOCHS = 2  # Validate every 2 epochs instead of every epoch!\n    \n    # Random seed\n    SEED = 42\n    \n    # Device\n    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# ============================================================================\n# OPTIMIZED DATASET WITH CACHING\n# ============================================================================\n\nclass OptimizedVesuviusDataset(Dataset):\n    \"\"\"Optimized dataset with memory caching and faster loading\"\"\"\n    \n    def __init__(self, data_dir, split='train', augment=False, subset_ratio=1.0):\n        self.data_dir = Path(data_dir)\n        self.split = split\n        self.augment = augment\n        \n        self.img_dir = self.data_dir / split / \"images\"\n        self.mask_dir = self.data_dir / split / \"masks\"\n        \n        image_files = sorted(list(self.img_dir.glob(\"*.npz\"))) or \\\n                     sorted(list(self.img_dir.glob(\"*.npy\")))\n        \n        # CRITICAL: Sample subset for faster training\n        if subset_ratio < 1.0:\n            n_samples = int(len(image_files) * subset_ratio)\n            indices = np.random.RandomState(42).choice(\n                len(image_files), n_samples, replace=False\n            )\n            self.image_files = [image_files[i] for i in sorted(indices)]\n            print(f\"[{split.upper()}] Sampled {len(self.image_files)}/{len(image_files)} samples ({subset_ratio*100:.0f}%)\")\n        else:\n            self.image_files = image_files\n            print(f\"[{split.upper()}] Found {len(self.image_files)} samples\")\n        \n        # Pre-cache file paths for faster access\n        self.mask_files = [self.mask_dir / f.name for f in self.image_files]\n        \n        # Check file format\n        self.use_npz = self.image_files[0].suffix == '.npz'\n    \n    def __len__(self):\n        return len(self.image_files)\n    \n    def __getitem__(self, idx):\n        # Optimized loading\n        if self.use_npz:\n            image = np.load(self.image_files[idx])['data']\n            mask = np.load(self.mask_files[idx])['data']\n        else:\n            image = np.load(self.image_files[idx])\n            mask = np.load(self.mask_files[idx])\n        \n        # Fast conversion\n        if image.dtype == np.uint8:\n            image = image.astype(np.float32) * (1.0 / 255.0)\n        \n        mask = (mask > 0).astype(np.float32)\n        \n        # Augmentation disabled by default for speed\n        if self.augment:\n            # Minimal augmentation\n            if np.random.random() > 0.5:\n                image = np.fliplr(image).copy()\n                mask = np.fliplr(mask).copy()\n        \n        # Convert to tensors\n        image = torch.from_numpy(image).unsqueeze(0)\n        mask = torch.from_numpy(mask).unsqueeze(0)\n        \n        return image, mask\n\n# ============================================================================\n# LIGHTWEIGHT U-NET++ MODEL\n# ============================================================================\n\nclass ConvBlock(nn.Module):\n    \"\"\"Optimized double convolution block\"\"\"\n    def __init__(self, in_ch, out_ch):\n        super().__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(in_ch, out_ch, 3, padding=1, bias=False),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_ch, out_ch, 3, padding=1, bias=False),\n            nn.BatchNorm2d(out_ch),\n            nn.ReLU(inplace=True)\n        )\n    \n    def forward(self, x):\n        return self.conv(x)\n\nclass LightweightUNetPlusPlus(nn.Module):\n    \"\"\"Lightweight U-Net++ for faster training\"\"\"\n    def __init__(self, in_channels=1, num_classes=1, base_filters=16):\n        super().__init__()\n        \n        filters = [base_filters, base_filters*2, base_filters*4, base_filters*8, base_filters*16]\n        \n        # Encoder\n        self.conv0_0 = ConvBlock(in_channels, filters[0])\n        self.conv1_0 = ConvBlock(filters[0], filters[1])\n        self.conv2_0 = ConvBlock(filters[1], filters[2])\n        self.conv3_0 = ConvBlock(filters[2], filters[3])\n        self.conv4_0 = ConvBlock(filters[3], filters[4])\n        \n        # Nested skip pathways\n        self.conv0_1 = ConvBlock(filters[0] + filters[1], filters[0])\n        self.conv0_2 = ConvBlock(filters[0]*2 + filters[1], filters[0])\n        self.conv0_3 = ConvBlock(filters[0]*3 + filters[1], filters[0])\n        self.conv0_4 = ConvBlock(filters[0]*4 + filters[1], filters[0])\n        \n        self.conv1_1 = ConvBlock(filters[1] + filters[2], filters[1])\n        self.conv1_2 = ConvBlock(filters[1]*2 + filters[2], filters[1])\n        self.conv1_3 = ConvBlock(filters[1]*3 + filters[2], filters[1])\n        \n        self.conv2_1 = ConvBlock(filters[2] + filters[3], filters[2])\n        self.conv2_2 = ConvBlock(filters[2]*2 + filters[3], filters[2])\n        \n        self.conv3_1 = ConvBlock(filters[3] + filters[4], filters[3])\n        \n        # Pooling & Upsampling\n        self.pool = nn.MaxPool2d(2)\n        self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        \n        # Output\n        self.final = nn.Conv2d(filters[0], num_classes, 1)\n    \n    def forward(self, x):\n        # Encoder\n        x0_0 = self.conv0_0(x)\n        x1_0 = self.conv1_0(self.pool(x0_0))\n        x2_0 = self.conv2_0(self.pool(x1_0))\n        x3_0 = self.conv3_0(self.pool(x2_0))\n        x4_0 = self.conv4_0(self.pool(x3_0))\n        \n        # Nested connections\n        x0_1 = self.conv0_1(torch.cat([x0_0, self.up(x1_0)], 1))\n        x1_1 = self.conv1_1(torch.cat([x1_0, self.up(x2_0)], 1))\n        x2_1 = self.conv2_1(torch.cat([x2_0, self.up(x3_0)], 1))\n        x3_1 = self.conv3_1(torch.cat([x3_0, self.up(x4_0)], 1))\n        \n        x0_2 = self.conv0_2(torch.cat([x0_0, x0_1, self.up(x1_1)], 1))\n        x1_2 = self.conv1_2(torch.cat([x1_0, x1_1, self.up(x2_1)], 1))\n        x2_2 = self.conv2_2(torch.cat([x2_0, x2_1, self.up(x3_1)], 1))\n        \n        x0_3 = self.conv0_3(torch.cat([x0_0, x0_1, x0_2, self.up(x1_2)], 1))\n        x1_3 = self.conv1_3(torch.cat([x1_0, x1_1, x1_2, self.up(x2_2)], 1))\n        \n        x0_4 = self.conv0_4(torch.cat([x0_0, x0_1, x0_2, x0_3, self.up(x1_3)], 1))\n        \n        return self.final(x0_4)\n\n# ============================================================================\n# LOSS FUNCTIONS (SAME)\n# ============================================================================\n\nclass DiceLoss(nn.Module):\n    def __init__(self, smooth=1e-6):\n        super().__init__()\n        self.smooth = smooth\n    \n    def forward(self, pred, target):\n        pred = torch.sigmoid(pred)\n        pred = pred.view(-1)\n        target = target.view(-1)\n        \n        intersection = (pred * target).sum()\n        dice = (2. * intersection + self.smooth) / (pred.sum() + target.sum() + self.smooth)\n        \n        return 1 - dice\n\nclass CombinedLoss(nn.Module):\n    def __init__(self, dice_weight=0.5, bce_weight=0.5):\n        super().__init__()\n        self.dice_weight = dice_weight\n        self.bce_weight = bce_weight\n        self.dice_loss = DiceLoss()\n        self.bce_loss = nn.BCEWithLogitsLoss()\n    \n    def forward(self, pred, target):\n        dice = self.dice_loss(pred, target)\n        bce = self.bce_loss(pred, target)\n        return self.dice_weight * dice + self.bce_weight * bce\n\n# ============================================================================\n# METRICS (SAME)\n# ============================================================================\n\ndef dice_coefficient(pred, target, smooth=1e-6):\n    pred = pred.view(-1)\n    target = target.view(-1)\n    intersection = (pred * target).sum()\n    dice = (2. * intersection + smooth) / (pred.sum() + target.sum() + smooth)\n    return dice\n\ndef iou_score(pred, target, smooth=1e-6):\n    pred = pred.view(-1)\n    target = target.view(-1)\n    intersection = (pred * target).sum()\n    union = pred.sum() + target.sum() - intersection\n    iou = (intersection + smooth) / (union + smooth)\n    return iou\n\nclass MetricTracker:\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.values = []\n    \n    def update(self, value):\n        self.values.append(value)\n    \n    def avg(self):\n        return np.mean(self.values) if self.values else 0\n\n# ============================================================================\n# OPTIMIZED TRAINING & VALIDATION\n# ============================================================================\n\ndef train_epoch(model, loader, criterion, optimizer, device, scaler=None, config=None):\n    \"\"\"Ultra-optimized training loop with gradient accumulation and progress bar\"\"\"\n    model.train()\n    total_loss = 0.0\n    total_dice = 0.0\n    total_iou = 0.0\n    num_batches = 0\n    \n    accumulation_steps = config.ACCUMULATION_STEPS if config else 1\n    \n    # Reset gradients at start\n    optimizer.zero_grad(set_to_none=True)\n    \n    # Progress bar like in the image\n    pbar = tqdm(enumerate(loader), total=len(loader), desc=\"Training\", \n                bar_format='{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}, loss={postfix[0]:.4f}, dice={postfix[1]:.4f}, iou={postfix[2]:.4f}]',\n                postfix=[0.0, 0.0, 0.0])\n    \n    for batch_idx, (images, masks) in pbar:\n        images, masks = images.to(device, non_blocking=True), masks.to(device, non_blocking=True)\n        \n        if scaler:\n            with torch.cuda.amp.autocast():\n                outputs = model(images)\n                loss = criterion(outputs, masks)\n                loss = loss / accumulation_steps  # Scale loss\n            \n            scaler.scale(loss).backward()\n            \n            # Update weights every accumulation_steps\n            if (batch_idx + 1) % accumulation_steps == 0:\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad(set_to_none=True)\n        else:\n            outputs = model(images)\n            loss = criterion(outputs, masks)\n            loss = loss / accumulation_steps\n            loss.backward()\n            \n            if (batch_idx + 1) % accumulation_steps == 0:\n                optimizer.step()\n                optimizer.zero_grad(set_to_none=True)\n        \n        # Calculate metrics every batch for accurate monitoring\n        with torch.no_grad():\n            pred = (torch.sigmoid(outputs) > 0.5).float()\n            dice = dice_coefficient(pred, masks)\n            iou = iou_score(pred, masks)  # Calculate REAL IoU\n            total_dice += dice.item()\n            total_iou += iou.item()\n        \n        total_loss += loss.item() * accumulation_steps  # Unscale for logging\n        num_batches += 1\n        \n        # Update progress bar\n        avg_loss = total_loss / num_batches\n        avg_dice = total_dice / num_batches\n        avg_iou = total_iou / num_batches\n        pbar.postfix[0] = avg_loss\n        pbar.postfix[1] = avg_dice\n        pbar.postfix[2] = avg_iou\n        \n        # Clear cache periodically to avoid OOM\n        if config and num_batches % config.EMPTY_CACHE_EVERY_N_BATCHES == 0:\n            torch.cuda.empty_cache()\n    \n    # Final gradient update if needed\n    if num_batches % accumulation_steps != 0:\n        if scaler:\n            scaler.step(optimizer)\n            scaler.update()\n        else:\n            optimizer.step()\n        optimizer.zero_grad(set_to_none=True)\n    \n    pbar.close()\n    \n    return {'loss': avg_loss, 'dice': avg_dice, 'iou': avg_iou}\n\n@torch.no_grad()\ndef validate_epoch(model, loader, criterion, device):\n    \"\"\"Ultra-fast validation with progress bar\"\"\"\n    model.eval()\n    total_loss = 0.0\n    total_dice = 0.0\n    total_iou = 0.0\n    num_batches = 0\n    \n    # Only validate on first 20 batches for speed!\n    max_batches = min(20, len(loader))\n    \n    # Progress bar for validation\n    pbar = tqdm(enumerate(loader), total=max_batches, desc=\"Validation\",\n                bar_format='{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}, loss={postfix[0]:.4f}, dice={postfix[1]:.4f}, iou={postfix[2]:.4f}]',\n                postfix=[0.0, 0.0, 0.0])\n    \n    for i, (images, masks) in pbar:\n        if i >= max_batches:\n            break\n            \n        images, masks = images.to(device, non_blocking=True), masks.to(device, non_blocking=True)\n        \n        outputs = model(images)\n        loss = criterion(outputs, masks)\n        \n        pred = (torch.sigmoid(outputs) > 0.5).float()\n        dice = dice_coefficient(pred, masks)\n        iou = iou_score(pred, masks)  # Calculate REAL IoU\n        \n        total_loss += loss.item()\n        total_dice += dice.item()\n        total_iou += iou.item()\n        num_batches += 1\n        \n        # Update progress bar\n        avg_loss = total_loss / num_batches\n        avg_dice = total_dice / num_batches\n        avg_iou = total_iou / num_batches\n        pbar.postfix[0] = avg_loss\n        pbar.postfix[1] = avg_dice\n        pbar.postfix[2] = avg_iou\n    \n    pbar.close()\n    \n    return {'loss': avg_loss, 'dice': avg_dice, 'iou': avg_iou}\n\n# ============================================================================\n# VISUALIZATION (SAME)\n# ============================================================================\n\ndef visualize_predictions(model, dataset, device, num_samples=5, save_path=None):\n    model.eval()\n    fig, axes = plt.subplots(num_samples, 3, figsize=(12, 4*num_samples))\n    if num_samples == 1:\n        axes = axes.reshape(1, -1)\n    \n    indices = np.random.choice(len(dataset), min(num_samples, len(dataset)), replace=False)\n    \n    with torch.no_grad():\n        for i, idx in enumerate(indices):\n            image, mask = dataset[idx]\n            img_input = image.unsqueeze(0).to(device)\n            \n            output = model(img_input)\n            pred = torch.sigmoid(output).cpu().numpy()[0, 0]\n            pred_binary = (pred > 0.5).astype(np.float32)\n            \n            img_np = image.cpu().numpy()[0]\n            mask_np = mask.cpu().numpy()[0]\n            \n            dice = dice_coefficient(\n                torch.from_numpy(pred_binary),\n                torch.from_numpy(mask_np)\n            ).item()\n            \n            axes[i, 0].imshow(img_np, cmap='gray')\n            axes[i, 0].set_title('Input')\n            axes[i, 0].axis('off')\n            \n            axes[i, 1].imshow(mask_np, cmap='hot')\n            axes[i, 1].set_title('Ground Truth')\n            axes[i, 1].axis('off')\n            \n            axes[i, 2].imshow(pred_binary, cmap='hot')\n            axes[i, 2].set_title(f'Prediction (Dice: {dice:.4f})')\n            axes[i, 2].axis('off')\n    \n    plt.tight_layout()\n    if save_path:\n        plt.savefig(save_path, dpi=150, bbox_inches='tight')\n        print(f\"✅ Saved: {save_path}\")\n    plt.close()\n\ndef plot_history(history, save_path=None):\n    fig, axes = plt.subplots(1, 3, figsize=(15, 4))\n    epochs = range(1, len(history['train_loss']) + 1)\n    \n    axes[0].plot(epochs, history['train_loss'], 'o-', label='Train')\n    axes[0].plot(epochs, history['val_loss'], 's-', label='Val')\n    axes[0].set_xlabel('Epoch')\n    axes[0].set_ylabel('Loss')\n    axes[0].set_title('Loss')\n    axes[0].legend()\n    axes[0].grid(True, alpha=0.3)\n    \n    axes[1].plot(epochs, history['train_dice'], 'o-', label='Train')\n    axes[1].plot(epochs, history['val_dice'], 's-', label='Val')\n    axes[1].set_xlabel('Epoch')\n    axes[1].set_ylabel('Dice Score')\n    axes[1].set_title('Dice Score')\n    axes[1].legend()\n    axes[1].grid(True, alpha=0.3)\n    \n    axes[2].plot(epochs, history['train_iou'], 'o-', label='Train')\n    axes[2].plot(epochs, history['val_iou'], 's-', label='Val')\n    axes[2].set_xlabel('Epoch')\n    axes[2].set_ylabel('IoU Score')\n    axes[2].set_title('IoU Score')\n    axes[2].legend()\n    axes[2].grid(True, alpha=0.3)\n    \n    plt.tight_layout()\n    if save_path:\n        plt.savefig(save_path, dpi=150, bbox_inches='tight')\n        print(f\"✅ Saved: {save_path}\")\n    plt.close()\n\n# ============================================================================\n# OPTIMIZED MAIN TRAINING\n# ============================================================================\n\ndef train_model_optimized(config=None):\n    \"\"\"Optimized training - TARGET: 5 min/epoch\"\"\"\n    if config is None:\n        config = OptimizedTrainingConfig()\n    \n    print(\"=\"*70)\n    print(\"VESUVIUS OPTIMIZED TRAINING - TARGET: 5 MIN/EPOCH\")\n    print(\"=\"*70)\n    \n    torch.manual_seed(config.SEED)\n    np.random.seed(config.SEED)\n    \n    # Enable cuDNN benchmarking for faster training\n    torch.backends.cudnn.benchmark = True\n    \n    # Clear CUDA cache before starting\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n        print(f\"🔧 GPU Memory: {torch.cuda.get_device_properties(0).total_memory / 1024**3:.1f} GB\")\n    \n    Path(config.OUTPUT_DIR).mkdir(parents=True, exist_ok=True)\n    \n    print(f\"\\n⚙️  Configuration:\")\n    print(f\"   • Model: {config.MODEL_NAME} (Lightweight)\")\n    print(f\"   • Base filters: {config.BASE_FILTERS}\")\n    print(f\"   • Batch size: {config.BATCH_SIZE} (effective: {config.BATCH_SIZE * config.ACCUMULATION_STEPS})\")\n    print(f\"   • Epochs: {config.EPOCHS}\")\n    print(f\"   • Learning rate: {config.LEARNING_RATE}\")\n    print(f\"   • Subset ratio: {config.TRAIN_SUBSET_RATIO*100:.0f}%\")\n    print(f\"   • Mixed precision: {config.MIXED_PRECISION}\")\n    print(f\"   • Device: {config.DEVICE}\")\n    \n    # Datasets with subset sampling\n    print(f\"\\n📦 Loading datasets...\")\n    train_dataset = OptimizedVesuviusDataset(\n        config.DATA_DIR, 'train', \n        augment=config.USE_AUGMENTATION,\n        subset_ratio=config.TRAIN_SUBSET_RATIO\n    )\n    val_dataset = OptimizedVesuviusDataset(\n        config.DATA_DIR, 'val', \n        augment=False,\n        subset_ratio=0.3  # Use 30% of val for faster validation\n    )\n    \n    train_loader = DataLoader(\n        train_dataset,\n        batch_size=config.BATCH_SIZE,\n        shuffle=True,\n        num_workers=0, # UBAH INI MENJADI NOL (0)\n        pin_memory=True,\n        drop_last=True\n    )\n\n    val_loader = DataLoader(\n        val_dataset,\n        batch_size=config.BATCH_SIZE,\n        shuffle=False,\n        num_workers=0, # UBAH INI MENJADI NOL (0)\n        pin_memory=False,\n        drop_last=False\n    )\n    \n    print(f\"   • Train batches: {len(train_loader)} (x{config.ACCUMULATION_STEPS} accum = {len(train_loader)//config.ACCUMULATION_STEPS} effective)\")\n    print(f\"   • Val batches: {len(val_loader)} (using {min(20, len(val_loader))} for speed)\")\n    print(f\"   • Estimated time/epoch: ~{len(train_loader) * 2.5 / 60:.1f} min\")\n    \n    # Lightweight model\n    print(f\"\\n🏗️  Building lightweight model...\")\n    model = LightweightUNetPlusPlus(\n        in_channels=config.IN_CHANNELS,\n        num_classes=config.NUM_CLASSES,\n        base_filters=config.BASE_FILTERS\n    ).to(config.DEVICE)\n    \n    total_params = sum(p.numel() for p in model.parameters())\n    print(f\"   • Parameters: {total_params:,}\")\n    \n    # Loss, optimizer, scheduler\n    criterion = CombinedLoss(config.DICE_WEIGHT, config.BCE_WEIGHT)\n    optimizer = torch.optim.AdamW(\n        model.parameters(), \n        lr=config.LEARNING_RATE, \n        weight_decay=config.WEIGHT_DECAY\n    )\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n        optimizer, T_max=config.T_MAX\n    )\n    scaler = torch.cuda.amp.GradScaler() if config.MIXED_PRECISION else None\n    \n    # Training loop\n    history = {\n        'train_loss': [], 'train_dice': [], 'train_iou': [],\n        'val_loss': [], 'val_dice': [], 'val_iou': []\n    }\n    \n    best_dice = 0.0\n    epochs_no_improve = 0\n    \n    print(f\"\\n{'='*70}\")\n    print(\"STARTING OPTIMIZED TRAINING\")\n    print(f\"{'='*70}\\n\")\n    \n    import time\n    total_training_time = 0\n    \n    for epoch in range(1, config.EPOCHS + 1):\n        print(f\"\\nEpoch {epoch}/{config.EPOCHS}\")\n        print(\"-\" * 70)\n        \n        epoch_start = time.time()\n        \n        train_metrics = train_epoch(\n            model, train_loader, criterion, optimizer, config.DEVICE, scaler, config\n        )\n        \n        # Validate only every N epochs for speed!\n        if epoch % config.VALIDATE_EVERY_N_EPOCHS == 0 or epoch == 1:\n            val_metrics = validate_epoch(\n                model, val_loader, criterion, config.DEVICE\n            )\n        else:\n            # Use last validation metrics if skipping\n            if history['val_loss']:\n                val_metrics = {\n                    'loss': history['val_loss'][-1],\n                    'dice': history['val_dice'][-1],\n                    'iou': history['val_iou'][-1]\n                }\n            else:\n                val_metrics = train_metrics.copy()\n        \n        epoch_time = time.time() - epoch_start\n        total_training_time += epoch_time\n        \n        # Print summary\n        print(f\"\\n📊 Epoch {epoch} Summary:\")\n        print(f\"   Train - Loss: {train_metrics['loss']:.4f} | Dice: {train_metrics['dice']:.4f} | IoU: {train_metrics['iou']:.4f}\")\n        if epoch % config.VALIDATE_EVERY_N_EPOCHS == 0 or epoch == 1:\n            print(f\"   Val   - Loss: {val_metrics['loss']:.4f} | Dice: {val_metrics['dice']:.4f} | IoU: {val_metrics['iou']:.4f}\")\n        else:\n            print(f\"   Val   - (validation skipped)\")\n        print(f\"   Time: {epoch_time/60:.2f} min | LR: {optimizer.param_groups[0]['lr']:.6f}\")\n        \n        history['train_loss'].append(train_metrics['loss'])\n        history['train_dice'].append(train_metrics['dice'])\n        history['train_iou'].append(train_metrics['iou'])\n        history['val_loss'].append(val_metrics['loss'])\n        history['val_dice'].append(val_metrics['dice'])\n        history['val_iou'].append(val_metrics['iou'])\n        \n        scheduler.step()\n        \n        # Only save if validation was performed\n        if epoch % config.VALIDATE_EVERY_N_EPOCHS == 0 or epoch == 1:\n            is_best = val_metrics['dice'] > best_dice\n            if is_best:\n                best_dice = val_metrics['dice']\n                epochs_no_improve = 0\n                torch.save({\n                    'epoch': epoch,\n                    'model_state_dict': model.state_dict(),\n                    'optimizer_state_dict': optimizer.state_dict(),\n                    'best_dice': best_dice\n                }, f\"{config.OUTPUT_DIR}/best_model.pth\")\n                print(f\"   ✅ Best model saved! (Dice: {best_dice:.4f})\")\n            else:\n                epochs_no_improve += 1\n        \n        if epoch % config.SAVE_EVERY_N_EPOCHS == 0:\n            torch.save(model.state_dict(), f\"{config.OUTPUT_DIR}/checkpoint_epoch_{epoch}.pth\")\n        \n        if epochs_no_improve >= config.EARLY_STOPPING_PATIENCE:\n            print(f\"\\n⚠️  Early stopping at epoch {epoch}\")\n            break\n    \n    print(f\"\\n{'='*70}\")\n    print(\"TRAINING COMPLETE!\")\n    print(f\"{'='*70}\")\n    print(f\"✅ Best Dice: {best_dice:.4f}\")\n    print(f\"⏱️  Total time: {total_training_time/60:.1f} minutes\")\n    print(f\"⏱️  Avg time/epoch: {total_training_time/epoch/60:.2f} minutes\")\n    \n    # Save history\n    with open(f\"{config.OUTPUT_DIR}/history.json\", 'w') as f:\n        json.dump(history, f, indent=2)\n    \n    # Plots\n    plot_history(history, f\"{config.OUTPUT_DIR}/training_curves.png\")\n    visualize_predictions(model, val_dataset, config.DEVICE, 5, f\"{config.OUTPUT_DIR}/predictions.png\")\n    \n    print(f\"\\n📁 Outputs saved to: {config.OUTPUT_DIR}\\n\")\n    \n    return model, history\n\n# ============================================================================\n# RUN OPTIMIZED TRAINING\n# ============================================================================\n\nif __name__ == \"__main__\":\n    # IMPORTANT: If you get OOM error, restart kernel first!\n    # This clears any GPU memory from previous runs\n    \n    config = OptimizedTrainingConfig()\n    \n    # Force garbage collection\n    import gc\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n    \n    model, history = train_model_optimized(config)\n    \n    print(\"\\n🎉 Training complete!\")\n    print(f\"📊 Best Dice: {max(history['val_dice']):.4f}\")\n    print(f\"📊 Best IoU: {max(history['val_iou']):.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom pathlib import Path\n\n# Ganti 'model' jika nama variabel model Anda berbeda\ntrained_model = model \n\n# Simpan di direktori /kaggle/working/\nNEW_MODEL_PATH = Path(\"/kaggle/working/\").resolve() / \"final_model_weights_only.pth\"\n\n# Menyimpan hanya state_dict\ntry:\n    torch.save(trained_model.state_dict(), NEW_MODEL_PATH)\n    print(f\"✅ Bobot model berhasil disimpan di: {NEW_MODEL_PATH}\")\nexcept NameError:\n    print(\"❌ ERROR: Variabel 'model' tidak ditemukan.\")\nexcept Exception as e:\n    print(f\"❌ ERROR saat menyimpan model: {e}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# KAGGLE VESUVIUS SCORE CALCULATOR\n# ============================================================================\n\nimport numpy as np\nimport torch\nfrom scipy.ndimage import label as connected_components\nfrom scipy.spatial.distance import cdist\nfrom skimage.measure import label as sk_label\nfrom sklearn.metrics import confusion_matrix\nfrom tqdm.auto import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ============================================================================\n# 1. SURFACE DICE @ τ\n# ============================================================================\n\ndef extract_surface_voxels(mask_3d):\n    \"\"\"Extract surface voxels from 3D binary mask\"\"\"\n    from scipy.ndimage import binary_erosion\n    \n    # Surface = original - eroded\n    eroded = binary_erosion(mask_3d)\n    surface = mask_3d & ~eroded\n    \n    return surface\n\ndef surface_dice(pred, gt, spacing=(1.0, 1.0, 1.0), tau=2.0):\n    \"\"\"\n    Calculate Surface Dice score\n    \n    Args:\n        pred: 3D binary prediction (H, W, D)\n        gt: 3D binary ground truth (H, W, D)\n        spacing: Physical spacing (z, y, x)\n        tau: Tolerance threshold (default 2.0)\n    \n    Returns:\n        Surface Dice score [0, 1]\n    \"\"\"\n    # Edge cases\n    pred_empty = pred.sum() == 0\n    gt_empty = gt.sum() == 0\n    \n    if pred_empty and gt_empty:\n        return 1.0\n    if pred_empty or gt_empty:\n        return 0.0\n    \n    # Extract surfaces\n    surf_pred = extract_surface_voxels(pred)\n    surf_gt = extract_surface_voxels(gt)\n    \n    # Get coordinates\n    coords_pred = np.array(np.where(surf_pred)).T * spacing\n    coords_gt = np.array(np.where(surf_gt)).T * spacing\n    \n    if len(coords_pred) == 0 or len(coords_gt) == 0:\n        return 0.0\n    \n    # Compute nearest distances (efficient batching)\n    def nearest_distances_batched(coords_a, coords_b, batch_size=10000):\n        n = len(coords_a)\n        distances = np.zeros(n)\n        \n        for i in range(0, n, batch_size):\n            end = min(i + batch_size, n)\n            batch = coords_a[i:end]\n            dists = cdist(batch, coords_b, metric='euclidean')\n            distances[i:end] = dists.min(axis=1)\n        \n        return distances\n    \n    # Distance from prediction to GT\n    dist_pred_to_gt = nearest_distances_batched(coords_pred, coords_gt)\n    matched_pred = (dist_pred_to_gt <= tau).sum()\n    \n    # Distance from GT to prediction\n    dist_gt_to_pred = nearest_distances_batched(coords_gt, coords_pred)\n    matched_gt = (dist_gt_to_pred <= tau).sum()\n    \n    # Surface Dice = average of both directions\n    surface_dice_score = (matched_pred + matched_gt) / (len(coords_pred) + len(coords_gt))\n    \n    return surface_dice_score\n\n# ============================================================================\n# 2. VOI SCORE\n# ============================================================================\n\ndef voi_score(pred, gt, alpha=0.3, connectivity=26):\n    \"\"\"\n    Calculate VOI (Variation of Information) score\n    \n    Args:\n        pred: 3D binary prediction\n        gt: 3D binary ground truth\n        alpha: Scaling parameter (default 0.3)\n        connectivity: Connectivity for connected components (26 or 6)\n    \n    Returns:\n        VOI score [0, 1]\n    \"\"\"\n    from scipy.stats import entropy\n    \n    # Get union foreground\n    union = pred | gt\n    \n    if union.sum() == 0:\n        return 1.0\n    \n    # Connected components (3D)\n    struct = None  # None = 26-connectivity\n    if connectivity == 6:\n        struct = np.array([[[0,0,0],[0,1,0],[0,0,0]],\n                          [[0,1,0],[1,1,1],[0,1,0]],\n                          [[0,0,0],[0,1,0],[0,0,0]]])\n    \n    pred_labels, n_pred = connected_components(pred & union, structure=struct)\n    gt_labels, n_gt = connected_components(gt & union, structure=struct)\n    \n    # Flatten for easier computation\n    pred_flat = pred_labels[union].flatten()\n    gt_flat = gt_labels[union].flatten()\n    \n    # Confusion matrix\n    n_pred += 1  # include background\n    n_gt += 1\n    \n    # Joint distribution\n    contingency = np.zeros((n_pred, n_gt))\n    for p, g in zip(pred_flat, gt_flat):\n        contingency[p, g] += 1\n    \n    # Normalize\n    contingency = contingency / contingency.sum()\n    \n    # Marginals\n    p_pred = contingency.sum(axis=1)\n    p_gt = contingency.sum(axis=0)\n    \n    # Conditional entropies\n    # H(GT | Pred) - split error\n    h_gt_given_pred = 0\n    for i in range(n_pred):\n        if p_pred[i] > 0:\n            conditional = contingency[i, :] / p_pred[i]\n            conditional = conditional[conditional > 0]\n            h_gt_given_pred += p_pred[i] * entropy(conditional, base=2)\n    \n    # H(Pred | GT) - merge error\n    h_pred_given_gt = 0\n    for j in range(n_gt):\n        if p_gt[j] > 0:\n            conditional = contingency[:, j] / p_gt[j]\n            conditional = conditional[conditional > 0]\n            h_pred_given_gt += p_gt[j] * entropy(conditional, base=2)\n    \n    # Total VOI\n    voi_total = h_gt_given_pred + h_pred_given_gt\n    \n    # Convert to bounded score\n    voi_score_value = 1.0 / (1.0 + alpha * voi_total)\n    \n    return voi_score_value\n\n# ============================================================================\n# 3. TOPO SCORE (Simplified Betti Number Matching)\n# ============================================================================\n\ndef compute_betti_numbers(mask_3d):\n    \"\"\"\n    Compute Betti numbers (simplified version)\n    b0 = number of connected components\n    b1 = approximate number of tunnels/handles\n    b2 = approximate number of cavities\n    \"\"\"\n    # b0: Connected components\n    labeled, b0 = sk_label(mask_3d, connectivity=3, return_num=True)\n    \n    # b1 and b2: Simplified approximation\n    # For full accuracy, you'd need proper persistent homology\n    # Here we use Euler characteristic approximation\n    \n    # Euler characteristic χ = b0 - b1 + b2\n    # For simple surfaces: χ ≈ 2 - 2g (g = genus/handles)\n    \n    # Simple heuristic: count \"holes\" in 2D slices\n    b1 = 0  # Tunnels (simplified)\n    b2 = 0  # Cavities (simplified)\n    \n    # Count potential tunnels by checking connectivity changes\n    for z in range(mask_3d.shape[0] - 1):\n        slice_curr = mask_3d[z]\n        slice_next = mask_3d[z + 1]\n        \n        # Components in current slice\n        _, n_curr = sk_label(slice_curr, return_num=True)\n        \n        # Components in intersection\n        intersection = slice_curr & slice_next\n        _, n_inter = sk_label(intersection, return_num=True)\n        \n        # If components merge/split, might indicate tunnel\n        if n_curr > n_inter:\n            b1 += (n_curr - n_inter)\n    \n    return b0, b1, b2\n\ndef topo_score(pred, gt, w0=0.34, w1=0.33, w2=0.33):\n    \"\"\"\n    Calculate Topological Score (simplified Betti matching)\n    \n    Args:\n        pred: 3D binary prediction\n        gt: 3D binary ground truth\n        w0, w1, w2: Weights for b0, b1, b2\n    \n    Returns:\n        Topo score [0, 1]\n    \"\"\"\n    if pred.sum() == 0 and gt.sum() == 0:\n        return 1.0\n    \n    # Compute Betti numbers\n    b0_pred, b1_pred, b2_pred = compute_betti_numbers(pred)\n    b0_gt, b1_gt, b2_gt = compute_betti_numbers(gt)\n    \n    # F1 score for each dimension\n    def betti_f1(pred_count, gt_count):\n        if pred_count == 0 and gt_count == 0:\n            return None  # Inactive dimension\n        if pred_count == 0 or gt_count == 0:\n            return 0.0\n        \n        # True positives = min\n        tp = min(pred_count, gt_count)\n        precision = tp / pred_count if pred_count > 0 else 0\n        recall = tp / gt_count if gt_count > 0 else 0\n        \n        if precision + recall == 0:\n            return 0.0\n        \n        f1 = 2 * precision * recall / (precision + recall)\n        return f1\n    \n    # Compute F1 for each dimension\n    f1_scores = []\n    weights = []\n    \n    f1_0 = betti_f1(b0_pred, b0_gt)\n    if f1_0 is not None:\n        f1_scores.append(f1_0)\n        weights.append(w0)\n    \n    f1_1 = betti_f1(b1_pred, b1_gt)\n    if f1_1 is not None:\n        f1_scores.append(f1_1)\n        weights.append(w1)\n    \n    f1_2 = betti_f1(b2_pred, b2_gt)\n    if f1_2 is not None:\n        f1_scores.append(f1_2)\n        weights.append(w2)\n    \n    # Weighted average (renormalize if some dimensions inactive)\n    if len(f1_scores) == 0:\n        return 1.0\n    \n    weights = np.array(weights)\n    weights = weights / weights.sum()  # Renormalize\n    \n    topo_score_value = np.average(f1_scores, weights=weights)\n    \n    return topo_score_value\n\n# ============================================================================\n# 4. COMBINED KAGGLE SCORE\n# ============================================================================\n\ndef kaggle_vesuvius_score(pred, gt, spacing=(1.0, 1.0, 1.0), \n                          tau=2.0, alpha=0.3,\n                          w_topo=0.30, w_surface=0.35, w_voi=0.35):\n    \"\"\"\n    Calculate complete Kaggle Vesuvius Challenge score\n    \n    Score = 0.30 × TopoScore + 0.35 × SurfaceDice@τ + 0.35 × VOI_score\n    \n    Args:\n        pred: 3D binary prediction array (D, H, W)\n        gt: 3D binary ground truth array (D, H, W)\n        spacing: Physical spacing (z, y, x) in mm\n        tau: Tolerance for Surface Dice\n        alpha: Scaling for VOI\n        w_topo, w_surface, w_voi: Weights (default competition weights)\n    \n    Returns:\n        dict with individual scores and final score\n    \"\"\"\n    print(\"Calculating Kaggle Vesuvius Score...\")\n    print(f\"  Volume shape: {pred.shape}\")\n    print(f\"  Spacing: {spacing}\")\n    \n    # Ensure binary\n    pred = (pred > 0.5).astype(bool)\n    gt = (gt > 0.5).astype(bool)\n    \n    # Calculate individual scores\n    print(\"\\n1️⃣  Computing TopoScore...\")\n    topo = topo_score(pred, gt)\n    \n    print(\"2️⃣  Computing SurfaceDice@τ...\")\n    surf_dice = surface_dice(pred, gt, spacing, tau)\n    \n    print(\"3️⃣  Computing VOI_score...\")\n    voi = voi_score(pred, gt, alpha)\n    \n    # Combined score\n    final_score = w_topo * topo + w_surface * surf_dice + w_voi * voi\n    \n    results = {\n        'TopoScore': topo,\n        'SurfaceDice': surf_dice,\n        'VOI_score': voi,\n        'Final_Score': final_score,\n        'weights': {\n            'topo': w_topo,\n            'surface': w_surface,\n            'voi': w_voi\n        }\n    }\n    \n    return results\n\n# ============================================================================\n# 5. EVALUATE MODEL ON VALIDATION SET\n# ============================================================================\n\ndef evaluate_model_kaggle_score(model, val_loader, device, spacing=(1.0, 1.0, 1.0)):\n    \"\"\"\n    Evaluate model on validation set with Kaggle scoring\n    \n    Args:\n        model: Trained model\n        val_loader: Validation DataLoader\n        device: torch device\n        spacing: Physical spacing\n    \n    Returns:\n        Average Kaggle score across all volumes\n    \"\"\"\n    model.eval()\n    \n    print(\"=\"*70)\n    print(\"EVALUATING MODEL WITH KAGGLE SCORE\")\n    print(\"=\"*70)\n    \n    all_scores = []\n    \n    with torch.no_grad():\n        for idx, (images, masks) in enumerate(tqdm(val_loader, desc=\"Processing batches\")):\n            images = images.to(device)\n            masks = masks.cpu().numpy()\n            \n            # Get predictions\n            outputs = model(images)\n            preds = (torch.sigmoid(outputs) > 0.5).cpu().numpy()\n            \n            # Process each sample in batch\n            for i in range(len(images)):\n                pred_3d = preds[i, 0]  # (H, W)\n                gt_3d = masks[i, 0]    # (H, W)\n                \n                # For 2D patches, create pseudo-3D by stacking\n                # In real evaluation, you'd reconstruct full 3D volumes\n                pred_volume = np.stack([pred_3d] * 3, axis=0)  # (3, H, W)\n                gt_volume = np.stack([gt_3d] * 3, axis=0)\n                \n                # Calculate score\n                try:\n                    score_dict = kaggle_vesuvius_score(\n                        pred_volume, gt_volume, \n                        spacing=spacing,\n                        tau=2.0, alpha=0.3\n                    )\n                    all_scores.append(score_dict['Final_Score'])\n                except Exception as e:\n                    print(f\"Error on sample {idx}-{i}: {e}\")\n                    continue\n            \n            # Only process first 10 batches for speed (remove this in production)\n            if idx >= 10:\n                break\n    \n    # Average score\n    avg_score = np.mean(all_scores)\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"RESULTS\")\n    print(\"=\"*70)\n    print(f\"Average Kaggle Score: {avg_score:.4f}\")\n    print(f\"Samples evaluated: {len(all_scores)}\")\n    \n    return avg_score, all_scores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# RUN KAGGLE SCORE EVALUATION\n# ============================================================================\n\n# Load model\nfrom pathlib import Path\n\nconfig = TrainingConfig()\n\n# Load best model\ncheckpoint = torch.load('/kaggle/working/training_output/best_model.pth')\nmodel = UNetPlusPlus(\n    in_channels=1, \n    num_classes=1, \n    base_filters=32\n).to(config.DEVICE)\n\nmodel.load_state_dict(checkpoint['model_state_dict'])\n\nprint(\"✅ Model loaded!\")\nprint(f\"📊 Best Dice from training: {checkpoint['best_dice']:.4f}\")\n\n# Evaluate\navg_score, all_scores = evaluate_model_kaggle_score(\n    model=model,\n    val_loader=val_loader,\n    device=config.DEVICE,\n    spacing=(1.0, 1.0, 1.0)  # Adjust based on actual spacing\n)\n\nprint(\"\\n\" + \"=\"*70)\nprint(\"🎯 ESTIMATED LEADERBOARD SCORE\")\nprint(\"=\"*70)\nprint(f\"Score: {avg_score:.4f}\")\nprint(\"\\n⚠️  Note: This is approximate - actual test set may differ\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}