{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":117682,"databundleVersionId":15062069},{"sourceType":"datasetVersion","sourceId":14688639,"datasetId":9383489,"databundleVersionId":15532246},{"sourceType":"datasetVersion","sourceId":14635971,"datasetId":9349395,"databundleVersionId":15474763}],"dockerImageVersionId":31259,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<!-- Python beginner 2025/12/24, for personal archive and reference -->","metadata":{}},{"cell_type":"code","source":"#Source: EduceLab-Scrolls (Parsons et al., 2023)\n!pip install /kaggle/input/imagecodecs/*.whl --no-index --find-links=/kaggle/input/imagecodecs/\nimport os\nimport zipfile\nimport pandas as pd\nimport numpy as np\nimport tifffile\nimport torch\nimport torch.nn as nn\nimport gc\nfrom tqdm import tqdm\nfrom scipy.ndimage import binary_closing\n# ====================================================\nclass Vesuvius3DUNet(nn.Module):\n    def __init__(self):\n        super().__init__()\n        def conv_block_3d(in_c, out_c):\n            return nn.Sequential(\n                nn.Conv3d(in_c, out_c, kernel_size=3, padding=1),\n                nn.BatchNorm3d(out_c),\n                nn.ReLU(inplace=True),\n                nn.Conv3d(out_c, out_c, kernel_size=3, padding=1),\n                nn.BatchNorm3d(out_c),\n                nn.ReLU(inplace=True)\n            )\n        self.enc1 = conv_block_3d(1, 16)\n        self.pool = nn.MaxPool3d(kernel_size=(1, 2, 2))\n        self.enc2 = conv_block_3d(16, 32)\n        self.up = nn.Upsample(scale_factor=(1, 2, 2), mode='trilinear', align_corners=True)\n        self.dec = conv_block_3d(32 + 16, 16)\n        self.final = nn.Sequential(\n            nn.Conv3d(16, 1, kernel_size=1),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        e1 = self.enc1(x)\n        e2 = self.enc2(self.pool(e1))\n        d = self.up(e2)\n        d = torch.cat([d, e1], dim=1)\n        out = self.dec(d)\n        return self.final(out)\n\n# ====================================================\n# 2. 사용자 보조 규칙 완벽 반영 (Rule 1 & 2 통합)\n# ====================================================\ndef apply_vesuvius_rules(vol):\n    Z, Y, X = vol.shape\n    out_vol = vol.copy()\n\n    # [Rational Bridge: 복원 로직]\n    for z in range(Z):\n        p1, p25 = vol[z, :-4, :-4], vol[z, 4:, 4:]\n        p5, p21 = vol[z, :-4, 4:], vol[z, 4:, :-4]\n        p7, p9 = vol[z, 1:-3, 1:-3], vol[z, 1:-3, 3:-1]\n        p17, p19 = vol[z, 3:-1, 1:-3], vol[z, 3:-1, 3:-1]\n        \n        restore_cond = ((p1 & p25) | (p5 & p21)) & ((p7 & p9) | (p17 & p19))\n        out_vol[z, 2:-2, 2:-2][restore_cond == 1] = 1\n\n    # [Rule 1 & 2: 제거 로직 (연속성 및 지지 판정)]\n    # z run length 계산을 위해 shift 연산 사용\n    for z in range(1, Z - 1):\n        # 1번 규칙: z 연속 길이 >= 3, xy 이웃 8방향 전부 0일 때 중간 제거\n        is_pillar = (vol[z-1] == 1) & (vol[z] == 1) & (vol[z+1] == 1)\n        \n        # 8방향 이웃 합 계산\n        neighbors_sum = (\n            vol[z, :-2, :-2] + vol[z, :-2, 1:-1] + vol[z, :-2, 2:] +\n            vol[z, 1:-1, :-2] +                    vol[z, 1:-1, 2:] +\n            vol[z, 2:, :-2] + vol[z, 2:, 1:-1] + vol[z, 2:, 2:]\n        )\n        # xy 지지 판정에 \"이전/다음 z\" 1칸 포함 (사용자 요청 사항)\n        z_neighbors = (\n            vol[z-1, 1:-1, 1:-1] + vol[z+1, 1:-1, 1:-1]\n        )\n        \n        # Rule 1 적용\n        rule1_mask = (is_pillar[1:-1, 1:-1]) & (neighbors_sum == 0) & (z_neighbors > 0)\n        out_vol[z, 1:-1, 1:-1][rule1_mask] = 0\n\n        # 2번 규칙: L >= 4일 때 중간 slice만 제거 (양 끝 유지)\n        if z > 1 and z < Z-2:\n            is_long_run = (vol[z-2]==1) & (vol[z-1]==1) & (vol[z]==1) & (vol[z+1]==1)\n            out_vol[z][is_long_run == 1] = 0\n\n    # 마무리 3x3 Closing\n    for z in range(Z):\n        out_vol[z] = binary_closing(out_vol[z], structure=np.ones((3,3))).astype(np.uint8)\n        \n    return out_vol\n\n# ====================================================\n# 3. 최적화된 실행 함수 (RAM 30GB 방어)\n# ====================================================\ndef run_gpu_inference_optimized():\n    INPUT_DIR = \"/kaggle/input/vesuvius-challenge-surface-detection\"\n    TEST_CSV = os.path.join(INPUT_DIR, \"test.csv\")\n    TEST_IMG_DIR = os.path.join(INPUT_DIR, \"test_images\")\n    DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    PATCH_SIZE = 256\n    Z_PATCH = 16 \n    THRESHOLD = 0.45 \n\n    model = Vesuvius3DUNet().to(DEVICE)\n    model.eval()\n    \n    test_df = pd.read_csv(TEST_CSV)\n    tif_files = []\n\n    for _, row in tqdm(test_df.iterrows(), total=len(test_df)):\n        img_id = str(row[\"id\"])\n        img_path = os.path.join(TEST_IMG_DIR, f\"{img_id}.tif\")\n        \n        with tifffile.TiffFile(img_path) as tif:\n            series = tif.series[0]\n            z_dim, h_img, w_img = series.shape\n            raw_pred_vol = np.zeros((z_dim, h_img, w_img), dtype=np.uint8)\n\n            for z_s in range(0, z_dim, Z_PATCH):\n                z_e = min(z_s + Z_PATCH, z_dim)\n                chunk = np.stack([series.asarray(key=zi) for zi in range(z_s, z_e)]).astype(np.float32)\n                chunk = (chunk - chunk.min()) / (chunk.max() - chunk.min() + 1e-6)\n\n                for h in range(0, h_img, PATCH_SIZE):\n                    for w in range(0, w_img, PATCH_SIZE):\n                        h_e, w_e = min(h+PATCH_SIZE, h_img), min(w+PATCH_SIZE, w_img)\n                        patch = torch.from_numpy(chunk[:, h:h_e, w:w_e]).unsqueeze(0).unsqueeze(0).to(DEVICE)\n                        \n                        with torch.no_grad():\n                            with torch.cuda.amp.autocast():\n                                pred = model(patch)\n                            raw_pred_vol[z_s:z_e, h:h_e, w:w_e] = (pred[0, 0] > THRESHOLD).cpu().numpy().astype(np.uint8)\n                \n                del chunk\n                gc.collect()\n\n            final_vol = apply_vesuvius_rules(raw_pred_vol)\n            final_vol *= 255\n            \n            out_name = f\"{img_id}.tif\"\n            tifffile.imwrite(out_name, final_vol, bigtiff=True)\n            tif_files.append(out_name)\n            \n            del raw_pred_vol, final_vol\n            torch.cuda.empty_cache()\n            gc.collect()\n\n    with zipfile.ZipFile(\"submission.zip\", \"w\", zipfile.ZIP_DEFLATED) as zf:\n        for f in tif_files:\n            zf.write(f, arcname=f)\n            os.remove(f)\n\nif __name__ == \"__main__\":\n    run_gpu_inference_optimized()\n    print(\"✅ 모든 에러를 극복하고 제출 파일 생성을 완료했습니다!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### MARKDOWN DATASETS \n# colab\n# first.pth\n# ves_25d_slice5.pth\n\n\"Please create a folder named 'ves' in Google Drive, and inside it, create two subfolders: 'train_image' and 'train_masks'. Then, selectively move only a subset of the .tif files from the existing folders into their respective new folders. Alternatively, you may randomly shuffle the files before moving them.\"\n\nfrom google.colab import drive\ndrive.mount('/content/drive')\n\n# 필요한 라이브러리 설치\n!pip install tifffile tqdm albumentations --quiet\n!pip install /content/drive/MyDrive/ves/imagecodecs-2026.1.14-cp311-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl\n\nimport os, gc, glob\nimport numpy as np\nimport tifffile\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm import tqdm\nfrom scipy.ndimage import binary_closing\n\n# ====================================================\n# 1. Dataset 정의\n# ====================================================\nclass Vesuvius25DDataset(Dataset):\n    def __init__(self, img_dir, mask_dir, z_radius=2):\n        self.img_dir = img_dir\n        self.mask_dir = mask_dir\n        self.zr = z_radius\n        \n        # 🔥 수정 핵심: 양쪽 폴더에 공통으로 존재하는 파일만 추출\n        img_files = set(os.listdir(img_dir))\n        mask_files = set(os.listdir(mask_dir))\n        self.ids = sorted([f for f in img_files if f in mask_files and f.endswith('.tif')])\n        \n        print(f\"✅ 학습 가능 파일 개수: {len(self.ids)}\")\n\n    def __len__(self):\n        return len(self.ids)\n\n    def __getitem__(self, idx):\n        name = self.ids[idx]\n        img = tifffile.imread(os.path.join(self.img_dir, name))  # (Z, H, W)\n        mask = tifffile.imread(os.path.join(self.mask_dir, name)) # (H, W)\n\n        Z, H, W = img.shape\n        zc = Z // 2\n\n        # z_radius 범위만큼 슬라이스 추출\n        slices = []\n        for dz in range(-self.zr, self.zr + 1):\n            zi = np.clip(zc + dz, 0, Z - 1)\n            slices.append(img[zi])\n\n        x = np.stack(slices, axis=0).astype(np.float32)\n        x = (x - x.min()) / (x.max() - x.min() + 1e-6) # 정규화\n        y = (mask > 0).astype(np.float32)\n\n        return torch.tensor(x), torch.tensor(y)\n\n# ====================================================\n# 2. 모델 정의 (UNet 2.5D)\n# ====================================================\nclass UNet25D(nn.Module):\n    def __init__(self, in_ch):\n        super().__init__()\n        def CBR(i, o):\n            return nn.Sequential(\n                nn.Conv2d(i, o, 3, padding=1),\n                nn.BatchNorm2d(o),\n                nn.ReLU(inplace=True)\n            )\n        self.enc1 = nn.Sequential(CBR(in_ch, 32), CBR(32, 32))\n        self.pool = nn.MaxPool2d(2)\n        self.enc2 = nn.Sequential(CBR(32, 64), CBR(64, 64))\n        self.up = nn.Upsample(scale_factor=2, mode=\"bilinear\", align_corners=False)\n        self.dec = nn.Sequential(CBR(64 + 32, 32), CBR(32, 32))\n        self.final = nn.Conv2d(32, 1, 1)\n\n    def forward(self, x):\n        e1 = self.enc1(x)\n        e2 = self.enc2(self.pool(e1))\n        d = self.up(e2)\n        d = torch.cat([d, e1], dim=1)\n        return self.final(d)\n\n# ====================================================\n# 3. 설정 및 학습 실행\n# ====================================================\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nIMG_DIR = \"/content/drive/MyDrive/ves/train_images\"\nMASK_DIR = \"/content/drive/MyDrive/ves/train_masks\"\n\n# 데이터 로더 (Batch size를 4~8로 올리면 더 빠릅니다)\ndataset = Vesuvius25DDataset(IMG_DIR, MASK_DIR, z_radius=2)\nloader = DataLoader(dataset, batch_size=4, shuffle=True)\n\nmodel = UNet25D(in_ch=5).to(DEVICE)\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-3)\ncriterion = nn.BCEWithLogitsLoss()\n\nEPOCHS = 6 \n\nprint(\"🚀 학습을 시작합니다...\")\nfor epoch in range(EPOCHS):\n    model.train()\n    loss_sum = 0\n    pbar = tqdm(loader, desc=f\"Epoch {epoch+1}/{EPOCHS}\")\n    \n    for x, y in pbar:\n        x = x.to(DEVICE)\n        y = y.unsqueeze(1).to(DEVICE)\n\n        pred = model(x)\n        loss = criterion(pred, y)\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        loss_sum += loss.item()\n        pbar.set_postfix(loss=f\"{loss.item():.4f}\")\n\n    print(f\"📊 Epoch {epoch+1} 완료 | 평균 Loss: {loss_sum/len(loader):.4f}\")\n    \n    torch.cuda.empty_cache()\n    gc.collect()\n\n# ====================================================\n# 4. 가중치 저장\n# ====================================================\nSAVE_PATH = \"/content/drive/MyDrive/ves/ves_25d_minimal.pth\"\ntorch.save(model.state_dict(), SAVE_PATH)\nprint(f\"✅ 모델 저장 완료: {SAVE_PATH}\")","metadata":{}}]}