{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":117682,"databundleVersionId":15062069},{"sourceType":"datasetVersion","sourceId":14982555,"datasetId":9590161,"databundleVersionId":15855963},{"sourceType":"kernelVersion","sourceId":290917305}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"try:\n    print(\"imagecodecs has lzw_decode:\", hasattr(imagecodecs, \"lzw_decode\"))\n    print(\"tifffile version:\", tifffile.__version__)\n    \nexcept:   \n    !pip install --no-index --find-links \"/kaggle/input/vsdetection-packages-offline-installer-only/whls\" imagecodecs\n    #/kaggle/input/vsdetection-packages-offline-installer-only/whls/imagecodecs-2026.1.1-cp311-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl\n    \n    import imagecodecs\n    import sys\n    # If tifffile was imported earlier (before imagecodecs), force a clean re-import:\n    for k in list(sys.modules):\n        if k.startswith(\"tifffile\"):\n            del sys.modules[k]\n    import tifffile  # re-import\n    \n    # Verify tifffile sees imagecodecs via its internal lazy import helper\n    try:\n        ic = tifffile.imagecodecs()\n        print(\"tifffile.imagecodecs() ->\", ic)\n    except Exception as e:\n        print(\"tifffile.imagecodecs() check failed:\", repr(e))\n    \n    print(\"imagecodecs has lzw_decode:\", hasattr(imagecodecs, \"lzw_decode\"))\n    print(\"tifffile version:\", tifffile.__version__)\n    \n'''\nimagecodecs has lzw_decode: True\ntifffile version: 2025.10.16\n'''\n\n\n#-------------------------------------\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport zipfile\nimport tifffile\nimport scipy.ndimage as ndi\nfrom skimage.morphology import remove_small_objects\nimport io\nfrom timeit import default_timer as timer\nfrom pathlib import Path\nimport torch\n\nfrom matplotlib import pyplot as plt\nimport matplotlib\n#matplotlib.use('TkAgg')\n\nfrom concurrent.futures import ThreadPoolExecutor\n\nimport sys\n#sys.path.append('/kaggle/input/hengck23-vesuvius-surface-01')\nsys.path.append('/kaggle/input/datasets/hengck23/hengck23-final-surface-det')\n\n \n\n\nfrom processing_public import do_postprocess_public\nfrom unet3d import Net as Unet3d\nfrom unet3d_modelling import do_predict_tta7 as do_predict_tta7_unet3d\n\nfrom refinenet import Net as RefineNet\nfrom refinenet_modelling import do_predict_tta7 as do_predict_tta7_refinenet\n\n\nfrom unetrppnet import Net as UNetrPP\nfrom unetrppnet_modelling import do_predict as do_predict_tta7_unetrpp\n\n\n#--------------------------------\n#helper\ndef time_to_str(t, mode='min'):\n    if mode=='min':\n        t  = int(t)/60\n        hr = t//60\n        min = t%60\n        return '%2d hr %02d min'%(hr,min)\n    elif mode=='sec':\n        t   = int(t)\n        min = t//60\n        sec = t%60\n        return '%2d min %02d sec'%(min,sec)\n    else:\n        raise NotImplementedError\n\nprint('IMPORT OK!!!')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-27T14:22:57.069007Z","iopub.execute_input":"2026-02-27T14:22:57.069706Z","iopub.status.idle":"2026-02-27T14:22:57.081445Z","shell.execute_reply.started":"2026-02-27T14:22:57.069676Z","shell.execute_reply":"2026-02-27T14:22:57.080845Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"OUT_DIR =\\\n    '/kaggle/working' \nKAGGLE_DIR = \\\n    '/kaggle/input/vesuvius-challenge-surface-detection'\n\n#configure \nCFG={\n    'num_id_per_thread': 5,\n    'batch_size': 4,\n    'float_type': torch.float16, \n} \nMULTI_THREAD = True #True False\nMODE = 'submit'  #local submit\n \n#check differet image size\nif MODE=='local':\n    IMAGE_DIR = f'{KAGGLE_DIR}/train_images'\n    valid_df=pd.DataFrame({'id':[\n        8862040, #320\n        3721853537,\n        70695797, #256\n        #--------\n        719197630, #320\n        722561671,\n        724175864,\n        725727411,\n        730065526,\n        # 744788585,\n        \n        # 746664827,\n        # 762867428,\n        # 765473271,\n        # 772457118,\n        # 776379178,\n        # 777463733,\n        # 785433964,\n        # 787804611,\n        # 797181487,\n        # 803213133,\n        \n        # 805297990,\n        #3866421231, #384\n    ]})\n    CFG['num_id_per_thread']=3 #speedup debug\n\nif MODE=='submit':\n    IMAGE_DIR = f'{KAGGLE_DIR}/test_images'\n    valid_df = pd.read_csv(f'{KAGGLE_DIR}/test.csv')\n\nprint(valid_df)\nprint('')\nprint('MULTI_THREAD', MULTI_THREAD) \nprint('MODE', MODE)\nprint('')\nprint('CONFIG OK!!!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-27T14:22:57.082641Z","iopub.execute_input":"2026-02-27T14:22:57.082897Z","iopub.status.idle":"2026-02-27T14:22:57.119915Z","shell.execute_reply.started":"2026-02-27T14:22:57.082875Z","shell.execute_reply":"2026-02-27T14:22:57.119279Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def show_result(img, mask):\n    D,H,W = img.shape\n    # Decide which slices to plot\n    n_slices = 5 + 2\n    fig, axes = plt.subplots(2, n_slices, figsize=(3*n_slices, 6))\n\n    slices = np.round(np.linspace(5, D-5, 5)).astype(int)\n    for i, s in enumerate(slices):\n        axes[0, i].imshow(img[s], cmap='gray')\n        axes[0, i].set_title(f\"Slice {s}\")\n        axes[0, i].axis('off')\n\n        axes[1, i].imshow(mask[s], cmap='gray')\n        axes[1, i].set_title(f\"Mask {s}\")\n        axes[1, i].axis('off')\n\n    if 1:\n        i=i+1\n        s=H//2\n        axes[0, i].imshow(img[:,s], cmap='gray')\n        axes[0, i].set_title(f\"y={s}\")\n        axes[0, i].axis('off')\n\n        axes[1, i].imshow(mask[:,s], cmap='gray')\n        axes[1, i].set_title(f\"y={s}\")\n        axes[1, i].axis('off')\n\n    if 1:\n        i=i+1\n        s = W // 2\n        axes[0, i].imshow(img[:,:, s], cmap='gray')\n        axes[0, i].set_title(f\"x={s}\")\n        axes[0, i].axis('off')\n\n        axes[1, i].imshow(mask[:,:, s], cmap='gray')\n        axes[1, i].set_title(f\"x={s}\")\n        axes[1, i].axis('off')\n\n    plt.tight_layout()\n    plt.show()\n\n####################################################################################\n#load net\ndef load_net(Net, checkpoint):\n    net = Net()\n    net.output_type=['infer']\n    f = torch.load(checkpoint, map_location=lambda storage, loc: storage, weights_only=False)\n    state_dict = f['state_dict']\n    print(net.load_state_dict(state_dict, strict=False))\n    net.eval()\n    return net\n\ncheckpoint = '/kaggle/input/datasets/hengck23/hengck23-final-surface-det/unet-fold0-ema-auxloss-00085670.pth'\nunet3d1  = load_net(Unet3d, checkpoint).to(f'cuda:{0}')\nunet3d2  = load_net(Unet3d, checkpoint).to(f'cuda:{1}')\n\n\ncheckpoint = '/kaggle/input/datasets/hengck23/hengck23-final-surface-det/refinenet-fold0-00010920.pth'\nrefinenet1  = load_net(RefineNet, checkpoint).to(f'cuda:{0}')\nrefinenet2  = load_net(RefineNet, checkpoint).to(f'cuda:{1}')\n\n\ncheckpoint='/kaggle/input/datasets/hengck23/hengck23-final-surface-det/unetrpp-fold0-ema-auxloss-00106320.pth'\nunetrpp1  = load_net(UNetrPP, checkpoint).to(f'cuda:{0}')\nunetrpp2  = load_net(UNetrPP, checkpoint).to(f'cuda:{1}')\n\ndef do_batch_image(id, device_no):\n\n    unet3d  = [unet3d1, unet3d2][device_no]\n    unetrpp  = [unetrpp1, unetrpp2][device_no]\n    refinenet  = [refinenet1, refinenet2][device_no]\n\n    predict = []\n    for i in range(len(id)):\n        tif_file = f'{IMAGE_DIR}/{id[i]}.tif'\n        image = tifffile.imread(tif_file)\n\n        #prob = do_predict_tta7_unet3d(unet3d, image, CFG)\n        #ks = [1, 2, 3], axes = [0, 1, 2]\n        prob1 = do_predict_tta7_unet3d(unet3d, image, CFG)\n        prob2 = do_predict_tta7_unetrpp(unetrpp, image, CFG)\n        prob=(prob1+prob2)/2\n        \n        for r in range(2):\n            guess = (prob*255).astype(np.uint8)\n            prob = do_predict_tta7_refinenet(refinenet, image, guess, CFG)\n\n        #------\n        pred = do_postprocess_public(prob, T_low=0.25, T_high=0.45, dust_min_size=10_000)\n        pred = pred.astype(np.uint8)\n        predict.append(pred)\n        assert (image.shape == pred.shape)\n        # print(np.unique(pred)) #[0,1]\n\n        # if i==0:\n        #     show_result(image, pred)\n\n    torch.cuda.empty_cache()\n    return predict\n\n\n#-------------------------------------------------------------------------\n\ndef do_submit(\n    submit_file,\n    temp_dir\n):\n    os.makedirs(temp_dir, exist_ok=True)\n\n    print('saving to:')\n    print('\\tsubmit_file:', submit_file)\n    print('\\ttemp_dir:', temp_dir)\n\n    valid_id = valid_df['id'].tolist()\n    num_valid_id = len(valid_id)\n\n    start_timer = timer()\n    with zipfile.ZipFile(\n            submit_file, 'w', compression=zipfile.ZIP_DEFLATED\n    ) as z:\n\n        for i in range(0, num_valid_id,CFG['num_id_per_thread']*2):\n            B = min(i+CFG['num_id_per_thread']*2, num_valid_id)-i\n            id1 = valid_id[i:i+B//2]\n            id2 = valid_id[i+B//2:i+B]\n\n            # id1 = []\n            # id2 = valid_id[i:i+B]\n\n            if MULTI_THREAD:\n                with ThreadPoolExecutor(max_workers=2) as ex:\n                    f1 = ex.submit(do_batch_image, id1, 0)\n                    f2 = ex.submit(do_batch_image, id2, 1)\n                    pred1 = f1.result()\n                    pred2 = f2.result()\n            else:\n                    pred1 = do_batch_image(id1, device_no=0)\n                    pred2 = do_batch_image(id2, device_no=0)\n\n            pred = pred1+pred2\n            id = id1+id2\n\n            # ---- write directly into zip ----\n            for j in range(len(id)):\n                buf = io.BytesIO()\n                tifffile.imwrite(buf, pred[j])\n                buf.seek(0)\n                z.writestr(f'{id[j]}.tif', buf.read())\n\n            # ---------------------------------\n            time_taken = time_to_str(timer() - start_timer, 'sec')\n            print(f'[{i},{i+B}]: {len(id1)}|{len(id2)} {id[-1]:10d} : {time_taken}')\n\n    print('')\n    total_time_taken = timer() - start_timer\n    print(f'Total time for {num_valid_id}:', time_to_str(total_time_taken, 'sec'))\n    if num_valid_id==1:\n        total_time_taken = total_time_taken/2\n    print(f'Expected submit time for {320}:', time_to_str(total_time_taken*320/num_valid_id, 'min'))\n    print('')\n    return pred, id #return last for debug\n    \nprint('MODEL OK!!!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-27T14:22:57.121127Z","iopub.execute_input":"2026-02-27T14:22:57.121412Z","iopub.status.idle":"2026-02-27T14:23:02.095647Z","shell.execute_reply.started":"2026-02-27T14:22:57.121392Z","shell.execute_reply":"2026-02-27T14:23:02.094891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit_file = f'{OUT_DIR}/submission.zip'\ntemp_dir = f'{OUT_DIR}/temp'\n\ndo_submit(submit_file, temp_dir)\n\n#---check------------------------------------\nfor p in Path(OUT_DIR).rglob('*'):\n    print(p)\n\n\nwith zipfile.ZipFile(submit_file, \"r\") as z:\n    save_file = z.namelist()\nprint('valid_df:',len(valid_df))\nprint('save_file:',len(save_file))\n# for f in save_file:\n#     print(f)\n\n\nprint('\\nSUBMIT OK!!!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-27T14:23:02.097078Z","iopub.execute_input":"2026-02-27T14:23:02.097324Z","iopub.status.idle":"2026-02-27T14:25:33.974356Z","shell.execute_reply.started":"2026-02-27T14:23:02.097303Z","shell.execute_reply":"2026-02-27T14:25:33.973679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\ntta16\nTotal time for 1:  1 min 09 sec\nExpected submit time for 320:  6 hr 12 min\n\n\nsaving to:\n\tsubmit_file: /kaggle/working//submission.zip\n\ttemp_dir: /kaggle/working//temp\n[0,1]: 0|1    1407735 :  2 min 26 sec\n\nTotal time for 1:  2 min 26 sec\nExpected submit time for 320: 13 hr 00 min\n\n\n\n\n\n\n\nsaving to:\n\tsubmit_file: /kaggle/working//submission.zip\n\ttemp_dir: /kaggle/working//temp\n[0,6]: 3|3  722561671 :  5 min 15 sec\n[6,8]: 1|1  725727411 :  7 min 00 sec\n\nTotal time for 8:  7 min 00 sec\nExpected submit time for 320:  4 hr 40 min\n\n/kaggle/working/temp\n/kaggle/working/submission.zip\n/kaggle/working/.virtual_documents\n/kaggle/working/.virtual_documents/__notebook_source__.ipynb\nvalid_df: 8\nsave_file: 8\n\n---------------------------\n\n\nsaving to:\n\tsubmit_file: /kaggle/working//submission.zip\n\ttemp_dir: /kaggle/working//temp\n 0 3866421231 (384, 384, 384):  3 min 05 sec\n 0 3866421231 (384, 384, 384):  3 min 30 sec\n\n \nP100, T4 = 16/15 GB percard\n\n#resnet3d-unet\nvolume = np.random.randint(0,256,(192,320,320)) #  14.6gb\n#volume = np.random.randint(0,256,(160,320,320)) #  9.9gb\n#volume = np.random.randint(0,256,(160,160,160)) #  2.8gb\n#volume = np.random.randint(0,256,(256,256,256)) # 10.4gb\n\n\n1x T4 gpu:\n(max mem 13.7gb)\n 0    8862040 (320, 320, 320):  0 min 41 sec\n 1 3721853537 (320, 320, 320):  1 min 25 sec\n 2   70695797 (256, 256, 256):  1 min 44 sec\n 3 3866421231 (384, 384, 384):  3 min 05 sec\n\n\n2xT4 gpu:\n(max mem 14.5/13.7gb)\n 0    8862040 (320, 320, 320):  0 min 26 sec\n 1 3721853537 (320, 320, 320):  0 min 52 sec\n 2   70695797 (256, 256, 256):  1 min 03 sec\n 3 3866421231 (384, 384, 384):  1 min 48 sec\n'''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-27T14:25:33.975302Z","iopub.execute_input":"2026-02-27T14:25:33.975596Z","iopub.status.idle":"2026-02-27T14:25:33.981389Z","shell.execute_reply.started":"2026-02-27T14:25:33.975567Z","shell.execute_reply":"2026-02-27T14:25:33.980750Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i, row in valid_df.iterrows():\n    if i ==5: break\n    \n    id = row['id']\n    tif_file = f'{IMAGE_DIR}/{id}.tif'\n    image = tifffile.imread(tif_file)\n\n    with zipfile.ZipFile(submit_file, \"r\") as z:\n        with z.open(f'{id}.tif') as f:\n            predict = tifffile.imread(io.BytesIO(f.read()))\n\n    show_result(image, predict)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-27T14:25:33.982725Z","iopub.execute_input":"2026-02-27T14:25:33.982946Z","iopub.status.idle":"2026-02-27T14:25:35.316429Z","shell.execute_reply.started":"2026-02-27T14:25:33.982925Z","shell.execute_reply":"2026-02-27T14:25:35.315386Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}