{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":61446,"databundleVersionId":6962461,"sourceType":"competition"},{"sourceId":6988238,"sourceType":"datasetVersion","datasetId":4016367},{"sourceId":7047232,"sourceType":"datasetVersion","datasetId":4009913},{"sourceId":7047382,"sourceType":"datasetVersion","datasetId":4055366},{"sourceId":7070094,"sourceType":"datasetVersion","datasetId":4071363},{"sourceId":7073418,"sourceType":"datasetVersion","datasetId":4072639},{"sourceId":7205725,"sourceType":"datasetVersion","datasetId":4055374},{"sourceId":7224027,"sourceType":"datasetVersion","datasetId":4181456},{"sourceId":7258395,"sourceType":"datasetVersion","datasetId":4206264},{"sourceId":7304002,"sourceType":"datasetVersion","datasetId":4237589},{"sourceId":7311364,"sourceType":"datasetVersion","datasetId":4242527},{"sourceId":7315836,"sourceType":"datasetVersion","datasetId":4245320},{"sourceId":7321059,"sourceType":"datasetVersion","datasetId":4245329},{"sourceId":7350318,"sourceType":"datasetVersion","datasetId":4266569},{"sourceId":7354364,"sourceType":"datasetVersion","datasetId":4271235},{"sourceId":7354626,"sourceType":"datasetVersion","datasetId":4271108},{"sourceId":7359335,"sourceType":"datasetVersion","datasetId":4272204},{"sourceId":7367471,"sourceType":"datasetVersion","datasetId":4279640},{"sourceId":7373862,"sourceType":"datasetVersion","datasetId":4284491},{"sourceId":7373875,"sourceType":"datasetVersion","datasetId":4284499},{"sourceId":7374032,"sourceType":"datasetVersion","datasetId":4284599},{"sourceId":7380046,"sourceType":"datasetVersion","datasetId":4288797},{"sourceId":7381525,"sourceType":"datasetVersion","datasetId":4289816},{"sourceId":7390847,"sourceType":"datasetVersion","datasetId":4296436},{"sourceId":7434865,"sourceType":"datasetVersion","datasetId":4322047},{"sourceId":7460678,"sourceType":"datasetVersion","datasetId":4342221},{"sourceId":7495928,"sourceType":"datasetVersion","datasetId":4364681},{"sourceId":7612672,"sourceType":"datasetVersion","datasetId":4433048},{"sourceId":150248402,"sourceType":"kernelVersion"},{"sourceId":150386064,"sourceType":"kernelVersion"}],"dockerImageVersionId":30580,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Prepare","metadata":{}},{"cell_type":"code","source":"!pip install -q --no-index --find-links /kaggle/input/triton-wheel /kaggle/input/triton-wheel/torch-2.1.1+cu118-cp310-cp310-linux_x86_64.whl\n!pip install -q /kaggle/input/connected-components-3d/connected_components_3d-3.12.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install -q /kaggle/input/pytorch-image-models-0-9-10/timm-0.9.10-py3-none-any.whl\n!pip install -q /kaggle/input/pip-download-for-segmentation-models-pytorch/munch-4.0.0-py2.py3-none-any.whl\n!pip install -q /kaggle/input/pip-download-for-segmentation-models-pytorch/pretrainedmodels-0.7.4.tar.gz\n!pip install -q /kaggle/input/pip-download-for-segmentation-models-pytorch/efficientnet_pytorch-0.7.1.tar.gz\n!pip install -q /kaggle/input/einops-0-7-0-wheel/einops-0.7.0-py3-none-any.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-05T12:07:29.628046Z","iopub.execute_input":"2024-02-05T12:07:29.628394Z","iopub.status.idle":"2024-02-05T12:12:28.85038Z","shell.execute_reply.started":"2024-02-05T12:07:29.628362Z","shell.execute_reply":"2024-02-05T12:12:28.849243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile /opt/conda/lib/python3.10/site-packages/triton/common/build.py\n# https://github.com/openai/triton/issues/2507\n\nimport contextlib\nimport functools\nimport io\nimport os\nimport shutil\nimport subprocess\nimport sys\nimport sysconfig\n\nimport setuptools\n\n\n# TODO: is_hip shouldn't be here\ndef is_hip():\n    import torch\n    return torch.version.hip is not None\n\n\n@functools.lru_cache()\ndef libcuda_dirs():\n    libs = subprocess.check_output([\"ldconfig\", \"-p\"]).decode()\n    # each line looks like the following:\n    # libcuda.so.1 (libc6,x86-64) => /lib/x86_64-linux-gnu/libcuda.so.1\n    locs = [line.split()[-1] for line in libs.splitlines() if \"libcuda.so\" in line]\n    dirs = [os.path.dirname(loc) for loc in locs]\n    env_ld_library_path = os.getenv(\"LD_LIBRARY_PATH\")\n    if env_ld_library_path and not dirs:\n        dirs = [dir for dir in env_ld_library_path.split(\":\") if os.path.exists(os.path.join(dir, \"libcuda.so\"))]\n    msg = 'libcuda.so cannot found!\\n'\n    if locs:\n        msg += 'Possible files are located at %s.' % str(locs)\n        msg += 'Please create a symlink of libcuda.so to any of the file.'\n    assert any(os.path.exists(os.path.join(path, 'libcuda.so')) for path in dirs), msg\n    return dirs\n\n\n@functools.lru_cache()\ndef rocm_path_dir():\n    return os.getenv(\"ROCM_PATH\", default=\"/opt/rocm\")\n\n\n@contextlib.contextmanager\ndef quiet():\n    old_stdout, old_stderr = sys.stdout, sys.stderr\n    sys.stdout, sys.stderr = io.StringIO(), io.StringIO()\n    try:\n        yield\n    finally:\n        sys.stdout, sys.stderr = old_stdout, old_stderr\n\n\n@functools.lru_cache()\ndef cuda_include_dir():\n    base_dir = os.path.join(os.path.dirname(__file__), os.path.pardir)\n    cuda_path = os.path.join(base_dir, \"third_party\", \"cuda\")\n    return os.path.join(cuda_path, \"include\")\n\n\ndef _build(name, src, srcdir):\n    if is_hip():\n        hip_lib_dir = os.path.join(rocm_path_dir(), \"lib\")\n        hip_include_dir = os.path.join(rocm_path_dir(), \"include\")\n    else:\n        cuda_lib_dirs = libcuda_dirs()\n        cu_include_dir = cuda_include_dir()\n    suffix = sysconfig.get_config_var('EXT_SUFFIX')\n    so = os.path.join(srcdir, '{name}{suffix}'.format(name=name, suffix=suffix))\n    # try to avoid setuptools if possible\n    cc = os.environ.get(\"CC\")\n    if cc is None:\n        # TODO: support more things here.\n        clang = shutil.which(\"clang\")\n        gcc = shutil.which(\"gcc\")\n        cc = gcc if gcc is not None else clang\n        if cc is None:\n            raise RuntimeError(\"Failed to find C compiler. Please specify via CC environment variable.\")\n    # This function was renamed and made public in Python 3.10\n    if hasattr(sysconfig, 'get_default_scheme'):\n        scheme = sysconfig.get_default_scheme()\n    else:\n        scheme = sysconfig._get_default_scheme()\n    # 'posix_local' is a custom scheme on Debian. However, starting Python 3.10, the default install\n    # path changes to include 'local'. This change is required to use triton with system-wide python.\n    if scheme == 'posix_local':\n        scheme = 'posix_prefix'\n    py_include_dir = sysconfig.get_paths(scheme=scheme)[\"include\"]\n\n    if is_hip():\n        ret = subprocess.check_call([cc, src, f\"-I{hip_include_dir}\", f\"-I{py_include_dir}\", f\"-I{srcdir}\", \"-shared\", \"-fPIC\", f\"-L{hip_lib_dir}\", \"-lamdhip64\", \"-o\", so])\n    else:\n        cc_cmd = [cc, src, \"-O3\", f\"-I{cu_include_dir}\", f\"-I{py_include_dir}\", f\"-I{srcdir}\", \"-shared\", \"-fPIC\", \"-lcuda\", \"-o\", so]\n        cc_cmd += [f\"-L{dir}\" for dir in cuda_lib_dirs]\n        ret = subprocess.check_call(cc_cmd)\n\n    if ret == 0:\n        return so\n    # fallback on setuptools\n    extra_compile_args = []\n    library_dirs = cuda_lib_dirs\n    include_dirs = [srcdir, cu_include_dir]\n    libraries = ['cuda']\n    # extra arguments\n    extra_link_args = []\n    # create extension module\n    ext = setuptools.Extension(\n        name=name,\n        language='c',\n        sources=[src],\n        include_dirs=include_dirs,\n        extra_compile_args=extra_compile_args + ['-O3'],\n        extra_link_args=extra_link_args,\n        library_dirs=library_dirs,\n        libraries=libraries,\n    )\n    # build extension module\n    args = ['build_ext']\n    args.append('--build-temp=' + srcdir)\n    args.append('--build-lib=' + srcdir)\n    args.append('-q')\n    args = dict(\n        name=name,\n        ext_modules=[ext],\n        script_args=args,\n    )\n    with quiet():\n        setuptools.setup(**args)\n    return so","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-05T12:12:28.852963Z","iopub.execute_input":"2024-02-05T12:12:28.853285Z","iopub.status.idle":"2024-02-05T12:12:28.862819Z","shell.execute_reply.started":"2024-02-05T12:12:28.85325Z","shell.execute_reply":"2024-02-05T12:12:28.861915Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"%%writefile inference.sh\n#!/bin/bash \n\nSEED=42\nBACKBONE=convnext_tiny\nCKPT_PATH=\"/kaggle/input/hoa-unet-convnext-custom-loss/convnext_tiny_1536_customloss_e20.pth|/kaggle/input/hoa-unet-convnext-custom-loss/convnext_tiny_1536_customloss_3drot_e30.pth.pth\"\nIN_CHANNELS=3\nNUM_CLASSES=3\nIMAGE_SIZE=3072\nBATCH_SIZE=2\nTHRESHOLD=0.4\nAXIS=\"z|y|x\"\nFLIP=5\nROT=3\n\nfor group in kidney_6 kidney_5; do\n    python inference.py \\\n    --seed $SEED \\\n    --group $group \\\n    --backbone $BACKBONE \\\n    --ckpt_path $CKPT_PATH \\\n    --in_channels $IN_CHANNELS \\\n    --num_classes $NUM_CLASSES \\\n    --image_size $IMAGE_SIZE \\\n    --batch_size $BATCH_SIZE \\\n    --axis $AXIS \\\n    --flip $FLIP \\\n    --rot $ROT \\\n    --overlap \\\n    --threshold $THRESHOLD\ndone","metadata":{"execution":{"iopub.status.busy":"2024-02-05T12:12:28.864032Z","iopub.execute_input":"2024-02-05T12:12:28.864408Z","iopub.status.idle":"2024-02-05T12:12:28.878957Z","shell.execute_reply.started":"2024-02-05T12:12:28.864377Z","shell.execute_reply":"2024-02-05T12:12:28.87815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile inference.py\n\nimport os\nimport sys\nimport cv2\nimport cc3d\nimport timm\nimport shutil\nimport argparse\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom tqdm import tqdm\nimport albumentations as A\nfrom functools import partial\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.distributed as dist\nimport torch.multiprocessing as mp\nimport torchvision.transforms as T\nfrom torch.utils.data import Dataset, DataLoader\n\nsys.path.append(\"/kaggle/input/segmentation-models-pytorch-extra-stem-2-5d\")\nimport segmentation_models_pytorch as smp\n\n############################################### helper functions ##################################################\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    rle = ' '.join(str(x) for x in runs)\n    if rle=='':\n        rle = '1 0'\n    return rle\n\ndef is_dist_avail_and_initialized():\n    if not dist.is_available():\n        return False\n    if not dist.is_initialized():\n        return False\n    return True\n\n\ndef get_world_size():\n    if not is_dist_avail_and_initialized():\n        return 1\n    return dist.get_world_size()\n\n\ndef get_rank():\n    if not is_dist_avail_and_initialized():\n        return 0\n    return dist.get_rank()\n\n\ndef is_main_process():\n    return get_rank() == 0\n\n\ndef build_model(backbone, in_channels, num_classes):\n    model = smp.Unet(\n        encoder_name=backbone,\n        encoder_weights=None,\n        encoder_args={\"in_channels\": in_channels},\n        decoder_norm_type=\"GN\",\n        decoder_act_type=\"GeLU\",\n        decoder_upsample_method=\"nearest\",\n        in_channels=in_channels,\n        classes=num_classes,\n        activation=None,\n    )\n    return model\n\n\ndef filter_checkpoint(state_dict):\n    new_state_dict = {}\n    for k, v in state_dict.items():\n        new_state_dict[k.replace(\"module.\", \"\")] = v\n    return new_state_dict\n\n\ndef load_model(backbone, in_channels, num_classes, path):\n    model = build_model(backbone, in_channels, num_classes)\n    state_dict = torch.load(path, map_location=\"cpu\")\n    state_dict = filter_checkpoint(state_dict)\n    model.load_state_dict(state_dict)\n    model.eval()\n    return model\n\n\nclass Ensemble(object):\n    def __init__(self, backbone, in_channels, num_classes, ckpts, device):\n        self.models = []\n        for ckpt_path in ckpts:\n            model = load_model(backbone, in_channels, num_classes, ckpt_path).to(device)\n            model = torch.compile(model)\n            self.models.append(model)\n            \n    def __call__(self, x):\n        out = None\n        for model in self.models:\n            if out is None:\n                out = model(x).sigmoid()\n            else:\n                out += model(x).sigmoid()\n        out /= len(self.models)\n        return out\n\n\nclass InferenceDataset(torch.utils.data.Dataset):\n    \n    axis2dim = {\"z\": 0, \"y\": 1, \"x\": 2}\n    \n    def __init__(self, volume_path, volume_shape, local_rank, world_size, in_channels=3, image_size=512, axis=\"z\"):\n        self.volume_path = volume_path\n        self.volume_shape = volume_shape\n        self.axis = axis\n        self.in_channels = in_channels\n        self.image_size = image_size\n        block = self.volume_shape[self.axis2dim[self.axis]] // world_size\n        if local_rank < world_size-1:\n            self.indexs = range(self.volume_shape[self.axis2dim[self.axis]])[local_rank*block:(local_rank+1)*block]\n        else:\n            self.indexs = range(self.volume_shape[self.axis2dim[self.axis]])[local_rank*block:]\n        \n    def __len__(self):\n        return len(self.indexs)\n    \n    def load_image(self, idx):\n        idx = self.indexs[idx]\n        volume = np.memmap(self.volume_path, shape=self.volume_shape, dtype=np.uint16, mode=\"r\")\n        idxs = np.clip(range(idx-self.in_channels//2, idx+self.in_channels//2+1), 0, self.volume_shape[self.axis2dim[self.axis]]-1)\n        if self.axis == \"z\":\n            image =  volume[idxs].transpose(1, 2, 0)\n        elif self.axis == \"x\":\n            image =  volume[:, :, idxs]\n        else:\n            image =  volume[:, idxs, :].transpose(0, 2, 1)\n        image = image.astype(np.float32)\n        image = image / 65535.0\n        return image\n    \n    def __getitem__(self, idx):\n        image = self.load_image(idx)\n        orig_size = image.shape\n        area = self.image_size**2\n        orig_area = orig_size[0]*orig_size[1]\n        scale = np.sqrt(area/orig_area)\n        new_h = int(orig_size[0]*scale) if int(orig_size[0]*scale) % 32 == 0 else int(orig_size[0]*scale) - (int(orig_size[0]*scale)%32) + 32\n        new_w = int(orig_size[1]*scale) if int(orig_size[1]*scale) % 32 == 0 else int(orig_size[1]*scale) - (int(orig_size[1]*scale)%32) + 32\n        # LANCZOS4 is slighter better than bilinear and bicubic\n        image = cv2.resize(image, (new_w, new_h), cv2.INTER_LANCZOS4)\n        image = torch.tensor(np.transpose(image, (2, 0, 1)))\n        return self.indexs[idx], image, torch.tensor(np.array([orig_size[0], orig_size[1]]))\n    \n\n############################################### main ##################################################\n\ndef main_worker(rank, args, queue):\n    \n    torch.backends.cudnn_benchmark = True\n    \n    # set device\n    device = torch.device(f\"cuda:{rank}\")\n    \n    # meta info\n    volume_shape = args.volume_shape\n    volume_path = args.volume_path\n        \n    # build model\n    model = Ensemble(args.backbone, args.in_channels, args.num_classes, args.ckpt_path, device)\n    \n    # inference \n    with torch.no_grad():\n        for size in args.image_size:\n            for axis in args.axis:\n                test_dataset = InferenceDataset(volume_path, volume_shape, rank, args.num_processes, args.in_channels, image_size=size, axis=axis)\n                test_loader = DataLoader(test_dataset, batch_size=args.batch_size, num_workers=4, pin_memory=True)\n                max_len = test_dataset.volume_shape[test_dataset.axis2dim[axis]]\n                pbar = tqdm(enumerate(test_loader), total=len(test_loader), desc=f'Inference {args.group} {axis}', ncols=150)\n                for step, (idx, images, shapes) in pbar:\n                    shape = shapes[0].numpy()\n                    idx = idx.numpy()\n                    images = images.to(device, non_blocking=True)\n                    bsz = images.size(0)\n                    batch_pred_mask = torch.zeros(bsz, args.num_classes, shape[0], shape[1]).to(device)\n                    for aug, flip in zip([torch.flip]*len(args.flip)+[partial(torch.rot90, dims=[2, 3])]*len(args.rot), args.flip+args.rot):\n                        with torch.cuda.amp.autocast(enabled=True):\n                            preds = model(aug(images, flip))\n                            flip = -flip if not isinstance(flip, list) else flip\n                            preds = F.interpolate(aug(preds, flip).float(), (int(shape[0]), int(shape[1])), mode='bicubic')\n                        batch_pred_mask += preds\n                    \n                    batch_pred_mask /= len(args.axis) * (len(args.flip)+len(args.rot)) * len(args.image_size)\n                    if args.overlap:\n                        batch_pred_mask /= args.num_classes\n                    masks = batch_pred_mask.to(torch.float16).cpu().numpy()\n                    queue.put((axis, idx, masks, max_len))\n                    pbar.set_postfix(shape=images.shape)\n                        \n                        \ndef write_worker(args, queue, write_lock):\n    while True:\n        axis, idx, masks, max_len = queue.get()\n        if idx is None:\n            break\n        with write_lock:\n            pred_masks = np.memmap(args.mask_path, shape=args.volume_shape, dtype=np.float16, mode=\"r+\")\n            if args.overlap:\n                for i in range(args.num_classes):\n                    mask = masks[:, i, ...]\n                    offset = i - args.num_classes // 2\n                    idxs = np.clip(idx+offset, 0, max_len-1)\n                    if axis == \"z\":\n                        pred_masks[idxs, :, :] += mask\n                    elif axis == \"y\":\n                        pred_masks[:, idxs, :] += mask.transpose(1, 0, 2)\n                    else:\n                        pred_masks[:, :, idxs] += mask.transpose(1, 2, 0)\n            else:\n                mask = masks[:, args.num_classes//2, ...]\n                idxs = np.clip(idx, 0, max_len-1)\n                if axis == \"z\":\n                    pred_masks[idxs, :, :] += mask\n                elif axis == \"y\":\n                    pred_masks[:, idxs, :] += mask.transpose(1, 0, 2)\n                else:\n                    pred_masks[:, :, idxs] += mask.transpose(1, 2, 0)\n            pred_masks.flush()\n            del pred_masks, masks, axis, idx, max_len\n                        \nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    \n    parser.add_argument(\"--seed\", type=int, default=42)\n    parser.add_argument(\"--group\", type=str, default=\"kidney_5\")\n    parser.add_argument(\"--backbone\", type=str, default=\"convnext_tiny\")\n    parser.add_argument(\"--in_channels\", type=int, default=3)\n    parser.add_argument(\"--num_classes\", type=int, default=3)\n    parser.add_argument(\"--ckpt_path\", type=str, default=\"\")\n    parser.add_argument(\"--batch_size\", type=int, default=3)\n    parser.add_argument(\"--image_size\", type=int, default=2560)\n    parser.add_argument(\"--axis\", type=str, default=\"z|y|x\")\n    parser.add_argument(\"--flip\", type=int, default=3)\n    parser.add_argument(\"--rot\", type=int, default=3)\n    parser.add_argument(\"--overlap\", action=\"store_true\", default=False)\n    parser.add_argument(\"--threshold\", type=float, default=0.5)\n    \n    args = parser.parse_args()\n    args.image_size = [args.image_size]\n    args.axis = args.axis.split(\"|\")\n    args.flip = [[], [1], [2], [3], [2,3]][:args.flip]\n    args.rot = [1, 2, 3][:args.rot]\n    args.ckpt_path = args.ckpt_path.split(\"|\")\n    args.num_processes = torch.cuda.device_count()\n    \n    ls_images = sorted(glob(os.path.join(\"/kaggle/input/blood-vessel-segmentation\", \"test\", args.group, \"*\", \"*.tif\")))\n    h, w = cv2.imread(ls_images[-1], cv2.IMREAD_UNCHANGED).shape\n    volume_shape = (len(ls_images), h, w)\n    volume_path = f\"/dev/shm/{args.group}.mmap\"\n    mask_path = f\"/dev/shm/{args.group}_mask.mmap\"\n    if not os.path.exists(volume_path):\n        volume = np.memmap(volume_path, shape=volume_shape, dtype=np.uint16, mode=\"w+\")\n        for i, path in enumerate(tqdm(ls_images, total=len(ls_images), desc=f\"Caching {args.group} images\")):\n            image = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n            volume[i] = image\n        volume.flush()\n        del volume\n    if not os.path.exists(mask_path):\n        mask = np.memmap(mask_path, shape=volume_shape, dtype=np.float16, mode=\"w+\")\n        mask.fill(0.0)\n        mask.flush()\n        del mask\n    \n    args.volume_shape = volume_shape\n    args.volume_path = volume_path\n    args.mask_path = mask_path\n    \n    # inference\n    queue = mp.Queue()\n    write_lock = mp.Lock()\n    inference_processes = []\n    write_processes = []\n    for rank in range(args.num_processes):\n        p = mp.Process(target=main_worker, args=(rank, args, queue))\n        p.start()\n        inference_processes.append(p)\n    for rank in range(args.num_processes*2):\n        p = mp.Process(target=write_worker, args=(args, queue, write_lock))\n        p.start()\n        write_processes.append(p)\n    for p in inference_processes:\n        p.join()\n    for _ in range(args.num_processes*2):\n        queue.put((None, None, None, None))\n    for p in write_processes:\n        p.join()\n        \n    # write to csv\n    rles, ids = [], []\n    pred_masks = np.memmap(args.mask_path, shape=args.volume_shape, dtype=np.float16, mode=\"r\")\n    for i in tqdm(range(len(ls_images)), total=len(ls_images)):\n        pred_mask = pred_masks[i, :, :]\n        pred_mask = (pred_mask > args.threshold).astype(np.uint8)\n        rle = rle_encode(pred_mask)\n        path = ls_images[i].split(os.path.sep)\n        dataset = path[-3]\n        slice_id, _ = os.path.splitext(path[-1])\n        rles.append(rle)\n        ids.append(f\"{dataset}_{slice_id}\")\n        \n    df = pd.DataFrame.from_dict({\n        \"id\": ids,\n        \"rle\": rles\n    })\n    df.to_csv(f\"{args.group}.csv\", index=False)\n    del pred_masks\n    \n    # clean up memmap files\n    os.remove(args.volume_path)\n    os.remove(args.mask_path)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T12:12:28.881185Z","iopub.execute_input":"2024-02-05T12:12:28.88147Z","iopub.status.idle":"2024-02-05T12:12:28.899756Z","shell.execute_reply.started":"2024-02-05T12:12:28.881434Z","shell.execute_reply":"2024-02-05T12:12:28.898841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom glob import glob\n\ndebug = False\n\nif len(glob(\"/kaggle/input/blood-vessel-segmentation/test/kidney_5/images/*.tif\")) == 3 and not debug:\n    ids = [f\"kidney_5_{i:04d}\" for i in range(3)] + [f\"kidney_6_{i:04d}\" for i in range(3)]\n    rles = [\"1 0\"] * 6\n    submission = pd.DataFrame.from_dict({\n        \"id\": ids,\n        \"rle\": rles\n    })\nelse:\n    !bash inference.sh\n    kidney_5 = pd.read_csv(\"kidney_5.csv\")\n    kidney_6 = pd.read_csv(\"kidney_6.csv\")\n    submission = pd.concat([kidney_5, kidney_6]).reset_index(drop=True)\n    \nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-05T12:13:32.108398Z","iopub.execute_input":"2024-02-05T12:13:32.109266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2024-02-05T12:12:29.282852Z","iopub.execute_input":"2024-02-05T12:12:29.283154Z","iopub.status.idle":"2024-02-05T12:12:29.300693Z","shell.execute_reply.started":"2024-02-05T12:12:29.283128Z","shell.execute_reply":"2024-02-05T12:12:29.299765Z"},"trusted":true},"execution_count":null,"outputs":[]}]}