{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":61446,"databundleVersionId":6962461,"sourceType":"competition"},{"sourceId":1760030,"sourceType":"datasetVersion","datasetId":1046169},{"sourceId":6089786,"sourceType":"datasetVersion","datasetId":3487378},{"sourceId":7523965,"sourceType":"datasetVersion","datasetId":4200177},{"sourceId":7570777,"sourceType":"datasetVersion","datasetId":4232099},{"sourceId":135925962,"sourceType":"kernelVersion"}],"dockerImageVersionId":30512,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"DEBUG = False","metadata":{"execution":{"iopub.status.busy":"2024-02-06T12:01:54.409134Z","iopub.execute_input":"2024-02-06T12:01:54.409585Z","iopub.status.idle":"2024-02-06T12:01:54.419376Z","shell.execute_reply.started":"2024-02-06T12:01:54.409559Z","shell.execute_reply":"2024-02-06T12:01:54.418513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Initialization","metadata":{}},{"cell_type":"markdown","source":"### Packages","metadata":{}},{"cell_type":"code","source":"!cp -r /kaggle/input/sennet-code/src ./","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-06T12:01:54.420691Z","iopub.execute_input":"2024-02-06T12:01:54.420971Z","iopub.status.idle":"2024-02-06T12:01:55.626810Z","shell.execute_reply.started":"2024-02-06T12:01:54.420948Z","shell.execute_reply":"2024-02-06T12:01:55.625520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n\nsys.path.append('src/')\nsys.path.append('src/inference/')\nsys.path.append(\n    '/kaggle/input/timm-smp/pytorch-image-models-main/pytorch-image-models-main'\n)\nsys.path.append(\n    \"/kaggle/input/timm-smp/segmentation_models.pytorch-master/segmentation_models.pytorch-master\"\n)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-06T12:01:55.629535Z","iopub.execute_input":"2024-02-06T12:01:55.629994Z","iopub.status.idle":"2024-02-06T12:01:55.635865Z","shell.execute_reply.started":"2024-02-06T12:01:55.629951Z","shell.execute_reply":"2024-02-06T12:01:55.634956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index --find-links /kaggle/input/contrails-wheels/ pretrainedmodels==0.7.4\n!pip install --no-index --find-links /kaggle/input/contrails-wheels/ efficientnet_pytorch==0.7.1\n!pip install --no-index /kaggle/input/einops-030/einops-0.3.0-py2.py3-none-any.whl","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-06T12:01:55.637168Z","iopub.execute_input":"2024-02-06T12:01:55.637529Z","iopub.status.idle":"2024-02-06T12:02:32.768909Z","shell.execute_reply.started":"2024-02-06T12:01:55.637500Z","shell.execute_reply":"2024-02-06T12:02:32.767889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Imports","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport sys\nimport glob\nimport json\nimport torch\nimport joblib\nimport operator\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom PIL import Image\nfrom tqdm.notebook import tqdm","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-06T12:02:32.772196Z","iopub.execute_input":"2024-02-06T12:02:32.772599Z","iopub.status.idle":"2024-02-06T12:02:36.254040Z","shell.execute_reply.started":"2024-02-06T12:02:32.772556Z","shell.execute_reply":"2024-02-06T12:02:36.252842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from params import *\n\nfrom data.dataset import KidneyDatasetInf\nfrom data.transforms import get_transfos\n\nfrom util.metrics import compute_surface_dice_score\nfrom util.torch import load_model_weights\n\nfrom predict import predict, predict_tta\nfrom model_zoo.models import define_model\n# from inference_main import Config","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-06T12:02:36.255514Z","iopub.execute_input":"2024-02-06T12:02:36.256134Z","iopub.status.idle":"2024-02-06T12:02:50.901990Z","shell.execute_reply.started":"2024-02-06T12:02:36.256096Z","shell.execute_reply":"2024-02-06T12:02:50.901209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Utils","metadata":{}},{"cell_type":"code","source":"def norm_by_percentile(volume, low=10, high=99.8):\n    xmin = np.percentile(volume, low)\n    xmax = np.percentile(volume, high)\n\n#     print(xmin, xmax)\n\n    imgs = []\n    batches = np.array_split(np.arange(len(volume)), 10)\n\n    for batch in tqdm(batches):\n        x = (volume[batch].astype(np.float32) - xmin) / (xmax - xmin)\n        x = np.clip(x, 0, 1)\n        imgs.append((x * 255).astype(np.uint8))\n        \n    del (volume, x)\n    gc.collect()\n\n    return np.concatenate(imgs, 0)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-06T12:02:50.903416Z","iopub.execute_input":"2024-02-06T12:02:50.904152Z","iopub.status.idle":"2024-02-06T12:02:50.911047Z","shell.execute_reply.started":"2024-02-06T12:02:50.904118Z","shell.execute_reply":"2024-02-06T12:02:50.910077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_encode(mask):\n    pixel = mask.flatten()\n    pixel = np.concatenate([[0], pixel, [0]])\n    run = np.where(pixel[1:] != pixel[:-1])[0] + 1\n    run[1::2] -= run[::2]\n    rle = ' '.join(str(r) for r in run)\n    if rle == '':\n        rle = '1 0'\n    return rle","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-06T12:02:50.912453Z","iopub.execute_input":"2024-02-06T12:02:50.912846Z","iopub.status.idle":"2024-02-06T12:02:50.921891Z","shell.execute_reply.started":"2024-02-06T12:02:50.912799Z","shell.execute_reply":"2024-02-06T12:02:50.920817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    \"\"\"\n    Placeholder to load a config from a saved json\n    \"\"\"\n    def __init__(self, dic):\n        for k, v in dic.items():\n            setattr(self, k, v)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-06T12:02:50.922909Z","iopub.execute_input":"2024-02-06T12:02:50.923180Z","iopub.status.idle":"2024-02-06T12:02:50.932656Z","shell.execute_reply.started":"2024-02-06T12:02:50.923148Z","shell.execute_reply":"2024-02-06T12:02:50.931893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_components(preds):\n    n_comps, comps, boxes = cv2.connectedComponentsWithStats(preds)[:3]\n    if n_comps < 256:\n        comps = comps.astype(np.uint8)\n    else:\n        assert n_comps < 2 ** 16\n        comps = comps.astype(np.uint16)\n    return n_comps, comps, boxes\n\n\ndef post_process(preds_pp, preds_other, axis=\"x\", comps_stats=None, offset=20, conf_th=0.5):\n    if axis == \"x\":\n        n = preds_pp.shape[0]\n    elif axis == \"y\":\n        n = preds_pp.shape[1]\n    else:\n        n = preds_pp.shape[2]\n\n    if comps_stats is None:\n        comps_stats = joblib.Parallel(n_jobs=2)(\n            joblib.delayed(get_components)(p) for p in tqdm(preds_pp)\n        )\n        \n    modified = {}\n    for i in tqdm(range(n)):\n        n_comps, comps, boxes = comps_stats[i]\n\n        pped = False\n        for j in range(1, n_comps):\n            bbox = boxes[j]\n            if (bbox[2] > 50 or bbox[3] > 50):\n\n                y0, y1, x0, x1 = bbox[0], bbox[0] + bbox[2], bbox[1], bbox[1] + bbox[3]\n                crop_cromps = comps[x0:x1, y0:y1]\n\n                if axis == \"x\":\n                    other = (\n                        preds_other[0][i][x0:x1, y0:y1] + preds_other[1][i][x0:x1, y0:y1]\n                    ).cpu().numpy()[crop_cromps == j]\n                elif axis == \"y\":\n                    other = (\n                        preds_other[0][:, i][x0:x1, y0:y1] + preds_other[1][:, i][x0:x1, y0:y1]\n                    ).cpu().numpy()[crop_cromps == j]\n                else:\n                    other = (\n                        preds_other[0][:, :, i][x0:x1, y0:y1] + preds_other[1][:, :, i][x0:x1, y0:y1]\n                    ).cpu().numpy()[crop_cromps == j]\n                \n                other_score = np.mean(other) / 2\n\n                if other_score < conf_th:\n                    for d in range(-offset, offset + 1):\n                        idx = np.clip(i + d, 0, n - 1)\n                        \n                        to_modif = modified.get(idx, None)\n                        if to_modif is None:\n                            if axis == \"x\":\n                                to_modif = preds_pp[idx].cpu().numpy()\n                            elif axis == \"y\":\n                                to_modif = preds_pp[:, idx].cpu().numpy()\n                            else:\n                                to_modif = preds_pp[:, :, idx].cpu().numpy()\n\n                        to_modif[x0:x1, y0:y1][crop_cromps == j] = 0\n\n                        # Find overalpping comps and set to 0\n                        comps_crop_d = comps_stats[idx][1][x0:x1, y0:y1]\n\n                        vs = np.unique(comps_crop_d[crop_cromps == j])\n                        to_remove = []\n                        for v in vs:\n                            if v > 0:\n                                iou = ((crop_cromps == j) * (comps_crop_d == v)).sum() # / ((crop_cromps == j).sum() + (comps_crop_d == v).sum())\n                                if iou > 0.:\n                                    to_remove.append(v)\n\n                        m = 10\n                        comps_crop_d = comps_stats[idx][1][x0 - m: x1 + m, y0 - m: y1 + m]  # extend\n                        for v in to_remove:\n                            to_modif [x0 - m: x1 + m, y0 - m: y1 + m][comps_crop_d == v] = 0\n                    \n                        modified[idx] = to_modif\n    return modified","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-06T12:02:50.933867Z","iopub.execute_input":"2024-02-06T12:02:50.934135Z","iopub.status.idle":"2024-02-06T12:02:50.954022Z","shell.execute_reply.started":"2024-02-06T12:02:50.934112Z","shell.execute_reply":"2024-02-06T12:02:50.953186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport numpy as np\nfrom tqdm import tqdm\nfrom torch.utils.data import DataLoader\n\ndef predict_multi(models, dataset, loss_config, batch_size=64, device=\"cuda\", use_fp16=False, num_workers=8, tile_size=None):\n    \"\"\"\n    Perform inference using a single model and generate predictions for the given dataset.\n\n    Args:\n        model (torch.nn.Module): Trained model for inference.\n        dataset (torch.utils.data.Dataset): Dataset for which to generate predictions.\n        loss_config (dict): Configuration for loss function and activation.\n        batch_size (int, optional): Batch size for prediction. Defaults to 64.\n        device (str, optional): Device for inference, 'cuda' or 'cpu'. Defaults to 'cuda'.\n        use_fp16 (bool, optional): Whether to use mixed-precision (FP16) inference. Defaults to False.\n        num_workers (int, optional): Number of worker threads for data loading. Defaults to 8.\n\n    Returns:\n        np array [N x C]: Predicted probabilities for each class for each sample.\n        list: Empty list, placeholder for the auxiliary task.\n    \"\"\"\n    preds = []\n\n    loader = DataLoader(\n        dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers\n    )\n\n    with torch.no_grad():\n        for img, _, _ in tqdm(loader):\n            with torch.cuda.amp.autocast(enabled=use_fp16):\n                pred_models = []\n                for model in models:\n                    pred, _ = model(img.cuda())\n\n                    # Get probabilities\n                    if loss_config[\"activation\"] == \"sigmoid\":\n                        pred = pred.sigmoid()\n                    elif loss_config[\"activation\"] == \"softmax\":\n                        pred = pred.softmax(1)\n                    pred_models.append(pred)\n#                     print(pred.shape)\n            pred_models = torch.stack(pred_models, 0).mean(0)\n#             print(pred_models.shape)\n            preds.append(pred_models.detach().cpu().numpy())\n\n    return np.concatenate(preds), []","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-06T12:12:18.283504Z","iopub.execute_input":"2024-02-06T12:12:18.284251Z","iopub.status.idle":"2024-02-06T12:12:18.294013Z","shell.execute_reply.started":"2024-02-06T12:12:18.284215Z","shell.execute_reply":"2024-02-06T12:12:18.293011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data","metadata":{}},{"cell_type":"code","source":"DATA_PATH = \"/kaggle/input/blood-vessel-segmentation/\"\n\nif DEBUG:\n    FOLDERS = [\n        DATA_PATH + \"train/kidney_1_dense/\",\n    ]\n    FOLDS = [\"1\"]\n\nelse:\n    FOLDERS = [\n        DATA_PATH + \"test/kidney_5/\",\n        DATA_PATH + \"test/kidney_6/\",\n    ]\n    FOLDS = [\"fullfit_0\"]\n#     FOLDS = [\"0\"]","metadata":{"execution":{"iopub.status.busy":"2024-02-06T12:12:21.880761Z","iopub.execute_input":"2024-02-06T12:12:21.881468Z","iopub.status.idle":"2024-02-06T12:12:21.886320Z","shell.execute_reply.started":"2024-02-06T12:12:21.881438Z","shell.execute_reply":"2024-02-06T12:12:21.885413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Models","metadata":{}},{"cell_type":"code","source":"EXP_FOLDERS = {\n    \"Unet_maxvit\":\n    [\n#         \"/kaggle/input/sennet-weights-1/2024-01-07_12/\",  # less noise k2 more ep   0.919 / 0.853  - LB 0.825\n        \"/kaggle/input/sennet-weights-1/2024-02-04_10/\",  # PL 0.5 30ep 0.919 / 0.848  -- 0.918 / 0.816   - LB 0.835\n        \"/kaggle/input/sennet-weights-1/2024-02-04_11/\",  # PL 0.5 40ep 0.919 / 0.842  -- 0.919 / 0.818\n    ],\n}","metadata":{"execution":{"iopub.status.busy":"2024-02-06T12:12:22.465770Z","iopub.execute_input":"2024-02-06T12:12:22.466143Z","iopub.status.idle":"2024-02-06T12:12:22.470808Z","shell.execute_reply.started":"2024-02-06T12:12:22.466115Z","shell.execute_reply":"2024-02-06T12:12:22.469751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"configs, models = {}, {}\n\nfor k in EXP_FOLDERS:\n    models_list = []\n    for exp_folder in EXP_FOLDERS[k]:\n        config = Config(json.load(open(exp_folder + \"config.json\", \"r\")))\n        configs[k] = config\n\n        for fold in FOLDS:\n            model = define_model(\n                config.decoder_name,\n                config.encoder_name,\n                num_classes=config.num_classes,\n                decoder_channels=config.decoder_channels,\n                n_channels=config.n_channels,\n                reduce_stride=config.reduce_stride,\n                use_pixel_shuffle=config.use_pixel_shuffle,\n                use_hypercolumns=config.use_hypercolumns,\n                center=config.center,\n                use_cls=config.loss_config['aux_loss_weight'] > 0,\n                frames=config.n_frames,\n                use_lstm=config.use_lstm,\n                bidirectional=config.bidirectional,\n                use_cnn=config.use_cnn,\n                kernel_size=config.kernel_size,\n                use_transfo=config.use_transfo,\n                two_layers=config.two_layers,\n#                 first_conv_3d=config.first_conv_3d if hasattr(config, \"first_conv_3d\") else False,\n                normalize=config.normalize_per_img if hasattr(config, \"normalize_per_img\") else False,\n                pretrained=False,\n            ).cuda()\n            model = model.cuda().eval()\n\n            weights = exp_folder + f\"{config.decoder_name}_{config.encoder_name}_{fold}.pt\"\n            model = load_model_weights(model, weights, verbose=1)\n\n            models_list.append(model)\n    models[k] = models_list","metadata":{"execution":{"iopub.status.busy":"2024-02-06T12:12:22.812272Z","iopub.execute_input":"2024-02-06T12:12:22.813142Z","iopub.status.idle":"2024-02-06T12:12:24.754404Z","shell.execute_reply.started":"2024-02-06T12:12:22.813112Z","shell.execute_reply":"2024-02-06T12:12:24.753479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Main","metadata":{}},{"cell_type":"code","source":"THRESHOLD = 0.5\n\nPCT = 0.0013\nUSE_PCT = False\n\nUSE_FP16 = True\nUSE_TTA = False\nBATCH_SIZE = 1\nINF_3D = True\n\npredict_fct = predict_multi  # predict_tta if USE_TTA else predict\n\nSAVE_FOLDER = \"/tmp/\"\nos.makedirs(SAVE_FOLDER, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-06T12:12:29.619853Z","iopub.execute_input":"2024-02-06T12:12:29.620213Z","iopub.status.idle":"2024-02-06T12:12:29.625608Z","shell.execute_reply.started":"2024-02-06T12:12:29.620182Z","shell.execute_reply":"2024-02-06T12:12:29.624731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = []\nfor folder in FOLDERS:\n    kidney = folder.split('/')[-2]\n    print(f\"\\n###   Kidney {kidney}   ###\\n\")\n    images = sorted(glob.glob(folder + \"images/*.tif\"))\n    \n    imgs = np.stack([np.array(Image.open(f)) for f in tqdm(images)])  # TODO : multiprocessing\n    \n    if \"pct\" in config.processed_folder:\n        imgs = norm_by_percentile(imgs)\n    else:\n        imgs = (imgs / 256).astype(np.uint8)\n    shape = imgs.shape\n    \n    transforms = get_transfos(\n        augment=False,\n        pad=int(config.encoder_name.split('_')[-1]) if \"maxvit\" in config.encoder_name else 32,\n        resize=(config.size, config.size) if (config.size is not None) else None,\n        scale=(config.aug_strength == 6),\n    )\n\n    dataset = KidneyDatasetInf(\n        imgs,\n        transforms=transforms,\n    )\n\n    del imgs\n    gc.collect()\n        \n#     preds = np.zeros(shape, dtype=np.float16)\n    for k in EXP_FOLDERS:\n        print(f\"-> Model {k} ({len(models[k])} model(s))\\n\")\n        \n        if not len(models[k]):\n            continue\n        \n        config = configs[k]\n        \n        dataset.frames_chanel = config.frames_chanel\n        dataset.n_frames = config.n_frames\n        dataset.stride = config.stride\n\n        pred, _ = predict_fct(\n            models[k],\n            dataset,\n            config.loss_config,\n            batch_size=BATCH_SIZE,\n            use_fp16=USE_FP16,\n            num_workers=2,\n        )\n        pred = pred[:, 0][:shape[0], :shape[1], :shape[2]]\n        np.save(SAVE_FOLDER +  f\"pred_{kidney}.npy\", pred)\n\n        del pred\n        gc.collect()\n        \n        if INF_3D and shape[0] > 50:\n            dataset.imgs = dataset.imgs.transpose(1, 0, 2)\n            pred, _ = predict_fct(\n                models[k],\n                dataset,\n                config.loss_config,\n                batch_size=BATCH_SIZE,\n                use_fp16=USE_FP16,\n                num_workers=2,\n            )\n            pred = pred[:, 0].transpose(1, 0, 2)[:shape[0], :shape[1], :shape[2]]\n            np.save(SAVE_FOLDER +  f\"pred_{kidney}_y.npy\", pred)\n\n            del pred\n            gc.collect()\n\n            dataset.imgs = dataset.imgs.transpose(1, 0, 2).transpose(2, 0, 1)\n            pred, _ = predict_fct(\n                models[k],\n                dataset,\n                config.loss_config,\n                batch_size=BATCH_SIZE,\n                use_fp16=USE_FP16,\n                num_workers=2,\n            )\n            pred = pred[:, 0].transpose(1, 2, 0)[:shape[0], :shape[1], :shape[2]]\n            np.save(SAVE_FOLDER +  f\"pred_{kidney}_z.npy\", pred)\n            \n            del pred\n            gc.collect()\n        else:\n            pred = np.zeros(shape, dtype=np.uint8)\n            np.save(SAVE_FOLDER +  f\"pred_{kidney}_y.npy\", pred)\n            np.save(SAVE_FOLDER +  f\"pred_{kidney}_z.npy\", pred)\n            \n        \n    del dataset\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-06T12:12:35.590290Z","iopub.execute_input":"2024-02-06T12:12:35.591161Z","iopub.status.idle":"2024-02-06T12:12:49.297354Z","shell.execute_reply.started":"2024-02-06T12:12:35.591127Z","shell.execute_reply":"2024-02-06T12:12:49.296143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### PP","metadata":{}},{"cell_type":"code","source":"PP = True","metadata":{"execution":{"iopub.status.busy":"2024-02-06T12:12:50.320415Z","iopub.execute_input":"2024-02-06T12:12:50.320813Z","iopub.status.idle":"2024-02-06T12:12:50.325268Z","shell.execute_reply.started":"2024-02-06T12:12:50.320776Z","shell.execute_reply":"2024-02-06T12:12:50.324314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for folder in FOLDERS:\n    if not PP:\n        continue\n\n    kidney = folder.split('/')[-2]\n    print(f\"\\n###   Kidney {kidney}   ###\\n\")\n    \n    print('- Load predictions')\n    pred_x = torch.from_numpy(np.load(SAVE_FOLDER + f\"pred_{kidney}.npy\")).cuda()\n    gc.collect()\n    \n    pred_y = torch.from_numpy(np.load(SAVE_FOLDER + f\"pred_{kidney}_y.npy\"))\n    gc.collect()\n\n    pred_z = torch.from_numpy(np.load(SAVE_FOLDER + f\"pred_{kidney}_z.npy\"))\n    gc.collect()\n\n    # x axis\n    print('\\n- Post-process x\\n')\n    comps_stats = joblib.Parallel(n_jobs=2)(\n        joblib.delayed(get_components)((p > THRESHOLD).byte().cpu().numpy()) for p in tqdm(pred_x)\n    )\n    pred_x = pred_x.cpu()\n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    pred_y = pred_y.cuda()\n    pred_z = pred_z.cuda()\n    gc.collect()\n\n    modified = post_process(\n        pred_x,\n        (pred_y, pred_z),\n        axis=\"x\",\n        comps_stats=comps_stats,\n        offset=20,\n        conf_th=THRESHOLD\n    )\n\n    del comps_stats\n    gc.collect()\n    \n    preds_pp_x = pred_x.clone().numpy()\n    for k in modified:\n        preds_pp_x[k] = torch.from_numpy(modified[k])\n    np.save(SAVE_FOLDER + f\"pred_{kidney}.npy\", preds_pp_x)\n    print(f' -> Saved predictions with {len(modified)} modified frames to', SAVE_FOLDER + f\"pred_{kidney}.npy\")\n    \n    del (preds_pp_x, modified)\n    gc.collect()\n    \n    # y axis\n    print('\\n- Post-process y\\n')\n    comps_stats = joblib.Parallel(n_jobs=2)(\n        joblib.delayed(get_components)((pred_y[:, i] > THRESHOLD).byte().cpu().numpy())\n        for i in tqdm(range(pred_y.shape[1]))\n    )\n    pred_y = pred_y.cpu()\n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    pred_x = pred_x.cuda()\n    gc.collect()\n    \n    modified = post_process(\n        pred_y,\n        (pred_x, pred_z),\n        axis=\"y\",\n        comps_stats=comps_stats,\n        offset=20,\n        conf_th=THRESHOLD\n    )\n    \n    del comps_stats\n    gc.collect()\n    \n    preds_pp_y = pred_y.clone().numpy()\n    for k in modified:\n        preds_pp_y[:, k] = torch.from_numpy(modified[k])\n    np.save(SAVE_FOLDER + f\"pred_{kidney}_y.npy\", preds_pp_y)\n    print(f' -> Saved predictions with {len(modified)} modified frames to', SAVE_FOLDER + f\"pred_{kidney}_y.npy\")\n    \n    del (preds_pp_y, modified)\n    gc.collect()\n    \n    # z axis\n    print('\\n- Post-process z\\n')\n    comps_stats = joblib.Parallel(n_jobs=2)(\n        joblib.delayed(get_components)((pred_z[:, :, i] > THRESHOLD).byte().cpu().numpy())\n        for i in tqdm(range(pred_z.shape[2]))\n    )\n    pred_z = pred_z.cpu()\n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    pred_y = pred_y.cuda()\n    gc.collect()\n    \n    modified = post_process(\n        pred_z,\n        (pred_y, pred_x),\n        axis=\"z\",\n        comps_stats=comps_stats,\n        offset=20,\n        conf_th=THRESHOLD\n    )\n    \n    del (comps_stats, pred_y, pred_x)\n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    preds_pp_z = pred_z.clone().numpy()\n    for k in modified:\n        preds_pp_z[:, :, k] = torch.from_numpy(modified[k])\n    np.save(SAVE_FOLDER + f\"pred_{kidney}_z.npy\", preds_pp_z)\n    print(f' -> Saved predictions with {len(modified)} frames to', SAVE_FOLDER + f\"pred_{kidney}_z.npy\")\n    \n    del (preds_pp_z, modified, pred_z)\n    torch.cuda.empty_cache()\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-06T12:13:05.873443Z","iopub.execute_input":"2024-02-06T12:13:05.874285Z","iopub.status.idle":"2024-02-06T12:13:05.893725Z","shell.execute_reply.started":"2024-02-06T12:13:05.874252Z","shell.execute_reply":"2024-02-06T12:13:05.892865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submit","metadata":{}},{"cell_type":"code","source":"sub = []\nfor folder in FOLDERS:\n    kidney = folder.split('/')[-2]\n    print(f\"\\n###   Kidney {kidney}   ###\\n\")\n    \n    print('- Load x predictions')\n    preds = np.load(SAVE_FOLDER + f\"pred_{kidney}.npy\")\n    print('- Load y predictions')\n    preds += np.load(SAVE_FOLDER + f\"pred_{kidney}_y.npy\")\n    print('- Load z predictions')\n    preds += np.load(SAVE_FOLDER + f\"pred_{kidney}_z.npy\")\n    \n    print('- Threshold and encode')\n    \n    if USE_PCT:\n        flat = preds[::2, ::2, ::2].flatten()\n        index = -int(len(flat) * PCT)\n        th = np.partition(flat, index)[index]\n\n    else:\n        th = THRESHOLD * 3\n\n    print(f\"-> Using threshold {th / 3 :.2f}\")\n    preds = (preds > th).astype(np.uint8)\n    gc.collect()\n        \n    for i, p in enumerate(preds):\n        sub.append(\n            pd.DataFrame({\n                'id': [f\"{kidney}_{i:04d}\"],\n                'rle': [rle_encode(p)],\n            })\n        )\n\n    del preds\n    gc.collect()\n\n    \n# Submission\nsub = pd.concat(sub, ignore_index=True)\nsubmission = pd.read_csv('/kaggle/input/blood-vessel-segmentation/sample_submission.csv')[['id']]\n\nassert len(sub) >= len(submission)\n\nsubmission = submission.merge(sub, how=\"left\")\n\nsubmission.to_csv('submission.csv', index=False)\nprint('\\n-> Saved submission to \"submission.csv\"\\n')\n\ndisplay(submission.head(10))","metadata":{"execution":{"iopub.status.busy":"2024-02-06T12:13:07.416723Z","iopub.execute_input":"2024-02-06T12:13:07.417083Z","iopub.status.idle":"2024-02-06T12:13:09.975099Z","shell.execute_reply.started":"2024-02-06T12:13:07.417058Z","shell.execute_reply":"2024-02-06T12:13:09.974103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Eval","metadata":{}},{"cell_type":"code","source":"if DEBUG:\n    gt = pd.read_csv('/kaggle/input/blood-vessel-segmentation/train_rles.csv')\n    gt = gt[gt['id'].apply(lambda x: 'kidney_1_dense' in x)].reset_index(drop=True)\n    gt['width'] = 912\n    gt['height'] = 1303\n\n    surf_dice = compute_surface_dice_score(sub, gt)\n    print(f' -> Surface Dice: {surf_dice :.3f}  -  th={THRESHOLD:.2f}')","metadata":{"execution":{"iopub.status.busy":"2024-02-06T12:13:25.145188Z","iopub.execute_input":"2024-02-06T12:13:25.145554Z","iopub.status.idle":"2024-02-06T12:13:25.151535Z","shell.execute_reply.started":"2024-02-06T12:13:25.145521Z","shell.execute_reply":"2024-02-06T12:13:25.150617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Done ! ","metadata":{}}]}