{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nsys.path.append('../input/hubmap-2022-env/env_v3/env_v3/')\nsys.path.append('../input/hubmap-2022-code/main_folder/main_folder/')","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:48:55.871532Z","iopub.execute_input":"2022-09-22T08:48:55.872164Z","iopub.status.idle":"2022-09-22T08:48:55.899514Z","shell.execute_reply.started":"2022-09-22T08:48:55.872056Z","shell.execute_reply":"2022-09-22T08:48:55.898631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport os\nimport torch\nimport cv2\nimport gc\nimport torchvision\nimport ttach as tta\nimport albumentations as A\n\nfrom glob import glob\nfrom tqdm import tqdm\nfrom matplotlib import pyplot as plt\nfrom itertools import chain\nfrom os.path import join as pjoin\nfrom copy import deepcopy\nfrom albumentations.pytorch import ToTensorV2\n\nfrom code_base.models import SMPWrapper, TransformerWrapper\nfrom code_base.datasets import HubMapDataset\nfrom code_base.utils.mask import rle_decode, overlay_mask\nfrom code_base.utils.other import imread_rgb\nfrom code_base.inference import apply_avarage_weights_on_swa_path, HubMapInference\nfrom code_base.constants import CLASSES, PIXEL_SCALE\nfrom code_base.utils.metrics import dice_coeff\nfrom code_base.models import SegmentationTTAWrapperKwargs","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:48:55.901541Z","iopub.execute_input":"2022-09-22T08:48:55.902195Z","iopub.status.idle":"2022-09-22T08:49:07.768120Z","shell.execute_reply.started":"2022-09-22T08:48:55.902159Z","shell.execute_reply":"2022-09-22T08:49:07.766872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/hubmap-organ-segmentation/sample_submission.csv\")\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:49:07.770694Z","iopub.execute_input":"2022-09-22T08:49:07.771805Z","iopub.status.idle":"2022-09-22T08:49:07.796744Z","shell.execute_reply.started":"2022-09-22T08:49:07.771772Z","shell.execute_reply":"2022-09-22T08:49:07.795701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_DATASET = \"hubmap-2022-models\"\nREAL_TEST = sample_submission.shape[0] > 1\nONLY_HUBMAP = False\nSELECTED_ORGANS = None\nREAL_TEST","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:49:07.798095Z","iopub.execute_input":"2022-09-22T08:49:07.799283Z","iopub.status.idle":"2022-09-22T08:49:07.807521Z","shell.execute_reply.started":"2022-09-22T08:49:07.799248Z","shell.execute_reply":"2022-09-22T08:49:07.806400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Possible exps:\\n\\n{}\".format(\"\\n\".join(os.listdir(f\"../input/{MODEL_DATASET}/\"))))","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:49:07.810086Z","iopub.execute_input":"2022-09-22T08:49:07.810464Z","iopub.status.idle":"2022-09-22T08:49:07.822008Z","shell.execute_reply.started":"2022-09-22T08:49:07.810387Z","shell.execute_reply":"2022-09-22T08:49:07.820642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EXP_NAME = \"unetpp_timmefficientnetb7NS_strongaugs_512_lr1e3_BceDiceFocalIoU_bs32_testscale_HMGTEXAndOldTrainP1_datax8_1ch_masksamplingP05_ConcatDataset_DenisPseudoScaledV3X16_denisx2scales_datav2_HpaPseudoV2X1\"\nprint(\"Possible checkpoints:\\n\\n{}\".format(\"\\n\".join(set([os.path.basename(el) for el in glob(\n    f\"../input/{MODEL_DATASET}/{EXP_NAME}/{EXP_NAME}/*/checkpoints/*.pt*\"\n) if \"train\" not in os.path.basename(el)]))))","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:49:07.825050Z","iopub.execute_input":"2022-09-22T08:49:07.825323Z","iopub.status.idle":"2022-09-22T08:49:07.850305Z","shell.execute_reply.started":"2022-09-22T08:49:07.825285Z","shell.execute_reply":"2022-09-22T08:49:07.849366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SLIDING_CONFIG = {\n    \"roi_size\": (1024, 1024),\n    \"sw_batch_size\": 4,\n    \"overlap\": 0.75,\n    \"padding_mode\": \"reflect\"\n}\n\nCONFIG = {\n    # Main\n    \"sliding_window_config\":\n    (\n        # Unet++ models\n        [None] * 5 +\n        # mitb3 models\n        [deepcopy(SLIDING_CONFIG) for _ in range(5)] + \n        # Unet models\n        [None] * 5 + \n        # Unet models\n        [None] * 5\n    ),\n    \"fill_binary_holes\": True,\n    \"test_tresh\": 0.5,\n#     {\n#         'Hubmap': {\n#             'kidney'        : 0.6,\n#             'prostate'      : 0.5,\n#             'largeintestine': 0.5,\n#             'spleen'        : 0.6,\n#             'lung'          : 0.4,\n#         },\n#         'HPA': {\n#             'kidney'        : 0.5,\n#             'prostate'      : 0.4,\n#             'largeintestine': 0.4,\n#             'spleen'        : 0.5,\n#             'lung'          : 0.4,\n#         },\n#     },\n    \"use_amp\": True,\n    # Class names refers to CLASSES.index\n    \"min_area\": [\n        0.001, 0.0005, 0.0001, \n        0.001, 1e-06\n    ],\n    \"is_relative_min_area\": True,\n#     \"mean_type\": \"gmean\",\n    # Data config\n    \"n_folds\":5,\n    \"train_data_root\":\"/home/vova/data/exps/hubmap_2022/train_images/\",\n    \"batch_size\": 1,\n    \"num_workers\": 0,\n    \"pad_config\": dict(\n        min_height=None, \n        min_width=None, \n        pad_height_divisor=32, \n        pad_width_divisor=32, \n    ),\n    \"use_one_channel_mask\": True,\n    \"to_rgb\": True,\n    \"additional_scalers\": {\n        'prostate': 0.15 * 2,\n        'spleen': 1 * 2,\n        'lung': 0.5 * 2,\n        'kidney': 1 * 2,\n        'largeintestine': 1 * 2\n    },\n    # Model config\n    \"exp_name\":EXP_NAME,\n    \"model_class\": SMPWrapper,\n    \"model_config\": { \n#         \"backbone_name\": \"timm-efficientnet-b5\",\n        \"backbone_name\": \"timm-efficientnet-b7\",\n#         \"num_classes\": len(CLASSES),\n        \"num_classes\": 1,\n#         \"arch_name\": \"Unet\",\n        \"arch_name\": \"UnetPP\",\n#         \"arch_name\": \"UnetGC\",\n#         \"arch_name\": \"FPN\",\n        \"pretrained\":False,\n#         \"use_slice_idx\": True,\n#         \"aux_params\": {\"classes\": len(CLASSES)},\n#         \"return_only_mask\": True\n        \n    },\n    \"tta_transforms\": tta.aliases.flip_transform(), #tta.Compose(\n#         [\n#             tta.HorizontalFlip(),\n#             tta.VerticalFlip()\n#         ]\n#     ),\n    \"chkp_name\":\"swa_models_valid_dice_score.pt\",\n    \"swa_checkpoint\": None,\n    \"distributed_chkp\": True,\n    \"use_sigmoid\": True,\n#     \"instance_norm_mode\": True\n}","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:52:43.444992Z","iopub.execute_input":"2022-09-22T08:52:43.445533Z","iopub.status.idle":"2022-09-22T08:52:43.456593Z","shell.execute_reply.started":"2022-09-22T08:52:43.445500Z","shell.execute_reply":"2022-09-22T08:52:43.455271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# solo tresh \nif isinstance(CONFIG[\"test_tresh\"], float):\n    humbap_tresh = CONFIG[\"test_tresh\"]\n    hpa_tresh = CONFIG[\"test_tresh\"]\n# multi tresh\nelif isinstance(CONFIG[\"test_tresh\"], dict):\n    humbap_tresh = [CONFIG[\"test_tresh\"][\"Hubmap\"][cls_name] for cls_name in CLASSES]\n    hpa_tresh = [CONFIG[\"test_tresh\"][\"HPA\"][cls_name] for cls_name in CLASSES]","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:54:17.218118Z","iopub.execute_input":"2022-09-22T08:54:17.218503Z","iopub.status.idle":"2022-09-22T08:54:17.224134Z","shell.execute_reply.started":"2022-09-22T08:54:17.218472Z","shell.execute_reply":"2022-09-22T08:54:17.223146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ADITIONAL_MODELS = [\n    {\n        \"exp_name\":\"unet_mitb5_strongaugs_1024_lr1e4_BceDiceFocalIoU_bs12_testscale_HMGTEXAndOldTrainP1_datax8_1ch_masksamplingP05_ConcatDataset_DenisPseudoScaledV4X16_denisx2scales_datav2_HpaPseudoV3FX1\",\n        \"model_config\": {\n            \"backbone_name\": \"mit_b5\",\n            \"num_classes\": 1, \n            \"arch_name\": \"Unet\",\n            \"pretrained\":False,\n        },\n        \"distributed_chkp\": True,\n        \"chkp_name\":\"swa_models_valid_dice_score.pt\",\n        \"tta_transforms\": tta.aliases.flip_transform()\n    },\n    {\n        \"exp_name\":\"deeplabv3p_pointrandX2loss_timmefficientnetb5NS_strongaugs_512_lr1e3_BceDiceFocalIoU_bs32_testscale_HMGTEXAndOldTrainP1_datax8_1ch_masksamplingP05_ConcatDataset_DenisPseudoScaledV3X16_denisx2scales_datav2_HpaPseudoV2X1\",\n        \"model_config\": {\n            \"backbone_name\": \"timm-efficientnet-b7\",\n            \"num_classes\": 1, \n            \"arch_name\": \"Unet\",\n            \"pretrained\":False,\n            \"point_rand_config\": {\"in_ch\": 161, \"num_classes\": 1, \"backbone_type\": \"effnet\"},\n        },\n        \"distributed_chkp\": True,\n        \"chkp_name\":\"swa_models_valid_dice_score.pt\",\n        \"tta_transforms\": tta.aliases.flip_transform()\n    },\n#     {\n#         \"exp_name\":\"unet_timmefficientnetb7NS_strongaugs_512_lr1e3_BceDiceFocalIoU_bs32_testscale_HMGTEXAndOldTrainP1_datax8_1ch_masksamplingP05_ConcatDataset_DenisPseudoScaledV3X16_denisx2scales_datav2_HpaPseudoV2X1\",\n#         \"model_config\": {\n#             \"backbone_name\": \"timm-efficientnet-b7\",\n#             \"num_classes\": 1, \n#             \"arch_name\": \"Unet\",\n#             \"pretrained\":False,\n#         },\n#         \"distributed_chkp\": True,\n#         \"chkp_name\":\"swa_models_valid_dice_score.pt\",\n#         \"tta_transforms\": tta.aliases.flip_transform()\n#     },\n    {\n        \"exp_name\":\"unet_pointrandX2loss_timmefficientnetb7NS_strongaugs_512_lr1e3_BceDiceFocalIoU_bs32_testscale_HMGTEXAndOldTrainP1_datax8_1ch_masksamplingP05_ConcatDataset_DenisPseudoScaledV3X16_denisx2scales_datav2_HpaPseudoV3FX1\",\n        \"model_config\": {\n            \"backbone_name\": \"timm-efficientnet-b7\",\n            \"num_classes\": 1, \n            \"arch_name\": \"Unet\",\n            \"pretrained\":False,\n            \"point_rand_config\": {\"in_ch\": 161, \"num_classes\": 1, \"backbone_type\": \"effnet\"},\n        },\n        \"distributed_chkp\": True,\n        \"chkp_name\":\"swa_models_valid_dice_score.pt\",\n        \"tta_transforms\": tta.aliases.flip_transform()\n    },\n]\n\n# ADITIONAL_MODELS = None","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:54:17.922607Z","iopub.execute_input":"2022-09-22T08:54:17.923479Z","iopub.status.idle":"2022-09-22T08:54:17.933340Z","shell.execute_reply.started":"2022-09-22T08:54:17.923435Z","shell.execute_reply":"2022-09-22T08:54:17.932032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(\"../input/hubmap-organ-segmentation/test.csv\")\ntrain_df = pd.read_csv(\"../input/hubmap-organ-segmentation/train.csv\").drop(columns=[\"rle\", \"age\", \"sex\"]).iloc[:10]\n\nif REAL_TEST:\n    hpa_df = test_df[test_df.data_source == \"HPA\"].reset_index(drop=True)\n    hubmap_df = test_df[test_df.data_source == \"Hubmap\"].reset_index(drop=True)\nelse:\n    hpa_df = train_df\n    hubmap_df = test_df","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:54:18.682909Z","iopub.execute_input":"2022-09-22T08:54:18.683646Z","iopub.status.idle":"2022-09-22T08:54:19.137574Z","shell.execute_reply.started":"2022-09-22T08:54:18.683611Z","shell.execute_reply":"2022-09-22T08:54:19.136618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SELECTED_ORGANS is not None:\n    hpa_df = hpa_df[hpa_df.organ.isin(SELECTED_ORGANS)].reset_index(drop=True)\n    hubmap_df = hubmap_df[hubmap_df.organ.isin(SELECTED_ORGANS)].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:54:19.342162Z","iopub.execute_input":"2022-09-22T08:54:19.342499Z","iopub.status.idle":"2022-09-22T08:54:19.347644Z","shell.execute_reply.started":"2022-09-22T08:54:19.342469Z","shell.execute_reply":"2022-09-22T08:54:19.346553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hpa_df","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:54:19.998940Z","iopub.execute_input":"2022-09-22T08:54:19.999869Z","iopub.status.idle":"2022-09-22T08:54:20.018265Z","shell.execute_reply.started":"2022-09-22T08:54:19.999827Z","shell.execute_reply":"2022-09-22T08:54:20.016232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hubmap_df","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:54:20.743010Z","iopub.execute_input":"2022-09-22T08:54:20.743365Z","iopub.status.idle":"2022-09-22T08:54:20.756564Z","shell.execute_reply.started":"2022-09-22T08:54:20.743335Z","shell.execute_reply":"2022-09-22T08:54:20.755476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loader_config = {\n    \"batch_size\": CONFIG[\"batch_size\"],\n    \"drop_last\": False,\n    \"shuffle\": False,\n    \"num_workers\": CONFIG[\"num_workers\"],\n}\nhpa_dataset_config = {\n    \"root\": (\n        # For not REAL_TEST run we are using train HPA images\n        \"../input/hubmap-organ-segmentation/test_images/\" if REAL_TEST else \"../input/hubmap-organ-segmentation/train_images/\"\n    ),\n    \"img_size\": None,\n    \"test_mode\": True,\n    \"precompute\": False,\n    \"dynamic_resize_mode\": \"scale_or\",\n    \"use_one_channel_mask\": CONFIG[\"use_one_channel_mask\"],\n    \"additional_scalers\": CONFIG.get(\"additional_scalers\", None),\n    \"to_rgb\": CONFIG[\"to_rgb\"],\n    \"transform\": [\n        A.PadIfNeeded(\n            border_mode=4, \n            value=None, \n            mask_value=None, \n            always_apply=True,\n            **CONFIG[\"pad_config\"]\n        ),\n        A.Normalize(), ToTensorV2(transpose_mask=True)\n    ]\n}\nhubmap_dataset_config = {\n    \"root\": \"../input/hubmap-organ-segmentation/test_images/\",\n    \"img_size\": None,\n    \"test_mode\": True,\n    \"precompute\": False,\n    \"dynamic_resize_mode\": None,\n    \"use_one_channel_mask\": CONFIG[\"use_one_channel_mask\"],\n    \"additional_scalers\": CONFIG.get(\"additional_scalers\", None),\n    \"to_rgb\": CONFIG[\"to_rgb\"],\n    \"transform\": [\n        A.PadIfNeeded(\n            border_mode=4, \n            value=None, \n            mask_value=None, \n            always_apply=True,\n            **CONFIG[\"pad_config\"]\n        ),\n        A.Normalize(), ToTensorV2(transpose_mask=True)\n    ],\n    \"apply_torchstain_norm\": CONFIG.get(\"torchstain_hubmap\", False)\n}\nhpa_loader = torch.utils.data.DataLoader(\n    HubMapDataset(df=hpa_df, **hpa_dataset_config),\n    **loader_config\n)\nhubmap_loader = torch.utils.data.DataLoader(\n    HubMapDataset(df=hubmap_df, **hubmap_dataset_config),\n    **loader_config\n)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:54:21.484917Z","iopub.execute_input":"2022-09-22T08:54:21.485274Z","iopub.status.idle":"2022-09-22T08:54:21.504749Z","shell.execute_reply.started":"2022-09-22T08:54:21.485243Z","shell.execute_reply":"2022-09-22T08:54:21.503834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hubmap_loader.dataset.apply_torchstain_norm","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:54:22.114874Z","iopub.execute_input":"2022-09-22T08:54:22.115246Z","iopub.status.idle":"2022-09-22T08:54:22.121592Z","shell.execute_reply.started":"2022-09-22T08:54:22.115215Z","shell.execute_reply":"2022-09-22T08:54:22.120458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def set_dropout_zero(model):\n    for name, child in model.named_children():\n        if isinstance(child, (torch.nn.Dropout, torch.nn.Dropout2d, torch.nn.Dropout3d)):\n            child.p = 0.0\n        set_dropout_zero(child)\n\ndef create_model_and_upload_chkp(\n    model_class,\n    model_config,\n    model_device,\n    model_chkp,\n    use_distributed=False,\n    swa_checkpoint=None,\n    tta_transform=None,\n    tta_merge_mode=\"mean\",\n    instance_norm_mode=False\n):\n    if \"swa\" in model_chkp:\n        print(\"swa by {}\".format(os.path.splitext(os.path.basename(model_chkp))[0]))\n        t_chkp = apply_avarage_weights_on_swa_path(model_chkp, use_distributed=use_distributed, take_best=swa_checkpoint)\n    else:\n        print(\"vanilla model\")\n        t_chkp = torch.load(model_chkp, map_location=\"cpu\")\n        \n    t_model = model_class(**model_config, device=model_device)\n    t_model.load_state_dict(t_chkp)\n    if instance_norm_mode:\n        t_model.train()\n        set_dropout_zero(t_model)\n    else:\n        t_model.eval()\n    if tta_transform is not None:\n        print(\"Wrapping model in TTA\")\n        t_model = SegmentationTTAWrapperKwargs(t_model, tta_transform, merge_mode=tta_merge_mode)\n    return t_model","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:54:22.596608Z","iopub.execute_input":"2022-09-22T08:54:22.596986Z","iopub.status.idle":"2022-09-22T08:54:22.606398Z","shell.execute_reply.started":"2022-09-22T08:54:22.596954Z","shell.execute_reply":"2022-09-22T08:54:22.605396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = [create_model_and_upload_chkp(\n        model_class=CONFIG[\"model_class\"],\n        model_config=CONFIG['model_config'],\n        model_device=\"cuda\",\n        model_chkp=f\"../input/{MODEL_DATASET}/{CONFIG['exp_name']}/{CONFIG['exp_name']}/fold_{m_i}/checkpoints/{CONFIG['chkp_name']}\",\n        swa_checkpoint=CONFIG['swa_checkpoint'],\n        use_distributed=CONFIG['distributed_chkp'],\n        tta_transform=CONFIG.get(\"tta_transforms\", None),\n        instance_norm_mode=CONFIG.get(\"instance_norm_mode\", False)\n) for m_i in range(CONFIG[\"n_folds\"])]","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:54:23.006446Z","iopub.execute_input":"2022-09-22T08:54:23.007269Z","iopub.status.idle":"2022-09-22T08:55:11.093209Z","shell.execute_reply.started":"2022-09-22T08:54:23.007235Z","shell.execute_reply":"2022-09-22T08:55:11.092139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if ADITIONAL_MODELS is not None:\n    for add_conf in ADITIONAL_MODELS:\n        model += [create_model_and_upload_chkp(\n                model_class=CONFIG[\"model_class\"],\n                model_config=add_conf['model_config'],\n                model_device=\"cuda\",\n                model_chkp=f\"../input/{MODEL_DATASET}/{add_conf['exp_name']}/{add_conf['exp_name']}/fold_{m_i}/checkpoints/{add_conf['chkp_name']}\",\n                swa_checkpoint=CONFIG['swa_checkpoint'],\n                use_distributed=add_conf['distributed_chkp'],\n                tta_transform=add_conf.get(\"tta_transforms\", None),\n        ) for m_i in range(CONFIG[\"n_folds\"])]\n","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:55:11.096410Z","iopub.execute_input":"2022-09-22T08:55:11.096679Z","iopub.status.idle":"2022-09-22T08:57:37.972170Z","shell.execute_reply.started":"2022-09-22T08:55:11.096653Z","shell.execute_reply":"2022-09-22T08:57:37.970231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Total models in Blend = {len(model)}\")\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:57:37.974219Z","iopub.execute_input":"2022-09-22T08:57:37.974955Z","iopub.status.idle":"2022-09-22T08:57:38.380142Z","shell.execute_reply.started":"2022-09-22T08:57:37.974904Z","shell.execute_reply":"2022-09-22T08:57:38.379280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"inference_class = HubMapInference(\n    device=\"cuda\",\n    verbose=True,\n    verbose_tqdm=True,\n    use_sigmoid=CONFIG[\"use_sigmoid\"],\n    use_amp=CONFIG[\"use_amp\"]\n)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:57:38.384893Z","iopub.execute_input":"2022-09-22T08:57:38.385523Z","iopub.status.idle":"2022-09-22T08:57:38.393750Z","shell.execute_reply.started":"2022-09-22T08:57:38.385487Z","shell.execute_reply":"2022-09-22T08:57:38.392713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hubmap_test_pred = inference_class.predict_test_loader(\n    nn_models=model,\n    test_loader=hubmap_loader,\n    tresh=humbap_tresh,\n    pad_config=CONFIG[\"pad_config\"],\n    min_area=CONFIG.get(\"min_area\", None),\n    is_relative_min_area=CONFIG.get(\"is_relative_min_area\", False),\n    # For HubMap we do not need to resize something. Just crop padding (to X32)\n    # In case of additional_scalers we have to rescale\n    use_rescaled=CONFIG.get(\"additional_scalers\", None) is not None,\n    scale_back=CONFIG.get(\"additional_scalers\", None) is not None,\n    fill_binary_holes=CONFIG[\"fill_binary_holes\"],\n    sliding_window_config=CONFIG[\"sliding_window_config\"],\n    print_input_shape=True,\n    mean_type=CONFIG.get(\"mean_type\", \"mean\")\n)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:57:38.397649Z","iopub.execute_input":"2022-09-22T08:57:38.400259Z","iopub.status.idle":"2022-09-22T08:58:01.828359Z","shell.execute_reply.started":"2022-09-22T08:57:38.400224Z","shell.execute_reply":"2022-09-22T08:58:01.827379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not ONLY_HUBMAP:\n    hpa_test_pred = inference_class.predict_test_loader(\n        nn_models=model,\n        test_loader=hpa_loader,\n        tresh=hpa_tresh,\n        pad_config=CONFIG[\"pad_config\"],\n        min_area=CONFIG.get(\"min_area\", None),\n        is_relative_min_area=CONFIG.get(\"is_relative_min_area\", False),\n        # For HPA we have to crop padding (to X32) in rescaled domain and then scale back to original sizes\n        use_rescaled=True,\n        scale_back=True,\n        fill_binary_holes=CONFIG[\"fill_binary_holes\"],\n        sliding_window_config=CONFIG[\"sliding_window_config\"],\n        print_input_shape=True,\n        mean_type=CONFIG.get(\"mean_type\", \"mean\")\n    )","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:58:01.829950Z","iopub.execute_input":"2022-09-22T08:58:01.830579Z","iopub.status.idle":"2022-09-22T08:58:01.837509Z","shell.execute_reply.started":"2022-09-22T08:58:01.830540Z","shell.execute_reply":"2022-09-22T08:58:01.836374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit","metadata":{}},{"cell_type":"code","source":"if ONLY_HUBMAP:\n    test_pred = hubmap_test_pred\nelse:\n    test_pred = pd.concat([hubmap_test_pred, hpa_test_pred]).reset_index(drop=True)\ntest_pred","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:58:01.839689Z","iopub.execute_input":"2022-09-22T08:58:01.840544Z","iopub.status.idle":"2022-09-22T08:58:01.857875Z","shell.execute_reply.started":"2022-09-22T08:58:01.840506Z","shell.execute_reply":"2022-09-22T08:58:01.856782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = sample_submission.drop(columns=\"rle\").merge(test_pred, on=\"id\", how=\"left\")\nif ONLY_HUBMAP or SELECTED_ORGANS is not None:\n    sample_submission[\"rle\"] = sample_submission[\"rle\"].fillna(\"\")\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:58:01.859334Z","iopub.execute_input":"2022-09-22T08:58:01.859688Z","iopub.status.idle":"2022-09-22T08:58:01.886305Z","shell.execute_reply.started":"2022-09-22T08:58:01.859654Z","shell.execute_reply":"2022-09-22T08:58:01.885257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not REAL_TEST:\n    img = imread_rgb(\n        f\"../input/hubmap-organ-segmentation/test_images/{sample_submission.id.iloc[0]}.tiff\"\n    )\n    mask = rle_decode(sample_submission.rle.iloc[0], shape=(test_df.iloc[0].img_width, test_df.iloc[0].img_height))\n    plt.figure(figsize=(10,10))\n    plt.title(\"Mask\")\n    plt.imshow(mask)\n    plt.show()\n    plt.figure(figsize=(10,10))\n    plt.title(\"Mask Overlay\")\n    plt.imshow(overlay_mask(img, mask, color_id=2))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:58:01.887953Z","iopub.execute_input":"2022-09-22T08:58:01.888316Z","iopub.status.idle":"2022-09-22T08:58:03.405000Z","shell.execute_reply.started":"2022-09-22T08:58:01.888265Z","shell.execute_reply":"2022-09-22T08:58:03.404102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv(\"submission.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T08:58:03.407618Z","iopub.execute_input":"2022-09-22T08:58:03.407968Z","iopub.status.idle":"2022-09-22T08:58:03.418981Z","shell.execute_reply.started":"2022-09-22T08:58:03.407929Z","shell.execute_reply":"2022-09-22T08:58:03.417752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}