{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This submission topped the first place solution on the private leaderboard (score: *0.9516*), even though it fared quite modestly on the public leaderboard (score: *0.9166*).\n\nThe approach taken in this notebook is:\n\n1. Use one single model trained with FPN and efficientnet-b7 back-end (Pytorch, using [segmentation models Pytorch](https://github.com/qubvel/segmentation_models.pytorch)), and with a window size of *1536* window size and *768* tile size. \n2. Perform inference on three grids, all with window size of *1536* window size and *768* tile size, but different overlaps with these sizes: *[32, 128, 256]*\n3. Take the predicted sets of masks and check if its average is above *0.49* and produce with that the final prediction set of masks.\n","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2021-01-31T14:06:05.025567Z","iopub.status.busy":"2021-01-31T14:06:05.024823Z","iopub.status.idle":"2021-01-31T14:06:05.027696Z","shell.execute_reply":"2021-01-31T14:06:05.027233Z"},"papermill":{"duration":0.024796,"end_time":"2021-01-31T14:06:05.027805","exception":false,"start_time":"2021-01-31T14:06:05.003009","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir efficientnet_pytorch-0.6.3\n!cp -R /kaggle/input/pytorch-segmentation-models-git/efficientnet_pytorch-0.6.3/efficientnet_pytorch-0.6.3/* efficientnet_pytorch-0.6.3\n!pip install -e efficientnet_pytorch-0.6.3\n!ln -s efficientnet_pytorch-0.6.3/efficientnet_pytorch efficientnet_pytorch","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:06:05.069631Z","iopub.status.busy":"2021-01-31T14:06:05.068857Z","iopub.status.idle":"2021-01-31T14:06:37.17294Z","shell.execute_reply":"2021-01-31T14:06:37.17239Z"},"papermill":{"duration":32.128829,"end_time":"2021-01-31T14:06:37.173043","exception":false,"start_time":"2021-01-31T14:06:05.044214","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf pretrained-models.pytorch-master","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir pretrained-models.pytorch-master\n!cp -R /kaggle/input/pytorch-segmentation-models-git/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/* pretrained-models.pytorch-master\n!pip install -e pretrained-models.pytorch-master\n!ln -s pretrained-models.pytorch-master/pretrainedmodels pretrainedmodels","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:06:37.221216Z","iopub.status.busy":"2021-01-31T14:06:37.218077Z","iopub.status.idle":"2021-01-31T14:07:07.87885Z","shell.execute_reply":"2021-01-31T14:07:07.877858Z"},"papermill":{"duration":30.68678,"end_time":"2021-01-31T14:07:07.878971","exception":false,"start_time":"2021-01-31T14:06:37.192191","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/pytorch-segmentation-models-git/timm-0.3.2-py3-none-any.whl","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:07:07.923744Z","iopub.status.busy":"2021-01-31T14:07:07.920881Z","iopub.status.idle":"2021-01-31T14:07:34.272673Z","shell.execute_reply":"2021-01-31T14:07:34.271708Z"},"papermill":{"duration":26.375318,"end_time":"2021-01-31T14:07:34.272822","exception":false,"start_time":"2021-01-31T14:07:07.897504","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install --no-index /kaggle/input/pytorch-segmentation-models-git/segmentation_models_pytorch-0.1.3-py3-none-any.whl\n!mkdir segmentation_models_pytorch_\n!cp -R /kaggle/input/pytorch-segmentation-models-git/segmentation-models-pytorch-0.1.3/segmentation-models-pytorch/* segmentation_models_pytorch_\n!pip install -e segmentation_models_pytorch_","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:07:34.322727Z","iopub.status.busy":"2021-01-31T14:07:34.321967Z","iopub.status.idle":"2021-01-31T14:07:41.023506Z","shell.execute_reply":"2021-01-31T14:07:41.022555Z"},"papermill":{"duration":6.728113,"end_time":"2021-01-31T14:07:41.023624","exception":false,"start_time":"2021-01-31T14:07:34.295511","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ln -s segmentation_models_pytorch_/segmentation_models_pytorch segmentation_models_pytorch\n!ls segmentation_models_pytorch","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pretrainedmodels","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:07:41.071021Z","iopub.status.busy":"2021-01-31T14:07:41.070006Z","iopub.status.idle":"2021-01-31T14:07:43.700592Z","shell.execute_reply":"2021-01-31T14:07:43.700076Z"},"papermill":{"duration":2.655367,"end_time":"2021-01-31T14:07:43.700696","exception":false,"start_time":"2021-01-31T14:07:41.045329","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\n\nfrom tqdm.notebook import tqdm\n\nimport sys, os, random, time, glob\nimport numba, cv2, gc\nimport pickle","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2021-01-31T14:07:43.747811Z","iopub.status.busy":"2021-01-31T14:07:43.747223Z","iopub.status.idle":"2021-01-31T14:07:44.719387Z","shell.execute_reply":"2021-01-31T14:07:44.718044Z"},"papermill":{"duration":0.997449,"end_time":"2021-01-31T14:07:44.719495","exception":false,"start_time":"2021-01-31T14:07:43.722046","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as D\n\nimport torchvision\nfrom torchvision import transforms as T\nfrom segmentation_models_pytorch import Unet\nfrom segmentation_models_pytorch import FPN","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:07:44.767738Z","iopub.status.busy":"2021-01-31T14:07:44.766792Z","iopub.status.idle":"2021-01-31T14:07:44.906566Z","shell.execute_reply":"2021-01-31T14:07:44.905715Z"},"papermill":{"duration":0.166042,"end_time":"2021-01-31T14:07:44.906671","exception":false,"start_time":"2021-01-31T14:07:44.740629","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import rasterio\nfrom rasterio.windows import Window\n\nimport albumentations as A","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:07:44.953128Z","iopub.status.busy":"2021-01-31T14:07:44.952481Z","iopub.status.idle":"2021-01-31T14:07:46.571313Z","shell.execute_reply":"2021-01-31T14:07:46.570806Z"},"papermill":{"duration":1.643555,"end_time":"2021-01-31T14:07:46.571408","exception":false,"start_time":"2021-01-31T14:07:44.927853","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seeds(seed = 42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    \nset_seeds()","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:07:46.622961Z","iopub.status.busy":"2021-01-31T14:07:46.622312Z","iopub.status.idle":"2021-01-31T14:07:46.628239Z","shell.execute_reply":"2021-01-31T14:07:46.627708Z"},"papermill":{"duration":0.035238,"end_time":"2021-01-31T14:07:46.628323","exception":false,"start_time":"2021-01-31T14:07:46.593085","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/input/hubmap-fpn-efficientnetb7-1536/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/models\n!mkdir /kaggle/working/models\n!cp /kaggle/input/hubmap-fpn-efficientnetb7-1536/*.pth /kaggle/working/models\n# !cp /kaggle/input/fpn-with-10-epochs//*.pth /kaggle/working/models","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:07:46.676662Z","iopub.status.busy":"2021-01-31T14:07:46.675946Z","iopub.status.idle":"2021-01-31T14:07:52.819782Z","shell.execute_reply":"2021-01-31T14:07:52.819272Z"},"papermill":{"duration":6.170474,"end_time":"2021-01-31T14:07:52.819899","exception":false,"start_time":"2021-01-31T14:07:46.649425","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/working/models","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_PATH = Path('../input/hubmap-kidney-segmentation')\nassert DATA_PATH.exists()\n\n# path to our training notebook.\nPATH_FOLD_MODELS = Path('/kaggle/working/models')\nassert PATH_FOLD_MODELS.exists()","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:07:52.868606Z","iopub.status.busy":"2021-01-31T14:07:52.867863Z","iopub.status.idle":"2021-01-31T14:07:52.870864Z","shell.execute_reply":"2021-01-31T14:07:52.870408Z"},"papermill":{"duration":0.028916,"end_time":"2021-01-31T14:07:52.87095","exception":false,"start_time":"2021-01-31T14:07:52.842034","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading data","metadata":{"papermill":{"duration":0.020832,"end_time":"2021-01-31T14:07:52.913013","exception":false,"start_time":"2021-01-31T14:07:52.892181","status":"completed"},"tags":[]}},{"cell_type":"code","source":"@numba.njit()\ndef rle_numba(pixels):\n    size = len(pixels)\n    points = []\n    if pixels[0] == 1: points.append(1)\n    for i in range(1, size):\n        if pixels[i] != pixels[i-1]:\n            if len(points) % 2 == 0:\n                points.append(i+1)\n            else:\n                points.append(i+1 - points[-1])\n    if pixels[-1] == 1: points.append(size-points[-1]+1)    \n    return points\n\ndef rle_numba_encode(image):\n    pixels = image.flatten(order = 'F')\n    points = rle_numba(pixels)\n    return ' '.join(str(x) for x in points)\n\ndef make_grid(shape, window=256, min_overlap=32):\n    \"\"\"\n        Return Array of size (N,4), where N - number of tiles,\n        2nd axis represente slices: x1,x2,y1,y2 \n    \"\"\"\n    x, y = shape\n    nx = x // (window - min_overlap) + 1\n    x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)\n    x1[-1] = x - window\n    x2 = (x1 + window).clip(0, x)\n    ny = y // (window - min_overlap) + 1\n    y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)\n    y1[-1] = y - window\n    y2 = (y1 + window).clip(0, y)\n    slices = np.zeros((nx,ny, 4), dtype=np.int64)\n    \n    for i in range(nx):\n        for j in range(ny):\n            slices[i,j] = x1[i], x2[i], y1[j], y2[j]    \n    return slices.reshape(nx*ny,4)","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:07:52.971696Z","iopub.status.busy":"2021-01-31T14:07:52.970853Z","iopub.status.idle":"2021-01-31T14:07:52.973162Z","shell.execute_reply":"2021-01-31T14:07:52.973657Z"},"papermill":{"duration":0.039615,"end_time":"2021-01-31T14:07:52.97377","exception":false,"start_time":"2021-01-31T14:07:52.934155","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HuBMAPModel(nn.Module):\n    def __init__(self, is_fpn=False, encoder_name='efficientnet-b7', encoder_weights='imagenet'):\n        super(HuBMAPModel, self).__init__()\n        print(f'encoder name: {encoder_name}')\n        if is_fpn:\n            self.model = FPN(encoder_name = encoder_name, \n                          encoder_weights = encoder_weights,\n                          classes = 1,\n                          activation = None)\n        else:\n            self.model = Unet(encoder_name = encoder_name, \n                          encoder_weights = encoder_weights,\n                          classes = 1,\n                          activation = None)\n        \n        \n    def forward(self, images):\n        return self.model(images)","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:07:53.056196Z","iopub.status.busy":"2021-01-31T14:07:53.055429Z","iopub.status.idle":"2021-01-31T14:07:53.058386Z","shell.execute_reply":"2021-01-31T14:07:53.057961Z"},"papermill":{"duration":0.030359,"end_time":"2021-01-31T14:07:53.058475","exception":false,"start_time":"2021-01-31T14:07:53.028116","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(is_fpn=False, encoder_name='efficientnet-b7'):\n    model = HuBMAPModel(is_fpn, encoder_name=encoder_name)\n    return model","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:07:53.105478Z","iopub.status.busy":"2021-01-31T14:07:53.104636Z","iopub.status.idle":"2021-01-31T14:07:53.10766Z","shell.execute_reply":"2021-01-31T14:07:53.107244Z"},"papermill":{"duration":0.027911,"end_time":"2021-01-31T14:07:53.107744","exception":false,"start_time":"2021-01-31T14:07:53.079833","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:07:53.494184Z","iopub.status.busy":"2021-01-31T14:07:53.49344Z","iopub.status.idle":"2021-01-31T14:07:53.496813Z","shell.execute_reply":"2021-01-31T14:07:53.496256Z"},"papermill":{"duration":0.367743,"end_time":"2021-01-31T14:07:53.496911","exception":false,"start_time":"2021-01-31T14:07:53.129168","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.cache/torch/hub/checkpoints/\n!cp /kaggle/input/pytorch-segmentation-models-git/efficientnet-b7-dcc49843.pth /root/.cache/torch/hub/checkpoints/\n!cp /kaggle/input/pytorch-segmentation-models-git/tf_efficientnet_b7_ra-6c08e654.pth /root/.cache/torch/hub/checkpoints/","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:07:53.54761Z","iopub.status.busy":"2021-01-31T14:07:53.546858Z","iopub.status.idle":"2021-01-31T14:07:56.278351Z","shell.execute_reply":"2021-01-31T14:07:56.278836Z"},"papermill":{"duration":2.759892,"end_time":"2021-01-31T14:07:56.278978","exception":false,"start_time":"2021-01-31T14:07:53.519086","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fold_models_paths = glob.glob(os.path.join(PATH_FOLD_MODELS, '*timm*.pth'))\nfold_models_paths = []\nfold_models_paths.extend(glob.glob(os.path.join(PATH_FOLD_MODELS, '/kaggle/working/models/*_best_model_fpn_efficientnetb7_1536_768_double_shift_efficientnet-b7-12b.pth')))\n# fold_models_paths.append('/kaggle/working/models/0_best_model_fpn_efficientnetb7_1536_768_double_shift_efficientnet-b7.pth')\nfold_models_paths","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold_models = []\n\nfor path in fold_models_paths:\n    try:\n        state_dict = torch.load(path)\n        model = get_model(path.find('fpn') > 0, encoder_name='timm-efficientnet-b7' if path.find('timm') > 0 else 'efficientnet-b7')\n        if 'model_state_dict' in state_dict:\n            model.load_state_dict(state_dict['model_state_dict'])\n        else:\n            model.load_state_dict(state_dict)\n        model.float()\n        model.to(DEVICE)\n        model.eval()\n\n        fold_models.append(model)\n    except Exception as e:\n        print(f'Failed to load {path}', e)","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:07:56.330469Z","iopub.status.busy":"2021-01-31T14:07:56.329762Z","iopub.status.idle":"2021-01-31T14:08:02.077332Z","shell.execute_reply":"2021-01-31T14:08:02.076794Z"},"papermill":{"duration":5.775973,"end_time":"2021-01-31T14:08:02.077451","exception":false,"start_time":"2021-01-31T14:07:56.301478","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(fold_models)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"identity = rasterio.Affine(1, 0, 0, 0, 1, 0)\n# identity = None\n\ndef read_from_slice(dataset, x1, x2, y1, y2):\n    image = dataset.read([1,2,3],\n                    window=Window.from_slices((x1,x2),(y1,y2)))\n    image = np.moveaxis(image, 0, -1)\n    return image","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:08:02.15148Z","iopub.status.busy":"2021-01-31T14:08:02.150003Z","iopub.status.idle":"2021-01-31T14:08:02.152521Z","shell.execute_reply":"2021-01-31T14:08:02.152994Z"},"papermill":{"duration":0.053039,"end_time":"2021-01-31T14:08:02.153107","exception":false,"start_time":"2021-01-31T14:08:02.100068","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WINDOW = 1536 # tile size\nMIN_OVERLAP = 32\nNEW_SIZE = 768 # size after re-size which are fed to the model","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:08:02.202372Z","iopub.status.busy":"2021-01-31T14:08:02.200702Z","iopub.status.idle":"2021-01-31T14:08:02.203147Z","shell.execute_reply":"2021-01-31T14:08:02.203568Z"},"papermill":{"duration":0.028502,"end_time":"2021-01-31T14:08:02.203668","exception":false,"start_time":"2021-01-31T14:08:02.175166","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predict","metadata":{"papermill":{"duration":0.021797,"end_time":"2021-01-31T14:08:02.247311","exception":false,"start_time":"2021-01-31T14:08:02.225514","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Affine transforms\nhorizontal_flip = A.HorizontalFlip(p = 1.0)\nvertical_flip = A.VerticalFlip(p = 1.0)\nrotate_cw = A.Rotate(limit = (-90, -90), p = 1.0)\nrotate_acw = A.Rotate(limit = (90, 90), p = 1.0)\n\n# List of augmentations for TTA\ntta_augs = [horizontal_flip,\n            vertical_flip,\n            rotate_cw,\n            rotate_acw]\n\n# List of deaugmentations corresponding to the above aug list\ntta_deaugs = [horizontal_flip,\n              vertical_flip,\n              rotate_acw,\n              rotate_cw]\n\n# # List of augmentations for TTA\n# tta_augs = [horizontal_flip,\n#             vertical_flip]\n\n# # List of deaugmentations corresponding to the above aug list\n# tta_deaugs = [horizontal_flip,\n#               vertical_flip]\n\n# List of augmentations for TTA\n# tta_augs = [horizontal_flip]\n\n# # List of deaugmentations corresponding to the above aug list\n# tta_deaugs = [horizontal_flip]","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:08:02.29898Z","iopub.status.busy":"2021-01-31T14:08:02.297236Z","iopub.status.idle":"2021-01-31T14:08:02.299589Z","shell.execute_reply":"2021-01-31T14:08:02.30002Z"},"papermill":{"duration":0.030777,"end_time":"2021-01-31T14:08:02.300117","exception":false,"start_time":"2021-01-31T14:08:02.26934","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls {DATA_PATH/'test'}","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:08:02.349299Z","iopub.status.busy":"2021-01-31T14:08:02.348722Z","iopub.status.idle":"2021-01-31T14:08:03.031705Z","shell.execute_reply":"2021-01-31T14:08:03.030797Z"},"papermill":{"duration":0.709379,"end_time":"2021-01-31T14:08:03.031836","exception":false,"start_time":"2021-01-31T14:08:02.322457","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef get_preprocessing():\n    _transform = [\n        A.Normalize(mean=[0.6276, 0.4468, 0.6769],\n                       std=[0.1446, 0.2113, 0.1233], \n                       max_pixel_value=255.0, always_apply=True, p=1.0)\n    ]\n    return A.Compose(_transform)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy import stats\nfrom scipy.stats import logistic","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_decode(mask_rle, shape=(256, 256)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    splits = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (splits[0:][::2], splits[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype='uint8')\n    for lo, hi in zip(starts, ends):\n        img[lo: hi] = 1\n    return img.reshape(shape, order='F') # Fortran order reshaping","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\npreprocess_input = get_preprocessing()\n\np = Path(DATA_PATH)\n\nsubmissions = []\noverlaps = [32, 128, 256]\n\nwith torch.no_grad():\n    for overlap in overlaps:\n        subm = {}\n        for i, filename in tqdm(enumerate(p.glob('test/*.tiff')), total = len(list(p.glob('test/*.tiff')))):\n            print(filename)\n\n            dataset = rasterio.open(filename.as_posix(), transform = identity)\n            slices = make_grid(dataset.shape, window=WINDOW, min_overlap=overlap)\n\n            preds = np.zeros(dataset.shape, dtype=np.uint8)\n            if dataset.count != 3:\n                print(f'Image file ({filename}) with subdatasets as channels')\n                layers = [rasterio.open(subd) for subd in dataset.subdatasets]\n\n            for (x1,x2,y1,y2) in tqdm(slices, total = len(slices)):\n                if dataset.count == 3:\n                    image = dataset.read([1,2,3],\n                                window=Window.from_slices((x1,x2),(y1,y2)))\n                    image = np.moveaxis(image, 0, -1)\n                else:\n                    image = np.zeros((WINDOW, WINDOW, 3), dtype=np.uint8)\n                    for fl in range(3):\n                        image[:,:,fl] = layers[fl].read(window=Window.from_slices((x1,x2),(y1,y2)))\n\n                image = preprocess_input(image = image)['image']\n                image = cv2.resize(image, (NEW_SIZE, NEW_SIZE))\n                image = np.moveaxis(image, -1, 0)\n                image = torch.from_numpy(image)\n                pred = np.zeros([len(fold_models), WINDOW, WINDOW])\n                for j, fold_model in enumerate(fold_models):\n                    score = fold_model(image.float().to(DEVICE)[None])\n                    score = score.squeeze().cpu().numpy()\n                    pred[j] = cv2.resize(score, (WINDOW, WINDOW))\n                pred = np.mean(pred, axis=0)\n                preds[x1:x2,y1:y2] = (pred > 0).astype(np.uint8)\n\n            subm[i] = {'id':filename.stem, 'predicted': rle_numba_encode(preds), 'shape': preds.shape}\n        submissions.append(subm)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert len(submissions) == len(overlaps)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import defaultdict\ngrouped_submissions = defaultdict(list)\n\nfor s_dict in submissions:\n    for index, v in s_dict.items():\n        grouped_submissions[v['id']].append({'predicted': v['predicted'], 'shape': v['shape']})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert len(grouped_submissions[list(grouped_submissions.keys())[0]]) == len(submissions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nsubm = {}\nfor i, (k, v) in enumerate(grouped_submissions.items()):\n    mean_mask = np.zeros([v[0]['shape'][0], v[0]['shape'][1]], dtype=np.float16)\n    print(f'{i + 1}. Processing {k} with {len(v)} masks')\n    for encoded in v:\n        mean_mask += rle_decode(encoded['predicted'], encoded['shape'])\n    print(f'Finished adding {len(v)} masks')\n    mean_mask = mean_mask / len(v)\n    mean_mask = (mean_mask > 0.49).astype(np.uint8)\n    subm[i] = {'id': k, 'predicted': rle_numba_encode(mean_mask)}\n    print(f'Finished encoding average')","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:52:25.272516Z","iopub.status.busy":"2021-01-31T14:52:25.272027Z","iopub.status.idle":"2021-01-31T14:52:26.10052Z","shell.execute_reply":"2021-01-31T14:52:26.099449Z"},"papermill":{"duration":0.873259,"end_time":"2021-01-31T14:52:26.100634","exception":false,"start_time":"2021-01-31T14:52:25.227375","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame.from_dict(subm, orient='index')\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:52:26.190269Z","iopub.status.busy":"2021-01-31T14:52:26.189446Z","iopub.status.idle":"2021-01-31T14:52:26.211105Z","shell.execute_reply":"2021-01-31T14:52:26.210631Z"},"papermill":{"duration":0.081237,"end_time":"2021-01-31T14:52:26.211199","exception":false,"start_time":"2021-01-31T14:52:26.129962","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!du -h submission.csv\n!du -k submission.csv","metadata":{"execution":{"iopub.execute_input":"2021-01-31T14:52:26.273656Z","iopub.status.busy":"2021-01-31T14:52:26.273077Z","iopub.status.idle":"2021-01-31T14:52:27.078018Z","shell.execute_reply":"2021-01-31T14:52:27.077544Z"},"papermill":{"duration":0.837668,"end_time":"2021-01-31T14:52:27.078123","exception":false,"start_time":"2021-01-31T14:52:26.240455","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm /kaggle/working/models/*.pth","metadata":{"papermill":{"duration":0.02996,"end_time":"2021-01-31T14:52:27.137831","exception":false,"start_time":"2021-01-31T14:52:27.107871","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}