{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"pip -q install ../input/smp-packages/pretrainedmodels-0.7.4-py3-none-any.whl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip -q install ../input/smp-packages/efficientnet_pytorch-0.6.3-py2.py3-none-any.whl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip -q install ../input/smp-packages/timm-0.3.2-py3-none-any.whl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip -q install ../input/smp-packages/segmentation_models_pytorch-0.1.3-py3-none-any.whl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport gc\nimport torch\nimport pathlib\nimport rasterio\nfrom rasterio.windows import Window\nimport segmentation_models_pytorch as smp\nfrom tqdm.notebook import tqdm\nimport cv2\nimport albumentations as albu\nfrom albumentations.pytorch import ToTensorV2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WINDOW = 1024\nimage_size = 512\nMIN_OVERLAP = 128\nTHRESHOLD = 0.3\nBATCH_SIZE = 8","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# best dice\nmodel_pths = [\n    '../input/hubmap-exp007/best_loss_fold0.pth',\n    '../input/hubmap-exp007/best_loss_fold1.pth',\n    '../input/hubmap-exp007/best_loss_fold2.pth',\n    '../input/hubmap-exp007/best_loss_fold3.pth',\n    '../input/hubmap-exp007/best_loss_fold4.pth'\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions","metadata":{}},{"cell_type":"code","source":"def rle_encode_less_memory(img):\n    pixels = img.T.flatten()\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef make_grid(shape, window=256, min_overlap=32):\n    \"\"\"\n        Return Array of size (N,4), where N - number of tiles,\n        2nd axis represente slices: x1,x2,y1,y2 \n    \"\"\"\n    x, y = shape\n    nx = x // (window - min_overlap) + 1\n    x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)\n    x1[-1] = x - window\n    x2 = (x1 + window).clip(0, x)\n    ny = y // (window - min_overlap) + 1\n    y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)\n    y1[-1] = y - window\n    y2 = (y1 + window).clip(0, y)\n    slices = np.zeros((nx,ny, 4), dtype=np.int64)\n    \n    for i in range(nx):\n        for j in range(ny):\n            slices[i,j] = x1[i], x2[i], y1[j], y2[j]    \n    return slices.reshape(nx*ny,4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_from_slice(dataset, x1, x2, y1, y2):\n    if dataset.count == 3:\n        image = dataset.read([1, 2, 3], window=Window.from_slices((x1, x2), (y1, y2)))\n        image = np.moveaxis(image, 0, -1)\n    else:\n        subdatasets = dataset.subdatasets\n        if len(subdatasets) > 0:\n            image = np.zeros((WINDOW, WINDOW, len(subdatasets)), dtype=np.uint8)\n            for i, subdataset in enumerate(subdatasets, 0):\n                with rasterio.open(subdataset) as layer:\n                    image[:,:,i] = layer.read(1, window=Window.from_slices((x1, x2), (y1, y2)))\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(model_pths):\n    models = []\n    for model_pth in model_pths:\n        model = smp.Unet(\n        'timm-efficientnet-b4', \n        encoder_weights=None, \n        in_channels=3, \n        classes=1, \n        activation=None,\n        decoder_use_batchnorm=True\n        )\n        state = torch.load(model_pth)\n        model.load_state_dict(state)\n        model.eval()\n        models.append(model)\n    return models","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = albu.Compose([\n    albu.Resize(image_size, image_size),\n    albu.Normalize(),\n    ToTensorV2()\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def infer(model, image):\n    with torch.no_grad():\n        outputs = model(image)\n        preds = torch.sigmoid(outputs.detach().cpu()).data.numpy()\n        \n    return preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# infer","metadata":{}},{"cell_type":"code","source":"p = pathlib.Path('../input/hubmap-kidney-segmentation')\nidentity = rasterio.Affine(1, 0, 0, 0, 1, 0)\n\nids = []\npredictions = []\nmodels = load_model(model_pths)\n\nfor i, filename in enumerate(p.glob('test/*.tiff')):\n    print(f'{i+1} Predicting {filename.stem}')\n    \n    dataset = rasterio.open(filename.as_posix(), transform = identity)\n    slices = make_grid(dataset.shape, window=WINDOW, min_overlap=MIN_OVERLAP)\n    preds = np.zeros(dataset.shape, dtype=np.uint8)\n    \n    batch_images_list = []\n    batch_cords_list = []\n    j = 0\n    for idx, (x1,x2,y1,y2) in enumerate(tqdm(slices)):\n        image = read_from_slice(dataset, x1, x2, y1, y2)\n        if image.sum() > 0:  # filter out black images\n            image = transform(image=image)['image']\n            batch_images_list.append(image)\n            batch_cords_list.append([x1, x2, y1, y2])\n            if (j+1) % BATCH_SIZE == 0 or (idx+1) == len(slices):\n                batch_images = torch.stack(batch_images_list)\n                pred = None\n                for model in models:\n                    if pred is None:\n                        pred = infer(model.cuda(), batch_images.cuda())\n                    else:\n                        pred += infer(model.cuda(), batch_images.cuda())\n                pred = pred / len(models)\n\n                for cord, prd in zip(batch_cords_list, pred):\n                    x1, x2, y1, y2 = cord[0], cord[1], cord[2], cord[3]\n                    prd = cv2.resize(prd.squeeze(), (WINDOW, WINDOW))\n                    preds[x1:x2, y1:y2] += (prd.squeeze() > THRESHOLD).astype(np.uint8)\n\n                del batch_images\n                del batch_images_list\n                del batch_cords_list\n                batch_images_list = []\n                batch_cords_list = []\n                torch.cuda.empty_cache()\n            j += 1\n            \n    preds = (preds > THRESHOLD).astype(np.uint8)\n    ids.append(filename.stem)\n    predictions.append(rle_encode_less_memory(preds))\n    #print(np.sum(preds))\n    del dataset\n    del slices\n    del preds\n    gc.collect();\n    #break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Making submission","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({\n    'id': ids,\n    'predicted': predictions\n})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}