{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/segmentation-models/segmentation_models.pytorch.0.1.3/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/segmentation-models/segmentation_models.pytorch.0.1.3/efficientnet_pytorch-0.6.3/efficientnet_pytorch-0.6.3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/segmentation-models/segmentation_models.pytorch.0.1.3/timm-0.3.2-py3-none-any.whl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/segmentation-models/segmentation_models.pytorch.0.1.3/segmentation_models.pytorch.0.1.3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport pdb\nimport glob\nimport pytz\nimport warnings\nimport pickle\nimport random\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm_notebook as tqdm\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, ExponentialLR\nfrom sklearn.model_selection import KFold\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nimport torch.backends.cudnn as cudnn\nfrom torch.utils.data import DataLoader, Dataset, sampler\nfrom albumentations.pytorch import ToTensorV2\nimport segmentation_models_pytorch as smp\n\nimport tifffile as tiff\nimport rasterio\nfrom rasterio.windows import Window","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for data agumentation\nfrom albumentations import (\n    Compose,\n    CenterCrop,\n    CLAHE,\n    Resize,\n    Normalize\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Initialize parameters\nheight, width = 1024, 1024\nreduce = 2\nTHRESHOLD = 0.40\nwindow = 2048\nmin_overlap = 256\nDATA = '/kaggle/input/hubmap-kidney-segmentation/test/'\nMODELS = [\"/kaggle/input/unet-timm-effnetb4/model_HuBMAP_Unet_timm_EffNetB4_NS_fold0.pth\"]\ndf_sample = pd.read_csv('/kaggle/input/hubmap-kidney-segmentation/sample_submission.csv')\nbatch_size = 16 #8\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/bguberfain/memory-aware-rle-encoding\n#with transposed mask\ndef rle_encode_less_memory(img):\n    #the image should be transposed\n    pixels = img.T.flatten()\n    \n    # This simplified method requires first and last pixel to be zero\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] -= runs[::2]\n    \n    return ' '.join(str(x) for x in runs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Imagenet statistics Mean and variance\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\nidentity = rasterio.Affine(1, 0, 0, 0, 1, 0)\n\ndef get_transforms(mean, std):\n    list_transforms = [Resize(height=height, width=width, interpolation=cv2.INTER_AREA, p=1.0)]\n    list_transforms.extend(\n        [\n            Normalize(mean=mean, std=std, p=1.0),\n            ToTensorV2(),\n        ]\n    )\n    list_trfms = Compose(list_transforms)\n    return list_trfms","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_grid(shape, window=256, min_overlap=32):\n    \"\"\"\n        Return Array of size (N,4), where N - number of tiles,\n        2nd axis represente slices: x1,x2,y1,y2 \n    \"\"\"\n    x, y = shape\n    nx = x // (window - min_overlap) + 1\n    x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)\n    x1[-1] = x - window\n    x2 = (x1 + window).clip(0, x)\n    ny = y // (window - min_overlap) + 1\n    y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)\n    y1[-1] = y - window\n    y2 = (y1 + window).clip(0, y)\n    slices = np.zeros((nx,ny, 4), dtype=np.int64)\n    \n    for i in range(nx):\n        for j in range(ny):\n            slices[i,j] = x1[i], x2[i], y1[j], y2[j]    \n    return slices.reshape(nx*ny,4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HuBMAPDataset(Dataset):\n    def __init__(self, data):\n        self.data = data\n        if self.data.count != 3:\n            subdatasets = self.data.subdatasets\n            self.layers = []\n            if len(subdatasets) > 0:\n                for i, subdataset in enumerate(subdatasets, 0):\n                    self.layers.append(rasterio.open(subdataset))\n        self.shape = self.data.shape\n        self.mask_grid = make_grid(self.data.shape, window=window, min_overlap=min_overlap)\n        self.transforms = get_transforms(mean, std)\n        \n    def __len__(self):\n        return len(self.mask_grid)\n    def __getitem__(self, idx):\n        x1, x2, y1, y2 = self.mask_grid[idx]\n        if self.data.count == 3:\n            img = data.read([1,2,3], window=Window.from_slices((x1, x2), (y1, y2)))\n            img = np.moveaxis(img, 0, -1)\n        else:\n            img = np.zeros((window, window, 3), dtype=np.uint8)\n            for i, layer in enumerate(self.layers):\n                img[:,:,i] = layer.read(window=Window.from_slices((x1,x2),(y1,y2)))\n        augmented = self.transforms(image=img)\n        img = augmented['image']\n        vetices = torch.tensor([x1, x2, y1, y2])\n        return img, vetices","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = []\nfor path in MODELS:\n    state_dict = torch.load(path, map_location=torch.device('cpu'))\n    model = smp.Unet('timm-efficientnet-b4', classes=1, encoder_weights=None)\n    model.load_state_dict(state_dict)\n    model.eval()\n    model.to(device)\n    models.append(model)\n\ndel state_dict\nprint(len(models))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Make_prediction(img, tta = True):\n    pred = None\n    with torch.no_grad():\n        for model in models:\n            p_tta = None\n            p = model(img)\n            p = torch.sigmoid(p).detach()\n            if p_tta is None:\n                p_tta = p\n            else:\n                p_tta += p\n            if tta:\n                #x,y,xy flips as TTA\n                flips = [[-1],[-2],[-2,-1]]\n                for f in flips:\n                    imgf = torch.flip(img, f)\n                    p = model(imgf)\n                    p = torch.flip(p, f)\n                    p_tta += torch.sigmoid(p).detach()\n                p_tta /= (1+len(flips))\n            if pred is None:\n                pred = p_tta\n            else:\n                pred += p_tta\n        pred /= len(models)\n    return pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"names, predictions = [],[]\nfor idx, row in tqdm(df_sample.iterrows(),total=len(df_sample)):\n    imageId = row['id']\n    data = rasterio.open(os.path.join(DATA, imageId+'.tiff'), transform = identity, num_threads='all_cpus')\n    preds = np.zeros(data.shape, dtype=np.uint8)\n    dataset = HuBMAPDataset(data)\n    dataloader = DataLoader(dataset, batch_size, num_workers=0, shuffle=False, pin_memory=True)\n    for i, (img, vertices) in enumerate(dataloader):\n        img = img.to(device)\n        pred = Make_prediction(img)\n        pred = pred.squeeze().cpu().numpy()\n        vertices = vertices.numpy()\n        for p, vert in zip(pred, vertices):\n            x1, x2, y1, y2 = vert\n            p = cv2.resize(p, (window, window))\n            preds[x1:x2,y1:y2] += (p > THRESHOLD).astype(np.uint8)\n    preds = (preds > 0.5).astype(np.uint8)\n    #convert to rle\n    rle = rle_encode_less_memory(preds)\n    names.append(imageId)\n    predictions.append(rle)\n    del preds, dataset, dataloader\n    gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print('names',names)\n#print('predictions',len(predictions))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({'id':names,'predicted':predictions})\ndf.to_csv('/kaggle/working/submission.csv', index=False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_sample=pd.read_csv('../input/hubmap-kidney-segmentation/sample_submission.csv')\n#display(df_sample)\n#df = pd.DataFrame({'id':names,'predicted':predictions})\n#df_sample.loc[df.index.values] = df.values  \n\n#df_sample.to_csv('/kaggle/working/submission.csv')\n#display(df_sample)\n#display(df_sample_1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}