{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# HuBMAP - Efficient Sampling Baseline (deepflash2, pytorch, fastai) [sub]\n\n> Submission kernel for model trained with efficient region based sampling. \n\n***\n\n## Overview\n\n1. Installation and package loading\n2. Functions and classes for prediction\n3. Configuration\n4. Prediction\n5. Submission\n\n#### Related Kernels\n\n- Train Notebook: https://www.kaggle.com/matjes/hubmap-efficient-sampling-deepflash2-train\n- Sampling Notebook: https://www.kaggle.com/matjes/hubmap-labels-pdf-0-5-0-25-0-01\n\n#### Versions\n- V12: Minor changes in deepflash2 API to support albumentations (changes `apply`in `DeformationField` slightly, see patch below)\n- V13: Adding prediction threshold 0.4\n- V14: Threshold 0.2 for d488c759a - see discussion https://www.kaggle.com/c/hubmap-kidney-segmentation/discussion/228993 \n- V15: **NEW PREDICTION** \n    - Using overlapping tiles and gaussian weighting from [nnunet](https://www.nature.com/articles/s41592-020-01008-z)/[github](https://github.com/MIC-DKFZ/nnUNet), which will also be part of the upcoming `deepflash2` release\n    - Supporting model ensembles\n    - Fixing submission to private LB using `rasterio` (thanks to @leighplt [kernel](https://www.kaggle.com/leighplt/pytorch-fcn-resnet50) and @iafoss ([kernel](https://www.kaggle.com/iafoss/hubmap-pytorch-fast-ai-starter-sub))","metadata":{"id":"OcsetTMwKXqC"}},{"cell_type":"markdown","source":"### Installation and package loading","metadata":{}},{"cell_type":"code","source":"# Install deepflash2 and dependencies\nimport sys\nsys.path.append(\"../input/segmentation-models-pytorch-install\")\n!pip install -q --no-deps ../input/deepflash2-lfs\nimport cv2, torch, gc, rasterio\nimport torch.nn.functional as F\nimport deepflash2.tta as tta\nimport matplotlib.pyplot as plt\nimport pandas as pd, numpy as np\nimport segmentation_models_pytorch as smp\nfrom pathlib import Path\nfrom rasterio.windows import Window\nfrom torch.utils.data import Dataset, DataLoader\nfrom scipy .ndimage.filters import gaussian_filter\nfrom tqdm.notebook import tqdm\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Functions and classes for prediction","metadata":{}},{"cell_type":"code","source":"#https://www.kaggle.com/bguberfain/memory-aware-rle-encoding\n#with transposed mask\ndef rle_encode_less_memory(img):\n    #the image should be transposed\n    pixels = img.T.flatten()\n    \n    # This simplified method requires first and last pixel to be zero\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] -= runs[::2]\n    \n    return ' '.join(str(x) for x in runs)\n\n# def load_model_weights(model, file, strict=True):\n#     state = torch.load(file, map_location='cpu')\n#     stats = state['stats']\n#     model_state = state['model']\n#     model.load_state_dict(model_state, strict=strict)\n#     return model, stats\n\n\ndef load_model_weights(model, file, strict=True):\n    state = torch.load(file, map_location='cpu')\n    try:\n        \n        model_state = state['model']\n    except:\n        model_state=state\n    model.load_state_dict(model_state, strict=strict)\n    model.eval()\n    return model\n\n\n# from https://github.com/MIC-DKFZ/nnUNet/blob/2fade8f32607220f8598544f0d5b5e5fa73768e5/nnunet/network_architecture/neural_network.py#L250\ndef _get_gaussian(patch_size, sigma_scale=1. / 8) -> np.ndarray:\n    tmp = np.zeros(patch_size)\n    center_coords = [i // 2 for i in patch_size]\n    sigmas = [i * sigma_scale for i in patch_size]\n    tmp[tuple(center_coords)] = 1\n    gaussian_importance_map = gaussian_filter(tmp, sigmas, 0, mode='constant', cval=0)\n    gaussian_importance_map = gaussian_importance_map / np.max(gaussian_importance_map) * 1\n    gaussian_importance_map = gaussian_importance_map.astype(np.float32)\n\n    # gaussian_importance_map cannot be 0, otherwise we may end up with nans!\n    gaussian_importance_map[gaussian_importance_map == 0] = np.min(\n        gaussian_importance_map[gaussian_importance_map != 0])\n\n    return gaussian_importance_map","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Some code adapted from https://www.kaggle.com/iafoss/hubmap-pytorch-fast-ai-starter-sub\nclass HubmapDataset(Dataset):\n    'HubmapDataset class that does not load the full tiff files.'\n    def __init__(self, file, stats, scale=3, shift=.8, output_shape=(512,512), s_th = 40):\n        \n        self.mean, self.std = stats\n        self.scale = scale\n        self.shift = shift\n        self.output_shape = output_shape\n        self.input_shape = tuple(int(t*scale) for t in self.output_shape)      \n        self.s_th = s_th #saturation blancking threshold\n        self.p_th = 1000*(self.output_shape[0]//256)**2 #threshold for the minimum number of pixels\n\n        identity = rasterio.Affine(1, 0, 0, 0, 1, 0)\n        self.data = rasterio.open(file, transform = identity, num_threads='all_cpus')\n        if self.data.count != 3:\n            subdatasets = self.data.subdatasets\n            self.layers = []\n            if len(subdatasets) > 0:\n                for i, subdataset in enumerate(subdatasets, 0):\n                    self.layers.append(rasterio.open(subdataset))\n            \n        # Tiling\n        self.slices = []\n        self.out_slices = []\n        self.out_data_shape = tuple(int(x//self.scale) for x in self.data.shape)\n        start_points = [o//2 for o in self.output_shape]\n        end_points = [(s - st) for s, st in zip(self.out_data_shape, start_points)]\n        n_points = [int(s//(o*self.shift))+1 for s, o in zip(self.out_data_shape, self.output_shape)]\n        center_points = [np.linspace(st, e, num=n, endpoint=True, dtype=np.int64) for st, e, n in zip(start_points, end_points, n_points)]\n        for cx in center_points[1]:\n            for cy in center_points[0]:\n                # Calculate output slices for whole image\n                slices = tuple(slice(int((c*self.scale - o/2).clip(0, s)), int((c*self.scale + o/2).clip(max=s)))\n                                 for (c, o, s) in zip((cy, cx), self.input_shape, self.data.shape))\n                self.slices.append(slices)\n                \n                out_slices = tuple(slice(int((c - o/2).clip(0, s)), int((c + o/2).clip(max=s)))\n                                 for (c, o, s) in zip((cy, cx), self.output_shape, self.out_data_shape))\n                self.out_slices.append(out_slices)\n                \n\n    def __len__(self):\n        return len(self.slices)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n            \n        slices = self.slices[idx]\n        if self.data.count == 3: # normal\n            img = self.data.read([1, 2, 3], \n                window=Window.from_slices(*slices)\n            )\n            img = np.moveaxis(img, 0, -1)\n        else: # with subdatasets/layers\n            img = np.zeros((*self.input_shape, 3), dtype=np.uint8)\n            for fl in range(3):\n                img[:, :, fl] = self.layers[fl].read(\n                    window=Window.from_slices(*slices)\n                )\n        \n        if self.scale!=1:\n            img = cv2.resize(img, self.output_shape, interpolation = cv2.INTER_AREA)\n        \n        #check for empty imges\n        hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)\n        h,s,v = cv2.split(hsv)\n        if (s>self.s_th).sum() <= self.p_th or img.sum() <= self.p_th:\n            # Remove if idx=-1\n            idx = -1\n        \n        img = (img/255.0 - self.mean)/self.std\n        img = img.transpose(2, 0, 1).astype('float32')\n        \n        return torch.from_numpy(img), idx\n    \n","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Configuration","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class CONFIG():\n    \n#     # data paths\n#     data_path = Path('../input/hubmap-kidney-segmentation')\n# #     model_file = '../input/deep-flash-resnest50d/unet_timm-resnest50d.pth'\n#     model_file = \"../input/hubmap-deepflash2/model_9545.pth\"\n    \n#     # deepflash2 dataset (https://matjesg.github.io/deepflash2/data.html#TileDataset)\n#     scale = 3 # zoom facor (zoom out)\n    \n#     tile_shape = (1024,1024)\n#     padding = (100,100) # Border overlap for prediction\n\n#     # pytorch model (https://github.com/qubvel/segmentation_models.pytorch)\n#     encoder_name = \"timm-regnetx_064\"\n#     encoder_weights = None\n#     in_channels = 3\n#     classes = 2\n    \n#     # dataloader \n#     batch_size = 7\n    \n#     # prediction threshold\n#     threshold = 0.45\n    \n# cfg = CONFIG()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sample submissions for ids\n# df_sample = pd.read_csv(cfg.data_path/'sample_submission.csv',  index_col='id')\n# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# # Models (see https://github.com/qubvel/segmentation_models.pytorch)\n# MODELS = [f for f in cfg.model_path.iterdir() if f.suffix=='.pth']\n# print(f'Found {len(MODELS)} models', *MODELS)\n\n# models = []\n# for i, m_path in enumerate(MODELS):\n#     #state_dict = torch.load(path,map_location=torch.device('cpu'))\n#     model = smp.Unet(encoder_name=cfg.encoder_name, \n#                      encoder_weights=cfg.encoder_weights, \n#                      in_channels=cfg.in_channels, \n#                      classes=cfg.classes)\n#     model, stats = load_model_weights(model, m_path)\n#     model.float()\n#     model.eval()\n#     model.to(device)\n#     models.append(model)\n#     models.append(model)\n\n# mp = Model_pred(models, use_tta=cfg.tta, batch_size=cfg.batch_size)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model_pred:\n    'Class for prediction with multiple models'\n    def __init__(self, models, use_tta=True, batch_size=32):\n        self.models = models\n        self.bs = batch_size\n        #self.tfms = [tta.HorizontalFlip()] if use_tta else [] #, tta.VerticalFlip()]  \n        self.tfms = [tta.HorizontalFlip(), tta.VerticalFlip(), tta.Rotate90(angles=[90,180,270])] if use_tta else [] #, tta.VerticalFlip()]  \n    def predict(self, ds):\n        #rasterio cannot be used with multiple workers\n        dl = DataLoader(ds, self.bs, num_workers=0, shuffle=False, pin_memory=True)\n        \n        # Create zero arrays\n        pred = np.zeros(ds.out_data_shape, dtype='float32')\n        merge_map = np.zeros(ds.out_data_shape, dtype='float32')\n        \n        # Gaussian weights\n        gw_numpy = _get_gaussian(ds.output_shape)\n        gw = torch.from_numpy(gw_numpy).to(device)\n        \n        with torch.no_grad():\n            for images, idxs in tqdm(iter(dl), total=len(dl)):\n                if ((idxs>=0).sum() > 0): #exclude empty images\n                    images = images[idxs>=0].to(device)\n                    idxs = idxs[idxs>=0]\n                    merger = tta.Merger()\n                    for t in tta.Compose(self.tfms):\n                        aug_images = t.augment_image(images)\n                        model_merger = tta.Merger()\n                        for model in self.models:\n                            out = model(aug_images)\n                            out = F.softmax(out, dim=1)\n                            model_merger.append(out)\n                        out = t.deaugment_mask(model_merger.result())\n                        merger.append(out)\n            \n                    # Apply gaussian weigthing\n                    batch_smx = merger.result()*gw.view(1,1,*gw.shape)\n                    batch_smx = [x for x in batch_smx.permute(0,2,3,1).cpu().numpy()]\n                    \n                    for smx, idx in zip(batch_smx, idxs):\n                        slcs = ds.out_slices[idx]\n                        # Only using positive class here\n                        pred[slcs] += smx[...,1]\n                        merge_map[slcs] += gw_numpy\n                    \n                    '''\n                    for i , (smx, idx) in enumerate(zip(batch_smx, idxs)):\n                        slcs = ds.out_slices[idx]\n                        img = images[i].detach().cpu().numpy()\n                        #img = np.moveaxis(img, 0, -1)\n                        #img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n                        mask = (smx[...,1] > 0.45).astype(np.uint8)\n                        #mask = (smx[...,1]*255).astype(np.uint8)\n                        #print(img.shape)\n                        #print(mask.shape)\n                        new_pred = crf(images[i],mask)\n                        \n                        #print(new_pred.shape)\n                        pred[slcs] += new_pred\n                        merge_map[slcs] += gw_numpy\n                    '''\n                    \n\n        pred /= merge_map\n        return pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.encoder","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.encoder","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n\nmodel = smp.DeepLabV3Plus(encoder_name=cfg.encoder_name, \n                 encoder_weights=cfg.encoder_weights, \n                 in_channels=cfg.in_channels, \n                 classes=cfg.classes,\n                 decoder_atrous_rates = (6, 12, 18),\n                  encoder_output_stride =8        ) #stage 4,5 wil have dil as 2 /4 else  for 16 stride its 2 for stage 5 last stage\n#model.decoder\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.decoder\n# depth convolution means in channel=Groups number so 1 0r out_channel/groups kernels will examine each group\n# where each group = In_channel/groups\n#point wise convolution - Instead of convolving  point matrix we convolve only one Point.3*3=1*1 in Point\n#spatial dim= mean H * W conv\n\nc1= torch.nn.Conv2d(3,3,(5,5),stride=1)\nc2= torch.nn.Conv2d(3,3,(1,1),stride=1,dilation =(1,3 ))\ni1=torch.randn(2,3,12,12)\nc2(c1(i1)).size()\n  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#state=torch.load('../input/hubmap-deeplabv3-v14/v14_stage3/model_6.pth',device)\n ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#state=torch.load('../input/hubmap-deeplabv3-v14/DeepLab_timm-regnetx_064_v14.pth',device)\n#state\n ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CONFIG():\n    \n    # data paths\n    data_path = Path('../input/hubmap-kidney-segmentation')\n    \n    #model_path = Path('../input/hubmap-deepflash2/model_9545.pth')\n    #model_path=[Path('../input/hubmap-deeplabv3-v14/v14_stage3/model_6.pth')\n      #          ,Path('../input/hubmap-deeplabv3-v14/v14_stage3/model_9.pth')\n                #,Path('../input/hubmap-deepflash2/model_9545.pth')\n     #          ]\n    \n    #model_path=[ Path('../input/hubmap-deeplabv3-v14/v14_stage3/model_9.pth') ]\n    model_path=[ Path('../input/hubmap-deeplabv3-v14/v14_stage3/model_6.pth'), \n                 Path('../input/hubmap-deeplabv3-v14/v14_stage3/model_9.pth'), \n                 Path('../input/hubmap-deepflash-1024-effnetb2-v06-pl2/model_9.pth'),\n                 Path('../input/hubmap-deepflash-1024-effnetb2-v07-nonpld48/model_11.pth'),\n                 Path('../input/hubmap-deepflash-1024-effnetb3-v08-nonpld48/model_11.pth'),\n                 Path('../input/hubmap-deepflash-1024-effnetb3-v09-pl2/model_12.pth')]\n                 \n    #            Path('../input/hubmap-deepflash2/model_9545.pth')]\n    #model_path=[ Path('../input/hubmap-deeplabv3-v14/v14_stage2/model_11.pth')  ]            \n    #Path('../input/hubmap-deeplabv3-v14/v14_stage2/model_11.pth')\n    #Path('../input/hubmap-deeplabv3-v14/DeepLab_timm-regnetx_064_v14.pth')\n    # zoom factor (e.g., 3 means downscaling from 1536 to 512)\n    scale =3 \n    # tile shift for prediction\n    shift = 0.8 \n#     tile_shape = (512, 512)\n    tile_shape = (1024, 1024)\n\n    # pytorch model (https://github.com/qubvel/segmentation_models.pytorch)\n    encoder_name = [\"timm-regnetx_064\",\"timm-regnetx_064\",  'efficientnet-b2',\n                    'efficientnet-b2', 'timm-efficientnet-b3','timm-efficientnet-b3']\n    encoder_weights = None\n    in_channels = 3\n    classes = 2\n    \n    # dataloader \n    batch_size = 8\n    \n    # test time augmentation\n    tta = True\n    # prediction threshold\n    threshold = 0.45\n    \ncfg = CONFIG()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for  x in cfg.encoder_name:\n    print(x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample = pd.read_csv(cfg.data_path/'sample_submission.csv',  index_col='id')\n#df_sample = pd.read_csv('../input/hubmap-kidney-segmentation/train.csv',  index_col='id')\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n\n# Model (see https://github.com/qubvel/segmentation_models.pytorch)\n'''\nmodel = smp.Unet(encoder_name=cfg.encoder_name, \n                 encoder_weights=cfg.encoder_weights, \n                 in_channels=cfg.in_channels, \n                 classes=cfg.classes)\n'''\nmodels=[]\n#for  path,name in  zip(cfg.model_path,cfg.encoder_name):\nfor  name,path in zip(cfg.encoder_name,cfg.model_path ) :\n    print(path,name)\n    if 'deeplab' in str(path):\n        print(path)\n        \n        model = smp.DeepLabV3Plus(encoder_name=name, \n                         encoder_weights=cfg.encoder_weights, \n                         in_channels=cfg.in_channels, \n                         classes=cfg.classes)\n        # model, stats = load_model_weights(model, cfg.model_file)\n        #model = torch.nn.DataParallel(model, device_ids=[0])\n        model = load_model_weights(model,path)\n        model.float()\n        model.eval()\n        model.to(device)\n        print('loaded',path)\n        models.append(model)\n    \n    else:\n        \n        \n        model= smp.Unet(encoder_name=name, \n                         encoder_weights=cfg.encoder_weights, \n                         in_channels=cfg.in_channels, \n                         classes=cfg.classes)\n        # model, stats = load_model_weights(model, cfg.model_file)\n        #model = torch.nn.DataParallel(model, device_ids=[0])\n        model = load_model_weights(model, path)\n        model.float()\n        model.eval()\n        model.to(device)\n        print('loaded 1',path)\n        models.append(model)\n\n     \n\n\nstats = np.array([0.61561477, 0.5179343 , 0.64067212]), np.array([0.2915353 , 0.31549066, 0.28647661])\n\n\n#mp = Model_pred([model], use_tta=cfg.tta, batch_size=cfg.batch_size)\nmp = Model_pred(models, use_tta=cfg.tta, batch_size=cfg.batch_size)\n\n\n# batch_tfms = [Normalize.from_stats(*stats)]\n# print(stats)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mp\nlen(models)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def area_threshold(img,min_area=200,top_threshold=0.45):\n    classification=img >top_threshold\n    #print(classification.shape,img.shape)\n    cont , heir = cv2.findContours(classification.astype('uint8'),cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)\n    \n    areas = []\n    for c in cont:\n        areas.append(cv2.contourArea(c))\n    \n    max_area = np.max(areas)\n    print(np.max(areas), np.min(areas), 'max',np.percentile(areas, 0.8), 'min',np.percentile(areas, 0.1))\n    ''''\n    if np.min(areas) > 600:\n        min_area = np.percentile(areas, 0.1) \n    else:\n        min_area = 500\n    ''' \n    \n    if np.min(areas) > 500:\n        min_area = np.percentile(areas, 0.1) \n    else:\n        min_area =  500\n    \n    \n    \n#     min_area = min(500, np.percentile(areas, 0.8))\n#     min_area = 500 \n#     print(min_area)\n    \n    for c in cont:\n        area=cv2.contourArea(c)\n        if area<min_area:\n            zero_mask= np.zeros(img.shape,np.uint8)\n#             cv2.drawContours(zero_mask,[c],0,255,-1)\n            cv2.drawContours(zero_mask, [c], 0, 255, -1)\n            \n            c0,c1=np.nonzero(zero_mask)\n            img[c0,c1]=0\n            \n    return img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction","metadata":{}},{"cell_type":"code","source":"#!mkdir -p /tmp/pip/cache/\n#!cp ../input/pydensecr/pydensecrf-1.0rc2-cp37-cp37m-linux_x86_64.whl /tmp/pip/cache/\n#!pip install --no-index --find-links /tmp/pip/cache/ pydensecrf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import pydensecrf.densecrf as dcrf\nfrom skimage.io import imread, imsave\n#from pydensecrf.utils import unary_from_labels, create_pairwise_bilateral\nfrom skimage.color import gray2rgb\nfrom skimage.color import rgb2gray","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Original_image = Image which has to labelled\n#Mask image = Which has been labelled by some technique..\ndef crf(original_image, mask_img):\n    \n    # Converting annotated image to RGB if it is Gray scale\n    if(len(mask_img.shape)<3):\n        mask_img = gray2rgb(mask_img)\n\n#     #Converting the annotations RGB color to single 32 bit integer\n    annotated_label = mask_img[:,:,0] + (mask_img[:,:,1]<<8) + (mask_img[:,:,2]<<16)\n    \n#     # Convert the 32bit integer color to 0,1, 2, ... labels.\n    colors, labels = np.unique(annotated_label, return_inverse=True)\n\n    n_labels = 2\n    \n    #Setting up the CRF model\n    #d = dcrf.DenseCRF2D(original_image.shape[1], original_image.shape[0], n_labels)\n    d = dcrf.DenseCRF2D(original_image.shape[1], original_image.shape[2], n_labels)\n    #print( original_image.shape)\n    # get unary potentials (neg log probability)\n    #print(len(labels),n_labels)\n    U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=False)\n    #print(U.shape)\n    d.setUnaryEnergy(U)\n\n    # This adds the color-independent term, features are the locations only.\n    d.addPairwiseGaussian(sxy=(3, 3), compat=3, kernel=dcrf.DIAG_KERNEL,\n                      normalization=dcrf.NORMALIZE_SYMMETRIC)\n        \n    #Run Inference for 30 steps \n    Q = d.inference(6)\n\n    # Find out the most probable class for each pixel.\n    MAP = np.argmax(Q, axis=0)\n\n    return MAP.reshape((original_image.shape[1],original_image.shape[2]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample.iloc[4]\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"names,preds = [],[]\nprint(df_sample.iloc[5])\nfor idx,row in tqdm(df_sample.iterrows(),total=len(df_sample)):\n    \n    print(f'###### File {idx} ######')\n    f = cfg.data_path/'test'/f'{idx}.tiff'\n    ds = HubmapDataset(f, stats, scale=cfg.scale, shift=cfg.shift, output_shape=cfg.tile_shape)\n    \n#     ds = TileDataset([f], scale=cfg.scale, tile_shape=cfg.tile_shape, padding=cfg.padding,**ds_kwargs)\n    \n    \n    print('Predicting...')   \n    pred = mp.predict(ds)\n    pred = area_threshold(pred)\n        \n    print('Rezising...')\n    shape = ds.data.shape\n    nan_array =~ np. isnan(pred)\n    \n    print(pred[nan_array] [np.where(pred[nan_array] >0.15)].mean(),\n          pred[nan_array] [np.where(pred[nan_array] >0.15)].std())\n    #break\n    pred = cv2.resize((pred*255).astype('uint8'), (shape[1], shape[0]))\n    \n    th = 0.2 if idx=='d488c759a' else cfg.threshold\n    \n    pred = (pred>th*255).astype(np.uint8)\n    \n    #convert to rle\n    #https://www.kaggle.com/bguberfain/memory-aware-rle-encoding\n    rle = rle_encode_less_memory(pred)\n    names.append(idx)\n    preds.append(rle)\n    \n    print('Plotting')\n    fig, ax = plt.subplots(figsize=(15,15))\n    ax.imshow(cv2.resize(pred, (1024, 1024*shape[0]//shape[1])))\n    plt.show()\n    \n    del pred\n    gc.collect()\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls  -l # 5534885 \ndf_sample.reset_index()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" \n\n#np.nansum(pred).mean()\n\n#nan_array =~ np. isnan(pred)\n#pred[nan_array] [np.where(pred[nan_array] >0.2)].std()\n\ndf_sample.reset_index().to_csv('submission.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_sample.iloc[5])\ndf = pd.DataFrame({'id':names,'predicted':preds})\ndf.to_csv('submission.csv',index=False)\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" ","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}