{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport tifffile as tiff\nimport cv2\nimport os\nimport gc\nfrom tqdm.notebook import tqdm\nimport zipfile","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MASKS = '../input/hubmap-kidney-segmentation/train.csv'\nDATA = '../input/hubmap-kidney-segmentation/train/'\nOUT_TRAIN1 = 'train1.zip'\nOUT_MASKS1 = 'masks1.zip'\nOUT_TRAIN2 = 'train2.zip'\nOUT_MASKS2 = 'masks2.zip'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_masks = pd.read_csv(MASKS).set_index('id')\ndf_masks.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def enc2mask(encs, shape):\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for m,enc in enumerate(encs):\n        if isinstance(enc,np.float) and np.isnan(enc): continue\n        s = enc.split()\n        for i in range(len(s)//2):\n            start = int(s[2*i]) - 1\n            length = int(s[2*i+1])\n            img[start:start+length] = 1 + m\n    return img.reshape(shape).T\n\ndef findpad(dim,crop_size):\n    \"\"\"\n    takes input dimension of image and returns the dimension to be padded\n    \"\"\"\n    out = []\n    for i in dim:\n        reminder = i%crop_size   #eg 1100%512= 76\n        if reminder > 0:\n            pad = crop_size - reminder  # pad = 512-76 = 436\n        else:\n            pad = 0  #if reminder = 0 no padding is required\n        out.append(pad)\n        \n    return out[0],out[1]\n\ndef dopadding(img,crop_size=1024,img_type='image'):\n    \"\"\"\n    find the padding size and do padding using np.pad on both y and x direction\n    crop_size: window/kernel size which is planned to use\n    \"\"\"\n    pad0,pad1 = findpad([img.shape[0],img.shape[1]],crop_size)\n    # img,y_padding,x_padding,color,filling value\n    if img_type=='image':\n        img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=0)\n    else:\n        img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2]],constant_values=0)\n        \n    return img\n\n# pad_img = dopadding(img,crop_size=WINDOW)\n# pad_img = cv2.resize(pad_img,(pad_img.shape[1]//reduce, pad_img.shape[0]//reduce), interpolation=cv2.INTER_AREA)\n# WINDOW = WINDOW/reduce\n\n\n\ndef make_inter_grid(img_shape,WINDOW=256):\n    \"\"\"\n    input: padded_image_dimension\n    \"\"\"\n\n    y,x = img_shape[0],img_shape[1]\n    start = int(WINDOW/2)\n    end_x = x - start\n    end_y = y - start\n    ny,nx = int(end_y/WINDOW),int(end_x/WINDOW)\n    base_y = np.linspace(start,end_y,ny+1,dtype=np.int64)\n    base_x = np.linspace(start,end_x,nx+1,dtype=np.int64)\n    \n    y1,y2 = base_y[0:-1],base_y[1:]\n    x1,x2 = base_x[0:-1],base_x[1:]\n    slices = np.zeros(shape=(len(x1),len(y1),4),dtype=np.int64)\n    for i in range(len(x1)):\n        for j in range(len(y1)):\n            slices[i, j] = x1[i], x2[i], y1[j], y2[j]\n    slices = slices.reshape(len(x1) * len(y1), 4)\n    return slices\n\n\ndef make_normal_grid(img_shape,WINDOW=256):\n    #padding\n    y,x = img_shape[0],img_shape[1]\n    nx,ny = int(x/WINDOW), int(y/WINDOW)\n\n    base_y = np.linspace(0,y,ny+1,dtype=np.int64)\n    base_x = np.linspace(0,x,nx+1,dtype=np.int64)\n    y1,y2 = base_y[0:-1],base_y[1:]\n    x1,x2 = base_x[0:-1],base_x[1:]\n    slices = np.zeros(shape=(len(x1),len(y1),4),dtype=np.int64)\n    for i in range(len(x1)):\n        for j in range(len(y1)):\n            slices[i, j] = x1[i], x2[i], y1[j], y2[j]\n    slices = slices.reshape(len(x1) * len(y1), 4)\n    return slices","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset 1 [Green box]\n\nSlides without overlaping\n\n![image.png](attachment:image.png)","attachments":{"image.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAdsAAAFDCAIAAAD9LTNNAAAAA3NCSVQICAjb4U/gAAAOi0lEQVR4Xu3d3ZEUVxIF4O6NMQIHNkJO0D6ADxiAJfOmB+GD8KFwggg5gBet3SViY36gpxGX6syTn96YLqoyv3PnTNMopOP5fD74hwABAgQKCPyrwAxGIECAAIH/Cmhk54AAAQJVBDRylSTMQYAAAY3sDBAgQKCKgEaukoQ5CBAgoJGdAQIECFQR0MhVkjAHAQIENLIzQIAAgSoCGrlKEuYgQICARnYGCBAgUEVAI1dJwhwECBDQyM4AAQIEqgho5CpJmIMAAQJ3lwmOH75cvsCrBAjsL3B+92r/h3riDgLeI++A7BEECBC4SkAjX8XkIgIECOwg8MKnFjtMMOQR/pj5YtA+InuRyAXxAho5PuKuC/oZ9jA5P666nuMfnPvHGtk3yfW8voWut3IlAQJfBXyO7CQQIECgioBGrpKEOQgQIKCRnQECBAhUEdDIVZIwBwECBDSyM0CAAIEqAhq5ShLmIECAgEZ2BggQIFBFQCNXScIcBAgQ0MjOAAECBKoIaOQqSZiDAAECGtkZIECAQBUBjVwlCXMQIEBAIzsDBAgQqCKgkaskYQ4CBAhoZGeAAAECVQQ0cpUkzEGAAAGN7AwQIECgioBGrpKEOQgQIKCRnQECBAhUEdDIVZIwBwECBDSyM0CAAIEqAj/2/6KuMrU5rhY4fnp79bU3v/D3hxO0mnwHuhvjnF//ucOSHtGgkdt+Z974W8jhJkCgnYBPLdpFZmACBGIFNHJstBYjQKCdgEZuF5mBCRCIFWjwOXKs/U0WO328yWOveugfjz55P9xk1O3No1FvMsM3sfbHeUxxPB6/OVfkF8/n8632atjIdb5JLoe2/7fQk3melMvlab1KgEABAZ9aFAjBCAQIEPifgEZ2EAgQIFBFQCNXScIcBAgQ0MjOAAECBKoIaOQqSZiDAAECGtkZIECAQBUBjVwlCXMQIEBAIzsDBAgQqCKgkaskYQ4CBAhoZGeAAAECVQQ0cpUkzEGAAAGN7AwQIECgioBGrpKEOQgQIKCRnQECBAhUEWj4X+N8iW7btpcu2eP10+dHT9l/qtPh/vme+4/xfIbvfeXmYv8Z7AlaHa79ccpSfO/8ZHzde+SMHG1BgECCgEZOSNEOBAhkCGjkjBxtQYBAgoBGTkjRDgQIZAho5IwcbUGAQIKARk5I0Q4ECGQIaOSMHG1BgECCgEZOSNEOBAhkCGjkjBxtQYBAgoBGTkjRDgQIZAho5IwcbUGAQIKARk5I0Q4ECGQIaOSMHG1BgECCgEZOSNEOBAhkCGjkjBxtQYBAgoBGTkjRDgQIZAho5IwcbUGAQIKARk5I0Q4ECGQIaOSMHG1BgECCgEZOSNEOBAhkCGjkjBxtQYBAgoBGTkjRDgQIZAho5IwcbUGAQIKARk5I0Q4ECGQIaOSMHG1BgECCgEZOSNEOBAhkCNz90BrHD19+6PpFF//+6D5/PP7ls2ecPj/7ki8QIECgg8ClRr5R/3ZgM+OvFzh9/vevf8jzJzz6eX96/rqvEPiVApca+Vc+d9y9b9Ev3/jDhIoZd/Is3ErA58it4jIsAQLRAho5Ol7LESDQSkAjt4rLsAQIRAtc9Tny+d2rGyIcP7199PTTx8vDbNt2+YIdXr3Fp8Y7rHWDR2y//bXnU0+H+4eP2w7v93y6ZxHwHtkZIECAQBWBq94jVxm27Rw7v9H76vTk7d7XL9Z/0+ePF22PucEXCHiPvADRLQgQILBEQCMvYXQTAgQILBDQyAsQ3YIAAQJLBDTyEkY3IUCAwAIBjbwA0S0IECCwREAjL2F0EwIECCwQ0MgLEN2CAAECSwQ08hJGNyFAgMACAY28ANEtCBAgsERAIy9hdBMCBAgsENDICxDdggABAksENPISRjchQIDAAgGNvADRLQgQILBEQCMvYXQTAgQILBDQyAsQ3YIAAQJLBDTyEkY3IUCAwAIBjbwA0S0IECCwREAjL2F0EwIECCwQ0MgLEN2CAAECSwQ08hJGNyFAgMACAY28ANEtCBAgsERAIy9hdBMCBAgsENDICxDdggABAksENPISRjchQIDAAgGNvADRLQgQILBE4G7JXXa9yfbm8uNOh/vLF+zy6u8Pn1JjpF329hACBH5CoGEj/8S2fmuHnw2Ffph14HKoowR8ahEVp2UIEGgtoJFbx2d4AgSiBDRyVJyWIUCgtUDDz5FPHy+Lb9t2+YIdXj09fsZ2eL/DQx8+wgegO4N7HIElAg0becnew26y/4+Efwxc7YdZI7p/bO431hHwqUWdLExCgMB0AY08/QTYnwCBOgIauU4WJiFAYLqARp5+AuxPgEAdAY1cJwuTECAwXUAjTz8B9idAoI6ARq6ThUkIEJguoJGnnwD7EyBQR0Aj18nCJAQITBfQyNNPgP0JEKgjoJHrZGESAgSmC2jk6SfA/gQI1BHQyHWyMAkBAtMFNPL0E2B/AgTqCGjkOlmYhACB6QIaefoJsD8BAnUENHKdLExCgMB0AY08/QTYnwCBOgIauU4WJiFAYLqARp5+AuxPgEAdAY1cJwuTECAwXUAjTz8B9idAoI6ARq6ThUkIEJguoJGnnwD7EyBQR0Aj18nCJAQITBfQyNNPgP0JEKgjoJHrZGESAgSmC2jk6SfA/gQI1BHQyHWyMAkBAtMFNPL0E2B/AgTqCGjkOlmYhACB6QIaefoJsD8BAnUENHKdLExCgMB0AY08/QTYnwCBOgLH8/n8vWmOH7587yVfJ7CPwPbbX/s86P9POR3uHz5xO7zfeQCPu7nA69evbzWD98i3kvdcAgQIPBXQyE9F/JoAAQK3EtDIt5L3XAIECDwV0MhPRfyaAAECtxK4u/Dg87tXF17d7aXjp7ePnnX6ePnR27ZdvmDCq/56akLKdswT8B45L1MbESDQVUAjd03O3AQI5Alo5LxMbUSAQFcBjdw1OXMTIJAnoJHzMrURAQJdBTRy1+TMTYBAnoBGzsvURgQIdBXQyF2TMzcBAnkCGjkvUxsRINBVQCN3Tc7cBAjkCWjkvExtRIBAVwGN3DU5cxMgkCegkfMytREBAl0FNHLX5MxNgECegEbOy9RGBAh0FdDIXZMzNwECeQIaOS9TGxEg0FVAI3dNztwECOQJaOS8TG1EgEBXAY3cNTlzEyCQJ6CR8zK1EQECXQU0ctfkzE2AQJ6ARs7L1EYECHQV0MhdkzM3AQJ5Aho5L1MbESDQVUAjd03O3AQI5Alo5LxMbUSAQFeBu36Db28uz3w63F++wKsECBCoKeA9cs1cTEWAwESBhu+RJ8b0szv7c8PPCvr9BHYR8B55F2YPIUCAwBUCGvkKJJcQIEBgFwGNvAuzhxAgQOAKAY18BZJLCBAgsIuAv9nbhfnWD9kO7289gucTIPCyQMNGPn28vNa2bZcvmPCqf7liQsp2zBPwqUVepjYiQKCrgEbumpy5CRDIE9DIeZnaiACBrgIauWty5iZAIE9AI+dlaiMCBLoKaOSuyZmbAIE8AY2cl6mNCBDoKqCRuyZnbgIE8gQ0cl6mNiJAoKuARu6anLkJEMgT0Mh5mdqIAIGuAhq5a3LmJkAgT0Aj52VqIwIEugpo5K7JmZsAgTwBjZyXqY0IEOgqoJG7JmduAgTyBDRyXqY2IkCgq4BG7pqcuQkQyBPQyHmZ2ogAga4CGrlrcuYmQCBPQCPnZWojAgS6CmjkrsmZmwCBPAGNnJepjQgQ6CqgkbsmZ24CBPIENHJepjYiQKCrgEbumpy5CRDIE9DIeZnaiACBrgIauWty5iZAIE9AI+dlaiMCBLoKaOSuyZmbAIE8AY2cl6mNCBDoKqCRuyZnbgIE8gQ0cl6mNiJAoKuARu6anLkJEMgT0Mh5mdqIAIGuAhq5a3LmJkAgT0Aj52VqIwIEugpo5K7JmZsAgTwBjZyXqY0IEOgqoJG7JmduAgTyBDRyXqY2IkCgq4BG7pqcuQkQyBPQyHmZ2ogAga4CGrlrcuYmQCBPQCPnZWojAgS6CmjkrsmZmwCBPAGNnJepjQgQ6CqgkbsmZ24CBPIENHJepjYiQKCrgEbumpy5CRDIE9DIeZnaiACBrgIauWty5iZAIE9AI+dlaiMCBLoKaOSuyZmbAIE8AY2cl6mNCBDoKqCRuyZnbgIE8gQ0cl6mNiJAoKuARu6anLkJEMgT0Mh5mdqIAIGuAhq5a3LmJkAgT0Aj52VqIwIEugpo5K7JmZsAgTwBjZyXqY0IEOgqoJG7JmduAgTyBDRyXqY2IkCgq4BG7pqcuQkQyBPQyHmZ2ogAga4Cd/0G395cnvl0uL98gVcJECBQU8B75Jq5mIoAgYkCGnli6nYmQKCmgEaumYupCBCYKKCRJ6ZuZwIEago0+Ju98+s/v9odj8drELdtu+Yy1xAgQKCagPfI1RIxDwECcwU08tzsbU6AQDUBjVwtEfMQIDBXQCPPzd7mBAhUE9DI1RIxDwECcwU08tzsbU6AQDUBjVwtEfMQIDBXQCPPzd7mBAhUE9DI1RIxDwECcwU08tzsbU6AQDUBjVwtEfMQIDBXQCPPzd7mBAhUE9DI1RIxDwECcwU08tzsbU6AQDUBjVwtEfMQIDBX4Hg+n+dub3MCBAhUEvAeuVIaZiFAYLaARp6dv+0JEKgkoJErpWEWAgRmC2jk2fnbngCBSgIauVIaZiFAYLaARp6dv+0JEKgkoJErpWEWAgRmC2jk2fnbngCBSgIauVIaZiFAYLaARp6dv+0JEKgkoJErpWEWAgRmC2jk2fnbngCBSgIauVIaZiFAYLaARp6dv+0JEKgkoJErpWEWAgRmC2jk2fnbngCBSgIauVIaZiFAYLaARp6dv+0JEKgkoJErpWEWAgRmC2jk2fnbngCBSgIauVIaZiFAYLaARp6dv+0JEKgkoJErpWEWAgRmC2jk2fnbngCBSgIauVIaZiFAYLaARp6dv+0JEKgkoJErpWEWAgRmC2jk2fnbngCBSgJ/A08Mi4UtynWBAAAAAElFTkSuQmCC"}}},{"metadata":{"trusted":true},"cell_type":"code","source":"WINDOW = 1024\nreduce = 4\ns_th = 40  #saturation blancking threshold\nsz=256\np_th = 200*sz//256 #threshold for the minimum number of pixels\n\nwith zipfile.ZipFile(OUT_TRAIN1, 'w') as img_out,zipfile.ZipFile(OUT_MASKS1, 'w') as mask_out:\n\n    for index, encs in tqdm(df_masks.iterrows(),total=len(df_masks)):\n        #read image and generate the mask\n        img = tiff.imread(os.path.join(DATA,index+'.tiff'))\n        if len(img.shape) == 5: \n            img = np.transpose(img.squeeze(), (1,2,0))\n        mask = enc2mask(encs,(img.shape[1],img.shape[0]))\n        print(f'processing image and mask of shapes: {img.shape},{mask.shape}')\n        #padding images/masks\n        pad_img = dopadding(img,crop_size=WINDOW,img_type='image')\n        pad_mask = dopadding(mask,crop_size=WINDOW,img_type='mask')\n        # downsizing whole image by reduce_factor\n        img = cv2.resize(pad_img,(pad_img.shape[1]//reduce, pad_img.shape[0]//reduce), interpolation=cv2.INTER_AREA)\n        mask = cv2.resize(pad_mask,(pad_img.shape[1]//reduce, pad_img.shape[0]//reduce), interpolation = cv2.INTER_NEAREST)\n        # generating grids for saving\n        reduced_window_size = int(WINDOW/reduce)\n        slices = make_normal_grid(img.shape,WINDOW=reduced_window_size)\n        for i,(x1,x2,y1,y2) in enumerate(tqdm(slices,desc=f'{len(slices)}')):\n            img_crop = img[y1:y2,x1:x2]\n            mask_crop = mask[y1:y2,x1:x2]\n            \n            #remove black or gray images based on saturation check\n            hsv = cv2.cvtColor(img_crop, cv2.COLOR_BGR2HSV)\n            h, s, v = cv2.split(hsv)\n            # if gray image continue to next pair of image/mask\n            if (s>s_th).sum() <= p_th or img_crop.sum() <= p_th: \n                    continue\n            \n            img_crop = cv2.cvtColor(img_crop, cv2.COLOR_RGB2BGR)\n        \n            im = cv2.imencode('.png',img_crop)[1]\n            img_out.writestr(f'{index}_{i}.png', im)\n            m = cv2.imencode('.png',mask_crop)[1]\n            mask_out.writestr(f'{index}_{i}.png', m)\n            gc.collect()\n            \n \ngc.collect()   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ncolumns, rows = 4,4\nidx0 = 20\nfig=plt.figure(figsize=(columns*4, rows*4))\nwith zipfile.ZipFile(OUT_TRAIN1, 'r') as img_arch, \\\n     zipfile.ZipFile(OUT_MASKS1, 'r') as msk_arch:\n    fnames = sorted(img_arch.namelist())[8:]\n    for i in range(rows):\n        for j in range(columns):\n            idx = i+j*columns\n            img = cv2.imdecode(np.frombuffer(img_arch.read(fnames[idx0+idx]), \n                                             np.uint8), cv2.IMREAD_COLOR)\n            img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n            mask = cv2.imdecode(np.frombuffer(msk_arch.read(fnames[idx0+idx]), \n                                              np.uint8), cv2.IMREAD_GRAYSCALE)\n    \n            fig.add_subplot(rows, columns, idx+1)\n            plt.axis('off')\n            plt.imshow(Image.fromarray(img))\n            plt.imshow(Image.fromarray(mask), alpha=0.2)\nplt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset2 [blue box]\n\nOverlaping slide covering 4 slides as given below:\n\n\n![image.png](attachment:image.png)","attachments":{"image.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAdsAAAFDCAIAAAD9LTNNAAAAA3NCSVQICAjb4U/gAAAOi0lEQVR4Xu3d3ZEUVxIF4O6NMQIHNkJO0D6ADxiAJfOmB+GD8KFwggg5gBet3SViY36gpxGX6syTn96YLqoyv3PnTNMopOP5fD74hwABAgQKCPyrwAxGIECAAIH/Cmhk54AAAQJVBDRylSTMQYAAAY3sDBAgQKCKgEaukoQ5CBAgoJGdAQIECFQR0MhVkjAHAQIENLIzQIAAgSoCGrlKEuYgQICARnYGCBAgUEVAI1dJwhwECBDQyM4AAQIEqgho5CpJmIMAAQJ3lwmOH75cvsCrBAjsL3B+92r/h3riDgLeI++A7BEECBC4SkAjX8XkIgIECOwg8MKnFjtMMOQR/pj5YtA+InuRyAXxAho5PuKuC/oZ9jA5P666nuMfnPvHGtk3yfW8voWut3IlAQJfBXyO7CQQIECgioBGrpKEOQgQIKCRnQECBAhUEdDIVZIwBwECBDSyM0CAAIEqAhq5ShLmIECAgEZ2BggQIFBFQCNXScIcBAgQ0MjOAAECBKoIaOQqSZiDAAECGtkZIECAQBUBjVwlCXMQIEBAIzsDBAgQqCKgkaskYQ4CBAhoZGeAAAECVQQ0cpUkzEGAAAGN7AwQIECgioBGrpKEOQgQIKCRnQECBAhUEdDIVZIwBwECBDSyM0CAAIEqAj/2/6KuMrU5rhY4fnp79bU3v/D3hxO0mnwHuhvjnF//ucOSHtGgkdt+Z974W8jhJkCgnYBPLdpFZmACBGIFNHJstBYjQKCdgEZuF5mBCRCIFWjwOXKs/U0WO328yWOveugfjz55P9xk1O3No1FvMsM3sfbHeUxxPB6/OVfkF8/n8632atjIdb5JLoe2/7fQk3melMvlab1KgEABAZ9aFAjBCAQIEPifgEZ2EAgQIFBFQCNXScIcBAgQ0MjOAAECBKoIaOQqSZiDAAECGtkZIECAQBUBjVwlCXMQIEBAIzsDBAgQqCKgkaskYQ4CBAhoZGeAAAECVQQ0cpUkzEGAAAGN7AwQIECgioBGrpKEOQgQIKCRnQECBAhUEWj4X+N8iW7btpcu2eP10+dHT9l/qtPh/vme+4/xfIbvfeXmYv8Z7AlaHa79ccpSfO/8ZHzde+SMHG1BgECCgEZOSNEOBAhkCGjkjBxtQYBAgoBGTkjRDgQIZAho5IwcbUGAQIKARk5I0Q4ECGQIaOSMHG1BgECCgEZOSNEOBAhkCGjkjBxtQYBAgoBGTkjRDgQIZAho5IwcbUGAQIKARk5I0Q4ECGQIaOSMHG1BgECCgEZOSNEOBAhkCGjkjBxtQYBAgoBGTkjRDgQIZAho5IwcbUGAQIKARk5I0Q4ECGQIaOSMHG1BgECCgEZOSNEOBAhkCGjkjBxtQYBAgoBGTkjRDgQIZAho5IwcbUGAQIKARk5I0Q4ECGQIaOSMHG1BgECCgEZOSNEOBAhkCNz90BrHD19+6PpFF//+6D5/PP7ls2ecPj/7ki8QIECgg8ClRr5R/3ZgM+OvFzh9/vevf8jzJzz6eX96/rqvEPiVApca+Vc+d9y9b9Ev3/jDhIoZd/Is3ErA58it4jIsAQLRAho5Ol7LESDQSkAjt4rLsAQIRAtc9Tny+d2rGyIcP7199PTTx8vDbNt2+YIdXr3Fp8Y7rHWDR2y//bXnU0+H+4eP2w7v93y6ZxHwHtkZIECAQBWBq94jVxm27Rw7v9H76vTk7d7XL9Z/0+ePF22PucEXCHiPvADRLQgQILBEQCMvYXQTAgQILBDQyAsQ3YIAAQJLBDTyEkY3IUCAwAIBjbwA0S0IECCwREAjL2F0EwIECCwQ0MgLEN2CAAECSwQ08hJGNyFAgMACAY28ANEtCBAgsERAIy9hdBMCBAgsENDICxDdggABAksENPISRjchQIDAAgGNvADRLQgQILBEQCMvYXQTAgQILBDQyAsQ3YIAAQJLBDTyEkY3IUCAwAIBjbwA0S0IECCwREAjL2F0EwIECCwQ0MgLEN2CAAECSwQ08hJGNyFAgMACAY28ANEtCBAgsERAIy9hdBMCBAgsENDICxDdggABAksENPISRjchQIDAAgGNvADRLQgQILBE4G7JXXa9yfbm8uNOh/vLF+zy6u8Pn1JjpF329hACBH5CoGEj/8S2fmuHnw2Ffph14HKoowR8ahEVp2UIEGgtoJFbx2d4AgSiBDRyVJyWIUCgtUDDz5FPHy+Lb9t2+YIdXj09fsZ2eL/DQx8+wgegO4N7HIElAg0becnew26y/4+Efwxc7YdZI7p/bO431hHwqUWdLExCgMB0AY08/QTYnwCBOgIauU4WJiFAYLqARp5+AuxPgEAdAY1cJwuTECAwXUAjTz8B9idAoI6ARq6ThUkIEJguoJGnnwD7EyBQR0Aj18nCJAQITBfQyNNPgP0JEKgjoJHrZGESAgSmC2jk6SfA/gQI1BHQyHWyMAkBAtMFNPL0E2B/AgTqCGjkOlmYhACB6QIaefoJsD8BAnUENHKdLExCgMB0AY08/QTYnwCBOgIauU4WJiFAYLqARp5+AuxPgEAdAY1cJwuTECAwXUAjTz8B9idAoI6ARq6ThUkIEJguoJGnnwD7EyBQR0Aj18nCJAQITBfQyNNPgP0JEKgjoJHrZGESAgSmC2jk6SfA/gQI1BHQyHWyMAkBAtMFNPL0E2B/AgTqCGjkOlmYhACB6QIaefoJsD8BAnUENHKdLExCgMB0AY08/QTYnwCBOgLH8/n8vWmOH7587yVfJ7CPwPbbX/s86P9POR3uHz5xO7zfeQCPu7nA69evbzWD98i3kvdcAgQIPBXQyE9F/JoAAQK3EtDIt5L3XAIECDwV0MhPRfyaAAECtxK4u/Dg87tXF17d7aXjp7ePnnX6ePnR27ZdvmDCq/56akLKdswT8B45L1MbESDQVUAjd03O3AQI5Alo5LxMbUSAQFcBjdw1OXMTIJAnoJHzMrURAQJdBTRy1+TMTYBAnoBGzsvURgQIdBXQyF2TMzcBAnkCGjkvUxsRINBVQCN3Tc7cBAjkCWjkvExtRIBAVwGN3DU5cxMgkCegkfMytREBAl0FNHLX5MxNgECegEbOy9RGBAh0FdDIXZMzNwECeQIaOS9TGxEg0FVAI3dNztwECOQJaOS8TG1EgEBXAY3cNTlzEyCQJ6CR8zK1EQECXQU0ctfkzE2AQJ6ARs7L1EYECHQV0MhdkzM3AQJ5Aho5L1MbESDQVUAjd03O3AQI5Alo5LxMbUSAQFeBu36Db28uz3w63F++wKsECBCoKeA9cs1cTEWAwESBhu+RJ8b0szv7c8PPCvr9BHYR8B55F2YPIUCAwBUCGvkKJJcQIEBgFwGNvAuzhxAgQOAKAY18BZJLCBAgsIuAv9nbhfnWD9kO7289gucTIPCyQMNGPn28vNa2bZcvmPCqf7liQsp2zBPwqUVepjYiQKCrgEbumpy5CRDIE9DIeZnaiACBrgIauWty5iZAIE9AI+dlaiMCBLoKaOSuyZmbAIE8AY2cl6mNCBDoKqCRuyZnbgIE8gQ0cl6mNiJAoKuARu6anLkJEMgT0Mh5mdqIAIGuAhq5a3LmJkAgT0Aj52VqIwIEugpo5K7JmZsAgTwBjZyXqY0IEOgqoJG7JmduAgTyBDRyXqY2IkCgq4BG7pqcuQkQyBPQyHmZ2ogAga4CGrlrcuYmQCBPQCPnZWojAgS6CmjkrsmZmwCBPAGNnJepjQgQ6CqgkbsmZ24CBPIENHJepjYiQKCrgEbumpy5CRDIE9DIeZnaiACBrgIauWty5iZAIE9AI+dlaiMCBLoKaOSuyZmbAIE8AY2cl6mNCBDoKqCRuyZnbgIE8gQ0cl6mNiJAoKuARu6anLkJEMgT0Mh5mdqIAIGuAhq5a3LmJkAgT0Aj52VqIwIEugpo5K7JmZsAgTwBjZyXqY0IEOgqoJG7JmduAgTyBDRyXqY2IkCgq4BG7pqcuQkQyBPQyHmZ2ogAga4CGrlrcuYmQCBPQCPnZWojAgS6CmjkrsmZmwCBPAGNnJepjQgQ6CqgkbsmZ24CBPIENHJepjYiQKCrgEbumpy5CRDIE9DIeZnaiACBrgIauWty5iZAIE9AI+dlaiMCBLoKaOSuyZmbAIE8AY2cl6mNCBDoKqCRuyZnbgIE8gQ0cl6mNiJAoKuARu6anLkJEMgT0Mh5mdqIAIGuAhq5a3LmJkAgT0Aj52VqIwIEugpo5K7JmZsAgTwBjZyXqY0IEOgqoJG7JmduAgTyBDRyXqY2IkCgq4BG7pqcuQkQyBPQyHmZ2ogAga4Cd/0G395cnvl0uL98gVcJECBQU8B75Jq5mIoAgYkCGnli6nYmQKCmgEaumYupCBCYKKCRJ6ZuZwIEago0+Ju98+s/v9odj8drELdtu+Yy1xAgQKCagPfI1RIxDwECcwU08tzsbU6AQDUBjVwtEfMQIDBXQCPPzd7mBAhUE9DI1RIxDwECcwU08tzsbU6AQDUBjVwtEfMQIDBXQCPPzd7mBAhUE9DI1RIxDwECcwU08tzsbU6AQDUBjVwtEfMQIDBXQCPPzd7mBAhUE9DI1RIxDwECcwU08tzsbU6AQDUBjVwtEfMQIDBX4Hg+n+dub3MCBAhUEvAeuVIaZiFAYLaARp6dv+0JEKgkoJErpWEWAgRmC2jk2fnbngCBSgIauVIaZiFAYLaARp6dv+0JEKgkoJErpWEWAgRmC2jk2fnbngCBSgIauVIaZiFAYLaARp6dv+0JEKgkoJErpWEWAgRmC2jk2fnbngCBSgIauVIaZiFAYLaARp6dv+0JEKgkoJErpWEWAgRmC2jk2fnbngCBSgIauVIaZiFAYLaARp6dv+0JEKgkoJErpWEWAgRmC2jk2fnbngCBSgIauVIaZiFAYLaARp6dv+0JEKgkoJErpWEWAgRmC2jk2fnbngCBSgIauVIaZiFAYLaARp6dv+0JEKgkoJErpWEWAgRmC2jk2fnbngCBSgJ/A08Mi4UtynWBAAAAAElFTkSuQmCC"}}},{"metadata":{"trusted":true},"cell_type":"code","source":"WINDOW = 1024\nreduce = 4\ns_th = 40  #saturation blancking threshold\nsz=256\np_th = 200*sz//256 #threshold for the minimum number of pixels\n\nwith zipfile.ZipFile(OUT_TRAIN2, 'w') as img_out,zipfile.ZipFile(OUT_MASKS2, 'w') as mask_out:\n\n    for index, encs in tqdm(df_masks.iterrows(),total=len(df_masks)):\n        #read image and generate the mask\n        img = tiff.imread(os.path.join(DATA,index+'.tiff'))\n        if len(img.shape) == 5: \n            img = np.transpose(img.squeeze(), (1,2,0))\n        mask = enc2mask(encs,(img.shape[1],img.shape[0]))\n        print(f'processing image and mask of shapes: {img.shape}, {mask.shape}')\n        #padding images/masks\n        pad_img = dopadding(img,crop_size=WINDOW,img_type='image')\n        pad_mask = dopadding(mask,crop_size=WINDOW,img_type='mask')\n        # downsizing whole image by reduce_factor\n        img = cv2.resize(pad_img,(pad_img.shape[1]//reduce, pad_img.shape[0]//reduce), interpolation=cv2.INTER_AREA)\n        mask = cv2.resize(pad_mask,(pad_img.shape[1]//reduce, pad_img.shape[0]//reduce), interpolation=cv2.INTER_AREA)\n        # generating grids for saving\n        reduced_window_size = int(WINDOW/reduce)\n        slices = make_inter_grid(img.shape,WINDOW=reduced_window_size)\n        for i,(x1,x2,y1,y2) in enumerate(tqdm(slices,desc=f'{len(slices)}')):\n            img_crop = img[y1:y2,x1:x2]\n            mask_crop = mask[y1:y2,x1:x2]\n        \n            #remove black or gray images based on saturation check\n            hsv = cv2.cvtColor(img_crop, cv2.COLOR_BGR2HSV)\n            h, s, v = cv2.split(hsv)\n            # if gray image continue to next pair of image/mask\n            if (s>s_th).sum() <= p_th or img_crop.sum() <= p_th: \n                    continue  \n            img_crop = cv2.cvtColor(img_crop, cv2.COLOR_RGB2BGR)\n            \n            im = cv2.imencode('_inter.png',img_crop)[1]\n            img_out.writestr(f'{index}_inter_{i}.png', im)\n            m = cv2.imencode('_inter.png',mask_crop)[1]\n            mask_out.writestr(f'{index}_inter_{i}.png', m)\n            gc.collect()\n            \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ncolumns, rows = 4,4\nidx0 = 20\nfig=plt.figure(figsize=(columns*4, rows*4))\nwith zipfile.ZipFile(OUT_TRAIN2, 'r') as img_arch, \\\n     zipfile.ZipFile(OUT_MASKS2, 'r') as msk_arch:\n    fnames = sorted(img_arch.namelist())[8:]\n    for i in range(rows):\n        for j in range(columns):\n            idx = i+j*columns\n            img = cv2.imdecode(np.frombuffer(img_arch.read(fnames[idx0+idx]), \n                                             np.uint8), cv2.IMREAD_COLOR)\n            img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n            mask = cv2.imdecode(np.frombuffer(msk_arch.read(fnames[idx0+idx]), \n                                              np.uint8), cv2.IMREAD_GRAYSCALE)\n    \n            fig.add_subplot(rows, columns, idx+1)\n            plt.axis('off')\n            plt.imshow(Image.fromarray(img))\n            plt.imshow(Image.fromarray(mask), alpha=0.2)\nplt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}