{"cells":[{"metadata":{},"cell_type":"markdown","source":"I provide a 256x256 image dataset for the initial prototyping. Based on the size of the detected features, I'd expect that the appropriate tile size for this data should be 1024x1024. However, it would be an overshoot for the initial model development. Therefore, I reduced the images by 4 times to 256x256.\n\n* The corresponding dataset: https://www.kaggle.com/iafoss/hubmap-256x256\n* The dataset with 512x512 tiles (reduction of 2): https://www.kaggle.com/iafoss/hubmap-512x512\n* The dataset with 1024x1024 tiles (no resolution reduction): https://www.kaggle.com/iafoss/hubmap-1024x1024\n\n\n* Update (11/17): remove gray background tiles based on saturation check\n* Update (11/19): fix a bug with dimentions in cv2.resize, please use the latest version"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport tifffile as tiff\nimport cv2\nimport os\nfrom tqdm.notebook import tqdm\nimport zipfile","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sz = 256   #the size of tiles\nreduce = 4 #reduce the original images by 4 times \nMASKS = '../input/hubmap-kidney-segmentation/train.csv'\nDATA = '../input/hubmap-kidney-segmentation/train/'\nOUT_TRAIN = 'train.zip'\nOUT_MASKS = 'masks.zip'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#functions to convert encoding to mask and mask to encoding\ndef enc2mask(encs, shape):\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for m,enc in enumerate(encs):\n        if isinstance(enc,np.float) and np.isnan(enc): continue\n        s = enc.split()\n        for i in range(len(s)//2):\n            start = int(s[2*i]) - 1\n            length = int(s[2*i+1])\n            img[start:start+length] = 1 + m\n    return img.reshape(shape).T\n\ndef mask2enc(mask, n=1):\n    pixels = mask.T.flatten()\n    encs = []\n    for i in range(1,n+1):\n        p = (pixels == i).astype(np.int8)\n        if p.sum() == 0: encs.append(np.nan)\n        else:\n            p = np.concatenate([[0], p, [0]])\n            runs = np.where(p[1:] != p[:-1])[0] + 1\n            runs[1::2] -= runs[::2]\n            encs.append(' '.join(str(x) for x in runs))\n    return encs\n\ndf_masks = pd.read_csv(MASKS).set_index('id')\ndf_masks.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"s_th = 40  #saturation blancking threshold\np_th = 200*sz//256 #threshold for the minimum number of pixels\n\nx_tot,x2_tot = [],[]\nwith zipfile.ZipFile(OUT_TRAIN, 'w') as img_out,\\\n zipfile.ZipFile(OUT_MASKS, 'w') as mask_out:\n    for index, encs in tqdm(df_masks.iterrows(),total=len(df_masks)):\n        #read image and generate the mask\n        img = tiff.imread(os.path.join(DATA,index+'.tiff'))\n        if len(img.shape) == 5: img = np.transpose(img.squeeze(), (1,2,0))\n        mask = enc2mask(encs,(img.shape[1],img.shape[0]))\n\n        #add padding to make the image dividable into tiles\n        shape = img.shape\n        pad0 = (reduce*sz - shape[0]%(reduce*sz))%(reduce*sz)\n        pad1 = (reduce*sz - shape[1]%(reduce*sz))%(reduce*sz)\n        img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                    constant_values=0)\n        mask = np.pad(mask,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2]],\n                    constant_values=0)\n\n        #split image and mask into tiles using the reshape+transpose trick\n        img = cv2.resize(img,(img.shape[1]//reduce,img.shape[0]//reduce),\n                         interpolation = cv2.INTER_AREA)\n        img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n        img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n\n        mask = cv2.resize(mask,(mask.shape[1]//reduce,mask.shape[0]//reduce),\n                          interpolation = cv2.INTER_NEAREST)\n        mask = mask.reshape(mask.shape[0]//sz,sz,mask.shape[1]//sz,sz)\n        mask = mask.transpose(0,2,1,3).reshape(-1,sz,sz)\n\n        #write data\n        for i,(im,m) in enumerate(zip(img,mask)):\n            #remove black or gray images based on saturation check\n            hsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)\n            h, s, v = cv2.split(hsv)\n            if (s>s_th).sum() <= p_th or im.sum() <= p_th: continue\n            \n            x_tot.append((im/255.0).reshape(-1,3).mean(0))\n            x2_tot.append(((im/255.0)**2).reshape(-1,3).mean(0))\n            \n            im = cv2.imencode('.png',cv2.cvtColor(im, cv2.COLOR_RGB2BGR))[1]\n            img_out.writestr(f'{index}_{i}.png', im)\n            m = cv2.imencode('.png',m)[1]\n            mask_out.writestr(f'{index}_{i}.png', m)\n\n#image stats\nimg_avr =  np.array(x_tot).mean(0)\nimg_std =  np.sqrt(np.array(x2_tot).mean(0) - img_avr**2)\nprint('mean:',img_avr, ', std:', img_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# columns, rows = 4,4\n# idx0 = 20\n# fig=plt.figure(figsize=(columns*4, rows*4))\n# with zipfile.ZipFile(OUT_TRAIN, 'r') as img_arch, \\\n#      zipfile.ZipFile(OUT_MASKS, 'r') as msk_arch:\n#     fnames = sorted(img_arch.namelist())[8:]\n#     for i in range(rows):\n#         for j in range(columns):\n#             idx = i+j*columns\n#             img = cv2.imdecode(np.frombuffer(img_arch.read(fnames[idx0+idx]), \n#                                              np.uint8), cv2.IMREAD_COLOR)\n#             img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n#             mask = cv2.imdecode(np.frombuffer(msk_arch.read(fnames[idx0+idx]), \n#                                               np.uint8), cv2.IMREAD_GRAYSCALE)\n    \n#             fig.add_subplot(rows, columns, idx+1)\n#             plt.axis('off')\n#             plt.imshow(Image.fromarray(img))\n#             plt.imshow(Image.fromarray(mask), alpha=0.2)\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.chdir('/kaggle/working/')\n!mkdir train\n!mkdir masks\n!unzip masks.zip\n!mv *.png masks\n!unzip train.zip\n!mv *.png train\n!ls train -l|wc -l\n!ls masks -l|wc -l\n!rm -rf *.zip","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pwd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}