{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport tifffile as tiff\nimport cv2\nimport os\nfrom tqdm.notebook import tqdm\nimport zipfile","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# image size information"},{"metadata":{"trusted":true},"cell_type":"code","source":"img_info = pd.read_csv('../input/hubmap-kidney-segmentation/HuBMAP-20-dataset_information.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for row in img_info.iterrows():\n    print('iamge number--'+row[1]['image_file'][:-5]+' width is',row[1]['width_pixels'],' height is',row[1]['height_pixels'])\n#     print(row[1]['image_file'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sz = 256   #the size of tiles\nreduce = 4 #reduce the original images by 4 times \n\nMASKS = '../input/hubmap-kidney-segmentation/train.csv'\nDATA = '../input/hubmap-kidney-segmentation/train/'\nOUT_TRAIN = 'train.zip'\nOUT_MASKS = 'masks.zip'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#functions to convert encoding to mask and mask to encoding\ndef enc2mask(encs, shape):\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for m,enc in enumerate(encs):\n        if isinstance(enc,np.float) and np.isnan(enc): continue\n        s = enc.split()\n        for i in range(len(s)//2):\n            start = int(s[2*i]) - 1\n            length = int(s[2*i+1])\n            img[start:start+length] = 1 + m\n    return img.reshape(shape).T\ndef mask2enc(mask, n=1):\n    pixels = mask.T.flatten()\n    encs = []\n    for i in range(1,n+1):\n        p = (pixels == i).astype(np.int8)\n        if p.sum() == 0: encs.append(np.nan)\n        else:\n            p = np.concatenate([[0], p, [0]])\n            runs = np.where(p[1:] != p[:-1])[0] + 1\n            runs[1::2] -= runs[::2]\n            encs.append(' '.join(str(x) for x in runs))\n    return encs\n\ndf_masks = pd.read_csv(MASKS).set_index('id')\ndf_masks.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"s_th = 40  #saturation blancking threshold\np_th = 200*sz//256 #threshold for the minimum number of pixels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"x_tot,x2_tot = [],[]\ncnt = 0\n\nwith zipfile.ZipFile(OUT_TRAIN, 'w') as img_out,\\\n zipfile.ZipFile(OUT_MASKS, 'w') as mask_out:\n    for index, encs in tqdm(df_masks.iterrows(),total=len(df_masks)):\n        #read image and generate the mask\n        img = tiff.imread(os.path.join(DATA,index+'.tiff')) \n        if len(img.shape) == 5: img = np.transpose(img.squeeze(), (1,2,0))\n        print(index + ' with shape ',img.shape)\n        \n        mask = enc2mask(encs,(img.shape[1],img.shape[0]))\n        #add padding to make the image dividable into tiles\n        shape = img.shape\n        pad0 = (reduce*sz - shape[0]%(reduce*sz))%(reduce*sz)\n        pad1 = (reduce*sz - shape[1]%(reduce*sz))%(reduce*sz)\n        \n        img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                    constant_values=0)\n        mask = np.pad(mask,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2]],\n                    constant_values=0)\n        \n        #split image and mask into tiles using the reshape+transpose trick\n        img = cv2.resize(img,(img.shape[1]//reduce,img.shape[0]//reduce),\n                         interpolation = cv2.INTER_AREA)\n        \n        img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n        img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n\n        mask = cv2.resize(mask,(mask.shape[1]//reduce,mask.shape[0]//reduce),\n                          interpolation = cv2.INTER_NEAREST)\n        mask = mask.reshape(mask.shape[0]//sz,sz,mask.shape[1]//sz,sz)\n        mask = mask.transpose(0,2,1,3).reshape(-1,sz,sz)\n        \n        \n        #write data\n        for i,(im,m) in enumerate(zip(img,mask)):\n            #remove black or gray images based on saturation check\n            hsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)\n            h, s, v = cv2.split(hsv)\n            if (s>s_th).sum() <= p_th or im.sum() <= p_th: continue # ------------------------ remove img with little pixels ------------------\n            \n            x_tot.append((im/255.0).reshape(-1,3).mean(0))\n            x2_tot.append(((im/255.0)**2).reshape(-1,3).mean(0))\n            \n            target_cnt = 0 # ---------------------- remove image/mask with little target pixels in mask ----------------------\n            for a in range(len(m)):\n                for b in range(len(m[0])):\n                    if m[a][b] == 1:\n                        target_cnt += 1\n            \n            if target_cnt >0 and target_cnt <=100: continue\n            \n            ################################################# -------- image out ------------ ####################################################\n            im = cv2.imencode('.png',cv2.cvtColor(im, cv2.COLOR_RGB2BGR))[1]\n            img_out.writestr(f'{index}_{i}.png', im)\n            m = cv2.imencode('.png',m)[1]\n            mask_out.writestr(f'{index}_{i}.png', m)\n            cnt += 1\n            \n\n        \n        \n#image stats\nimg_avr =  np.array(x_tot).mean(0)\nimg_std =  np.sqrt(np.array(x2_tot).mean(0) - img_avr**2)\nprint('mean:',img_avr, ', std:', img_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(cnt)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train2"},{"metadata":{},"cell_type":"markdown","source":"shiffted"},{"metadata":{"trusted":true},"cell_type":"code","source":"s_th = 40  #saturation blancking threshold\np_th = 200*sz//256 #threshold for the minimum number of pixels\norig = 1024 # padding value -----------------------\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_tot,x2_tot = [],[]\ncnt = 0\n\nwith zipfile.ZipFile('2'+OUT_TRAIN, 'w') as img_out,\\\n zipfile.ZipFile('2'+OUT_MASKS, 'w') as mask_out:\n    for index, encs in tqdm(df_masks.iterrows(),total=len(df_masks)):\n        #read image and generate the mask\n        img = tiff.imread(os.path.join(DATA,index+'.tiff')) \n        if len(img.shape) == 5: img = np.transpose(img.squeeze(), (1,2,0))\n        print(index + ' with shape ',img.shape)\n        \n        mask = enc2mask(encs,(img.shape[1],img.shape[0]))\n        #add padding to make the image dividable into tiles\n        shape = img.shape\n        pad0 = (reduce*sz - shape[0]%(reduce*sz))%(reduce*sz)\n        pad1 = (reduce*sz - shape[1]%(reduce*sz))%(reduce*sz)\n        \n        pad0 += orig\n        pad1 += orig\n        \n        img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                    constant_values=0)\n        mask = np.pad(mask,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2]],\n                    constant_values=0)\n        \n        #split image and mask into tiles using the reshape+transpose trick\n        img = cv2.resize(img,(img.shape[1]//reduce,img.shape[0]//reduce),\n                         interpolation = cv2.INTER_AREA)\n        \n        img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n        img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n\n        mask = cv2.resize(mask,(mask.shape[1]//reduce,mask.shape[0]//reduce),\n                          interpolation = cv2.INTER_NEAREST)\n        mask = mask.reshape(mask.shape[0]//sz,sz,mask.shape[1]//sz,sz)\n        mask = mask.transpose(0,2,1,3).reshape(-1,sz,sz)\n        \n        \n        #write data\n        for i,(im,m) in enumerate(zip(img,mask)):\n            #remove black or gray images based on saturation check\n            hsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)\n            h, s, v = cv2.split(hsv)\n            if (s>s_th).sum() <= p_th or im.sum() <= p_th: continue # ------------------------ remove img with little pixels ------------------\n            \n            x_tot.append((im/255.0).reshape(-1,3).mean(0))\n            x2_tot.append(((im/255.0)**2).reshape(-1,3).mean(0))\n            \n            target_cnt = 0 # ---------------------- remove image/mask with little target pixels in mask ----------------------\n            for a in range(len(m)):\n                for b in range(len(m[0])):\n                    if m[a][b] == 1:\n                        target_cnt += 1\n            \n            if target_cnt >0 and target_cnt <=100: continue\n            \n            ################################################# -------- image out ------------ ####################################################\n            im = cv2.imencode('.png',cv2.cvtColor(im, cv2.COLOR_RGB2BGR))[1]\n            img_out.writestr(f'{index}_{i}.png', im)\n            m = cv2.imencode('.png',m)[1]\n            mask_out.writestr(f'{index}_{i}.png', m)\n            cnt += 1\n            \n\n#image stats\nimg_avr =  np.array(x_tot).mean(0)\nimg_std =  np.sqrt(np.array(x2_tot).mean(0) - img_avr**2)\nprint('mean:',img_avr, ', std:', img_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(cnt)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}