{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport tifffile as tiff\nimport cv2\nimport os\nfrom tqdm import tqdm\n\n# Crop the original dataset into 256X256 figures\n\nclass TrainDataTransformer(object):\n    def __init__(self,\n                 mask_path='../input/hubmap-kidney-segmentation/train.csv',\n                 data_path='../input/hubmap-kidney-segmentation/train',\n                 size=512,\n                 reduce=2,\n                 seed=42):\n        # size of tiles\n        self.size = size\n        # scale\n        self.reduce = reduce\n        self.mask_path = mask_path\n        self.data_path = data_path\n        self.seed = seed\n\n    def __call__(self,img_output_path,mask_output_path):\n        if not os.path.exists(img_output_path):\n            os.mkdir(img_output_path)\n        if not os.path.exists(mask_output_path):\n            os.mkdir(mask_output_path)\n\n        # For statistics\n        x_tot, x_tot2 = [], []\n        df_masks = pd.read_csv(self.mask_path).set_index('id')\n        for index, encs in tqdm(df_masks.iterrows(), total=len(df_masks)):\n            img = tiff.imread(os.path.join(self.data_path, index+'.tiff'))\n            if len(img.shape) == 5:\n                # Normal case: HWC\n                # Weird case: 1,1,C,H,W\n                img = np.transpose(img.squeeze(), (1,2,0))\n            mask = self.enc2mask(encs, [img.shape[1], img.shape[0]])\n\n            # add padding to make image dividable into tiles\n            shape = img.shape\n            nums = self.reduce*self.size\n            # split into small block, then reduce the scale\n            pad0 = (nums - shape[0]%nums) % nums\n            pad1 = (nums - shape[1]%nums) % nums\n            img = np.pad(img, [[pad0//2,pad0-pad0//2], [pad1//2,pad1-pad1//2], [0,0]],\n                         constant_values=0)\n            mask = np.pad(mask, [[pad0 // 2, pad0 - pad0 // 2], [pad1 // 2, pad1 - pad1 // 2]],\n                          constant_values=0)\n\n            # split the image and the mask into tiles\n            img = cv2.resize(img, (img.shape[1]//self.reduce, img.shape[0]//self.reduce),\n                             interpolation=cv2.INTER_AREA)\n            # Transform into small block\n            # W,W_s,H,H_s,C\n            img = img.reshape(img.shape[0]//self.size, self.size, img.shape[1]//self.size, self.size, 3)\n            img = img.transpose(0,2,1,3,4).reshape(-1, self.size, self.size, 3)\n\n            mask = cv2.resize(mask, (mask.shape[1] // self.reduce, mask.shape[0]//self.reduce),\n                              interpolation=cv2.INTER_NEAREST)\n            mask = mask.reshape(mask.shape[0]//self.size, self.size, mask.shape[1]//self.size, self.size)\n            mask = mask.transpose(0,2,1,3).reshape(-1, self.size, self.size)\n\n            for i,(im,m) in enumerate(zip(img,mask)):\n                # Saturation threshold\n                s_thresh = 40\n                # Minimum number threshold\n                p_thresh = 200*self.size//256\n                hsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)\n                h, s, v = cv2.split(hsv)\n                if (s>s_thresh).sum() <= p_thresh or im.sum() < p_thresh:\n                    continue\n\n                x_tot.append((im/255.0).reshape(-1,3).mean(0))\n                x_tot2.append(((im/255.0)**2).reshape(-1,3).mean(0))\n\n                im_path = os.path.join(img_output_path, str(index)+'_'+str(i)+'.png')\n                m_path = os.path.join(mask_output_path, str(index)+'_'+str(i)+'.png')\n                im = cv2.imencode('.png', cv2.cvtColor(im, cv2.COLOR_RGB2BGR))[1].tofile(im_path)\n                m = cv2.imencode('.png', m)[1].tofile(m_path)\n\n        im_avg = np.array(x_tot).mean(0)\n        im_std = np.sqrt(np.array(x_tot2).mean(0) - im_avg**2)\n        print('avg: {}, std: {}'.format(im_avg, im_std))\n\n\n    def enc2mask(self, encs, shape):\n        # Converting encoding to mask\n        # See: RLE Encoding\n        # Encoding is formatted as [RLEEnc, length, RLEEnc, length ...]\n        # Since it's Column-major here, length is add to the column.\n        # Split with space\n        img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n        for i,enc in enumerate(encs):\n            if isinstance(enc, np.float) and np.isnan(enc):\n                continue\n            enc_split = enc.split()\n            for idx in range(len(enc_split)//2):\n                start = int(enc_split[2*idx]) - 1\n                length = int(enc_split[2*idx+1])\n                img[start:start+length] = 1 + i\n        # Note that encoding is Column-major and in Python it's Row-major\n        return img.reshape(shape).T\n\n\n    def mask2enc(self, mask):\n        pixels = mask.T.flatten()\n\n        pixels[0] = 0\n        pixels[-1] = 0\n        runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n        runs[1::2] -= runs[::2]\n        return ' '.join(str(x) for x in runs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.mkdir(img_output_path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DT = TrainDataTransformer(size=256, reduce=2)\nDT(img_output_path='train_img_256_r2', mask_output_path='mask_img_256_r2')\n# DT = TrainDataTransformer(size=512, reduce=2)\n# DT(img_output_path='./train_img_512_r2', mask_output_path='./mask_img_512_r2')\n# DT = TrainDataTransformer(size=1024, reduce=1)\n# DT(img_output_path='./train_img_1024_r1', mask_output_path='./mask_img_1024_r1')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}