{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport tifffile as tiff\nimport cv2\nimport os\nimport gc\nfrom tqdm.notebook import tqdm\nimport zipfile","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-23T12:07:53.812355Z","iopub.execute_input":"2022-08-23T12:07:53.813424Z","iopub.status.idle":"2022-08-23T12:07:53.825082Z","shell.execute_reply.started":"2022-08-23T12:07:53.813392Z","shell.execute_reply":"2022-08-23T12:07:53.823552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MASKS = '../input/hubmap-organ-segmentation/train.csv'\nDATA = '../input/hubmap-organ-segmentation/train_images/'\nOUT_TRAIN = 'train.zip'\nOUT_MASKS = 'masks.zip'\n","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:56.105998Z","iopub.execute_input":"2022-08-23T12:07:56.106845Z","iopub.status.idle":"2022-08-23T12:07:56.115556Z","shell.execute_reply.started":"2022-08-23T12:07:56.106790Z","shell.execute_reply":"2022-08-23T12:07:56.113243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ndf_masks = pd.read_csv(MASKS).set_index('id')\ndf_masks.head()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:58.075485Z","iopub.execute_input":"2022-08-23T12:07:58.076006Z","iopub.status.idle":"2022-08-23T12:07:58.563621Z","shell.execute_reply.started":"2022-08-23T12:07:58.075965Z","shell.execute_reply":"2022-08-23T12:07:58.562184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def enc2mask(encs, shape):\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for m,enc in enumerate(encs):\n        if isinstance(enc,np.float) and np.isnan(enc): continue\n        s = str(enc).split()\n        for i in range(len(s)//2):\n            start = int(s[2*i]) - 1\n            length = int(s[2*i+1])\n            img[start:start+length] = 1 + m\n    return img.reshape(shape).T\n\ndef findpad(dim,crop_size):\n    \"\"\"\n    takes input dimension of image and returns the dimension to be padded\n    \"\"\"\n    out = []\n    for i in dim:\n        reminder = i%crop_size   #eg 1100%512= 76\n        if reminder > 0:\n            pad = crop_size - reminder  # pad = 512-76 = 436\n        else:\n            pad = 0  #if reminder = 0 no padding is required\n        out.append(pad)\n        \n    return out[0],out[1]\n\ndef dopadding(img,crop_size=1024,img_type='image'):\n    \"\"\"\n    find the padding size and do padding using np.pad on both y and x direction\n    crop_size: window/kernel size which is planned to use\n    \"\"\"\n    pad0,pad1 = findpad([img.shape[0],img.shape[1]],crop_size)\n    # img,y_padding,x_padding,color,filling value\n    if img_type=='image':\n        img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=0)\n    else:\n        img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2]],constant_values=0)\n        \n    return img\n\n# pad_img = dopadding(img,crop_size=WINDOW)\n# pad_img = cv2.resize(pad_img,(pad_img.shape[1]//reduce, pad_img.shape[0]//reduce), interpolation=cv2.INTER_AREA)\n# WINDOW = WINDOW/reduce\n\n\n\ndef make_inter_grid(img_shape,WINDOW=256):\n    \"\"\"\n    input: padded_image_dimension\n    \"\"\"\n\n    y,x = img_shape[0],img_shape[1]\n    start = int(WINDOW/2)\n    end_x = x - start\n    end_y = y - start\n    ny,nx = int(end_y/WINDOW),int(end_x/WINDOW)\n    base_y = np.linspace(start,end_y,ny+1,dtype=np.int64)\n    base_x = np.linspace(start,end_x,nx+1,dtype=np.int64)\n    \n    y1,y2 = base_y[0:-1],base_y[1:]\n    x1,x2 = base_x[0:-1],base_x[1:]\n    slices = np.zeros(shape=(len(x1),len(y1),4),dtype=np.int64)\n    for i in range(len(x1)):\n        for j in range(len(y1)):\n            slices[i, j] = x1[i], x2[i], y1[j], y2[j]\n    slices = slices.reshape(len(x1) * len(y1), 4)\n    return slices\n\n\ndef make_normal_grid(img_shape,WINDOW=256):\n    #padding\n    y,x = img_shape[0],img_shape[1]\n    nx,ny = int(x/WINDOW), int(y/WINDOW)\n\n    base_y = np.linspace(0,y,ny+1,dtype=np.int64)\n    base_x = np.linspace(0,x,nx+1,dtype=np.int64)\n    y1,y2 = base_y[0:-1],base_y[1:]\n    x1,x2 = base_x[0:-1],base_x[1:]\n    slices = np.zeros(shape=(len(x1),len(y1),4),dtype=np.int64)\n    for i in range(len(x1)):\n        for j in range(len(y1)):\n            slices[i, j] = x1[i], x2[i], y1[j], y2[j]\n    slices = slices.reshape(len(x1) * len(y1), 4)\n    return slices","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:01.974022Z","iopub.execute_input":"2022-08-23T12:09:01.974909Z","iopub.status.idle":"2022-08-23T12:09:01.998549Z","shell.execute_reply.started":"2022-08-23T12:09:01.974876Z","shell.execute_reply":"2022-08-23T12:09:01.997165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WINDOW = 1024\nreduce = 4\ns_th = 40  #saturation blancking threshold\nsz=256\np_th = 200*sz//256 #threshold for the minimum number of pixels\n\nwith zipfile.ZipFile(OUT_TRAIN, 'w') as img_out,zipfile.ZipFile(OUT_MASKS, 'w') as mask_out:\n\n    for index, encs in tqdm(df_masks.iterrows(),total=len(df_masks)):\n        #read image and generate the mask\n        img = tiff.imread(os.path.join(DATA,str(index)+'.tiff'))\n        if len(img.shape) == 5: \n            img = np.transpose(img.squeeze(), (1,2,0))\n        mask = enc2mask(encs,(img.shape[1],img.shape[0]))\n        print(f'processing image and mask of shapes: {img.shape},{mask.shape}')\n        #padding images/masks\n        pad_img = dopadding(img,crop_size=WINDOW,img_type='image')\n        pad_mask = dopadding(mask,crop_size=WINDOW,img_type='mask')\n        # downsizing whole image by reduce_factor\n        img = cv2.resize(pad_img,(pad_img.shape[1]//reduce, pad_img.shape[0]//reduce), interpolation=cv2.INTER_AREA)\n        mask = cv2.resize(pad_mask,(pad_img.shape[1]//reduce, pad_img.shape[0]//reduce), interpolation = cv2.INTER_NEAREST)\n        # generating grids for saving\n        reduced_window_size = int(WINDOW/reduce)\n        slices = make_normal_grid(img.shape,WINDOW=reduced_window_size)\n        for i,(x1,x2,y1,y2) in enumerate(tqdm(slices,desc=f'{len(slices)}')):\n            img_crop = img[y1:y2,x1:x2]\n            mask_crop = mask[y1:y2,x1:x2]\n            \n            #remove black or gray images based on saturation check\n            hsv = cv2.cvtColor(img_crop, cv2.COLOR_BGR2HSV)\n            h, s, v = cv2.split(hsv)\n            # if gray image continue to next pair of image/mask\n            if (s>s_th).sum() <= p_th or img_crop.sum() <= p_th: \n                    continue\n            \n            img_crop = cv2.cvtColor(img_crop, cv2.COLOR_RGB2BGR)\n        \n            im = cv2.imencode('.png',img_crop)[1]\n            img_out.writestr(f'{index}_{i}.png', im)\n            m = cv2.imencode('.png',mask_crop)[1]\n            mask_out.writestr(f'{index}_{i}.png', m)\n            gc.collect()\n            \n            ","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:06.479033Z","iopub.execute_input":"2022-08-23T12:09:06.482420Z","iopub.status.idle":"2022-08-23T12:09:29.661853Z","shell.execute_reply.started":"2022-08-23T12:09:06.482325Z","shell.execute_reply":"2022-08-23T12:09:29.659794Z"},"trusted":true},"execution_count":null,"outputs":[]}]}