{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**About this notebook**\n* Images and their respective masks are sliced\n* Only those images are kept that have masks\n* Before using the images and masks, please check the shape of each image and mask. Although they should be 224x224 but still you know.. :D","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\nimport tifffile as tiff \nfrom tqdm import tqdm\n\nimport pickle","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-07T06:08:58.240731Z","iopub.execute_input":"2022-07-07T06:08:58.241146Z","iopub.status.idle":"2022-07-07T06:08:58.249161Z","shell.execute_reply.started":"2022-07-07T06:08:58.241113Z","shell.execute_reply":"2022-07-07T06:08:58.247845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images_path='../input/hubmap-organ-segmentation/train_images'\ntest_images_path='../input/hubmap-organ-segmentation/test_images'","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:48:26.039538Z","iopub.execute_input":"2022-07-07T05:48:26.041169Z","iopub.status.idle":"2022-07-07T05:48:26.047052Z","shell.execute_reply.started":"2022-07-07T05:48:26.041107Z","shell.execute_reply":"2022-07-07T05:48:26.045584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=pd.read_csv('../input/hubmap-organ-segmentation/train.csv')\ntest_df=pd.read_csv('../input/hubmap-organ-segmentation/test.csv')\nsample_sub=pd.read_csv('../input/hubmap-organ-segmentation/sample_submission.csv')\n\nprint('Training data size: ',train_df.shape)\nprint('Test data size',test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:48:26.049379Z","iopub.execute_input":"2022-07-07T05:48:26.050015Z","iopub.status.idle":"2022-07-07T05:48:26.419839Z","shell.execute_reply.started":"2022-07-07T05:48:26.049902Z","shell.execute_reply":"2022-07-07T05:48:26.418514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:48:26.424103Z","iopub.execute_input":"2022-07-07T05:48:26.427056Z","iopub.status.idle":"2022-07-07T05:48:26.506605Z","shell.execute_reply.started":"2022-07-07T05:48:26.426977Z","shell.execute_reply":"2022-07-07T05:48:26.505365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_height_unique=train_df['img_height'].unique()\nimages_height_unique","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:48:26.509054Z","iopub.execute_input":"2022-07-07T05:48:26.509913Z","iopub.status.idle":"2022-07-07T05:48:26.530126Z","shell.execute_reply.started":"2022-07-07T05:48:26.509857Z","shell.execute_reply":"2022-07-07T05:48:26.528830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets Print a sample\nimg=tiff.imread(os.path.join(train_images_path,str(train_df.loc[0,'id'])+'.tiff'))\nimg.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:48:26.531819Z","iopub.execute_input":"2022-07-07T05:48:26.532579Z","iopub.status.idle":"2022-07-07T05:48:27.042592Z","shell.execute_reply.started":"2022-07-07T05:48:26.532504Z","shell.execute_reply":"2022-07-07T05:48:27.040745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Plot image\nplt.figure(figsize=(10,10))\nplt.imshow(cv2.cvtColor(img,cv2.COLOR_BGR2RGB))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:48:27.044496Z","iopub.execute_input":"2022-07-07T05:48:27.045057Z","iopub.status.idle":"2022-07-07T05:48:28.533495Z","shell.execute_reply.started":"2022-07-07T05:48:27.045001Z","shell.execute_reply":"2022-07-07T05:48:28.532484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#RLE to Mask\ndef rleToMask(rleString,height,width):\n    rows,cols = height,width\n    rleNumbers = [int(numstring) for numstring in rleString.split(' ')]\n    rlePairs = np.array(rleNumbers).reshape(-1,2)\n    img = np.zeros(rows*cols,dtype=np.uint8)\n    for index,length in rlePairs:\n        index -= 1\n        img[index:index+length] = 255\n    img = img.reshape(cols,rows)\n    img = img.T\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:48:28.534594Z","iopub.execute_input":"2022-07-07T05:48:28.535048Z","iopub.status.idle":"2022-07-07T05:48:28.545183Z","shell.execute_reply.started":"2022-07-07T05:48:28.535012Z","shell.execute_reply":"2022-07-07T05:48:28.544005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask=rleToMask(train_df.loc[0,'rle'],3000,3000)\n\n#Plot Image with mask\nplt.figure(figsize=(10,10))\nplt.imshow(cv2.cvtColor(img,cv2.COLOR_BGR2RGB))\nplt.imshow(mask,alpha=0.4)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:48:28.546782Z","iopub.execute_input":"2022-07-07T05:48:28.547282Z","iopub.status.idle":"2022-07-07T05:48:31.001744Z","shell.execute_reply.started":"2022-07-07T05:48:28.547231Z","shell.execute_reply":"2022-07-07T05:48:31.000245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Get images and masks\nimage_tiles,mask_tiles=[],[]\nM=N=224\nfor ix in tqdm(range(train_df.shape[0])):\n    \n    #Get Image information\n    image_h=train_df.loc[ix,'img_height']\n    image_w=train_df.loc[ix,'img_width']\n    image_rle=train_df.loc[ix,'rle']\n    image_id=train_df.loc[ix,'id']\n    \n    #Read image\n    img=tiff.imread(os.path.join(train_images_path,str(train_df.loc[0,'id'])+'.tiff'))\n    img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n    \n    #read mask\n    mask=rleToMask(image_rle,image_h,image_w)\n    \n    #Create tiles\n    img_tiles=[img[x:x+M,y:y+N] for x in range(0,img.shape[0],M) for y in range(0,img.shape[1],N)]\n    msk_tiles=[mask[x:x+M,y:y+N] for x in range(0,mask.shape[0],M) for y in range(0,mask.shape[1],N)]\n    \n    image_tiles.append(img_tiles)\n    mask_tiles.append(msk_tiles)\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:48:31.005756Z","iopub.execute_input":"2022-07-07T05:48:31.006314Z","iopub.status.idle":"2022-07-07T05:48:45.185063Z","shell.execute_reply.started":"2022-07-07T05:48:31.006260Z","shell.execute_reply":"2022-07-07T05:48:45.183661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Now from each sliced image, we remove those slice that do not have any mask in them\nfinal_images,final_masks=[],[]\nfor ix in tqdm(range(len(image_tiles))):\n    for image,mask in zip(image_tiles[ix],mask_tiles[ix]):\n        if mask.max()==255:\n            final_images.append(image)\n            final_masks.append(mask)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:04:18.367508Z","iopub.execute_input":"2022-07-07T06:04:18.367922Z","iopub.status.idle":"2022-07-07T06:04:22.465911Z","shell.execute_reply.started":"2022-07-07T06:04:18.367890Z","shell.execute_reply":"2022-07-07T06:04:22.464438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of final images: ',len(final_images))\nprint('Number of final masks: ',len(final_masks))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:04:23.713333Z","iopub.execute_input":"2022-07-07T06:04:23.713810Z","iopub.status.idle":"2022-07-07T06:04:23.721879Z","shell.execute_reply.started":"2022-07-07T06:04:23.713775Z","shell.execute_reply":"2022-07-07T06:04:23.719950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets print some images and their masks\nr=c=4\nfig=plt.figure(figsize=(12,12))\nfor i in range(1,r*c+1):\n    fig.add_subplot(r,c,i)\n    plt.imshow(final_images[i])\n    plt.imshow(final_masks[i],alpha=0.4)\n    plt.xticks([])\n    plt.yticks([])\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:08:03.072828Z","iopub.execute_input":"2022-07-07T06:08:03.073262Z","iopub.status.idle":"2022-07-07T06:08:04.263399Z","shell.execute_reply.started":"2022-07-07T06:08:03.073227Z","shell.execute_reply":"2022-07-07T06:08:04.262147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Pickle Dump**","metadata":{}},{"cell_type":"code","source":"with open('images_dump.pkl','wb') as img_dump:\n    pickle.dump(final_images,img_dump)\n    \nwith open('masks_dump.pkl','wb') as msk_dump:\n    pickle.dump(final_masks,msk_dump)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:11:13.311396Z","iopub.execute_input":"2022-07-07T06:11:13.311841Z","iopub.status.idle":"2022-07-07T06:11:40.188728Z","shell.execute_reply.started":"2022-07-07T06:11:13.311794Z","shell.execute_reply":"2022-07-07T06:11:40.187790Z"},"trusted":true},"execution_count":null,"outputs":[]}]}