{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Generate Mask Data\n\nrefer: https://www.kaggle.com/code/awsaf49/uwmgi-mask-data/notebook","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\npd.options.plotting.backend = \"plotly\"\nimport random\nfrom glob import glob\nimport os, shutil\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nimport time\nimport copy\nimport joblib\nimport gc\nfrom IPython import display as ipd\nfrom joblib import Parallel, delayed\n\n# visualization\nimport cv2\nimport matplotlib.pyplot as plt\nfrom matplotlib.patches import Rectangle\n\nimport tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:02:28.043022Z","iopub.execute_input":"2022-07-17T06:02:28.044274Z","iopub.status.idle":"2022-07-17T06:02:28.052288Z","shell.execute_reply.started":"2022-07-17T06:02:28.044212Z","shell.execute_reply":"2022-07-17T06:02:28.051373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\ndef rle_decode(mask_rle, shape):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = np.asarray(mask_rle.split(), dtype=int)\n    starts = s[0::2] - 1\n    lengths = s[1::2]\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T  # Needed to align to RLE direction\n\n\n# ref.: https://www.kaggle.com/stainsby/fast-tested-rle\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:02:28.221656Z","iopub.execute_input":"2022-07-17T06:02:28.222212Z","iopub.status.idle":"2022-07-17T06:02:28.232337Z","shell.execute_reply.started":"2022-07-17T06:02:28.222166Z","shell.execute_reply":"2022-07-17T06:02:28.231219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def path2info(row):\n    path = row['image_path']\n    data = path.split('/')\n    id_ = int(data[-1].split('.')[0])\n    row['id'] = id_\n    return row","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:02:28.621587Z","iopub.execute_input":"2022-07-17T06:02:28.621956Z","iopub.status.idle":"2022-07-17T06:02:28.627933Z","shell.execute_reply.started":"2022-07-17T06:02:28.621928Z","shell.execute_reply":"2022-07-17T06:02:28.626681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/hubmap-organ-segmentation/train.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:02:28.773312Z","iopub.execute_input":"2022-07-17T06:02:28.774395Z","iopub.status.idle":"2022-07-17T06:02:29.088757Z","shell.execute_reply.started":"2022-07-17T06:02:28.774351Z","shell.execute_reply":"2022-07-17T06:02:29.087872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_paths = glob('/kaggle/input/hubmap-organ-segmentation/train_images/*')\ntest_paths = glob('/kaggle/input/hubmap-organ-segmentation/test_images/*')\npath_train_df = pd.DataFrame(train_paths, columns=['image_path'])\npath_train_df = path_train_df.progress_apply(path2info, axis=1)\ndf = df.merge(path_train_df, on='id')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:02:29.090584Z","iopub.execute_input":"2022-07-17T06:02:29.091647Z","iopub.status.idle":"2022-07-17T06:02:29.370049Z","shell.execute_reply.started":"2022-07-17T06:02:29.091597Z","shell.execute_reply":"2022-07-17T06:02:29.368887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Mask","metadata":{}},{"cell_type":"code","source":"def id2mask(id_, df=None):\n    idf = df[df['id']==id_]\n    wh = idf[['img_height','img_width']].iloc[0]\n    shape = (wh.img_height, wh.img_width, 3)\n    mask = np.zeros(shape, dtype=np.uint8)\n    rle = idf.rle.squeeze()\n    if len(idf) and not pd.isna(rle):\n        mask = rle_decode(rle, shape[:2])\n    return mask\n\ndef rgb2gray(mask):\n    pad_mask = np.pad(mask, pad_width=[(0,0),(0,0),(1,0)])\n    gray_mask = pad_mask.argmax(-1)\n    return gray_mask\n\ndef gray2rgb(mask):\n    rgb_mask = tf.keras.utils.to_categorical(mask, num_classes=4)\n    return rgb_mask[..., 1:].astype(mask.dtype)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:02:29.371673Z","iopub.execute_input":"2022-07-17T06:02:29.372086Z","iopub.status.idle":"2022-07-17T06:02:29.380266Z","shell.execute_reply.started":"2022-07-17T06:02:29.372054Z","shell.execute_reply":"2022-07-17T06:02:29.379456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image","metadata":{}},{"cell_type":"code","source":"def load_img(path):\n    img = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n    img = img.astype('float32') # original is uint16\n    img = (img - img.min())/(img.max() - img.min())*255.0 # scale image to [0, 255]\n    img = img.astype('uint8')\n    return img\n\ndef show_img(img, mask=None):\n#     clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\n#     img = clahe.apply(img)\n#     plt.figure(figsize=(10,10))\n    plt.imshow(img, cmap='bone')\n    \n    if mask is not None:\n        # plt.imshow(np.ma.masked_where(mask!=1, mask), alpha=0.5, cmap='autumn')\n        plt.imshow(mask, alpha=0.5)\n        handles = [Rectangle((0,0),1,1, color=_c) for _c in [(0.667,0.0,0.0), (0.0,0.667,0.0), (0.0,0.0,0.667)]]\n    plt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:02:29.381448Z","iopub.execute_input":"2022-07-17T06:02:29.382531Z","iopub.status.idle":"2022-07-17T06:02:29.396905Z","shell.execute_reply.started":"2022-07-17T06:02:29.382490Z","shell.execute_reply":"2022-07-17T06:02:29.395916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Check mask","metadata":{}},{"cell_type":"code","source":"row=1; col=2\nplt.figure(figsize=(5*col,5*row))\nfor i, id_ in enumerate(df[~df.rle.isna()].sample(frac=1.0)['id'].unique()[:row*col]):\n    img = load_img(df[df['id']==id_].image_path.iloc[0])\n    mask = id2mask(id_,df=df)*255\n    plt.subplot(row, col, i+1)\n    i+=1\n    show_img(img, mask=mask)\n    plt.tight_layout()\n# id_1 = 10044\n# img = load_img(df[df['id']==id_1].image_path.iloc[0])\n# mask = id2mask(id_1, df=df)*255\n# plt.figure(figsize=(15, 15))\n# plt.imshow(img)\n# plt.imshow(mask,cmap='coolwarm', alpha=0.5)\n# plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:02:29.413279Z","iopub.execute_input":"2022-07-17T06:02:29.413703Z","iopub.status.idle":"2022-07-17T06:02:33.365824Z","shell.execute_reply.started":"2022-07-17T06:02:29.413666Z","shell.execute_reply":"2022-07-17T06:02:33.364654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Write Mask","metadata":{}},{"cell_type":"code","source":"def save_mask(id_,df=None, count=0):\n    idf = df[df['id']==id_]\n    mask = id2mask(id_, df=df)*255 # mask from [0, 1] to [0, 255]\n    image_path = idf.image_path.iloc[0]\n    img = load_img(image_path) # load image\n    mask_path = image_path.replace('/kaggle/input/','/tmp/png/')\n    mask_folder = mask_path.rsplit('/',1)[0]\n    os.makedirs(mask_folder, exist_ok=True)\n    cv2.imwrite(mask_path, mask, [cv2.IMWRITE_PNG_COMPRESSION, 1]) # write mask as .png\n    mask_path2 = image_path.replace('/kaggle/input/','/tmp/np/').replace('.tiff','.npy')\n    mask_folder2 = mask_path2.rsplit('/',1)[0]\n    os.makedirs(mask_folder2, exist_ok=True)\n    np.save(mask_path2, mask*255) # write mask as .npy\n    return mask_path","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:02:33.367537Z","iopub.execute_input":"2022-07-17T06:02:33.368081Z","iopub.status.idle":"2022-07-17T06:02:33.375104Z","shell.execute_reply.started":"2022-07-17T06:02:33.368043Z","shell.execute_reply":"2022-07-17T06:02:33.374171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save Mask\ntmp_df = df.copy()\ntmp_df = tmp_df[~df.rle.isna()]\nids = tmp_df['id'].unique()\n_ = Parallel(n_jobs = -1, backend = 'threading')(delayed(save_mask)(id_, df=tmp_df, count=i)\\\n                                                for i, id_ in enumerate(tqdm(ids, total=len(ids))))\n","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:02:33.376603Z","iopub.execute_input":"2022-07-17T06:02:33.377133Z","iopub.status.idle":"2022-07-17T06:03:27.198991Z","shell.execute_reply.started":"2022-07-17T06:02:33.377100Z","shell.execute_reply":"2022-07-17T06:03:27.197710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check Saved Mask\ni = 200\nimg = load_img(df.image_path.iloc[i])\nmask_path = df['image_path'].iloc[i].replace('/kaggle/input/','/tmp/png/')\nmask = cv2.imread(mask_path, cv2.IMREAD_UNCHANGED)\nplt.figure(figsize=(5,5))\nshow_img(img, mask=mask)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:03:27.201727Z","iopub.execute_input":"2022-07-17T06:03:27.202089Z","iopub.status.idle":"2022-07-17T06:03:29.398195Z","shell.execute_reply.started":"2022-07-17T06:03:27.202056Z","shell.execute_reply":"2022-07-17T06:03:29.397303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['mask_path'] = df.image_path.str.replace('/kaggle/input','/kaggle/input/hubmap-organ-segmentation-mask/png/')\ndf.to_csv('train.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:03:29.399485Z","iopub.execute_input":"2022-07-17T06:03:29.400325Z","iopub.status.idle":"2022-07-17T06:03:29.719036Z","shell.execute_reply.started":"2022-07-17T06:03:29.400281Z","shell.execute_reply":"2022-07-17T06:03:29.717984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.make_archive('/kaggle/working/png',\n                    'zip',\n                    '/tmp/png/',\n                   'hubmap-organ-segmentation')","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:03:29.720159Z","iopub.execute_input":"2022-07-17T06:03:29.721253Z","iopub.status.idle":"2022-07-17T06:03:31.480413Z","shell.execute_reply.started":"2022-07-17T06:03:29.721208Z","shell.execute_reply":"2022-07-17T06:03:31.479068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.make_archive('/kaggle/working/np',\n                    'zip',\n                    '/tmp/np/',\n                   'hubmap-organ-segmentation')","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:03:31.482372Z","iopub.execute_input":"2022-07-17T06:03:31.483066Z"},"trusted":true},"execution_count":null,"outputs":[]}]}