{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Generating HuBMAP Images and Masks\n\n**1. Generating the data**\n\nWe will generate HuBMAP images and masks in png format and the size will keep in original ones. So you can feel free to resize them after loading.\nThe dataset is public and the total size is about 5G. \nIt will run about 30 mins to generate but you could check **[this link](https://www.kaggle.com/datasets/dingyan/hubmap-data)** to directly use it.\n    \n**2. Building the model**\n\nHere is [a notebook](https://www.kaggle.com/code/dingyan/hubmap-unet-like-model-with-keras) to quickly build an **image segmentation model** using this dataset. \n\nIf you feel this notebook is helpful, please **upvote**. Thank you.","metadata":{}},{"cell_type":"markdown","source":"**1. Read dataframe from train.csv file**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tifffile as tiff\n\ntrain_file_path = '../input/hubmap-organ-segmentation/train.csv'\ndf = pd.read_csv(train_file_path)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-06T02:58:38.526390Z","iopub.execute_input":"2022-08-06T02:58:38.526724Z","iopub.status.idle":"2022-08-06T02:58:38.691473Z","shell.execute_reply.started":"2022-08-06T02:58:38.526700Z","shell.execute_reply":"2022-08-06T02:58:38.690449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2. We will use tifffile to read the tiff images and parse rle to masks.**","metadata":{}},{"cell_type":"code","source":"import os\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\ntrain_img_path = '../input/hubmap-organ-segmentation/train_images'\nids = df['id'].values\n\ndef load_img(id):\n    img = tiff.imread(os.path.join(train_img_path, f\"{id}.tiff\"))\n    return img\n\ndef rle2mask(id, shape):\n    rle = df[df['id'] == id]['rle'].iloc[-1]\n    s = rle.split()\n    starts, lengths = [np.asarray(x, dtype='int') for x in (s[0:][::2], s[1:][::2])]\n    starts = starts - 1\n    mask = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for s, l in zip(starts, lengths):\n        mask[s:s+l] = 1\n    mask = mask.reshape((shape[0], shape[1])).T\n#     we need at least three channels to save an image so here expand mask to (3000, 3000, 1)\n    mask = mask[:, :, tf.newaxis]\n    return mask","metadata":{"execution":{"iopub.status.busy":"2022-08-06T02:58:40.504833Z","iopub.execute_input":"2022-08-06T02:58:40.505187Z","iopub.status.idle":"2022-08-06T02:58:40.515629Z","shell.execute_reply.started":"2022-08-06T02:58:40.505162Z","shell.execute_reply":"2022-08-06T02:58:40.514925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**3. Creating the target directory \"HuBMAP_train\". The images are in \"images\" folder and masks in \"masks\" folder**","metadata":{}},{"cell_type":"code","source":"import os\ntrain_img_output_path = './HuBMAP_train/images'\ntrain_mask_output_path = './HuBMAP_train/masks'\nif not os.path.exists(train_img_output_path):\n    os.makedirs(train_img_output_path)\nif not os.path.exists(train_mask_output_path):\n    os.makedirs(train_mask_output_path)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T02:58:42.725167Z","iopub.execute_input":"2022-08-06T02:58:42.726037Z","iopub.status.idle":"2022-08-06T02:58:42.731959Z","shell.execute_reply.started":"2022-08-06T02:58:42.726009Z","shell.execute_reply":"2022-08-06T02:58:42.730840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**4. Generating images and masks in original sizes.**","metadata":{}},{"cell_type":"code","source":"for id in tqdm(ids):\n    img = load_img(id)\n    keras.utils.save_img(f'{train_img_output_path}/{id}.png', img)\n    mask = rle2mask(id, img.shape)\n    keras.utils.save_img(f'{train_mask_output_path}/{id}.png', mask, scale=False) # Here we use scale=False because we want keep the mask value in 0 and 1. Otherwise they will be rescaled to 0-255","metadata":{"execution":{"iopub.status.busy":"2022-08-06T02:58:45.286615Z","iopub.execute_input":"2022-08-06T02:58:45.286936Z","iopub.status.idle":"2022-08-06T02:58:55.960492Z","shell.execute_reply.started":"2022-08-06T02:58:45.286913Z","shell.execute_reply":"2022-08-06T02:58:55.959047Z"},"trusted":true},"execution_count":null,"outputs":[]}]}