{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# HuBMAP + HPA 1024x1024 png dataset\nThis notebook produces a downsized HuBMAP + HPA competition dataset of 1024x1024 images. Both the training images and their respective masks are produced and saved in png format.","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport cv2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-03T17:50:33.717152Z","iopub.execute_input":"2023-04-03T17:50:33.717731Z","iopub.status.idle":"2023-04-03T17:50:33.725595Z","shell.execute_reply.started":"2023-04-03T17:50:33.717683Z","shell.execute_reply":"2023-04-03T17:50:33.724075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_PATH = '/kaggle/input/hubmap-organ-segmentation'\nOUTPUT_PATH = '/kaggle/working/HuBMAP-HPA-1024x1024-PNG'\nRESOLUTION = 1024\ntrain_images_path = os.path.join(INPUT_PATH, 'train_images')","metadata":{"execution":{"iopub.status.busy":"2023-04-03T17:50:33.731126Z","iopub.execute_input":"2023-04-03T17:50:33.731845Z","iopub.status.idle":"2023-04-03T17:50:33.743080Z","shell.execute_reply.started":"2023-04-03T17:50:33.731799Z","shell.execute_reply":"2023-04-03T17:50:33.741758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(os.path.join(INPUT_PATH, 'train.csv'))\ndf.sample(2)","metadata":{"execution":{"iopub.status.busy":"2023-04-03T17:50:33.762149Z","iopub.execute_input":"2023-04-03T17:50:33.762596Z","iopub.status.idle":"2023-04-03T17:50:33.931562Z","shell.execute_reply.started":"2023-04-03T17:50:33.762559Z","shell.execute_reply":"2023-04-03T17:50:33.930291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Helper functions for converting the RLEncoding to an array and for loading and resizing the images","metadata":{}},{"cell_type":"code","source":"def rle2mask(mask_rle, shape):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return\n    Returns numpy array, 1 - mask, 0 - background\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T","metadata":{"execution":{"iopub.status.busy":"2023-04-03T17:50:33.933622Z","iopub.execute_input":"2023-04-03T17:50:33.934057Z","iopub.status.idle":"2023-04-03T17:50:33.941915Z","shell.execute_reply.started":"2023-04-03T17:50:33.934021Z","shell.execute_reply":"2023-04-03T17:50:33.940677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_image_mask(image_id, new_size=None):\n    image = cv2.imread(os.path.join(train_images_path, f\"{image_id}.tiff\"), cv2.IMREAD_UNCHANGED)\n    mask = rle2mask(\n        df[df.id == image_id][\"rle\"].values[0], \n        (image.shape[1], image.shape[0]))\n    if new_size:\n        image = cv2.resize(image, new_size, interpolation=cv2.INTER_CUBIC)\n        mask = cv2.resize(mask, new_size, interpolation=cv2.INTER_AREA)\n\n    return image, mask","metadata":{"execution":{"iopub.status.busy":"2023-04-03T17:50:33.943464Z","iopub.execute_input":"2023-04-03T17:50:33.943801Z","iopub.status.idle":"2023-04-03T17:50:33.953496Z","shell.execute_reply.started":"2023-04-03T17:50:33.943768Z","shell.execute_reply":"2023-04-03T17:50:33.952427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# A sample image for testing the helper functions\nim_id = 10044\nimg, mask = get_image_mask(im_id)\nprint(img.shape, mask.shape, img.mean(axis=(0,1)), img.std(axis=(0,1)))\nimg_d, mask_d = get_image_mask(im_id, new_size=(RESOLUTION, RESOLUTION))\nprint(img_d.shape, mask_d.shape, img_d.mean(axis=(0,1)), img_d.std(axis=(0,1)))","metadata":{"execution":{"iopub.status.busy":"2023-04-03T17:50:33.955638Z","iopub.execute_input":"2023-04-03T17:50:33.956106Z","iopub.status.idle":"2023-04-03T17:50:35.375117Z","shell.execute_reply.started":"2023-04-03T17:50:33.956047Z","shell.execute_reply":"2023-04-03T17:50:35.373697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_sample(image, mask):\n    fig = plt.figure(figsize=(16, 5))\n    fig.suptitle('Original image and mask', fontsize=20)\n    fig.patch.set_facecolor('xkcd:taupe')\n    ax = plt.subplot(1, 3, 1)\n    plt.imshow(image)\n    plt.axis(\"off\")\n    ax = plt.subplot(1, 3, 2)\n    plt.imshow(mask, cmap=\"binary\")\n    plt.axis(\"off\")\n    ax = plt.subplot(1, 3, 3)\n    plt.imshow(image)\n    plt.imshow(mask, cmap=\"BrBG\", alpha=0.5)\n    plt.axis(\"off\")\n    \nRGBimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\ndisplay_sample(RGBimg, mask)","metadata":{"execution":{"iopub.status.busy":"2023-04-03T17:50:35.377774Z","iopub.execute_input":"2023-04-03T17:50:35.378226Z","iopub.status.idle":"2023-04-03T17:50:41.296640Z","shell.execute_reply.started":"2023-04-03T17:50:35.378179Z","shell.execute_reply":"2023-04-03T17:50:41.295549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display the two images side by side with their respective masks  \nfig = plt.figure(figsize=(10, 5))\nfig.suptitle('Original and resized image', fontsize=20)\nfig.patch.set_facecolor('xkcd:taupe')\nax = plt.subplot(1, 2, 1)\nplt.title(\"Original image\")\nplt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))\nplt.imshow(mask, cmap=\"BrBG\", alpha=0.5)\nplt.axis(\"off\")\nax = plt.subplot(1, 2, 2)\nplt.title(\"Resized image 1024x1024\")\nplt.imshow(cv2.cvtColor(img_d, cv2.COLOR_BGR2RGB))\nplt.imshow(mask_d, cmap=\"BrBG\", alpha=0.5)\nplt.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-03T17:50:41.297976Z","iopub.execute_input":"2023-04-03T17:50:41.298446Z","iopub.status.idle":"2023-04-03T17:50:43.724223Z","shell.execute_reply.started":"2023-04-03T17:50:41.298411Z","shell.execute_reply":"2023-04-03T17:50:43.722989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create the dataset with resolution 1024x1024","metadata":{}},{"cell_type":"markdown","source":"While loading every picture and resizing it, we will also calculate the mean value and standard deviation of evey channel (RGB) of all the images in the dataset.  \nThere is a very detailed article of how this should be done properly by Nikita Kozodoi in this article: [Computing Mean & STD in Image Dataset](https://kozodoi.me/blog/20210308/compute-image-stats)","metadata":{}},{"cell_type":"code","source":"train_dir = os.path.join(OUTPUT_PATH, \"train\") \nmask_dir = os.path.join(OUTPUT_PATH, \"masks\")\ntry: \n    os.makedirs(train_dir)\nexcept OSError as error: \n    print(error)  \n\ntry: \n    os.makedirs(mask_dir)\nexcept OSError as error: \n    print(error)  ","metadata":{"execution":{"iopub.status.busy":"2023-04-03T17:50:43.726792Z","iopub.execute_input":"2023-04-03T17:50:43.727177Z","iopub.status.idle":"2023-04-03T17:50:43.735930Z","shell.execute_reply.started":"2023-04-03T17:50:43.727139Z","shell.execute_reply":"2023-04-03T17:50:43.734285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pixels = 0.0\npsum = np.zeros(3).astype(float)\npsum_sq = np.zeros(3).astype(float)\n\nfor im_id in tqdm(df.id, file=sys.stdout, colour=\"blue\"):\n    img, mask = get_image_mask(im_id, new_size=(RESOLUTION, RESOLUTION))\n    pixels += img.shape[0]*img.shape[1]\n    psum += np.sum(img.astype(float), axis=(0,1))\n    psum_sq += np.power(img.astype(float), 2).sum(axis=(0,1))\n    # Save image and mask\n    cv2.imwrite(os.path.join(train_dir, f\"{im_id}.png\"), img)\n    cv2.imwrite(os.path.join(mask_dir, f\"{im_id}.png\"), mask)\n    # mask should only have 1 channel\n    assert(len(mask.shape) == 2)\n    # mask should contain only 0's and 1's\n    assert(np.array_equal(np.unique(mask), np.array([0, 1])))        ","metadata":{"execution":{"iopub.status.busy":"2023-04-03T17:50:43.737669Z","iopub.execute_input":"2023-04-03T17:50:43.738056Z","iopub.status.idle":"2023-04-03T17:54:36.879352Z","shell.execute_reply.started":"2023-04-03T17:50:43.738021Z","shell.execute_reply":"2023-04-03T17:54:36.877091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check if mask was saved as 2D grayscale image with only 0's and 1's\nsample_mask = cv2.imread(os.path.join(mask_dir, \"10044.png\"), cv2.IMREAD_UNCHANGED)\n# mask should only have 1 channel\nassert(len(sample_mask.shape) == 2)\n# mask should contain only 0's and 1's\nassert(np.array_equal(np.unique(sample_mask), np.array([0, 1])))","metadata":{"execution":{"iopub.status.busy":"2023-04-03T17:54:36.882775Z","iopub.execute_input":"2023-04-03T17:54:36.883272Z","iopub.status.idle":"2023-04-03T17:54:36.916973Z","shell.execute_reply.started":"2023-04-03T17:54:36.883211Z","shell.execute_reply":"2023-04-03T17:54:36.915841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate total mean and std\ntotal_mean = psum/pixels\ntotal_std = np.sqrt((psum_sq/pixels) - total_mean**2)\n# transpose from BGR to RGB\ntemp = total_mean[2]; total_mean[2] = total_mean[0]; total_mean[0] = temp;\ntemp = total_std[2]; total_std[2] = total_std[0]; total_std[0] = temp;\nprint(f\"Total mean of dataset: {total_mean}\")\nprint(f\"Total std of dataset: {total_std}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-03T17:54:36.918696Z","iopub.execute_input":"2023-04-03T17:54:36.919058Z","iopub.status.idle":"2023-04-03T17:54:36.927613Z","shell.execute_reply.started":"2023-04-03T17:54:36.919024Z","shell.execute_reply":"2023-04-03T17:54:36.926253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -l /kaggle/working/HuBMAP-HPA-1024x1024-PNG/train | grep png | wc -l\n!ls -l /kaggle/working/HuBMAP-HPA-1024x1024-PNG/masks | grep png | wc -l\n# Should show 351 files twice","metadata":{"execution":{"iopub.status.busy":"2023-04-03T17:54:36.929610Z","iopub.execute_input":"2023-04-03T17:54:36.930022Z","iopub.status.idle":"2023-04-03T17:54:39.137149Z","shell.execute_reply.started":"2023-04-03T17:54:36.929981Z","shell.execute_reply":"2023-04-03T17:54:39.135617Z"},"trusted":true},"execution_count":null,"outputs":[]}]}