{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nfrom tqdm.notebook import tqdm\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt\nimport cv2\nimport skimage.io","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = '/kaggle/input/hubmap-kidney-segmentation'\nsplit = 'train' # Change this to use test\ntile_size = 256\next = 'png' # Change to jpg for smaller files","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Helper function"},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/paulorzp/rle-functions-run-lenght-encode-decode\ndef mask2rle(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels= img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n \ndef rle2mask(mask_rle, shape=(1600,256)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load the CSVs"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(f'{data_dir}/train.csv')\nsub_df = pd.read_csv(f'{data_dir}/sample_submission.csv')\n\ntrain_df['enc_dec_enc'] = ''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Creating a dataset - https://www.kaggle.com/xhlulu/hubmap-break-down-images-into-512x512-tiles"},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# Those folders will store our images\nos.makedirs(f'{split}_tiles/images', exist_ok=True)\nos.makedirs(f'{split}_tiles/masks', exist_ok=True)\n\n# This list will contain information about all our images\nmeta_ls = []\n\n# Choose a dataframe based on the split\nif split == 'train':\n    df = train_df\nelse:\n    df = sub_df\n\n# The break down starts here\nfor ix in range(df.shape[0]):\n    img_id = df.id[ix]\n    path = f\"{data_dir}/{split}/{img_id}.tiff\"\n    img = skimage.io.imread(path).squeeze()\n    mask = rle2mask(df.encoding[ix], shape=img.shape[1::-1])\n\n    x_max, y_max = img.shape[:2]\n\n    for x0 in tqdm(range(0, x_max, tile_size)):\n        x1 = min(x_max, x0 + tile_size)\n        for y0 in range(0, y_max, tile_size):\n            y1 = min(y_max, y0 + tile_size)\n\n            img_tile = img[x0:x1, y0:y1]\n            mask_tile = mask[x0:x1, y0:y1]\n\n            img_tile_path = f\"{split}_tiles/images/{img_id}_{x0}-{x1}x_{y0}-{y1}y.{ext}\"\n            mask_tile_path = f\"{split}_tiles/masks/{img_id}_{x0}-{x1}x_{y0}-{y1}y.png\"\n\n            cv2.imwrite(img_tile_path, cv2.cvtColor(img_tile, cv2.COLOR_RGB2BGR))\n            cv2.imwrite(mask_tile_path, mask_tile)\n\n            meta_ls.append([\n                img_id, x0, x1, y0, y1, img_tile.min(), img_tile.max(), \n                mask_tile.max(), img_tile_path, mask_tile_path, x_max, y_max\n            ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"meta_df = pd.DataFrame(meta_ls, columns=['image_id', 'x0', 'x1', 'y0', 'y1', 'min_pixel_value', 'max_pixel_value', 'max_mask_value', 'image_tile_path', 'mask_tile_path', 'w', 'h'])\nmeta_df.to_csv(f'{split}_metadata.csv', index=False)\nmeta_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### reconstruction of the full image and coordinate-wise RLE encoding"},{"metadata":{"trusted":true},"cell_type":"code","source":"# adapted from https://stackoverflow.com/questions/53327999/creating-a-run-length-for-a-number-matrix-in-python\ndef run_length_encoding(matrix):\n    # List for storing run length encoding\n    encoding = []\n\n    # Counts the number of occurrences\n    count = 0\n    true_count = 0\n    false_counts = 1\n    \n    # Initialize previous element to first element in matrix\n    previous_element = matrix[0][0]\n\n    for row in matrix:\n        for current_element in row:\n            if current_element == previous_element:\n                count += 1\n            else:\n                if previous_element == 1:\n                    encoding.append(count)\n                    true_count += count\n                else:   \n                    false_counts += count\n                    encoding.append(false_counts+true_count)\n        \n                # Reset counter and update previous element\n                count = 1\n                previous_element = current_element\n\n    return ' '.join(str(x) for x in encoding)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor id_ in meta_df['image_id'].value_counts().index.tolist():\n    w = meta_df.loc[meta_df['image_id'] ==  id_, 'w'].values[0]\n    h = meta_df.loc[meta_df['image_id'] == id_, 'h'].values[0]\n    full_mask = np.zeros((w, h), dtype=np.int8)\n    \n    # full image reconstruction\n    for _, row in meta_df.loc[meta_df['image_id'] == id_].iterrows():\n        tile_mask = cv2.imread(row['mask_tile_path'],0)\n        cmin, cmax, rmin, rmax = row['x0'], row['x1'], row['y0'], row['y1']\n        full_mask[cmin:cmax, rmin:rmax] = tile_mask[:cmax, :rmax].astype(np.int8)\n        \n    # run-length encoding\n    rle = run_length_encoding(full_mask.T)\n    df.loc[df['id'] == id_, 'enc_dec_enc'] = rle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2, figsize=(30, 15))\n\nmask = rle2mask(df.loc[df['id'] == id_, 'enc_dec_enc'].values[0], full_mask.shape[::-1])\nax[0].imshow(full_mask)\nax[0].set_title(\"Enocoded-Decoded-Encoded\", fontsize=30);\n\nmask = rle2mask(df.loc[df['id'] == id_, 'encoding'].values[0], full_mask.shape[::-1])\nax[1].imshow(mask)\nax[1].set_title(\"Enocoded\", fontsize=30);\n\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### reconstruction of the full image directly into a flat array (not implemented)"},{"metadata":{},"cell_type":"markdown","source":"```\n%%time\n\nfor id_ in meta_df['image_id'].value_counts().index.tolist():\n    w = meta_df.loc[meta_df['image_id'] == id_, 'w'].values[0]\n    h = meta_df.loc[meta_df['image_id'] == id_, 'h'].values[0]\n    \n    full_mask = np.zeros((w, h), dtype=np.int8).T.flatten()\n\n    start = 0\n\n    # full image reconstruction\n    for _, row in meta_df.loc[meta_df['image_id'] == id_].iterrows():\n\n        tile_mask = cv2.imread(row['mask_tile_path'], 0).astype(\"int8\")\n        cmin, cmax, rmin, rmax = row['x0'], row['x1'], row['y0'], row['y1']\n        \n        #y * width + x\n        tile_mask = tile_mask[:cmax-cmin, :rmax-rmin].T.flatten()\n        \n        # Convert 2d index to 1d \n        index = cmin * w + rmin \n\n        full_mask[index:index+len(tile_mask)] = tile_mask\n    \n    # rle from here - https://www.kaggle.com/bguberfain/memory-aware-rle-encoding\n    full_mask[0] = 0\n    full_mask[-1] = 0\n    full_mask = np.where(full_mask[1:] != full_mask[:-1])[0] + 2\n\n    full_mask[1::2] -= full_mask[::2]\n\n    df.loc[df['id'] == id_, 'enc_dec_enc'] = ' '.join(str(x) for x in full_mask)\n```"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}