{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport json\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport cv2 as cv\nimport tifffile as tiff\nfrom tqdm.notebook import tqdm\nfrom PIL import Image\n\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n        \n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_DIR = '/kaggle/input/hubmap-kidney-segmentation/'\nTRAIN_DATA_DIR = '/kaggle/input/hubmap-kidney-segmentation/train/'\nTEST_DATA_DIR = '/kaggle/input/hubmap-kidney-segmentation/test/'\nTRAIN_SAVE_DIR = \"/kaggle/working/train_tiles/\"\nTEST_SAVE_DIR = \"/kaggle/working/test_tiles/\"\nMODEL_SAVE_DIR = \"/kaggle/working/\"\nTILE_SIZE = 256\nREDUCE_RATE = 4\n\nif not os.path.exists(TRAIN_SAVE_DIR):\n    os.mkdir(TRAIN_SAVE_DIR)\n\nif not os.path.exists(TEST_SAVE_DIR):\n    os.mkdir(TEST_SAVE_DIR)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helpers"},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_pil_images(\n    images, \n    masks=None,\n    labels=None,\n    columns=5, width=20, height=8, max_images=15, \n    label_wrap_length=50, label_font_size=9):\n\n    if len(images) > max_images:\n        print(f\"Showing {max_images} images of {len(images)}:\")\n        images=images[0:max_images]\n        if masks is not None:\n            masks= masks[0:max_images]\n\n    height = max(height, int(len(images)/columns) * height)\n    plt.figure(figsize=(width, height))\n    \n    if masks is not None:\n        for i, (image, mask) in enumerate(zip(images,masks)):\n            plt.subplot(len(images) / columns + 1, columns, i + 1)\n            plt.imshow(image)\n            plt.imshow(mask, cmap='coolwarm', alpha=0.5)\n            \n            if labels is not None:\n                plt.title(labels[i], fontsize=label_font_size); \n            \n    else:\n        for i, image in enumerate(images):\n            plt.subplot(len(images) / columns + 1, columns, i + 1)\n            plt.imshow(image)\n        \n            if labels is not None:\n                plt.title(labels[i], fontsize=label_font_size); \n\n        \ndef mask2rle(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels= img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n \ndef rle2mask(mask_rle, shape=(1600,256)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T\n\n\ndef is_tile_contains_info(img, pixel_limits, content_threshold, expected_shape):\n    \"\"\"\n    img: np.array\n    pixel_limits: tuple\n    content_threshold: float percents\n    expected_shape: tuple\n    \"\"\"\n    \n    left_limit = np.prod(img > pixel_limits[0], axis=-1)\n    right_limit =  np.prod(img < pixel_limits[1], axis=-1)\n\n    if img.shape != expected_shape:\n        return False, 0.\n\n    percent_of_pixels = np.sum(left_limit*right_limit) / (img.shape[0] * img.shape[1])\n    return  percent_of_pixels > content_threshold, percent_of_pixels\n\ndef extract_train_tiles(sample_img_path, rle_mask_sample, idx):\n    print(idx)\n    sample_image = tiff.imread(sample_img_path)\n    \n    if idx in ['e79de561c', '095bf7a1f', '54f2eec69', '1e2425f28']:\n        sample_image = np.transpose(sample_image.squeeze(), (1,2,0))\n\n        \n    sample_mask = rle2mask(rle_mask_sample, (sample_image.shape[1], sample_image.shape[0]))\n    print(f\"Original Tiff image shape: {sample_image.shape}\")\n    \n    pad0 = (REDUCE_RATE*TILE_SIZE - sample_image.shape[0]%(REDUCE_RATE*TILE_SIZE))%(REDUCE_RATE*TILE_SIZE)\n    pad1 = (REDUCE_RATE*TILE_SIZE - sample_image.shape[1]%(REDUCE_RATE*TILE_SIZE))%(REDUCE_RATE*TILE_SIZE)\n    \n    sample_image = np.pad(sample_image,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                   constant_values=0)\n    sample_mask = np.pad(sample_mask,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2]],\n                  constant_values=0)\n        \n    sample_image = cv.resize(sample_image,(sample_image.shape[1]//REDUCE_RATE,sample_image.shape[0]//REDUCE_RATE),\n                             interpolation = cv.INTER_AREA)\n    \n    sample_mask = cv.resize(sample_mask,(sample_mask.shape[1]//REDUCE_RATE,sample_mask.shape[0]//REDUCE_RATE),\n                             interpolation = cv.INTER_AREA)\n    \n    print(f\"Reduced Tiff image shape: {sample_image.shape}\")\n    \n    tiles, masks, paths = [], [], []\n    for x in range(0,sample_image.shape[0],TILE_SIZE):\n        for y in range(0,sample_image.shape[1],TILE_SIZE):\n            sub_image = np.float32(sample_image[x:x+TILE_SIZE,y:y+TILE_SIZE])\n            sub_mask = sample_mask[x:x+TILE_SIZE,y:y+TILE_SIZE]\n            if is_tile_contains_info(sub_image, (50, 220), 0.7, (TILE_SIZE,TILE_SIZE, 3))[0]:\n                tiles.append(sub_image)\n                masks.append(sub_mask)\n            else:\n                continue\n    \n    if not os.path.exists(os.path.join(TRAIN_SAVE_DIR, idx)):\n        os.mkdir(os.path.join(TRAIN_SAVE_DIR, idx))\n\n    count = 0\n    for tile,mask in zip(tiles,masks):\n        cv.imwrite(os.path.join(TRAIN_SAVE_DIR, idx, f\"img_{count}.png\"), tile)\n        cv.imwrite(os.path.join(TRAIN_SAVE_DIR, idx, f\"mask_{count}.png\"), mask)\n        paths.append((os.path.join(TRAIN_SAVE_DIR, idx, f\"img_{count}.png\"), \n                      os.path.join(TRAIN_SAVE_DIR, idx, f\"mask_{count}.png\")))\n\n        count += 1\n            \n    print(f\"Length tiles\", len(tiles))\n    gc.collect()\n    \n    return paths\n\ndef extract_test_tiles(sample_img_path, idx):\n    print(idx)\n    sample_image = tiff.imread(sample_img_path)\n    \n    if idx in ['26dc41664', 'c68fe75ea']:\n        sample_image = np.transpose(sample_image.squeeze(), (1,2,0))\n\n    print(f\"Original Tiff image shape: {sample_image.shape}\")\n    \n    pad0 = (REDUCE_RATE*TILE_SIZE - sample_image.shape[0]%(REDUCE_RATE*TILE_SIZE))%(REDUCE_RATE*TILE_SIZE)\n    pad1 = (REDUCE_RATE*TILE_SIZE - sample_image.shape[1]%(REDUCE_RATE*TILE_SIZE))%(REDUCE_RATE*TILE_SIZE)\n    \n    sample_image = np.pad(sample_image,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                   constant_values=0)\n    \n    sample_image = cv.resize(sample_image,(sample_image.shape[1]//REDUCE_RATE,sample_image.shape[0]//REDUCE_RATE),\n                             interpolation = cv.INTER_AREA)\n    \n    print(f\"Reduced Tiff image shape: {sample_image.shape}\")\n    \n    tiles, paths = [], []\n    for x in range(0,sample_image.shape[0],TILE_SIZE):\n        for y in range(0,sample_image.shape[1],TILE_SIZE):\n            sub_image = np.float32(sample_image[x:x+TILE_SIZE,y:y+TILE_SIZE])\n            tiles.append(sub_image)\n    \n    if not os.path.exists(os.path.join(TEST_SAVE_DIR, idx)):\n        os.mkdir(os.path.join(TEST_SAVE_DIR, idx))\n\n    count = 0\n    for tile in tiles:\n        cv.imwrite(os.path.join(TEST_SAVE_DIR, idx, f\"img_{count}.png\"), tile)\n        paths.append(os.path.join(TEST_SAVE_DIR, idx, f\"img_{count}.png\"))\n        count += 1\n            \n    print(f\"Length tiles\", len(tiles))\n    gc.collect()\n    \n    return paths\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data exploration"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(os.path.join(DATA_DIR, 'train.csv'), encoding='utf-8')\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv('../input/hubmap-kidney-segmentation/sample_submission.csv')\nsample_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(os.path.join(DATA_DIR, 'train/2f6ecfcdf-anatomical-structure.json'), mode='r', encoding='utf-8') as f:\n    sample_anatomy = json.load(f)\n\nsample_anatomy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_img_paths = [os.path.join(TRAIN_DATA_DIR, item + '.tiff') for item in train_df['id']]\ntest_img_paths = [os.path.join(TEST_DATA_DIR, item + '.tiff') for item in sample_df['id']]\n\nprint(train_img_paths)\nprint(test_img_paths)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nIMAGE_IDX = 4\n\nsample_image = tiff.imread(train_img_paths[IMAGE_IDX])\n\nsample_image = np.transpose(sample_image.squeeze(), (1,2,0))\nimg_id = train_df['id'].values[0]\nprint(\"This image's id:\", img_id)\nprint(f\"Sample image shape: {sample_image.shape}\")\n      \nsample_mask = rle2mask(train_df['encoding'].values[IMAGE_IDX], (sample_image.shape[1], sample_image.shape[0]))\nprint(f\"Sample image shape: {sample_mask.shape}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,10))\nplt.imshow(sample_image)\nplt.imshow(sample_mask, cmap='coolwarm', alpha=0.5);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make patches"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nall_train_paths = []\nfor idx, train_img_path in enumerate(train_img_paths):\n    paths = extract_train_tiles(train_img_path, \n                                train_df['encoding'].values[idx], \n                                train_df['id'].values[idx])\n    all_train_paths.extend(paths)\n    \nprint(\"Length of all samples:\", len(all_train_paths))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nall_test_paths = []\nfor idx, test_img_path in enumerate(test_img_paths):\n    paths = extract_test_tiles(test_img_path, \n                               sample_df['id'].values[idx])\n    all_test_paths.extend(paths)\n    \nprint(\"Length of all samples:\", len(all_test_paths))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Result"},{"metadata":{"trusted":true},"cell_type":"code","source":"imgs = [Image.open(img_path) for img_path, _ in all_train_paths]\nmasks = [Image.open(mask_path) for _, mask_path in all_train_paths]\ndisplay_pil_images(imgs[:15], masks[:15])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imgs = [Image.open(img_path) for img_path in all_test_paths]\ndisplay_pil_images(imgs[450:500], labels=all_test_paths[450:500])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!zip -r train_tiles_256.zip train_tiles\n!zip -r test_tiles_256.zip test_tiles\n!rm -r train_tiles\n!rm -r test_tiles","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}