{"cells":[{"metadata":{},"cell_type":"markdown","source":"# HuBMAP Quick Reference\nSo finally we have the new data and annotations. I decided to put together a \"quick reference\" with the images, and some basic statistics about the glomeruli.  \nFirst we will install [pyvips](https://pypi.org/project/pyvips/) for handling the large images. It is a lenghty install process (6-7min)."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"%%capture\n!wget -O vips-8.10.5.tar.gz https://github.com/libvips/libvips/releases/download/v8.10.5/vips-8.10.5.tar.gz\n!sleep .5\n!tar xf ./vips-8.10.5.tar.gz\n!rm -fr ./vips-8.10.5.tar.gz\n%cd vips-8.10.5\n!./configure\n!make\n!make install\n!ldconfig\n%cd /kaggle/working\n!pip install --user pyvips","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"A few helper functions below to plot images."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom skimage import io\nimport json\nimport cv2\nimport pyvips\n%matplotlib inline  \n\n# a few helper functions below to plot the images and creates masks\nSCALE = 0.1\nTRAIN_PATH = '../input/hubmap-kidney-segmentation/train/'\nTEST_PATH = '../input/hubmap-kidney-segmentation/test/'\n\ndef get_h_w(poly):\n    pol = poly.astype('int32')\n    width = cv2.boundingRect(pol)[3]\n    height = cv2.boundingRect(pol)[2]\n    return width, height\n\ndef read_tif_file(fname):\n    img = io.imread(fname)\n    img = np.squeeze(img)\n    if img.shape[0] == 3: # swap axes as required\n        img = img.swapaxes(0,1)\n        img = img.swapaxes(1,2)\n    return img\n\ndef read_mask_file(fname, mshape):\n    with open(fname) as f:\n        mdata = json.load(f)\n        polys = []\n        gcnt = 0\n        pmin = [1e6,1e6]\n        pmax = [0,0]\n        for index in range(mdata.__len__()):\n            if mdata[index]['properties']['classification']['name'] == 'glomerulus':\n                geom = np.array(mdata[index]['geometry']['coordinates'])\n                if geom.shape[0] == 1:\n                    poly = geom[0] * SCALE\n                    poly = poly.astype('int32')\n                    polys.append(poly)\n                    gcnt += 1\n                    h, w = get_h_w(geom[0].astype('int32'))\n                    if h*w > pmax[0] *pmax[1]:\n                        pmax = [h,w]\n                    if h*w < pmin[0] *pmin[1]:\n                        pmin = [h,w]\n        mask = np.zeros(mshape, dtype=np.int8)\n        cv2.fillPoly(mask, polys, 1)\n        mask = mask.astype(bool, copy=False)\n    return mask, gcnt, pmin, pmax\n\nstats = []\n        \ndef show_file(path, file, train=True):\n    img = pyvips.Image.tiffload(path+file, access='sequential')\n    org_dims = np.array((img.width, img.height))\n    if img.get('n-pages') == 3: # multi-page file\n        img_r = pyvips.Image.tiffload(path+file, page=0, access='sequential')\n        img_r = img_r.resize(SCALE)\n        dims = np.array((img_r.height, img_r.width))\n        img_g = pyvips.Image.tiffload(path+file, page=1, access='sequential')\n        img_g = img_g.resize(SCALE)\n        img_b = pyvips.Image.tiffload(path+file, page=2, access='sequential')\n        img_b = img_b.resize(SCALE)\n        img_r = np.ndarray(buffer=img_r.write_to_memory(),\n                           dtype=np.uint8,\n                           shape=[img_r.height, img_r.width, img_r.bands])\n        img_g = np.ndarray(buffer=img_g.write_to_memory(),\n                           dtype=np.uint8,\n                           shape=[img_g.height, img_g.width, img_g.bands])\n        img_b = np.ndarray(buffer=img_b.write_to_memory(),\n                           dtype=np.uint8,\n                           shape=[img_b.height, img_b.width, img_b.bands])\n        img = np.concatenate((img_r, img_g, img_b), axis=2)\n    else:\n        img = img.resize(SCALE)\n        dims = np.array((img.height, img.width))\n        img = np.ndarray(buffer=img.write_to_memory(),\n                         dtype=np.uint8,\n                         shape=[dims[0], dims[1], 3])\n    if train:\n        mask, gcnt, pmin, pmax = read_mask_file(path+file.replace('.tiff','.json'), dims)\n        stats.append(['train', file, org_dims[0], org_dims[1], gcnt, '{}x{}'.format(pmin[0], pmin[1]), '{}x{}'.format(pmax[0], pmax[1])])\n    else:\n        stats.append(['test', file, org_dims[0], org_dims[1], 0, 'N/A', 'N/A'])\n    plt.figure(figsize=(20,20))\n    plt.imshow(img)\n    if train:\n        plt.title('Glomeruli:{}   Smallest:{}x{}   Largest:{}x{}'.format(gcnt, pmin[0], pmin[1], pmax[0], pmax[1]), fontdict={'fontsize': 20})\n        plt.imshow(mask, alpha=0.25);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The size of the individual glomeruli is determined using cv2.boundingRect on the polyline. Example here:"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"poly = np.array([[2787, 7396],[2725, 7426],[2706, 7458],[2704, 7468],[2700, 7478],\n                [2699, 7510],[2698, 7521],[2699, 7522],[2699, 7525],[2735, 7566],\n                [2744, 7577],[2745, 7577],[2756, 7590],[2844, 7616],[2882, 7589],\n                [2908, 7559],[2927, 7536],[2934, 7515],[2934, 7507],[2889, 7445],\n                [2876, 7424],[2872, 7421],[2870, 7420],[2849, 7405],[2838, 7403],\n                [2826, 7397],[2792, 7396],[2787, 7396]])\n\nwidth, height = get_h_w(poly)\npoly[:,0] -= cv2.boundingRect(poly)[0]\npoly[:,1] -= cv2.boundingRect(poly)[1]\nmask = np.zeros([width,height], dtype=np.int8)\ncv2.fillPoly(mask, [poly], (255,255,255))\nplt.imshow(mask);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Note that the images are scaled down with a factor of 0.1, which means that the x- and y-axis ticks are  also 1/10th of original size. The glomeruli sizes are original though."},{"metadata":{},"cell_type":"markdown","source":"# Train: 0486052bb.tiff"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_file(TRAIN_PATH, '0486052bb.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train: 095bf7a1f.tiff"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_file(TRAIN_PATH, '095bf7a1f.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train: 1e2425f28.tiff"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_file(TRAIN_PATH, '1e2425f28.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train: 26dc41664.tiff"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_file(TRAIN_PATH, '26dc41664.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train: 2f6ecfcdf.tiff"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_file(TRAIN_PATH, '2f6ecfcdf.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train: 4ef6695ce.tiff"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_file(TRAIN_PATH, '4ef6695ce.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train: 54f2eec69.tiff"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_file(TRAIN_PATH, '54f2eec69.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train: 8242609fa.tiff"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_file(TRAIN_PATH, '8242609fa.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train: aaa6a05cc.tiff"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_file(TRAIN_PATH, 'aaa6a05cc.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train: afa5e8098.tiff"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"show_file(TRAIN_PATH, 'afa5e8098.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train: b2dc8411c.tiff"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"show_file(TRAIN_PATH, 'b2dc8411c.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train: b9a3865fc.tiff"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"show_file(TRAIN_PATH, 'b9a3865fc.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train: c68fe75ea.tiff"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"show_file(TRAIN_PATH, 'c68fe75ea.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train: cb2d976f4.tiff"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"show_file(TRAIN_PATH, 'cb2d976f4.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train: e79de561c.tiff"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"show_file(TRAIN_PATH, 'e79de561c.tiff')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test: 2ec3f1bb9.tiff"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"show_file(TEST_PATH, '2ec3f1bb9.tiff', False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test: 3589adb90.tiff"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"show_file(TEST_PATH, '3589adb90.tiff', False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test: 57512b7f1.tiff"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"show_file(TEST_PATH, '57512b7f1.tiff', False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test: aa05346ff.tiff"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"show_file(TEST_PATH, 'aa05346ff.tiff', False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test: d488c759a.tiff"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"show_file(TEST_PATH, 'd488c759a.tiff', False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Summary"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"df = pd.DataFrame(stats, columns=['dataset', 'file', 'x-size', 'y-size', 'glomeruli', 'smallest', 'largest'])\ndf.to_pickle('./image_stats.pkl')\ndf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}