{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CPTAC-PDA Pancreas masks and probability density function\n\n> Proof of concept and generalization of HuBMAP Efficient Sampling II","metadata":{}},{"cell_type":"markdown","source":"## Implementation (masks and probability density function)\n\n> - Create segmentations masks from JSON\n> - Create probability density function (PDF) for efficient sampling from mask and anatmical structure\n\n\n**Inputs**\n- https://www.kaggle.com/matjes/cptacpda dataset comprising three whole slice images \\[[1](https://pathology.cancerimagingarchive.net/pathdata/cptac_camicroscope/osdCamicroscope.php?tissueId=C3L-03371-25), [2](https://pathology.cancerimagingarchive.net/pathdata/cptac_camicroscope/osdCamicroscope.php?tissueId=C3L-01158-25), [3](https://pathology.cancerimagingarchive.net/pathdata/cptac_camicroscope/osdCamicroscope.php?tissueId=C3L-03350-24)] from [cancerimagingarchive](https://www.cancerimagingarchive.net/).\n    \n\n**Settings**\n\n- Sampling probability for pancreas regions (`pancreas_p`): 0.9 \n- Sampling probability for background regions (`bg_p`): 0.1      \n","metadata":{}},{"cell_type":"code","source":"# Install zarr and load packages\n!pip install -qq zarr\nimport cv2, zarr, json, rasterio\nimport matplotlib.pyplot as plt, numpy as np, pandas as pd\nfrom shapely.geometry import shape, GeometryCollection\nfrom rasterio import features\nfrom pathlib import Path","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_polys(file, classification):\n    with open(file) as jsonfile:\n        data = json.load(jsonfile)   \n        geom_list = []\n        for features in data:\n            try:\n                if features['properties']['classification']['name']==classification:\n                    geom_list.append(shape(features[\"geometry\"]))\n            except:\n                print('Classification Missing')\n                print(features)\n        return GeometryCollection(geom_list)  ","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Settings","metadata":{}},{"cell_type":"code","source":"class CONFIG:\n    path = Path('../input/cptacpda')\n    \n    scale = 2        # Downscale final mask by factor 2\n    cdf_size = 512   # Downscale CDF for memory efficient loading during training\n    bg_p = 0.1       # Background Probability\n    pancreas_p = 0.9  # Pancreas Probability\n\ncfg = CONFIG()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load inputs and define outputs**","metadata":{}},{"cell_type":"code","source":"# Input \ndf_info = pd.read_csv(cfg.path/\"dataset_information.csv\", index_col='image_file')\n\n# Output\nroot = zarr.group(f'/kaggle/working/masks_scale{cfg.scale}')\n# Saving cdf in 'pdfs' due to naming conventions for sampling during training in deepflash2\ng_msk, g_pdf, g_cdf = root.create_groups('labels', 'pdfs', 'cdfs', overwrite=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Loop over files to create...**\n1. The segmentation mask (.zarr)\n1. A list of pancreatic islands\n1. The probability density function for *region sampling*","metadata":{}},{"cell_type":"code","source":"identity = rasterio.Affine(1, 0, 0, 0, 1, 0)\ndf_list = []\nfor idx, row in df_info.iterrows():\n    \n    print(f'##### {idx} ####')\n    f = cfg.path/f'{idx}.json'\n    out_shape = (row.width_pixels, row.height_pixels)\n    geoms_msk = read_polys(f, classification='done')\n    msk = features.geometry_mask(geoms_msk, out_shape, identity, invert=True).astype('uint8')\n    \n    # Plot\n    fig, ax = plt.subplots(ncols=2, figsize=(15,15))\n    resize_w = int((msk.shape[1]/msk.shape[0])*cfg.cdf_size)\n    ax[0].imshow(cv2.resize(msk, dsize=(resize_w, cfg.cdf_size)))\n    ax[0].set_title('Mask')\n    ax[0].set_axis_off()\n    \n    geoms_regions = read_polys(f, classification='pancreas')\n    pdf = features.geometry_mask(geoms_regions, out_shape, identity, invert=True).astype('uint8')\n    \n    if cfg.scale!=1:\n        new_size = (msk.shape[1] // cfg.scale, msk.shape[0] // cfg.scale)\n        print('Scaling to', new_size)\n        msk = cv2.resize(msk, new_size)\n        pdf = cv2.resize(pdf, new_size)\n        \n    pdf = pdf.astype('float32')          \n    pdf[pdf==0] = cfg.bg_p/np.sum(pdf==0)\n    pdf[msk>0] = 0\n    pdf[pdf==1] = cfg.pancreas_p/np.sum(pdf==1)\n      \n    print('Getting pancreatic islets stats')\n    nb_components, output, stats, centroids = cv2.connectedComponentsWithStats(msk, connectivity=4)\n    print(f'Found {nb_components} pancreas')\n    df_centroids = pd.DataFrame(centroids[1:], columns=['cy', 'cx'])\n    df_centroids = df_centroids.join(pd.DataFrame(stats[1:], columns=['left', 'top', 'width', 'height', 'area']))\n    df_centroids['idx'] = idx \n    df_centroids.reset_index(inplace=True)\n    df_centroids.set_index(['idx', 'index'], inplace=True)\n    df_list.append(df_centroids)\n    \n    # Saving \n    g_msk[idx] = msk\n    g_pdf[idx] = pdf\n    \n    # Saving cdf\n    pdf = cv2.resize(pdf, dsize=(resize_w, cfg.cdf_size))      \n    g_cdf[idx] = np.cumsum(pdf/np.sum(pdf)) \n            \n    ax[1].imshow(pdf)\n    ax[1].set_title('Probability density function for sampling')\n    ax[1].set_axis_off() \n    plt.show()\n\n\ndf_stats = pd.concat(df_list)\ndf_stats.to_csv(f'/kaggle/working/masks_scale{cfg.scale}/roi_stats.csv')\ndf_stats","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}