{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Acknowledgment\n\nCopied from https://www.kaggle.com/wrrosa/hubmap-tf-with-tpu-efficientunet-512x512-subm by Wojtek Rosa.\n\nThe main differences from Wojtek's work:\n1. Omit case where pre-tiled Test dataset is used; instead, read data directly from .tiff images\n2. Instead of constructing entire mask image before run length encoding (RLE), do prediction and encoding one \"tile column\" at a time\n3. Add reflected border to image before prediction, trim border off results before encoding"},{"metadata":{"trusted":true},"cell_type":"code","source":"LAVIN = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ! pip install ../input/kerasapplications/keras-team-keras-applications-3b180cb -f ./ --no-index -q\n# ! pip install ../input/efficientnet/efficientnet-1.1.0/ -f ./ --no-index -q\nimport numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport gc\nimport sys\nimport re\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n# import tifffile\nimport rasterio\nfrom rasterio.windows import Window\n\nimport pathlib\n# from tqdm.notebook import tqdm\nimport cv2\n\n# import tensorflow as tf\n# import efficientnet as efn\n# import efficientnet.tfkeras\n\nimport yaml\nimport pprint\nimport json","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Run parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"trained_model_dirname = \"/kaggle/input/hubmap-models/\"\ntiff_image_dirname = \"/kaggle/input/hubmap-kidney-segmentation/test/\"\nCSV_filename = \"/kaggle/working/submission.csv\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read and Print Parameters\nRead parameters and matrics set in Training notebook:"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(trained_model_dirname+'params.yaml') as file:\n    P = yaml.load(file, Loader=yaml.FullLoader)\n    pprint.pprint(P)\n    \nP[ 'THRESHOLD' ] = 0.4 # preds > THRESHOLD\n\nwith open(trained_model_dirname + 'metrics.json') as json_file:\n    M = json.load(json_file)\nprint('Model run datetime: '+M['datetime'])\nprint('OOF val_dice_coe: ' + str(M['oof_dice_coe']))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Run length encoding (RLE) Functions\nBased on https://www.kaggle.com/friedchips/fully-correct-hubmap-rle-encoding-and-decoding:\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def encode_RLE( mask, column_pixel_offset = 0 ):\n    '''\n    Given a predicted binary image tile column \"mask\" and a starting offset in\n    column-major ordered pixels, calculate and return the string RLE, which will be\n    concatenated with RLEs from other columns, to construct RLE for the entire image:\n    '''\n    mask = mask.T.reshape(-1) # make 1D, column-first\n    mask = np.pad(mask, 1) # make sure that the 1d mask starts and ends with a 0\n    starts = np.nonzero((~mask[:-1]) & mask[1:])[0] # start points\n    ends = np.nonzero(mask[:-1] & (~mask[1:]))[0] # end points\n    rle = np.empty(2 * starts.size, dtype=int) # interlacing...\n    rle[0::2] = starts  + column_pixel_offset # ...starts...\n    rle[1::2] = ends - starts # ...and lengths\n    rle = ' '.join([ str(elem) for elem in rle ]) # turn into space-separated string\n    return rle\n\ndef rle2mask(rle, mask_shape):\n    ''' takes a space-delimited RLE string in column-first order\n    and turns it into a 2d boolean numpy array of shape mask_shape '''\n    \n    mask = np.zeros(np.prod(mask_shape), dtype=bool) # 1d mask array\n    rle = np.array(rle.split()).astype(int) # rle values to ints\n    starts = rle[::2]\n    lengths = rle[1::2]\n    for s, l in zip(starts, lengths):\n        mask[s:s+l] = True\n    return mask.reshape(np.flip(mask_shape)).T # flip because of column-first order\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualizing results"},{"metadata":{"trusted":true},"cell_type":"code","source":"# From https://www.kaggle.com/friedchips/fully-correct-hubmap-rle-encoding-and-decoding\n\ndef visualize_mask_and_image( tiff_image_dirname, CSV_filename ):\n\n    image_RLEs = pd.read_csv( CSV_filename )\n\n    for tiff_image_filename in glob.glob( tiff_image_dirname + \"*.tiff\" ):\n        \n        if \"26dc\" not in tiff_image_filename:  # ### REMOVE THIS!!! ###\n            continue\n        \n        image_id = pathlib.Path( tiff_image_filename ).stem\n\n        RLE = image_RLEs.predicted[ image_RLEs.id == image_id ]\n\n        if ( len( RLE ) == 0 ):\n            print( \"For image_id\", image_id, \"no prediction\", file = sys.stderr )\n        else:\n            RLE = RLE.values[ 0 ]   # Extract from 1-long Pandas Sequence\n            print( \"image_id\", image_id, \"len(RLE)\", len( RLE ), file = sys.stderr )\n            with rasterio.open( tiff_image_filename, transform = None) as dataset:\n                image_pixel_rows, image_pixel_cols = dataset.shape\n                print( \"tiff image dataset.shape\", dataset.shape, file = sys.stderr )\n                \n                mask = rle2mask( RLE, ( image_pixel_rows, image_pixel_cols ) )\n                plt.title( image_id )\n                plt.imshow( mask )\n                del mask\n                del RLE\n                plt.show()\n                gc.collect()\n                '''  # ### COMMENTED OUT, => Out Of Memory (OOM) ###\n                plt.title( image_id )\n                image = np.squeeze( dataset.read( [1, 2, 3 ] ) )  # Channels\n                print( \"image.shape\", image.shape, file = sys.stderr )\n                image = np.moveaxis( image, 0, -1 )\n                plt.imshow( image )\n                # del image\n                plt.show()\n                '''\n\ntiff_image_dirname = \"/kaggle/input/hubmap-kidney-segmentation/test/\"\nCSV_filename = \"/kaggle/input/version-17-columnwise-submission/submission.csv\"\n# visualize_mask_and_image( tiff_image_dirname, CSV_filename )\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# analyze_RLE\nChecks for inconsistencies in the run-length encoding (RLE) of glom masks from \"test\" set.  Looks for \"start\" \npositions out of order, or two run-length segments overlapping or abutting.\n\nAlso does morphological operations:  openings (which tend to remove small isolated 1's) of various sizes and \nclosings (which tend to fill in small gaps or concavities) and measures how many 1-pixels are removed or added.\nThe numbers listed (e.g., \"3-opening\") refer to the \"radius\" of a structuring element; thus, a 3-opening uses\na 7 x 7 array of 1's for its structuring element."},{"metadata":{"trusted":true},"cell_type":"code","source":"def analyze_RLE( tiff_image_dirname, CSV_filename ):\n\n    image_RLEs = pd.read_csv( CSV_filename )\n\n    for tiff_image_filename in glob.glob( tiff_image_dirname + \"*.tiff\" ):\n        \n        image_id = pathlib.Path( tiff_image_filename ).stem\n\n        RLE = image_RLEs.predicted[ image_RLEs.id == image_id ]\n        \n        if len( RLE ) == 0:\n            print( \"for image_id\", image_id, \"RLE is empty\" )\n        \n        else:\n            RLE = RLE.values[ 0 ]   # Extract from 1-long Pandas Sequence\n            print( \"\\nimage_id\", image_id, \"len(RLE)\", len( RLE ) )\n            tokens = RLE.split()\n            ntokens = len( tokens )\n            npairs = ntokens // 2\n            if ntokens % 2 != 0:\n                print( f\"  ERROR:  Odd number of tokens {ntokens} in RLE string\" )\n            else:\n                print( f\"  Number of RLE pairs is {npairs}\" )\n                \n            with rasterio.open( tiff_image_filename, transform = None) as dataset:\n                image_pixel_rows, image_pixel_cols = dataset.shape\n                print( \"  tiff image dataset.shape\", dataset.shape)\n                last_start, last_length = -1, 0\n                for start_str, length_str in zip( tokens[ 0 : : 2 ], tokens[ 1 : : 2 ] ):\n                    start = int( start_str )\n                    length = int( length_str )\n                    if last_start >= start:\n                        print( f\"  ERROR:  start {start} <= last_start {last_start }\" )\n                    elif last_start + last_length > start:\n                        print( f\"  ERROR:  last run {(last_start,last_length)} overlaps current run {(start,length)}\" )\n                    elif last_start + last_length == start:\n                        print( f\"  WARNING:  last run {(last_start,last_length)} abuts current run {(start,length)}\" )\n                    last_start, last_length = start, length\n                if last_start + length > image_pixel_rows * image_pixel_cols:\n                    print( f\"ERROR:  Overall last run goes past image size\" )\n                mask = rle2mask( RLE, ( image_pixel_rows, image_pixel_cols ) )\n                original_count = np.sum(mask)\n                print( f\"  mask 1's count {original_count}\" )\n                for i in range( 1, 11, 2 ):\n                    opened_mask = cv2.morphologyEx( mask.astype( np.uint8 ), cv2.MORPH_OPEN, np.ones( ( 2 * i + 1, 2 * i + 1 ) ) )\n                    count = np.sum( opened_mask )\n                    print( f\"  mask 1's with {i}-opening is {count} ({int(original_count-count)} 1's removed)\" )\n                for i in range( 1, 11, 2 ):\n                    opened_mask = cv2.morphologyEx( mask.astype( np.uint8 ), cv2.MORPH_CLOSE, np.ones( ( 2 * i + 1, 2 * i + 1 ) ) )\n                    count = np.sum( opened_mask )\n                    print( f\"  mask 1's with {i}-closing is {count} ({int(count-original_count)} 0's removed)\" )\n                \nanalyze_RLE( tiff_image_dirname, CSV_filename )\n   \n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}