{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction\n\nNotebook to run Prediction, Encoding and Saving of Training Set TIFF images and DICE comparison with\nground truth masks"},{"metadata":{},"cell_type":"markdown","source":"# Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install ../input/kerasapplications/keras-team-keras-applications-3b180cb -f ./ --no-index -q\n! pip install ../input/efficientnet/efficientnet-1.1.0/ -f ./ --no-index -q\nimport numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport gc\nimport sys\nimport re\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport tifffile\nimport rasterio\nfrom rasterio.windows import Window\n\nimport pathlib\nfrom tqdm.notebook import tqdm\nimport cv2\n\nimport tensorflow as tf\nimport efficientnet as efn\nimport efficientnet.tfkeras\n\nimport yaml\nimport pprint\nimport json","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Run parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"trained_model_dirname = \"/kaggle/input/hubmap-models/\"\ntiff_image_dirname = \"/kaggle/input/hubmap-kidney-segmentation/train/\"\nCSV_filename = \"/kaggle/working/submission.csv\"\npredicted_csv_filename = \"/kaggle/input/predict-encode-and-save-for-train-images/submission.csv\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! ls -al /kaggle/input","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read and Print Parameters\nRead parameters and matrics set in Training notebook:"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(trained_model_dirname+'params.yaml') as file:\n    P = yaml.load(file, Loader=yaml.FullLoader)\nP[ 'THRESHOLD' ] = 0.3 # preds > THRESHOLD\nP[ 'DIM' ] = 1024  # ### REMOVE ###\npprint.pprint(P)\n    \nwith open(trained_model_dirname + 'metrics.json') as json_file:\n    M = json.load(json_file)\nprint('Model run datetime: '+M['datetime'])\nprint('OOF val_dice_coe: ' + str(M['oof_dice_coe']))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Miscellaneous Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def running_on_TPU():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        return True\n    except:\n        return False\n    \nprint( \"running_on_TPU\", running_on_TPU(), file = sys.stderr )\n\ndef load_models( mod_path ):\n    '''\n    Return trained models, one per training \"fold\"\n    '''\n    fold_models = []\n    for fold_model_path in glob.glob(mod_path+'*.h5'):\n        fold_models.append(tf.keras.models.load_model(fold_model_path,compile = False))\n\n    return fold_models\n\ndef tiff_image_shape( image_pathname ):\n    with rasterio.open( image_pathname ) as image:\n        return image.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Run length encoding (RLE) Functions\nBased on https://www.kaggle.com/friedchips/fully-correct-hubmap-rle-encoding-and-decoding:\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def encode_RLE( mask, column_pixel_offset = 0 ):\n    '''\n    Given a predicted binary image tile column \"mask\" and a starting offset in\n    column-major ordered pixels, calculate and return the string RLE, which will be\n    concatenated with RLEs from other columns, to construct RLE for the entire image:\n    '''\n    mask = mask.T.reshape(-1) # make 1D, column-first\n    mask = np.pad(mask, 1) # make sure that the 1d mask starts and ends with a 0\n    starts = np.nonzero((~mask[:-1]) & mask[1:])[0] # start points\n    ends = np.nonzero(mask[:-1] & (~mask[1:]))[0] # end points\n    rle = np.empty(2 * starts.size, dtype=int) # interlacing...\n    rle[0::2] = starts  + column_pixel_offset # ...starts...\n    rle[1::2] = ends - starts # ...and lengths\n    rle = ' '.join([ str(elem) for elem in rle ]) # turn into space-separated string\n    return rle\n\ndef rle2mask(rle, mask_shape):\n    ''' takes a space-delimited RLE string in column-first order\n    and turns it into a 2d boolean numpy array of shape mask_shape '''\n    \n    mask = np.zeros(np.prod(mask_shape), dtype=bool) # 1d mask array\n    rle = np.array(rle.split()).astype(int) # rle values to ints\n    starts = rle[::2]\n    lengths = rle[1::2]\n    for s, l in zip(starts, lengths):\n        mask[s:s+l] = True\n    return mask.reshape(np.flip(mask_shape)).T # flip because of column-first order\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Low-memory glom Prediction and Encoding\nThis version of the code above reduces the memory by doing model prediction and Run Length Encoding (RLE)\ncolumn by column rather than for the entire image."},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_encode_save_testcases( image_dirname, model_dirname, CSV_filename, P, noheader = False ):\n    models = load_models( model_dirname )\n    results = {}\n    for image_index, image_pathname in enumerate( glob.glob( image_dirname + \"*.tiff\" ) ):\n        print( \"predict_encode_save_testcases for image\", image_pathname )\n        image_id = pathlib.Path( image_pathname ).stem\n        if image_index in range( 100 ):   # All TIFF images\n            try:\n                RLE = predict_encode_image( image_pathname, models, P )\n            except:\n                image_rows, image_cols = tiff_image_shape( image_pathname )\n                RLE = \"0 \" + str( image_rows * image_cols - 1 )   # all 1's image\n        else:\n            RLE = \"0 1\"  # single 1 image\n        results[ image_index ] = { \"id\": image_id, \"predicted\" : RLE }\n        print( \"image_id\", image_id, \"len(RLE)\", len( RLE ) )\n        del RLE\n    \n    # Write results to CSV_filename\n    results_df = pd.DataFrame.from_dict( results, orient='index' )\n    results_df.to_csv( CSV_filename, index=False )\n    del results\n\n    \n        \ndef predict_encode_image( image_pathname, models, P ):\n    '''\n    Predict the mask for image in \"image_pathname\" and return its Run Length Encoding (RLE)\n    '''\n    with rasterio.open( image_pathname ) as image:\n        image_pixel_rows, image_pixel_cols = image.shape\n        # print( \"image\", image_pathname, \"has shape\", image.shape )\n        DIM = P['DIM']\n        RLE = ''\n        for image_pixel_col in range( 0, image_pixel_cols // DIM * DIM - DIM + 1, DIM ):\n            RLE += predict_encode_column( image, models, image_pixel_col, P ) + \" \"\n\n        return RLE.strip()   # Remove leading, trailing whitespace\n\ndef predict_encode_column( image, models, start_image_col, P ):\n    '''\n    Predict the mask column corresponding to column in \"image\" starting at \"start_image_col\"\n    and return its run-length encoding\n    '''\n    image_pixel_rows, image_pixel_cols = image.shape\n    DIM = P['DIM']\n    OVL = P['PIXEL_OVERLAP']\n    # print( \"predict_encode_column starting at\", start_image_col, \"/\", image_pixel_cols )\n    column_mask = np.zeros( ( image_pixel_rows, DIM ), dtype = np.uint8 )\n    for image_pixel_row in range( 0, image_pixel_rows // DIM * DIM - DIM + 1, DIM ):\n        image_tile = get_bordered_tile( image, image_pixel_row, start_image_col, DIM, OVL )\n        assert image_tile.shape == ( DIM + 2 * OVL, DIM + 2 * OVL, 3 )\n        mask_tile = predict_tile( image_tile, models, P )\n        assert mask_tile.shape == ( DIM, DIM )\n        column_mask[ image_pixel_row : image_pixel_row + DIM, 0 : DIM ] = mask_tile\n        del mask_tile\n\n    column_offset = start_image_col * image_pixel_rows\n    RLE = encode_RLE( column_mask, column_offset )\n    del column_mask\n    # print( \"sum(column_mask)\", np.sum( column_mask ), \"len(RLE) for start_image_col\", start_image_col, \"is\", len( RLE ) )\n    return RLE\n    \n    \ndef predict_tile( image_tile, models, P ):\n    '''\n    Perform prediction for \"image_tile\", averaging over \"models\", and return thresholded version\n    NOTE \"image_tile\" is DIM x DIM PLUS PIXEL_OVERLAP border on all four sides, predicted_mask_tile\n    is trimmed to DIM x DIM\n    '''\n    THRESHOLD = P['THRESHOLD']\n    DIM = P['DIM']\n    OVL = P['PIXEL_OVERLAP']\n    assert image_tile.shape == ( DIM + 2 * OVL, DIM + 2 * OVL, 3 )\n    image_tile_batch = np.expand_dims( image_tile, axis = 0 )\n    predicted_mask_tile = None\n    for model in models:\n        model_prediction = model.predict( image_tile_batch )\n        predicted_mask_tile = model_prediction if predicted_mask_tile is None else predicted_mask_tile + model_prediction\n    if len( models ) > 1:\n        predicted_mask_tile /= len( models )\n    predicted_mask_tile = ( np.squeeze( predicted_mask_tile ) > P['THRESHOLD'] ).astype( np.uint8 )\n    # print( \"thresholded predicted_mask_tile.sum()\", predicted_mask_tile.sum(), file = sys.stderr )\n    predicted_mask_tile = predicted_mask_tile[ OVL : -OVL, OVL : -OVL ]\n    assert predicted_mask_tile.shape == ( DIM, DIM )\n\n    return predicted_mask_tile\n\ndef get_bordered_tile( image, pixel_row, pixel_col, DIM, OVL ):\n    '''\n    Extract a DIM x DIM pixel tile from \"image\" with upper left corner at (pixel_row, pixel_col )\n    and add an OVL-pixel border around it by extending the sides or, on tiles at the image\n    boundary, by reflection about the boundary.  Return DIM + 2*OVL x DIM + 2*OVL tile\n    '''\n    image_pixel_rows, image_pixel_cols = image.shape\n    # Calculate top, bottom, left and right extensions and reflections\n    top_shift = OVL if pixel_row > OVL else -OVL\n    bot_shift = OVL if pixel_row < ( image_pixel_rows // DIM * DIM - OVL - DIM + 1 ) else -OVL\n    lft_shift = OVL if pixel_col > OVL else -OVL\n    rgt_shift = OVL if pixel_col < ( image_pixel_cols // DIM * DIM - OVL - DIM + 1 ) else -OVL\n    shifts = [ top_shift, bot_shift, lft_shift, rgt_shift ]\n    extents = list( map( lambda x : max( 0, x ), shifts ) )\n    reflects = list( map( lambda x : max( 0, -x ), shifts ) )\n    tile_window = Window.from_slices( ( pixel_row - extents[ 0 ], pixel_row + DIM + extents[ 1 ] ),\n                                      ( pixel_col - extents[ 2 ], pixel_col + DIM + extents[ 3 ] ) )\n    tile = image.read( [1, 2, 3 ], window = tile_window )\n    tile = np.moveaxis( tile, 0, -1 )\n    ts = tile.shape\n    if np.any( reflects ):\n        tile = cv2.copyMakeBorder( tile, reflects[ 0 ], reflects[ 1 ], reflects[ 2 ], reflects[ 3 ], cv2.BORDER_REFLECT )\n    return tile","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Compare predicted \"train\" glom mask against ground truth"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_mask_image_from_json( json_filename, mask_shape ):\n    '''\n    Reads polygonal representation of mask image from \"json_filename\" and converts to\n    binary image, which is returned.\n    '''\n    with open( json_filename, \"r\") as read_file:\n        mask_data = json.load(read_file)\n        \n    polys = []\n    for index in range(mask_data.__len__()):\n        geom = np.array(mask_data[index]['geometry']['coordinates'])\n        polys.append(geom)\n\n    mask = np.zeros(mask_shape, dtype = np.uint8 )\n    cv2.fillPoly(mask, polys, 1)\n    mask = mask.astype(bool)\n    return mask\n\ndef read_mask_from_csv( csv_filename, image_id, mask_shape ):\n    '''\n    Reads Run Length Encoded (RLE) for \"image_id\" and converts to binary image, which\n    is returned.\n    '''\n    CSV = pd.read_csv( csv_filename )\n    RLE = CSV.predicted[ CSV[ \"id\" ] == image_id ].values[ 0 ]\n    return rle2mask( RLE, mask_shape )\n    \ndef calculate_DICE_for_two_masks( predicted_mask, ground_truth_mask ):\n    '''\n    Calculates numerator and denominator of DICE coefficient for the binary glom mark images\n    in two arguments and returns them\n    '''\n    npredicted = np.sum( predicted_mask )\n    ngroundtruth = np.sum( ground_truth_mask  )\n    nint = np.sum( predicted_mask & ground_truth_mask )\n    return 2 * nint, npredicted + ngroundtruth\n\ndef DICE_compare_two_masks( image_id, tiff_image_dirname, predicted_csv_filename ):\n    image_shape = tiff_image_shape( tiff_image_dirname + image_id + \".tiff\" )\n    ground_truth_mask = read_mask_image_from_json( tiff_image_dirname + image_id + \".json\", image_shape )\n    predicted_mask = read_mask_from_csv( predicted_csv_filename, image_id, image_shape )\n    # predicted_mask = clean_binary_image( predicted_mask, 0 )  # ### DON'T BOTHER, CLEANING DOESN'T HELP\n    # predicted_mask = expand( predicted_mask, -1 )   # ### ONLY IMPROVES SCORE FROM 0.94938 to .94985\n    DICE = calculate_DICE_for_two_masks( predicted_mask, ground_truth_mask )\n    return DICE\n\ndef DICE_compare_predicted_ground_truth( tiff_image_dirname, predicted_csv_filename ):\n    tiff_image_filenames = glob.glob( tiff_image_dirname + \"*.tiff\" )\n    DICE_scores = {}\n    sum_num = 0.0\n    sum_den = 0.0\n    for tiff_image_filename in tiff_image_filenames:\n        image_id = pathlib.Path( tiff_image_filename ).stem\n        print( \"predicting mask for image\", image_id )\n        num, den = DICE_compare_two_masks( image_id, tiff_image_dirname, predicted_csv_filename )\n        DICE_score = num / den\n        DICE_scores[ image_id ] = DICE_score\n        sum_num += num\n        sum_den += den\n        # print( \"after\", image_id, \"DICE_scores are\", DICE_scores )\n    print( \"weighted average DICE score\", sum_num / sum_den )\n    return DICE_scores\n\ndef clean_binary_image( image, radius ):\n    if radius > 0 :\n        kernel = np.ones( ( 2 * radius + 1, 2 * radius + 1 ) )\n        image = cv2.morphologyEx( image.astype( np.uint8 ), cv2.MORPH_OPEN, kernel )\n        image = cv2.morphologyEx( image, cv2.MORPH_CLOSE, kernel ).astype( np.bool )\n    return image\n\ndef expand( image, radius ):\n    if radius > 0:\n        kernel = np.ones( ( 2 * radius + 1, 2 * radius + 1 ) )\n        image = cv2.dilate( image.astype( np.uint8 ), kernel )\n    elif radius < 0:\n        kernel = np.ones( ( -2 * radius + 1, -2 * radius + 1 ) )\n        image = cv2.erode( image.astype( np.uint8 ), kernel )\n    elif radius == 0:\n        pass\n    return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Do predictions for all \"train\" images; \n\nThis step generates glom mask predictions for all eight \"train\" images.\nThis is, of course, a cheat because we're using the models that were trained on \"trained\".  This will take several hours.  Put resulting RLE's in \"predict-encode-and-save-for-train-images/train_submission.csv\"\n\nThis step was executed once, and the results were saved in the dataset https://www.kaggle.com/markalavin/predict-encode-and-save-for-train-images.  After that,\nthis step was commented out, and comparisons (DICE, XOR) were made between predicted::ground truth pairs of glom masks."},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\npredict_encode_save_testcases( tiff_image_dirname, trained_model_dirname, CSV_filename, P )\n'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Do the DICE comparison of predicted and ground truth\nThis step is intended to calculate the DICE coefficient (https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient), which measures the similarity of two binary images and ranges from 0 (all pixels differ between predicted and ground truth images) to 1 (predicted and ground truth images are identical).\n\nThe overall DICE coefficient is calculated between the composite of all eight train image predictions and ground truth images, that is, it's the average the DICE scores for each train image, weighted by the number of 1's in the trained + predicted image."},{"metadata":{"trusted":true},"cell_type":"code","source":"DICE_scores = DICE_compare_predicted_ground_truth( tiff_image_dirname, predicted_csv_filename )\nprint( \"DICE_scores\", DICE_scores )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Windowed DICE Comparison\nBreak predicted and ground truth mask images into DIM x DIM windows, calculate a DICE score for each window, show histogram of results.  NOTE that all-0's windows are ignored, rather than treating as if they had DICE scores of 1.0."},{"metadata":{"trusted":true},"cell_type":"code","source":"def DICE_window_compare_two_masks( image_id, tiff_image_dirname, predicted_csv_filename,\n                                 threshold = None ):\n    '''\n    Returns { ( start_row, start_col ) : DICE_score, ... }, but omits cases\n    where both predicted and ground-truth masks are all 0's:\n    '''\n    DIM = P[ 'DIM' ]\n    result = {}\n    image_shape = tiff_image_shape( tiff_image_dirname + image_id + \".tiff\" )\n    ground_truth_mask = read_mask_image_from_json( tiff_image_dirname + image_id + \".json\", image_shape )\n    predicted_mask = read_mask_from_csv( predicted_csv_filename, image_id, image_shape )\n    for start_row in range( 0, image_shape[ 0 ] // DIM * DIM - DIM + 1, DIM ):\n        for start_col in range( 0, image_shape[ 1 ] // DIM * DIM - DIM + 1, DIM ):\n            predicted_mask_window = predicted_mask[ start_row : start_row + DIM, start_col : start_col + DIM ]\n            ground_truth_mask_window = ground_truth_mask[ start_row : start_row + DIM, start_col : start_col + DIM ]\n            DICE_num, DICE_denom = calculate_DICE_for_two_masks( predicted_mask_window, ground_truth_mask_window )\n            if ( DICE_num > 0 ) or ( DICE_denom > 0 ):\n                result[ ( start_row, start_col ) ] = DICE_num / DICE_denom\n    # print( \"result\", result )\n    return result\n\ndef DICE_window_compare_predicted_ground_truth( tiff_image_dirname, predicted_csv_filename ):\n    tiff_image_filenames = glob.glob( tiff_image_dirname + \"*.tiff\" )\n    for tiff_image_filename in tiff_image_filenames:\n        image_id = pathlib.Path( tiff_image_filename ).stem\n        print( \"for image\", image_id )\n        DICE_window_scores = DICE_window_compare_two_masks( image_id, tiff_image_dirname, predicted_csv_filename )\n        plt.hist( DICE_window_scores.values(), bins = 32, range = ( 0, 1.0 ) )\n        plt.show()\n    return\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DICE_window_compare_predicted_ground_truth( tiff_image_dirname, predicted_csv_filename )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Windowed XOR comparison\nSimilar to Windowed DICE comparison, but just counts the number of discrepant pixels between each of the predicted/ground truth image pairs.  Thus, while higher DICE score\n(max. 1.0) is desirable, we would like the windowed XOR score to be minimal, and 0.0\nmeans the predicted and ground truth windowed images are identical.\n\nFor purposes of comparison, we define a threshold number of discrepant pixels, that is, we only consider windows where the predicted and ground truth images have at least the threshold number of discrepancies.\n\nResult is reported in two ways:   First, we plot a histogram of XOR scores, which ranges from the threshold to the window size (DIM x DIM).  Second, for each window with at least the threshold number of discrepancies, we plot four binary images:\n\n   predicted_image,  ground_truth_image, pNOTg, and gNOTp\n   \nwhere the latter two show the two 1-sided differences, e.g., all the pixels that are 1 in the predicted image and 0 in the ground-truth image."},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_XOR_for_two_masks( predicted_mask, ground_truth_mask ):\n    '''\n    Calculates ( pNOTg, gNOTp ) the counts of discrepancies for the binary glom\n    mark images in two arguments and returns them.\n    '''\n    pNOTg = np.sum( np.logical_and( predicted_mask, np.logical_not( ground_truth_mask ) ) )\n    gNOTp = np.sum( np.logical_and( ground_truth_mask, np.logical_not( predicted_mask ) ) )\n    return pNOTg, gNOTp\n\ndef XOR_window_compare_two_masks( image_id, tiff_image_dirname, predicted_csv_filename,\n                                 threshold = None ):\n    '''\n    Returns { ( start_row, start_col ) : XOR_score, ... }, but omits cases\n    where both predicted and ground-truth masks are all 0's:\n    '''\n    DIM = P[ 'DIM' ]\n    result = {}\n    image_shape = tiff_image_shape( tiff_image_dirname + image_id + \".tiff\" )\n    ground_truth_mask = read_mask_image_from_json( tiff_image_dirname + image_id + \".json\", image_shape )\n    predicted_mask = read_mask_from_csv( predicted_csv_filename, image_id, image_shape )\n    for start_row in range( 0, image_shape[ 0 ] // DIM * DIM - DIM + 1, DIM ):\n        for start_col in range( 0, image_shape[ 1 ] // DIM * DIM - DIM + 1, DIM ):\n            predicted_mask_window = predicted_mask[ start_row : start_row + DIM, start_col : start_col + DIM ]\n            ground_truth_mask_window = ground_truth_mask[ start_row : start_row + DIM, start_col : start_col + DIM ]\n            DICE_num, DICE_denom = calculate_DICE_for_two_masks( predicted_mask_window, ground_truth_mask_window )\n            if ( DICE_num > 0 ) or ( DICE_denom > 0 ):\n                pNOTg, gNOTp = calculate_XOR_for_two_masks( predicted_mask_window, ground_truth_mask_window )\n                result[ ( start_row, start_col ) ] = pNOTg, gNOTp\n                if (threshold is not None ) & ( ( pNOTg + gNOTp ) >= threshold ):\n                    fig, axes = plt.subplots( 1, 4, figsize = ( 12.0, 6.0 ) )\n                    fig.suptitle( \"Image \" + image_id + \" at \" + str( start_row ) + \", \" + str( start_col ) )\n                    axes[ 0 ].imshow( predicted_mask_window )\n                    axes[ 1 ].imshow( ground_truth_mask_window )\n                    axes[ 2 ].imshow( np.logical_and( predicted_mask_window, np.logical_not( ground_truth_mask_window ) ) )\n                    axes[ 3 ].imshow( np.logical_and( ground_truth_mask_window, np.logical_not( predicted_mask_window ) ) )\n                    plt.show()\n    return result\n\ndef XOR_window_compare_predicted_ground_truth( tiff_image_dirname, predicted_csv_filename ):\n    tiff_image_filenames = glob.glob( tiff_image_dirname + \"*.tiff\" )\n    for tiff_image_filename in tiff_image_filenames:\n        image_id = pathlib.Path( tiff_image_filename ).stem\n        print( \"for image\", image_id )\n        XOR_window_scores = XOR_window_compare_two_masks( image_id, tiff_image_dirname, predicted_csv_filename, threshold = 5000 )\n        plt.hist( XOR_window_scores.values(), bins = 32 )\n        plt.show()\n    return\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"XOR_window_compare_predicted_ground_truth( tiff_image_dirname, predicted_csv_filename )","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}