{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Inference with Keras U-Net+MobileNetV2 \nThis is the final notebook in a series of three, the two first being:\n  * [[data] HuBMAP TIF 2 JPG+TFRecords 128,256,512,1024](https://www.kaggle.com/mistag/data-hubmap-tif-2-jpg-tfrecords-128-256-512-1024), generating training data\n  * [[train] Keras U-Net+MobileNetV2](https://www.kaggle.com/mistag/train-keras-u-net-mobilenetv2/output?scriptVersionId=48294593), training a U-Net model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport cv2\nimport glob\nimport tensorflow as tf\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import Model, model_from_json\nfrom tensorflow.keras.utils import CustomObjectScope\nfrom skimage import io\nimport json\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get hyperparameters from the training notebook\nwith open('../input/train-keras-u-net-mobilenetv2/hparams.json') as json_file:\n    hparams = json.load(json_file)\nhparams","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper functions\nInput images are downscaled with a scaling factor, and the predicted mask will be upscaled again. The TIF files are really big, and to save memory the images are mapped to disk using numpy.memmap(). A little slower than keeping the whole image in memory, but frees up memory for other things!"},{"metadata":{"trusted":true},"cell_type":"code","source":"SCALE_FACTOR = hparams['SCALE_FACTOR']\nIMG_SIZE = hparams['IMG_SIZE']\nK_SPLITS = hparams['K_SPLITS'] # number of folds\nP_THRESHOLD = 0.5\n\ndef read_tif_file(fname):\n    img = io.imread(fname)\n    img = np.squeeze(img)\n    if img.shape[0] == 3: # swap axes as required\n        img = img.swapaxes(0,1)\n        img = img.swapaxes(1,2)\n    return img\n\n# map image to file(s)\ndef map_img2file(fname):\n    img = read_tif_file(fname)\n    dims = np.array(img.shape)\n    ch = 1 if len(dims) == 2 else dims[2]\n    for i in range(ch):\n        f = np.memmap('img{}.dat'.format(i), dtype=np.uint8, mode='w+', shape=(dims[0], dims[1]))\n        f[:] = img[:,:,i] if ch > 1 else img[:,:]\n        del f\n    return dims\n\n# read part of image from file\ndef get_patch_from_file(dims, pos, psize):\n    ch = 1 if len(dims) == 2 else dims[2]\n    patch = np.zeros([psize[0], psize[1]], dtype=np.uint8) if ch == 1 else np.zeros([psize[0], psize[1], ch], dtype=np.uint8)\n    for i in range(ch):\n        f = np.memmap('img{}.dat'.format(i), dtype=np.uint8, mode='r', shape=(dims[0], dims[1]))\n        p = f[pos[0]:pos[0]+psize[0], pos[1]:pos[1]+psize[1]]\n        if ch == 1:\n            patch[0:psize[0], 0:psize[1]] = p\n        else:\n            patch[0:psize[0], 0:psize[1], i] = p\n        del f\n    return patch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Using the RLE-encoder from [this notebook](https://www.kaggle.com/iafoss/hubmap-pytorch-fast-ai-starter-sub):"},{"metadata":{"trusted":true},"cell_type":"code","source":"##https://www.kaggle.com/bguberfain/memory-aware-rle-encoding\n#with bug fix\ndef rle_encode_less_memory(img):\n    #watch out for the bug\n    pixels = img.T.flatten()\n    \n    # This simplified method requires first and last pixel to be zero\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] -= runs[::2]\n    \n    return ' '.join(str(x) for x in runs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inference time\nThe models were trained using K-folds technique, and here we create an ensemble of all the trained models, and also throw in some (optional) TTA (Test Time Augmentation).\nThe predicted masks are big, so we write them to disk immediately after each image - no point making a DataFrame here. "},{"metadata":{"trusted":true},"cell_type":"code","source":"AUGS = 3 # number of augmentations\n\ndef create_TTA_batch(img):\n    batch=np.zeros((AUGS,IMG_SIZE,IMG_SIZE,3), dtype=np.float32)\n    orig = tf.keras.preprocessing.image.img_to_array(img)/255. \n    batch[0,:,:,:] = orig\n    batch[1,:,:,:] = cv2.rotate(orig, cv2.ROTATE_90_CLOCKWISE) \n    batch[2,:,:,:] = cv2.rotate(orig, cv2.ROTATE_90_COUNTERCLOCKWISE)\n    return batch\n\ndef create_TTA_mask(preds):\n    # de-augment mask where needed\n    preds[1,:,:] = np.expand_dims(cv2.rotate(preds[1,:,:], cv2.ROTATE_90_COUNTERCLOCKWISE), axis = 2)\n    preds[2,:,:] = np.expand_dims(cv2.rotate(preds[2,:,:], cv2.ROTATE_90_CLOCKWISE), axis = 2)\n    # sum up\n    pred = np.sum(preds, axis=0) / AUGS\n    return pred > P_THRESHOLD","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = '../input/hubmap-kidney-segmentation/test/'\nfilelist = glob.glob(PATH+'*.tiff')\n# create submission file\nSUB_FILE = './submission.csv'\nwith open(SUB_FILE, 'w') as f:\n    f.write(\"id,predicted\\n\")\n    \nsize = int(IMG_SIZE * SCALE_FACTOR) # tile size that will be processed\n\ns_th = 45  # saturation threshold\np_th = IMG_SIZE*IMG_SIZE//32 # pixel count threshold\nTTA = True\n\nfor file in filelist:\n    fid = file.replace('\\\\','.').replace('/','.').split('.')[-2]\n    print(fid)\n    dims = map_img2file(file)\n    pmask = np.zeros(dims[:2], dtype=np.uint8)\n    for fold in range(K_SPLITS):\n        # load model\n        with open('../input/train-keras-u-net-mobilenetv2/model{}.json'.format(fold), 'r') as m:\n            lm = m.read()\n            model = model_from_json(lm)\n        model.load_weights('../input/train-keras-u-net-mobilenetv2/model{}.h5'.format(fold))\n        print(\"Model {}\".format(fold))\n        # process image\n        for x in range(dims[0]//size):\n            for y in range(dims[1]//size):\n                patch = cv2.resize(get_patch_from_file(np.array(dims),\n                                                                [x*size,y*size],\n                                                                [size,size]),\n                                   dsize=(IMG_SIZE, IMG_SIZE),\n                                   interpolation = cv2.INTER_AREA)\n                # determine if patch should be predicted or not\n                _, s, _ = cv2.split(cv2.cvtColor(patch, cv2.COLOR_BGR2HSV))\n                if (s>s_th).sum() > p_th:\n                    if TTA:\n                        batch = create_TTA_batch(patch)\n                        predictions = model.predict(batch)\n                        mask = create_TTA_mask(predictions)\n                    else:\n                        batch = np.array([patch])/255.\n                        predictions = model.predict(batch)\n                        mask = predictions[0] >= P_THRESHOLD\n                    # update total mask\n                    pint = cv2.resize(mask.astype(int), dsize=(size, size), interpolation = cv2.INTER_NEAREST) #upsample to original\n                    pmask[x*size:(x+1)*size, y*size:(y+1)*size] += pint.astype(np.uint8)\n    # save mask to submission file\n    pmask = pmask > K_SPLITS/2 # threshold across folds\n    with open(SUB_FILE, 'a') as f:\n        f.write(\"{},\".format(fid))\n        f.write(rle_encode_less_memory(pmask))\n        f.write(\"\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%rm -f *.dat","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}