{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Inference with Keras U-Net (512x512)\nThis is the final notebook in a series of three, the two first being:\n  * [[data] HuBMAP TIF 2 JPG+TFRecords 128,256,512,1024](https://www.kaggle.com/mistag/data-hubmap-tif-2-jpg-tfrecords-128-256-512-1024), generating training data\n  * [[train] Keras U-Net with TFRecords input](https://www.kaggle.com/mistag/train-keras-u-net-with-tfrecords-input), training a U-Net model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport cv2\nimport glob\nimport tensorflow as tf\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import Model, model_from_json\nfrom tensorflow.keras.utils import CustomObjectScope\nfrom skimage import io\nimport json\nimport gc\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get hyperparameters from the training notebook\nwith open('../input/train-keras-u-net-with-tfrecords-input/hparams.json') as json_file:\n    hparams = json.load(json_file)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper functions\nInput images are downscaled with a scaling factor, and the predicted mask will be upscaled again. "},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_SIZE = hparams['IMG_SIZE']\nSCALE_FACTOR = hparams['SCALE_FACTOR']\nP_THRESHOLD = 0.5\n\ndef read_tif_file(fname):\n    img = io.imread(fname)\n    img = np.squeeze(img)\n    if img.shape[0] == 3: # swap axes as required\n        img = img.swapaxes(0,1)\n        img = img.swapaxes(1,2)\n    return img\n\ndef read_mask_file(fname, mshape):\n    with open(fname) as f:\n        mdata = json.load(f)\n        polys = []\n        for index in range(mdata.__len__()):\n            if (mdata[index]['properties']['classification']['name'] == 'Cortex') or (mdata[index]['properties']['classification']['name'] == 'glomerulus'):\n                geom = np.array(mdata[index]['geometry']['coordinates'])\n                if geom.shape[0] == 1:\n                    polys.append(geom)\n                else:\n                    for j in range(geom.shape[0]):\n                        polys.append(np.array([geom[j][0]]).astype(int))\n        mask = np.zeros(mshape, dtype=np.int8)\n        cv2.fillPoly(mask, polys, 1)\n        mask = mask.astype(bool, copy=False)\n    return mask","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Using the RLE-encoder from [this notebook](https://www.kaggle.com/iafoss/hubmap-pytorch-fast-ai-starter-sub):"},{"metadata":{"trusted":true},"cell_type":"code","source":"##https://www.kaggle.com/bguberfain/memory-aware-rle-encoding\n#with bug fix\ndef rle_encode_less_memory(img):\n    #watch out for the bug\n    pixels = img.T.flatten()\n    \n    # This simplified method requires first and last pixel to be zero\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] -= runs[::2]\n    \n    return ' '.join(str(x) for x in runs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inference time\nRebuild model from the training notebook and load weights. Since we used a custom loss function and metrics, using keras.models.load_model() will not work here. The predicted masks are big, so we write them to disk immediately - no point making a DataFrame here. "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Augmentations\n\nAUGS = 3\n\ndef create_TTA_batch(img):\n    batch=np.zeros((AUGS,IMG_SIZE,IMG_SIZE,3), dtype=np.float32)\n    orig = tf.keras.preprocessing.image.img_to_array(img)/255. # un-augmented\n    batch[0,:,:,:] = orig\n    batch[1,:,:,:] = cv2.rotate(orig, cv2.ROTATE_90_CLOCKWISE) \n    batch[2,:,:,:] = cv2.rotate(orig, cv2.ROTATE_90_COUNTERCLOCKWISE)\n    #batch[3,:,:,:] = cv2.flip(orig, 1) # horizontal flip\n    #batch[4,:,:,:] = cv2.flip(orig, 0) # vertical flip\n    #batch[5,:,:,:] = np.clip(orig *1.1, 0, 1)\n    #batch[6,:,:,:] = orig *0.9\n    return batch\n\ndef create_TTA_mask(preds):\n    # de-augment mask where needed\n    preds[1,:,:] = np.expand_dims(cv2.rotate(preds[1,:,:], cv2.ROTATE_90_COUNTERCLOCKWISE), axis = 2)\n    preds[2,:,:] = np.expand_dims(cv2.rotate(preds[2,:,:], cv2.ROTATE_90_CLOCKWISE), axis = 2)\n    #preds[3,:,:] = np.expand_dims(cv2.flip(preds[3,:,:], 1), axis = 2)\n    #preds[4,:,:] = np.expand_dims(cv2.flip(preds[4,:,:], 0), axis = 2)\n    # sum up\n    pred = np.sum(preds, axis=0) / AUGS\n    return pred > P_THRESHOLD\n\n# get files to process\nPATH = '../input/hubmap-kidney-segmentation/test/'\nfilelist = glob.glob(PATH+'*.tiff')\nfilelist\nSUB_FILE = './submission.csv'\nwith open(SUB_FILE, 'w') as f:\n    f.write(\"id,predicted\\n\")\n\n# load model\nwith open('../input/train-keras-u-net-with-tfrecords-input/model.json', 'r') as m:\n    lm = m.read()\n    model = model_from_json(lm)\nmodel.load_weights('../input/train-keras-u-net-with-tfrecords-input/model.h5')\n    \ns_th = 45 # saturation blanking threshold\np_th = IMG_SIZE*IMG_SIZE//32   # pixel count threshold\n\nsize = IMG_SIZE * SCALE_FACTOR\nfor file in filelist:\n    fid = file.replace('\\\\','.').replace('/','.').split('.')[-2]\n    print(fid)\n    #img, pmask = 0,0\n    img = read_tif_file(file)\n    dims = np.array(img.shape[:2])\n    pmask = np.zeros(dims, dtype=np.uint8)\n    for x in range(img.shape[0]//size):\n        for y in range(img.shape[1]//size):\n            patch = cv2.resize(img[x*size:(x+1)*size, y*size:(y+1)*size], dsize=(IMG_SIZE, IMG_SIZE), \n                               interpolation = cv2.INTER_AREA)\n            _, s, _ = cv2.split(cv2.cvtColor(patch, cv2.COLOR_BGR2HSV))\n            go = False\n            if (s>s_th).sum() > p_th:\n                go = True\n            if go:\n                # TTA\n                batch = create_TTA_batch(patch)\n                predictions = model.predict(batch)\n                mask = create_TTA_mask(predictions)\n                pint = cv2.resize(mask.astype(int), dsize=(size, size), interpolation = cv2.INTER_NEAREST) #upscale to original\n                pmask[x*size:(x+1)*size, y*size:(y+1)*size] = pint.astype(np.uint8)\n        # clean up some memory\n        del s, patch\n        try:\n            del batch, predictions, mask, pint\n        except:\n            pass\n        gc.collect()\n    # save mask\n    with open(SUB_FILE, 'a') as f:\n        f.write(\"{},\".format(fid))\n        f.write(rle_encode_less_memory(pmask.astype(np.uint8)))\n        f.write(\"\\n\")\n    del pmask, img\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}