{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_DIR = \"/kaggle/input/hubmap-kidney-segmentation/\"\nTMP_DIR = \"/kaggle/temp/\"\nMODEL_FILE = \"/kaggle/input/vgg-unet/vgg_unet.h5\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sz = 1024   #the size of tiles\ninput_sz = 64\nvram = 16\nbatch_size = 1 * (vram // 8) * (1024 // input_sz) ** 2\nprint(\"Batch size\", batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport tifffile\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mask2rle(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels= img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef rle_encode_less_memory(img):\n    #watch out for the bug\n    pixels = img.T.flatten()\n    \n    # This simplified method requires first and last pixel to be zero\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] -= runs[::2]\n    \n    return ' '.join(str(x) for x in runs)\n\ndef rle2mask(mask_rle, shape=(1600,256)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -fR {TMP_DIR}/test_tiles/\n!mkdir -p {TMP_DIR}/test_tiles/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ids = []\nfor dirname, _, filenames in os.walk(f\"{INPUT_DIR}/test/\"):\n    for filename in filenames:\n        if filename.endswith(\".tiff\"):\n            image_id = filename[:-len(\".tiff\")]\n            test_ids.append(image_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_tiles = {}\npads = {}\nboxes = {}\nfor id in test_ids:\n    gc.collect()\n    image = tifffile.imread(f\"{INPUT_DIR}/test/{id}.tiff\")\n    print(id, image.shape)\n    image = np.squeeze(image)\n    if image.shape[0] == 3:\n        image = image.transpose(1,2,0)   \n    image = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)\n    gc.collect()\n    print(\"converted to gray\")\n        \n    shape = image.shape\n    pad0,pad1 = (sz - shape[0] % sz), (sz - shape[1] % sz) \n    pads[id] = (pad0, pad1)\n    print(\"before padding\")    \n    image = np.pad(image, [[pad0 // 2, pad0 - pad0 // 2], [pad1 // 2, pad1 - pad1 // 2]], constant_values=0) \n    gc.collect()\n    print(\"image padded\")\n    num_tiles0 = image.shape[0] // sz\n    num_tiles1 = image.shape[1] // sz\n    num_tiles[id] = (num_tiles0, num_tiles1)\n    image = image.reshape(num_tiles0, sz, num_tiles1, sz)\n    image = image.transpose(0, 2, 1, 3).reshape(-1, sz, sz)\n    gc.collect()\n    print(\"tiles created\")      \n    for i,im in enumerate(image):    \n        if im.std() == 0:\n            continue\n        im2 = cv2.resize(im, (input_sz, input_sz), interpolation=cv2.INTER_NEAREST)\n        cv2.imwrite(f'{TMP_DIR}/test_tiles/{id}_{i}.png', im2)  \n        del im2\n        gc.collect()\n    del image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\n\ndef dice(gt, pr):    \n    intersections = tf.reduce_sum(gt[:,:,1] * pr[:,:,1])\n    unions = tf.reduce_sum(gt[:,:,1] + pr[:,:,1])\n    eps = 10 ** -6\n    dice_scores = 2.0 * (intersections + eps/2) / (unions + eps)\n    return dice_scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_ORDERING_CHANNELS_LAST = \"channels_last\"\nIMAGE_ORDERING_CHANNELS_FIRST = \"channels_first\"\n\n# Default IMAGE_ORDERING = channels_last\nIMAGE_ORDERING = IMAGE_ORDERING_CHANNELS_LAST\n\nclass DataLoaderError(Exception):\n    pass\n\ndef get_image_array(image_input,\n                    width, height,\n                    imgNorm=\"sub_mean\", ordering='channels_first'):\n    \"\"\" Load image array from input \"\"\"\n\n    if not os.path.isfile(image_input):\n        raise DataLoaderError(\"get_image_array: path {0} doesn't exist\".format(image_input))\n    img = cv2.imread(image_input, 1)   \n\n    if imgNorm == \"sub_and_divide\":\n        img = np.float32(cv2.resize(img, (width, height))) / 127.5 - 1\n    elif imgNorm == \"sub_mean\":\n        img = cv2.resize(img, (width, height))\n        img = img.astype(np.float32)\n        img[:, :, 0] -= 103.939\n        img[:, :, 1] -= 116.779\n        img[:, :, 2] -= 123.68\n        img = img[:, :, ::-1]\n    elif imgNorm == \"divide\":\n        img = cv2.resize(img, (width, height))\n        img = img.astype(np.float32)\n        img = img/255.0\n\n    if ordering == 'channels_first':\n        img = np.rollaxis(img, 2, 0)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import load_model\nmodel = load_model(MODEL_FILE, custom_objects={\"dice\": dice})\n    \no_shape = model.output_shape\ni_shape = model.input_shape\n\nif IMAGE_ORDERING == 'channels_first':\n    output_height = i_shape[2] // 2\n    output_width = i_shape[3] // 2\n    input_height = i_shape[2]\n    input_width = i_shape[3]\n    n_classes = o_shape[1]  \nelif IMAGE_ORDERING == 'channels_last':\n    output_height = i_shape[1] // 2\n    output_width = i_shape[2] // 2\n    input_height = i_shape[1]\n    input_width = i_shape[2]\n    n_classes = o_shape[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"total = 0\nfor dirname, _, filenames in os.walk(f\"{TMP_DIR}/test_tiles/\"):\n    for filename in filenames:\n        if filename.endswith(\".png\"):\n            total += 1\nsteps_per_epoch = total // batch_size      \nprint(\"Total: \", total)   \nprint(\"Steps per epoch: \", steps_per_epoch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.DataFrame(columns = ['id','predicted'])\nfor id in test_ids:\n    gc.collect()\n    print(id)\n    predicted = np.zeros(shape = (num_tiles[id][0] * num_tiles[id][1], sz, sz), dtype=np.uint8)    \n    print(\"prediction placeholder\")\n    names = []\n    orig = []\n    for i in range(predicted.shape[0]):\n        name = f'{TMP_DIR}/test_tiles/{id}_{i}.png'\n        if not os.path.exists(name):\n            continue\n        names.append(name)\n        orig.append(i)\n    indices = np.array(range(len(names)))\n    add = batch_size - (len(names) % batch_size)\n    if add != batch_size:\n        adds = np.array(range(add))\n        indices = np.concatenate((indices, adds))\n    indices = indices.reshape(-1, batch_size)\n    \n    for idxs in indices:        \n        x = np.array([get_image_array(names[i], input_width, input_height, ordering=IMAGE_ORDERING) for i in idxs]) \n        pr = model.predict(x)\n        pr = pr.reshape((batch_size, output_height,  output_width, n_classes))\n        if n_classes == 1:\n            pr = np.squeeze(np.where(pr > 0.5, 1, 0))\n        else:\n            pr = pr.argmax(axis=3)    \n        gc.collect()          \n        out_res = np.array([cv2.resize(p, (sz, sz), interpolation=cv2.INTER_NEAREST) for p in pr])         \n        for k,i in enumerate(idxs):\n            j = orig[i]\n            predicted[j] = out_res[k].astype(np.uint8)        \n\n        del out_res        \n        gc.collect()\n    print(\"tiles predicted\")\n    predicted = predicted.reshape(num_tiles[id][0], num_tiles[id][1], sz, sz)\n    predicted = predicted.transpose(0,2,1,3)\n    predicted = predicted.reshape(num_tiles[id][0]*sz, num_tiles[id][1]*sz)\n    predicted = predicted[pads[id][0] // 2 : -(pads[id][0] - pads[id][0] // 2), pads[id][1] // 2 : -(pads[id][1] - pads[id][1] // 2)]\n    print(id, predicted.shape)\n    gc.collect()\n    rle_string = rle_encode_less_memory(predicted) \n    print(\"mask rle encoded\")\n    del predicted\n    submission_df = submission_df.append({'id':id, 'predicted':rle_string}, ignore_index=True)\n    print(\"submission updated\")    \n    del rle_string\nsubmission_df.to_csv('submission.csv',index=False)   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}