{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --no-index --find-links=../input/preinstall efficientnet","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport gc\nfrom functools import partial\nimport json\nimport rasterio\nfrom rasterio.windows import Window\nimport yaml\nimport pprint\n\nimport pathlib\nfrom tqdm.notebook import tqdm\nimport cv2\n\nimport tensorflow as tf\nimport efficientnet as efn\nimport efficientnet.tfkeras\nimport time","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mod_paths = ['../input/hubmap-ensamble-model1/','../input/hubmap-ensamble-model2/']\n    \nTHRESHOLD = 0.5 # preds > THRESHOLD\nBATCH_SIZE = 256\n\nCHECKSUM = False # compute mask sum for each image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"identity = rasterio.Affine(1, 0, 0, 0, 1, 0)\nfold_models = []\nfor mod_path in mod_paths:\n    with open(mod_path+'params.yaml') as file:\n        P = yaml.load(file, Loader=yaml.FullLoader)\n        pprint.pprint(P)\n    with open(mod_path + 'metrics.json') as json_file:\n        M = json.load(json_file)\n        print('Model run datetime: '+M['datetime'])\n        print('OOF val_dice_coef: ' + str(M['oof_dice_coef']))\n        \n    for fold_model_path in glob.glob(mod_path+'*.h5'):\n        fold_models.append(tf.keras.models.load_model(fold_model_path,compile = False))\nprint(len(fold_models))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WINDOW = P['TILE'] if 'TILE' in P.keys() else P['DIM_FROM']\nCROP_SIZE = WINDOW//2\nINPUT_SIZE = P['INPUT_SIZE']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MIN_OVERLAP = WINDOW - CROP_SIZE\nBOARD_CUT = (WINDOW - CROP_SIZE)//2","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_encode_less_memory(img):\n    pixels = np.concatenate([[False], img.T.flatten(), [False]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\n\ndef make_grid(shape, window, min_overlap=0, board_cut = 0):\n    \"\"\"\n        Return Array of size (N,4), where N - number of tiles,\n        2nd axis represente slices: x1,x2,y1,y2 \n    \"\"\"\n    step = window - min_overlap\n    x, y = shape\n    start_x = np.arange(0, max(x-window+board_cut,1), step, dtype=np.int64)\n    start_y = np.arange(0, max(y-window+board_cut,1), step, dtype=np.int64)\n    if start_x[-1] < x-window+board_cut and x-window+board_cut > 0:\n        start_x = np.concatenate([start_x,[x-window+board_cut]])\n    if start_y[-1] < y-window+board_cut and y-window+board_cut > 0:\n        start_y = np.concatenate([start_y,[y-window+board_cut]])\n        \n    slices = np.zeros((len(start_x),len(start_y), 4), dtype=np.int64)\n    \n    for i in range(len(start_x)):\n        for j in range(len(start_y)):\n            slices[i,j] = start_x[i], min(start_x[i]+window,x), start_y[j], min(start_y[j]+window,y)   \n    return slices.reshape(len(start_x)*len(start_y),4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nimage_feature = {\n    'image': tf.io.FixedLenFeature([], tf.string),\n    'x1': tf.io.FixedLenFeature([], tf.int64),\n    'y1': tf.io.FixedLenFeature([], tf.int64)\n}\ndef _parse_image(example_proto):\n    example = tf.io.parse_single_example(example_proto, image_feature)\n    image = tf.reshape( tf.io.decode_raw(example['image'],out_type=np.dtype('uint8')), (P['DIM'],P['DIM'], 3))\n    return image, example['x1'], example['y1']\n\ndef load_dataset(filenames, ordered=True):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(_parse_image)\n    return dataset\n\ndef get_dataset(FILENAME):\n    dataset = load_dataset(FILENAME)\n    dataset  = dataset.batch(64)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def crop_resize(data,windows,size):\n    images = []\n    ori_h,ori_w = data.shape\n    for window in windows:\n        window_x1,window_x2,window_y1,window_y2 =  window\n        pad_x1,pad_x2,pad_y1,pad_y2 =  [0,0,0,0]\n        if window_x1 < 0:\n            window_x1 = 0\n            pad_x1 = window_x1 - window[0]\n        if window_y1 < 0:\n            window_y1 = 0\n            pad_y1 = window_y1 - window[2]\n        if window_x2 > ori_h:\n            window_x2 = ori_h\n            pad_x2 = window[1]-window_x2\n        if window_y2 > ori_w:\n            window_y2 = ori_w\n            pad_y2 = window[3]-window_y2\n            \n        if data.count != 3:\n            layers = [rasterio.open(subd) for subd in data.subdatasets]\n            image = np.zeros((int(window_x2-window_x1), int(window_y2-window_y1), 3), dtype=np.uint8)\n            for i in range(3):\n                image[:,:,i] = layers[i].read(window=Window.from_slices((window_x1,window_x2),(window_y1,window_y2)))\n        else:\n            image = data.read([1,2,3],\n                            window=Window.from_slices((window_x1,window_x2),(window_y1,window_y2)))\n            image = np.moveaxis(image, 0, -1)\n            \n        image = np.pad(image,((pad_x1,pad_x2),(pad_y1,pad_y2),(0,0)),'constant',constant_values = 0) \n        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n        image = cv2.resize(image,size)\n        images.append(image)\n    images = np.stack(images, axis=0)\n    return images","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p = pathlib.Path('../input/hubmap-kidney-segmentation')\nsubm = {}\n\ncnt = 0\nfor i, filename in tqdm(enumerate(p.glob('test/*.tiff')), \n                        total = len(list(p.glob('test/*.tiff')))):\n    \n    print(f'{i+1} Predicting {filename.stem}') \n    \n    data = rasterio.open(filename)\n    slices = make_grid(data.shape, window=WINDOW, min_overlap=MIN_OVERLAP, board_cut=BOARD_CUT)\n    dataset = tf.data.Dataset.from_tensor_slices(slices).batch(BATCH_SIZE)\n    \n    print('window num:{}'.format(len(slices)))\n    \n    image_h, image_w = data.shape\n    \n\n                \n    preds = np.zeros(data.shape, dtype=np.bool)\n        \n    for windows in dataset:\n        images = crop_resize(data,windows.numpy(),(INPUT_SIZE,INPUT_SIZE))\n                \n        pred_batch = None\n        \n        for fold_model in fold_models:\n            if pred_batch is None:\n                pred_batch = fold_model.predict(images)\n            else:\n                pred_batch += fold_model.predict(images)\n            \n        pred_batch = pred_batch/len(fold_models)\n            \n        for j,pred in enumerate(pred_batch):\n            x1,x2,y1,y2 = windows[j]\n            window_h, window_w = x2-x1, y2-y1\n            preds_start_x = x1+BOARD_CUT\n            preds_end_x = x2-BOARD_CUT\n            preds_start_y = y1+BOARD_CUT\n            preds_end_y = y2-BOARD_CUT\n            \n            pred_start_x = BOARD_CUT\n            pred_end_x = window_h-BOARD_CUT\n            pred_start_y = BOARD_CUT\n            pred_end_y = window_w-BOARD_CUT\n                \n            pred = cv2.resize(pred, (window_w, window_h))\n            preds[preds_start_x:preds_end_x,preds_start_y:preds_end_y] += (pred[pred_start_x:pred_end_x,pred_start_y:pred_end_y] > THRESHOLD)\n        \n        del pred,pred_batch,images\n        gc.collect();\n        \n    subm[i] = {'id':filename.stem, 'predicted': rle_encode_less_memory(preds)}\n    \n    del preds\n    gc.collect();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame.from_dict(subm, orient='index')\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{},"execution_count":null,"outputs":[]}]}