{"cells":[{"metadata":{},"cell_type":"markdown","source":"********Ref.\n\nhttps://www.kaggle.com/wrrosa/hubmap-tf-with-tpu-efficientunet-512x512-tfrecs\nhttps://www.kaggle.com/wrrosa/hubmap-tf-with-tpu-efficientunet-512x512-train\nhttps://www.kaggle.com/wrrosa/hubmap-tf-with-tpu-efficientunet-512x512-subm\nhttps://www.kaggle.com/tivfrvqhs5/global-mask-shift"},{"metadata":{},"cell_type":"markdown","source":"TFrecord 생성 \nhttps://www.kaggle.com/wrrosa/hubmap-tf-with-tpu-efficientunet-512x512-tfrecs"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport tifffile as tiff\nimport cv2\nimport os\nfrom tqdm.notebook import tqdm\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orig = 1024\nsz = 512 #128 #256 #the size of tiles\nreduce = orig//sz  #reduce the original images by 'reduce' times \nMASKS = '../input/hubmap-kidney-segmentation/train.csv'\nDATA = '../input/hubmap-kidney-segmentation/train/'\n#어디에 쓰이는 threshold ?  \ns_th = 40  #saturation blancking threshold ==> hsv에서 s split 된것 기준 tfRecord만들 때 필터링 \np_th = 200*sz//256 #threshold for the minimum number of pixels ==> 400","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#functions to convert encoding to mask and mask to encoding\ndef enc2mask(encs, shape):\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for m,enc in enumerate(encs):\n        if isinstance(enc,np.float) and np.isnan(enc): continue\n        s = enc.split()\n        for i in range(len(s)//2):\n            start = int(s[2*i]) - 1\n            length = int(s[2*i+1])\n            img[start:start+length] = 1 + m\n    return img.reshape(shape).T\n\ndef mask2enc(mask, n=1):\n    pixels = mask.T.flatten()\n    encs = []\n    for i in range(1,n+1):\n        p = (pixels == i).astype(np.int8)\n        if p.sum() == 0: encs.append(np.nan)\n        else:\n            p = np.concatenate([[0], p, [0]])\n            runs = np.where(p[1:] != p[:-1])[0] + 1\n            runs[1::2] -= runs[::2]\n            encs.append(' '.join(str(x) for x in runs))\n    return encs\n\ndf_masks = pd.read_csv(MASKS).set_index('id')\ndf_masks.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The following function can be used to convert a value to a type compatible\n# with tf.train.Example.\n\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef serialize_example(image, mask):\n  \"\"\"\n  Creates a tf.train.Example message ready to be written to a file.\n  \"\"\"\n  # Create a dictionary mapping the feature name to the tf.train.Example-compatible\n  # data type.\n  feature = {\n      'image': _bytes_feature(image),\n      'mask': _bytes_feature(mask),\n  }\n\n  # Create a Features message using tf.train.Example.\n\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Train\nos.makedirs('train')\n\nx_tot,x2_tot = [],[]\nfor index, encs in tqdm(df_masks.iterrows()):\n    #read image and generate the mask\n    img = tiff.imread(os.path.join(DATA,index+'.tiff'))\n    if len(img.shape) == 5:img = np.transpose(img.squeeze(), (1,2,0))\n    mask = enc2mask(encs,(img.shape[1],img.shape[0]))\n\n    #add padding to make the image dividable into tiles\n    shape = img.shape\n    pad0 = (reduce*sz - shape[0]%(reduce*sz))%(reduce*sz)\n    pad1 = (reduce*sz - shape[1]%(reduce*sz))%(reduce*sz)\n    img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                constant_values=0)\n    mask = np.pad(mask,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2]],\n                constant_values=0)\n\n    #split image and mask into tiles using the reshape+transpose trick\n    img = cv2.resize(img,(img.shape[1]//reduce,img.shape[0]//reduce),\n                     interpolation = cv2.INTER_AREA)\n    img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n    img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n\n    mask = cv2.resize(mask,(mask.shape[1]//reduce,mask.shape[0]//reduce),\n                      interpolation = cv2.INTER_NEAREST)\n    mask = mask.reshape(mask.shape[0]//sz,sz,mask.shape[1]//sz,sz)\n    mask = mask.transpose(0,2,1,3).reshape(-1,sz,sz)\n\n    filename = 'train/'+ index + '.tfrec'\n    cnt = 0\n    with tf.io.TFRecordWriter(filename) as writer:\n\n        for i,(im,m) in enumerate(zip(img,mask)):\n            #remove black or gray images based on saturation check\n            hsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)\n            h, s, v = cv2.split(hsv)\n            if (s>s_th).sum() <= p_th or im.sum() <= p_th: continue\n\n            x_tot.append((im/255.0).reshape(-1,3).mean(0))\n            x2_tot.append(((im/255.0)**2).reshape(-1,3).mean(0))\n\n            im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR)\n            example = serialize_example(im.tobytes(),m.tobytes())\n            writer.write(example)\n\n            cnt +=1\n    #os.rename(filename,'train/'+ index + '-'+str(img.shape[0]) +'.tfrec') # old dataset version with issue         \n    os.rename(filename,'train/'+ index + '-'+str(cnt) +'.tfrec')\n    \n            \n#image stats\nimg_avr =  np.array(x_tot).mean(0)\nimg_std =  np.sqrt(np.array(x2_tot).mean(0) - img_avr**2)\nprint('mean:',img_avr, ', std:', img_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\nimport glob\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\ntrain_images = glob.glob('train/*.tfrec')\nctraini = count_data_items(train_images)\nprint(f'Num train images: {ctraini}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DIM = sz\nmini_size = 64\ndef _parse_image_function(example_proto):\n    image_feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'mask': tf.io.FixedLenFeature([], tf.string)\n    }\n    single_example = tf.io.parse_single_example(example_proto, image_feature_description)\n    image = tf.reshape( tf.io.decode_raw(single_example['image'],out_type=np.dtype('uint8')), (DIM,DIM, 3))\n    mask =  tf.reshape(tf.io.decode_raw(single_example['mask'],out_type='bool'),(DIM,DIM,1))\n    \n    image = tf.image.resize(image,(mini_size,mini_size))/255.0\n    mask = tf.image.resize(tf.cast(mask,'uint8'),(mini_size,mini_size))\n    return image, mask\n\n\ndef load_dataset(filenames):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(lambda ex: _parse_image_function(ex))\n    return dataset\n\nN = 8\ndef get_dataset(FILENAME):\n    dataset = load_dataset(FILENAME)\n    dataset = dataset.batch(N*N)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nfrom skimage.segmentation import mark_boundaries\nfor imgs, masks in get_dataset(train_images[0]).take(1):\n    pass\n\nplt.figure(figsize = (N,N))\ngs1 = gridspec.GridSpec(N,N)\n\nfor i in range(N*N):\n   # i = i + 1 # grid spec indexes from 0\n    ax1 = plt.subplot(gs1[i])\n    plt.axis('on')\n    ax1.set_xticklabels([])\n    ax1.set_yticklabels([])\n    ax1.set_aspect('equal')\n    #ax1.imshow(imgs[i])\n    #ax1.imshow(masks[i])\n    ax1.imshow(mark_boundaries(imgs[i], masks[i].numpy().squeeze().astype('bool')))\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Train 2\n#Same idea as train, but shifted by padding.\nos.makedirs('train2')\nfor index, encs in tqdm(df_masks.iterrows()):\n    #read image and generate the mask\n    img = tiff.imread(os.path.join(DATA,index+'.tiff'))\n    if len(img.shape) == 5:img = np.transpose(img.squeeze(), (1,2,0))\n    mask = enc2mask(encs,(img.shape[1],img.shape[0]))\n\n    #add padding to make the image dividable into tiles\n    shape = img.shape\n    pad0 = (reduce*sz - shape[0]%(reduce*sz))%(reduce*sz)\n    pad1 = (reduce*sz - shape[1]%(reduce*sz))%(reduce*sz)\n    print('before shift:',pad0,pad1)\n    ##############\n    pad0 += orig\n    pad1 += orig\n    print('after shift:',pad0,pad1)\n    ##############\n    \n    img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                constant_values=0)\n    mask = np.pad(mask,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2]],\n                constant_values=0)\n\n    #split image and mask into tiles using the reshape+transpose trick\n    img = cv2.resize(img,(img.shape[1]//reduce,img.shape[0]//reduce),\n                     interpolation = cv2.INTER_AREA)\n    img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n    img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n\n    mask = cv2.resize(mask,(mask.shape[1]//reduce,mask.shape[0]//reduce),\n                      interpolation = cv2.INTER_NEAREST)\n    mask = mask.reshape(mask.shape[0]//sz,sz,mask.shape[1]//sz,sz)\n    mask = mask.transpose(0,2,1,3).reshape(-1,sz,sz)\n\n    #write data\n    filename = 'train2/'+ index + '.tfrec'\n    cnt = 0\n    with tf.io.TFRecordWriter(filename) as writer:\n\n        for i,(im,m) in enumerate(zip(img,mask)):\n            #remove black or gray images based on saturation check\n            hsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)\n            h, s, v = cv2.split(hsv)\n            if (s>s_th).sum() <= p_th or im.sum() <= p_th: continue\n                \n            im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR)\n            example = serialize_example(im.tobytes(),m.tobytes())\n            writer.write(example)\n            \n            cnt +=1\n    #os.rename(filename,'train2/'+ index + '-'+str(img.shape[0]) +'.tfrec') # old dataset version with issue        \n    os.rename(filename,'train2/'+ index + '-'+str(cnt) +'.tfrec')  \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train2_images = glob.glob('train2/*.tfrec')\nctrain2i = count_data_items(train2_images)\nprint(f'Num train2 images: {ctrain2i}')\n#왜 train1이랑 길이가 다른지 .. ? 원본사이즈 shift 의미가 뭔지 ? ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for imgs, masks in get_dataset(train2_images[0]).take(1):\n    pass\nplt.figure(figsize = (N,N))\nfor i in range(N*N):\n   # i = i + 1 # grid spec indexes from 0\n    ax1 = plt.subplot(gs1[i])\n    plt.axis('on')\n    ax1.set_xticklabels([])\n    ax1.set_yticklabels([])\n    ax1.set_aspect('equal')\n    ax1.imshow(mark_boundaries(imgs[i], masks[i].numpy().squeeze().astype('bool')))\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#inference \n#The approach presented above is superfast and elegant, but it does not contain the coordinates of the image (x1, y1) so (without modification) it is useless for inference. Now @leighplt https://www.kaggle.com/leighplt/pytorch-fcn-resnet50 presented how to use rasterio - but it doesn't support batching. So idea is to create tfrecords using rasterio and use them in inference - should be faster.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"WINDOW = orig #1024\nMIN_OVERLAP = 300\nNEW_SIZE = sz #512\n\nimport numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport gc\n\nimport rasterio\nfrom rasterio.windows import Window\n\nimport pathlib\nfrom tqdm.notebook import tqdm\nimport cv2\n\nimport tensorflow as tf\n\ndef make_grid(shape, window=256, min_overlap=32):\n    \"\"\"\n        Return Array of size (N,4), where N - number of tiles,\n        2nd axis represente slices: x1,x2,y1,y2 \n    \"\"\"\n    x, y = shape\n    nx = x // (window - min_overlap) + 1\n    x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)\n    x1[-1] = x - window\n    x2 = (x1 + window).clip(0, x)\n    ny = y // (window - min_overlap) + 1\n    y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)\n    y1[-1] = y - window\n    y2 = (y1 + window).clip(0, y)\n    slices = np.zeros((nx,ny, 4), dtype=np.int64)\n    \n    for i in range(nx):\n        for j in range(ny):\n            slices[i,j] = x1[i], x2[i], y1[j], y2[j]    \n    return slices.reshape(nx*ny,4)\n\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_example(image, x1, y1):\n  feature = {\n      'image': _bytes_feature(image),\n      'x1': _int64_feature(x1),\n      'y1': _int64_feature(y1)\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p = pathlib.Path('../input/hubmap-kidney-segmentation')\nidentity = rasterio.Affine(1, 0, 0, 0, 1, 0)\nos.makedirs('test', exist_ok = True)\n\nfor i, filename in tqdm(enumerate(p.glob('test/*.tiff')), \n                        total = len(list(p.glob('test/*.tiff')))):\n    \n    print(f'{i+1} Creating tfrecords for image: {filename.stem}')\n    #rasterio as posix ?? transform ?? \n    dataset = rasterio.open(filename.as_posix(), transform = identity)\n    slices = make_grid(dataset.shape, window=WINDOW, min_overlap=MIN_OVERLAP)\n    \n    print(slices.shape[0])\n    cnt = 0\n    part = 0 \n    fname = f'test/{filename.stem}-part{part}.tfrec'\n    writer = tf.io.TFRecordWriter(fname) \n    for (x1,x2,y1,y2) in slices:\n        if cnt>999:\n            writer.close()\n            os.rename(fname, f'test/{filename.stem}-part{part}-{cnt}.tfrec')\n            part += 1\n            fname = f'test/{filename.stem}-part{part}.tfrec'\n            writer = tf.io.TFRecordWriter(fname)\n            cnt = 0\n        \n        image = dataset.read([1,2,3],\n                    window=Window.from_slices((x1,x2),(y1,y2)))\n        image = np.moveaxis(image, 0, -1)\n        image = cv2.resize(image, (NEW_SIZE, NEW_SIZE),interpolation = cv2.INTER_AREA)\n        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n        example = serialize_example(image.tobytes(),x1,y1)\n        writer.write(example)\n        cnt+=1\n    writer.close()\n    del writer\n    os.rename(fname, f'test/{filename.stem}-part{part}-{cnt}.tfrec')\n    gc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_images = glob.glob('test/*.tfrec')\nctesti = count_data_items(test_images)\nprint(f'Num test images: {ctesti}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DIM = sz\nmini_size = 64\ndef _parse_image_function(example_proto):\n    image_feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'x1': tf.io.FixedLenFeature([], tf.int64),\n        'y1': tf.io.FixedLenFeature([], tf.int64)\n    }\n    single_example = tf.io.parse_single_example(example_proto, image_feature_description)\n    image = tf.reshape( tf.io.decode_raw(single_example['image'],out_type=np.dtype('uint8')), (DIM,DIM, 3))\n    x1 = single_example['x1']\n    y1 = single_example['y1']\n    image = tf.image.resize(image,(mini_size,mini_size))/255.0\n    return image, x1, y1\n\n\ndef load_dataset(filenames):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(lambda ex: _parse_image_function(ex))\n    return dataset\n\nN = 8\ndef get_dataset(FILENAME):\n    dataset = load_dataset(FILENAME)\n    dataset = dataset.batch(N*N)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for imgs, x1, y1 in get_dataset(test_images[0]).take(1):\n    pass\n\nplt.figure(figsize = (N,N))\ngs1 = gridspec.GridSpec(N,N)\n\nfor i in range(N*N):\n   # i = i + 1 # grid spec indexes from 0\n    ax1 = plt.subplot(gs1[i])\n    plt.axis('on')\n    ax1.set_xticklabels([])\n    ax1.set_yticklabels([])\n    ax1.set_aspect('equal')\n    ax1.set_title(f'{x1[i]}; {y1[i]}', fontsize=6)\n    ax1.imshow(imgs[i])\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Train**\nhttps://www.kaggle.com/wrrosa/hubmap-tf-with-tpu-efficientunet-512x512-train"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Init - parameters, packages, gcs_paths, tpu\nP = {}\nP['EPOCHS'] = 30\nP['BACKBONE'] = 'efficientnetb4' \nP['NFOLDS'] = 4\nP['SEED'] = 0\nP['VERBOSE'] = 0\nP['DISPLAY_PLOT'] = True \nP['BATCH_COE'] = 8 # BATCH_SIZE = P['BATCH_COE'] * strategy.num_replicas_in_sync\n\nP['TILING'] = [1024,512] # 1024,512 1024,256 1024,128 1536,512 768,384\nP['DIM'] = P['TILING'][1] \nP['DIM_FROM'] = P['TILING'][0]\n\nP['LR'] = 5e-4 \nP['OVERLAPP'] = True\nP['STEPS_COE'] = 3\n\nimport yaml\nwith open(r'params.yaml', 'w') as file:\n    yaml.dump(P, file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install segmentation_models -q\n%matplotlib inline\n\nimport os\nos.environ['SM_FRAMEWORK'] = 'tf.keras'\nimport glob\nimport segmentation_models as sm\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import KFold\n\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.utils import get_custom_objects\n\nfrom kaggle_datasets import KaggleDatasets\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept ValueError: # no TPU found, detect GPUs\n    #strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nBATCH_SIZE = P['BATCH_COE'] * strategy.num_replicas_in_sync\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)\nprint(\"BATCH_SIZE: \", str(BATCH_SIZE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path(f'hubmap-tfrecords-1024-{P[\"DIM\"]}')\nALL_TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nALL_TRAINING_FILENAMES","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if P['OVERLAPP']:\n    ALL_TRAINING_FILENAMES2 = tf.io.gfile.glob(GCS_PATH + '/train2/*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\nprint('NUM_TRAINING_IMAGES:' )\nif P['OVERLAPP']:\n    print(count_data_items(ALL_TRAINING_FILENAMES2)+count_data_items(ALL_TRAINING_FILENAMES))\nelse:\n    print(count_data_items(ALL_TRAINING_FILENAMES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Datasets pipeline \nDIM = P['DIM']\ndef _parse_image_function(example_proto,augment = True):\n    image_feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'mask': tf.io.FixedLenFeature([], tf.string)\n    }\n    single_example = tf.io.parse_single_example(example_proto, image_feature_description)\n    image = tf.reshape( tf.io.decode_raw(single_example['image'],out_type=np.dtype('uint8')), (DIM,DIM, 3))\n    mask =  tf.reshape(tf.io.decode_raw(single_example['mask'],out_type='bool'),(DIM,DIM,1))\n    \n    if augment: # https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.flip_left_right(image)\n            mask = tf.image.flip_left_right(mask)\n\n        if tf.random.uniform(()) > 0.4:\n            image = tf.image.flip_up_down(image)\n            mask = tf.image.flip_up_down(mask)\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.rot90(image, k=1)\n            mask = tf.image.rot90(mask, k=1)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_saturation(image, 0.7, 1.3)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_contrast(image, 0.8, 1.2)\n    \n    return tf.cast(image, tf.float32),tf.cast(mask, tf.float32)\n\ndef load_dataset(filenames, ordered=False, augment = True):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda ex: _parse_image_function(ex, augment = augment), num_parallel_calls=AUTO)\n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(128, seed = P['SEED'])\n    dataset = dataset.batch(BATCH_SIZE,drop_remainder=True)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(ordered=True):\n    dataset = load_dataset(VALIDATION_FILENAMES, ordered=ordered, augment = False)\n    dataset = dataset.batch(BATCH_SIZE,drop_remainder=True)\n    #dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model \n# https://tensorlayer.readthedocs.io/en/latest/_modules/tensorlayer/cost.html#dice_coe\ndef dice_coe(output, target, axis = None, smooth=1e-10):\n    output = tf.dtypes.cast( tf.math.greater(output, 0.5), tf. float32 )\n    target = tf.dtypes.cast( tf.math.greater(target, 0.5), tf. float32 )\n    inse = tf.reduce_sum(output * target, axis=axis)\n    l = tf.reduce_sum(output, axis=axis)\n    r = tf.reduce_sum(target, axis=axis)\n\n    dice = (2. * inse + smooth) / (l + r + smooth)\n    dice = tf.reduce_mean(dice, name='dice_coe')\n    return dice\n\n# https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\ndef tversky(y_true, y_pred, alpha=0.7, beta=0.3, smooth=1):\n    y_true_pos = K.flatten(y_true)\n    y_pred_pos = K.flatten(y_pred)\n    true_pos = K.sum(y_true_pos * y_pred_pos)\n    false_neg = K.sum(y_true_pos * (1 - y_pred_pos))\n    false_pos = K.sum((1 - y_true_pos) * y_pred_pos)\n    return (true_pos + smooth) / (true_pos + alpha * false_neg + beta * false_pos + smooth)\ndef tversky_loss(y_true, y_pred):\n    return 1 - tversky(y_true, y_pred)\ndef focal_tversky_loss(y_true, y_pred, gamma=0.75):\n    tv = tversky(y_true, y_pred)\n    return K.pow((1 - tv), gamma)\n\nget_custom_objects().update({\"focal_tversky\": focal_tversky_loss})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model fit \nM = {}\nmetrics = ['loss','dice_coe','accuracy']\nfor fm in metrics:\n    M['val_'+fm] = []\n\nfold = KFold(n_splits=P['NFOLDS'], shuffle=True, random_state=P['SEED'])\nfor fold,(tr_idx, val_idx) in enumerate(fold.split(ALL_TRAINING_FILENAMES)):\n    \n    print('#'*35); print('############ FOLD ',fold+1,' #############'); print('#'*35);\n    print(f'Image Size: {DIM}, Batch Size: {BATCH_SIZE}')\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    TRAINING_FILENAMES = [ALL_TRAINING_FILENAMES[fi] for fi in tr_idx]\n    if P['OVERLAPP']:\n        TRAINING_FILENAMES += [ALL_TRAINING_FILENAMES2[fi] for fi in tr_idx]\n    \n    VALIDATION_FILENAMES = [ALL_TRAINING_FILENAMES[fi] for fi in val_idx]\n    STEPS_PER_EPOCH = P['STEPS_COE'] * count_data_items(TRAINING_FILENAMES) // BATCH_SIZE\n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():   \n        model = sm.Unet(P['BACKBONE'], encoder_weights='imagenet')\n        model.compile(optimizer = tf.keras.optimizers.Adam(lr = P['LR']),\n                      loss = tf.keras.losses.BinaryCrossentropy(),#'focal_tversky',\n                      metrics=[dice_coe,'accuracy'])\n        \n    # CALLBACKS\n    checkpoint = tf.keras.callbacks.ModelCheckpoint('/kaggle/working/model-fold-%i.h5'%fold,\n                                 verbose=P['VERBOSE'],monitor='val_dice_coe',patience = 10,\n                                 mode='max',save_best_only=True)\n    \n    early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_dice_coe',mode = 'max', patience=10, restore_best_weights=True)\n    reduce = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=8, min_lr=0.00001)\n        \n    print(f'Training Model Fold {fold+1}...')\n    history = model.fit(\n        get_training_dataset(),\n        epochs = P['EPOCHS'],\n        steps_per_epoch = STEPS_PER_EPOCH,\n        callbacks = [checkpoint, reduce,early_stop],\n        validation_data = get_validation_dataset(),\n        verbose=P['VERBOSE']\n    )   \n    \n    #with strategy.scope():\n    #    model = tf.keras.models.load_model('/kaggle/working/model-fold-%i.h5'%fold, custom_objects = {\"dice_coe\": dice_coe})\n    \n    # SAVE METRICS\n    m = model.evaluate(get_validation_dataset(),return_dict=True)\n    for fm in metrics:\n        M['val_'+fm].append(m[fm])\n    \n    # PLOT TRAINING\n    # https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n    if P['DISPLAY_PLOT']:        \n        plt.figure(figsize=(15,5))\n        n_e = np.arange(len(history.history['dice_coe']))\n        plt.plot(n_e,history.history['dice_coe'],'-o',label='Train dice_coe',color='#ff7f0e')\n        plt.plot(n_e,history.history['val_dice_coe'],'-o',label='Val dice_coe',color='#1f77b4')\n        x = np.argmax( history.history['val_dice_coe'] ); y = np.max( history.history['val_dice_coe'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max dice_coe\\n%.2f'%y,size=14)\n        plt.ylabel('dice_coe',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        plt2 = plt.gca().twinx()\n        plt2.plot(n_e,history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(n_e,history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        plt.ylabel('Loss',size=14)\n        plt.legend(loc=3)\n        plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### WRITE METRICS\nimport json\nfrom datetime import datetime\nM['datetime'] = str(datetime.now())\nfor fm in metrics:\n    M['oof_'+fm] = np.mean(M['val_'+fm])\n    print('OOF '+ fm + ' '+ str(M['oof_'+fm]))\nwith open('metrics.json', 'w') as outfile:\n    json.dump(M, outfile)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Inference with submission \nhttps://www.kaggle.com/wrrosa/hubmap-tf-with-tpu-efficientunet-512x512-subm"},{"metadata":{"trusted":true},"cell_type":"code","source":"#params \nmod_path = '../input/hubmap-tf-with-tpu-efficientunet-512x512-train/'\nimport yaml\nimport pprint\nwith open(mod_path+'params.yaml') as file:\n    P = yaml.load(file, Loader=yaml.FullLoader)\n    pprint.pprint(P)\n    \nTHRESHOLD = 0.4 # preds > THRESHOLD\nWINDOW = 1024\nMIN_OVERLAP = 300\nNEW_SIZE = P['DIM']\n\nSUBMISSION_MODE = 'PUBLIC_TFREC' \n# 'PUBLIC_TFREC' = use created tfrecords for public test set with MIN_OVERLAP = 300 tiling 1024-512, ignore other (private test) data\n# 'FULL' do not use tfrecords, just full submission \n\nCHECKSUM = False # compute mask sum for each image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#metrics \nimport json\n\nwith open(mod_path + 'metrics.json') as json_file:\n    M = json.load(json_file)\nprint('Model run datetime: '+M['datetime'])\nprint('OOF val_dice_coe: ' + str(M['oof_dice_coe']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#packages \n! pip install ../input/kerasapplications/keras-team-keras-applications-3b180cb -f ./ --no-index -q #?? Error \n! pip install ../input/efficientnet/efficientnet-1.1.0/ -f ./ --no-index -q\nimport numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport gc\n\nimport rasterio\nfrom rasterio.windows import Window\n\nimport pathlib\nfrom tqdm.notebook import tqdm\nimport cv2\n\nimport tensorflow as tf\nimport efficientnet as efn\nimport efficientnet.tfkeras","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#functions \ndef rle_encode_less_memory(img):\n    pixels = img.T.flatten()\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef make_grid(shape, window=256, min_overlap=32):\n    \"\"\"\n        Return Array of size (N,4), where N - number of tiles,\n        2nd axis represente slices: x1,x2,y1,y2 \n    \"\"\"\n    x, y = shape\n    nx = x // (window - min_overlap) + 1\n    x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)\n    x1[-1] = x - window\n    x2 = (x1 + window).clip(0, x)\n    ny = y // (window - min_overlap) + 1\n    y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)\n    y1[-1] = y - window\n    y2 = (y1 + window).clip(0, y)\n    slices = np.zeros((nx,ny, 4), dtype=np.int64)\n    \n    for i in range(nx):\n        for j in range(ny):\n            slices[i,j] = x1[i], x2[i], y1[j], y2[j]    \n    return slices.reshape(nx*ny,4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#models \nidentity = rasterio.Affine(1, 0, 0, 0, 1, 0)\nfold_models = []\nfor fold_model_path in glob.glob(mod_path+'*.h5'):\n    fold_models.append(tf.keras.models.load_model(fold_model_path,compile = False))\nprint(len(fold_models))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#tfrecords func. \nAUTO = tf.data.experimental.AUTOTUNE\nimage_feature = {\n    'image': tf.io.FixedLenFeature([], tf.string),\n    'x1': tf.io.FixedLenFeature([], tf.int64),\n    'y1': tf.io.FixedLenFeature([], tf.int64)\n}\ndef _parse_image(example_proto):\n    example = tf.io.parse_single_example(example_proto, image_feature)\n    image = tf.reshape( tf.io.decode_raw(example['image'],out_type=np.dtype('uint8')), (P['DIM'],P['DIM'], 3))\n    return image, example['x1'], example['y1']\n\ndef load_dataset(filenames, ordered=True):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(_parse_image)\n    return dataset\n\ndef get_dataset(FILENAME):\n    dataset = load_dataset(FILENAME)\n    dataset  = dataset.batch(64)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#results \np = pathlib.Path('../input/hubmap-kidney-segmentation')\nsubm = {}\n\nfor i, filename in tqdm(enumerate(p.glob('test/*.tiff')), \n                        total = len(list(p.glob('test/*.tiff')))):\n    \n    print(f'{i+1} Predicting {filename.stem}')\n    \n    dataset = rasterio.open(filename.as_posix(), transform = identity)\n    preds = np.zeros(dataset.shape, dtype=np.uint8)    \n    \n    if SUBMISSION_MODE == 'PUBLIC_TFREC' and MIN_OVERLAP == 300 and WINDOW == 1024 and NEW_SIZE == 512:\n        print('SUBMISSION_MODE: PUBLIC_TFREC')\n        fnames = glob.glob('/kaggle/input/hubmap-tfrecords-1024-512-test/test/'+filename.stem+'*.tfrec')\n        \n        if len(fnames)>0: # PUBLIC TEST SET\n            for FILENAME in fnames:\n                pred = None\n                for fold_model in fold_models:\n                    tmp = fold_model.predict(get_dataset(FILENAME))/len(fold_models)\n                    if pred is None:\n                        pred = tmp\n                    else:\n                        pred += tmp\n                    del tmp\n                    gc.collect()\n\n                pred = tf.cast((tf.image.resize(pred, (WINDOW,WINDOW)) > THRESHOLD),tf.bool).numpy().squeeze()\n\n                idx = 0\n                for img, X1, Y1 in get_dataset(FILENAME):\n                    for fi in range(X1.shape[0]):\n                        x1 = X1[fi].numpy()\n                        y1 = Y1[fi].numpy()\n                        preds[x1:(x1+WINDOW),y1:(y1+WINDOW)] += pred[idx]\n                        idx += 1\n                        \n        else: # IGNORE PRIVATE TEST SET (CREATE TFRECORDS IN FUTURE)\n            pass\n    else:\n        print('SUBMISSION_MODE: FULL')\n        slices = make_grid(dataset.shape, window=WINDOW, min_overlap=MIN_OVERLAP)\n\n\n        for (x1,x2,y1,y2) in slices:\n            image = dataset.read([1,2,3],\n                        window=Window.from_slices((x1,x2),(y1,y2)))\n            image = np.moveaxis(image, 0, -1)\n            image = cv2.resize(image, (NEW_SIZE, NEW_SIZE),interpolation = cv2.INTER_AREA)\n            image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n            image = np.expand_dims(image, 0)\n\n            pred = None\n\n            for fold_model in fold_models:\n                if pred is None:\n                    pred = np.squeeze(fold_model.predict(image))\n                else:\n                    pred += np.squeeze(fold_model.predict(image))\n\n            pred = pred/len(fold_models)\n\n            pred = cv2.resize(pred, (WINDOW, WINDOW))\n            preds[x1:x2,y1:y2] += (pred > THRESHOLD).astype(np.uint8)\n\n    preds = (preds > 0.5).astype(np.uint8)\n    \n    subm[i] = {'id':filename.stem, 'predicted': rle_encode_less_memory(preds)}\n    \n    if CHECKSUM:\n        print('Checksum: '+ str(np.sum(preds)))\n    \n    del preds\n    gc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame.from_dict(subm, orient='index')\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()\n#?? empty --> efficientnet pip install error 때문 ? ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**** Global Mask Shift\nhttps://www.kaggle.com/tivfrvqhs5/global-mask-shift"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle2mask(mask_rle, shape=(1600,256)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T\n\n\n#https://www.kaggle.com/bguberfain/memory-aware-rle-encoding\ndef rle_encode_less_memory(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    This simplified method requires first and last pixel to be zero\n    '''\n    pixels = img.T.flatten()\n    \n    # This simplified method requires first and last pixel to be zero\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] -= runs[::2]\n    \n    return ' '.join(str(x) for x in runs)\n\ndef image_size_dict(img_id, x, y):\n    image_id = [thing[:-5] for thing in img_id]\n    x_y = [(x[i], y[i]) for i in range(0, len(x))]    \n    return dict(zip(image_id, x_y))\n\n\ndef global_shift_mask(maskpred1, y_shift, x_shift):\n    \"\"\"\n    applies a global shift to a mask by padding one side and cropping from the other\n    \"\"\"\n    if y_shift <0 and x_shift >=0:\n        maskpred2 = np.pad(maskpred1, [(0,abs(y_shift)), (abs(x_shift), 0)], mode='constant', constant_values=0)\n        maskpred3 = maskpred2[abs(y_shift):, :maskpred1.shape[1]]\n    elif y_shift >=0 and x_shift <0:\n        maskpred2 = np.pad(maskpred1, [(abs(y_shift),0), (0, abs(x_shift))], mode='constant', constant_values=0)\n        maskpred3 = maskpred2[:maskpred1.shape[0], abs(x_shift):]\n    elif y_shift >=0 and x_shift >=0:\n        maskpred2 = np.pad(maskpred1, [(abs(y_shift),0), (abs(x_shift), 0)], mode='constant', constant_values=0)\n        maskpred3 = maskpred2[:maskpred1.shape[0], :maskpred1.shape[1]]\n    elif y_shift < 0 and x_shift < 0:\n        maskpred2 = np.pad(maskpred1, [(0, abs(y_shift)), (0, abs(x_shift))], mode='constant', constant_values=0)\n        maskpred3 = maskpred2[abs(y_shift):, abs(x_shift):]\n    return maskpred3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfpred = pd.read_csv('../input/best-public-hubmap/submission_public_TPU.csv')\nTARGET_ID = 'afa5e8098'\ny_shift = -40\nx_shift = -24","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#get image sizes \n\ndfinfo = pd.read_csv('../input/hubmap-kidney-segmentation/HuBMAP-20-dataset_information.csv')\n\nsize_dict = image_size_dict(dfinfo.image_file, dfinfo.width_pixels, dfinfo.height_pixels)  #dict which contains image sizes mapped to id's\nmask_shape = size_dict.get(TARGET_ID)\n\ntaridx = dfpred[dfpred['id']==TARGET_ID].index.values[0]  #row of TARGET_ID in dfpred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"maskpred = rle2mask(dfpred.iloc[taridx]['predicted'], mask_shape)\n\nmaskpred1 = maskpred.copy()\nmaskpred1[maskpred1>0]=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mask_shifted = global_shift_mask(maskpred1, y_shift, x_shift)  #apply specified shift to mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"newrle = rle_encode_less_memory(mask_shifted)  #rle encode shifted mask\n\ndfpred.at[taridx, 'predicted'] = newrle\n\ndfsample = pd.read_csv('../input/hubmap-kidney-segmentation/sample_submission.csv')\n\nmydict = dict(zip(dfpred['id'], dfpred['predicted']))\n\ndfsample['predicted'] = dfsample['id'].map(mydict).fillna(dfsample['predicted'])\n\ndfsample = dfsample.replace(np.nan, '', regex=True)\n\ndfsample.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfsample.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}