{"cells":[{"metadata":{},"cell_type":"markdown","source":"# HUBMAP TPU Train Phase"},{"metadata":{},"cell_type":"markdown","source":"### References\n\n\n[Getting Started: TPUs + Cassava Leaf Disease](https://www.kaggle.com/jessemostipak/getting-started-tpus-cassava-leaf-disease)\n\n[CutMix and MixUp on GPU/TPU](https://www.kaggle.com/cdeotte/cutmix-and-mixup-on-gpu-tpu)\n\n[Getting started with 100+ flowers on TPU](https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu)\n\n[Triple Stratified KFold with TFRecords](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords)\n\nAnd will add accordingly...\n"},{"metadata":{},"cell_type":"markdown","source":"### other notebooks\n\n[Make Tfrecords of 512x512 or other tiles](https://www.kaggle.com/itsuki9180/make-tfrecords-of-512x512-or-other-tiles)\n\nHUBMAP TPU Train Phase (This notebook)\n\n[HUBMAP GPU Inference Phase](https://www.kaggle.com/itsuki9180/hubmap-gpu-inference-phase)\n"},{"metadata":{},"cell_type":"markdown","source":"# Set up environment"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import math, re, os, gc\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import Model, load_model\nfrom keras.utils.generic_utils import get_custom_objects\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow import keras\nfrom functools import partial\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nprint(\"Tensorflow version \" + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_PATH = KaggleDatasets().get_gcs_path('hubmaptfrecords512')\n\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nAUG_BATCH = BATCH_SIZE\nIMAGE_SIZE = [512, 512]\nDIM = IMAGE_SIZE[0]\n\nEPOCHS = 40\nEFUN = 3\nFOLDS = 4\nPHASE = 'train'\ndebug = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define TF functions for TPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"# this code will convert our test image data to a float32 \ndef to_float32(image, label):\n    return tf.cast(image, tf.float32), tf.cast(label, tf.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image_jpeg(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image_png(image):\n    image = tf.image.decode_png(image, channels=1)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [*IMAGE_SIZE, 1])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"mask\": tf.io.FixedLenFeature([], tf.string),\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"mask\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image_jpeg(example['image'])\n    mask = decode_image_png(example['mask'])\n    image = tf.cast(image, tf.float32)\n    mask = tf.cast(mask, tf.float32)\n    if labeled:\n        label = mask#tf.cast(example['mask'], tf.float32)\n        return image, label\n    idnum = mask\n    return image, idnum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if strategy.num_replicas_in_sync==1:\n    TRAINING_FILENAMES = tf.io.gfile.glob('../input/hubmaptfrecords512/' + 'train*.tfrec')\nelse:\n    TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = int( count_data_items(TRAINING_FILENAMES) * (FOLDS-1.)/FOLDS )\nNUM_VALIDATION_IMAGES = int( count_data_items(TRAINING_FILENAMES) * (1./FOLDS) )\n\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\nprint('Dataset: {} training images, {} validation images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"def flip_lr(img, mask):\n    if tf.random.uniform(()) > 0.5:\n        img = tf.image.flip_left_right(img)\n        mask = tf.image.flip_left_right(mask)\n    return img, mask\n\ndef flip_ud(img, mask):\n    if tf.random.uniform(()) > 0.5:\n        img = tf.image.flip_up_down(img)\n        mask = tf.image.flip_up_down(mask)\n    return img, mask\n\ndef flip_rot(img, mask):\n    p = tf.random.uniform(())\n    if  p > 2./3:\n        img=tf.image.rot90(img,k=1)\n        mask=tf.image.rot90(mask,k=1)\n    elif p > 1./3:\n        img=tf.image.rot90(img,k=3)\n        mask=tf.image.rot90(mask,k=3)\n    return img, mask\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cutmix(image, mask, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = IMAGE_SIZE[0]\n    \n    imgs = []; masks = []\n    for j in range(AUG_BATCH):\n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        # MAKE CUTMIX MASK\n        one = mask[j,ya:yb,0:xa,:]\n        two = mask[k,ya:yb,xa:xb,:]\n        three = mask[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        msk = tf.concat([mask[j,0:ya,:,:],middle,mask[j,yb:DIM,:,:]],axis=0)\n        masks.append(msk)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    mask2 = tf.reshape(tf.stack(masks),(AUG_BATCH,DIM,DIM,1))\n    return image2,mask2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform(image,mask):\n    # THIS FUNCTION APPLIES BOTH CUTMIX AND MIXUP\n    DIM = IMAGE_SIZE[0]\n    SWITCH = 0.5\n    CUTMIX_PROB = 0.5\n    # FOR SWITCH PERCENT OF TIME WE DO CUTMIX AND (1-SWITCH) WE DO MIXUP\n    image1 = []\n    mask1 = []\n    for j in range(AUG_BATCH):\n        \n        #print(image,mask)\n        img, msk = image[j,], mask[j,]\n        img, msk = flip_lr(img,msk)\n        img, msk = flip_ud(img,msk)\n        img, msk = flip_rot(img,msk)\n        \n        #img,msk = transform_mat(image[j,],mask[j,])\n        img = tf.image.random_saturation(img, 0.7, 1.3)\n        img = tf.image.random_contrast(img, 0.8, 1.2)\n        img = tf.image.random_brightness(img, 0.1)\n        img = tf.image.random_hue(img, 0.15, seed=None)\n        image1.append(img)\n        mask1.append(msk)\n        \n    image1 = tf.reshape(tf.stack(image1),(AUG_BATCH,DIM,DIM,3))\n    mask1 = tf.reshape(tf.stack(mask1),(AUG_BATCH,DIM,DIM,1))\n    image2, mask2 = cutmix(image1, mask1, CUTMIX_PROB)\n    imgs = []; masks = []\n    for j in range(AUG_BATCH):\n        #P = tf.cast( tf.random.uniform([],0,1)<=SWITCH, tf.float32)\n        #imgs.append(P*image2[j,]+(1-P)*image3[j,])\n        imgs.append(image2[j,])\n        #labs.append(P*label2[j,]+(1-P)*label3[j,])\n        masks.append(mask2[j,])\n        \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    mask4 = tf.reshape(tf.stack(masks),(AUG_BATCH,DIM,DIM,1))\n    return to_float32(image4, mask4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset(dataset=TRAINING_FILENAMES, do_aug=True):\n    #print(dataset)\n    #dataset = load_dataset(dataset, labeled=True)  \n    #dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.batch(AUG_BATCH)\n    if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTOTUNE) # note we put AFTER batching\n    dataset = dataset.unbatch()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_dataset(dataset, do_onehot=True):\n    dataset = dataset.batch(BATCH_SIZE)\n    #if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTOTUNE) # we must use one hot like augmented train data\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTOTUNE) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Show Samples"},{"metadata":{"trusted":true},"cell_type":"code","source":"if debug:\n    row = 16; col = 2;\n    row = min(row,AUG_BATCH//col)\n    all_elements = get_training_dataset(load_dataset(TRAINING_FILENAMES),do_aug=False).unbatch()\n    augmented_element = all_elements.repeat().batch(AUG_BATCH).map(transform)\n\n    for (img,mask) in augmented_element:\n        plt.figure(figsize=(15,int(15*row/col)))\n        #for j in range(row*col):\n        i=0\n        j=1\n        while j<=row*col:\n            plt.subplot(row,col,j)\n            plt.axis('off')\n            plt.imshow(img[i,])\n            j+=1\n            plt.subplot(row,col,j)\n            j+=1\n            plt.axis('off')\n            msk = tf.reshape(mask[i,],[DIM,DIM])\n            plt.imshow(tf.stack([msk,msk,msk],axis=-1))\n            i+=1\n            \n        plt.show()\n        break\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    # Thanks to the dataset.prefetch(AUTO) statement in the following function this happens essentially for free on TPU. \n    # Data pipeline code is executed on the \"CPU\" part of the TPU while the TPU itself is computing gradients.\n    #image = tf.image.random_flip_left_right(image)\n    return image, label\ndef get_training_dataset(dataset=TRAINING_FILENAMES, do_aug=True):\n    #print(dataset)\n    #dataset = load_dataset(dataset, labeled=True)  \n    #dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.batch(AUG_BATCH)\n    if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTOTUNE) # note we put AFTER batching\n    dataset = dataset.unbatch()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EfficientUNet(using efficientnet as encoder)\n\nmost of codes from [zhoudaxia233/EfficientUnet](https://github.com/zhoudaxia233/EfficientUnet).\n\nand see also [qubvel/efficientnet](https://github.com/qubvel/efficientnet)."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import re\nfrom collections import namedtuple\nfrom tensorflow.keras import layers\nimport tensorflow.keras.backend as K\nimport tensorflow as tf\nimport math\nimport numpy as np\n\nGlobalParams = namedtuple('GlobalParams', ['batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate', 'num_classes',\n                                           'width_coefficient', 'depth_coefficient', 'depth_divisor', 'min_depth',\n                                           'drop_connect_rate'])\nGlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields)\n\nBlockArgs = namedtuple('BlockArgs', ['kernel_size', 'num_repeat', 'input_filters', 'output_filters', 'expand_ratio',\n                                     'id_skip', 'strides', 'se_ratio'])\nBlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields)\n\nIMAGENET_WEIGHTS = {\n\n    'efficientnet-b0': {\n        'name': 'efficientnet-b0_imagenet_1000.h5',\n        'url': 'https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b0_imagenet_1000.h5',\n        'md5': 'bca04d16b1b8a7c607b1152fe9261af7',\n    },\n\n    'efficientnet-b1': {\n        'name': 'efficientnet-b1_imagenet_1000.h5',\n        'url': 'https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b1_imagenet_1000.h5',\n        'md5': 'bd4a2b82f6f6bada74fc754553c464fc',\n    },\n\n    'efficientnet-b2': {\n        'name': 'efficientnet-b2_imagenet_1000.h5',\n        'url': 'https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b2_imagenet_1000.h5',\n        'md5': '45b28b26f15958bac270ab527a376999',\n    },\n\n    'efficientnet-b3': {\n        'name': 'efficientnet-b3_imagenet_1000.h5',\n        'url': 'https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b3_imagenet_1000.h5',\n        'md5': 'decd2c8a23971734f9d3f6b4053bf424',\n    },\n\n    'efficientnet-b4': {\n        'name': 'efficientnet-b4_imagenet_1000.h5',\n        'url': 'https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b4_imagenet_1000.h5',\n        'md5': '01df77157a86609530aeb4f1f9527949',\n    },\n\n    'efficientnet-b5': {\n        'name': 'efficientnet-b5_imagenet_1000.h5',\n        'url': 'https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b5_imagenet_1000.h5',\n        'md5': 'c31311a1a38b5111e14457145fccdf32',\n    }\n\n}\n\n\ndef round_filters(filters, global_params):\n    \"\"\"Round number of filters.\"\"\"\n    multiplier = global_params.width_coefficient\n    divisor = global_params.depth_divisor\n    min_depth = global_params.min_depth\n    if not multiplier:\n        return filters\n\n    filters *= multiplier\n    min_depth = min_depth or divisor\n    new_filters = max(min_depth, int(filters + divisor / 2) // divisor * divisor)\n    # Make sure that round down does not go down by more than 10%.\n    if new_filters < 0.9 * filters:\n        new_filters += divisor\n    return int(new_filters)\n\n\ndef round_repeats(repeats, global_params):\n    \"\"\"Round number of repeats.\"\"\"\n    multiplier = global_params.depth_coefficient\n    if not multiplier:\n        return repeats\n    return int(math.ceil(multiplier * repeats))\n\n\ndef get_efficientnet_params(model_name, override_params=None):\n    \"\"\"Get efficientnet params based on model name.\"\"\"\n    params_dict = {\n        # (width_coefficient, depth_coefficient, resolution, dropout_rate)\n        # Note: the resolution here is just for reference, its values won't be used.\n        'efficientnet-b0': (1.0, 1.0, 224, 0.2),\n        'efficientnet-b1': (1.0, 1.1, 240, 0.2),\n        'efficientnet-b2': (1.1, 1.2, 260, 0.3),\n        'efficientnet-b3': (1.2, 1.4, 300, 0.3),\n        'efficientnet-b4': (1.4, 1.8, 380, 0.4),\n        'efficientnet-b5': (1.6, 2.2, 456, 0.4),\n        'efficientnet-b6': (1.8, 2.6, 528, 0.5),\n        'efficientnet-b7': (2.0, 3.1, 600, 0.5),\n    }\n    if model_name not in params_dict.keys():\n        raise KeyError('There is no model named {}.'.format(model_name))\n\n    width_coefficient, depth_coefficient, _, dropout_rate = params_dict[model_name]\n\n    blocks_args = [\n        'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',\n        'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',\n        'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',\n        'r1_k3_s11_e6_i192_o320_se0.25',\n    ]\n    global_params = GlobalParams(\n        batch_norm_momentum=0.99,\n        batch_norm_epsilon=1e-3,\n        dropout_rate=dropout_rate,\n        drop_connect_rate=0.2,\n        num_classes=1000,\n        width_coefficient=width_coefficient,\n        depth_coefficient=depth_coefficient,\n        depth_divisor=8,\n        min_depth=None)\n\n    if override_params:\n        global_params = global_params._replace(**override_params)\n\n    decoder = BlockDecoder()\n    return decoder.decode(blocks_args), global_params\n\n\nclass BlockDecoder(object):\n    \"\"\"Block Decoder for readability.\"\"\"\n\n    @staticmethod\n    def _decode_block_string(block_string):\n        \"\"\"Gets a block through a string notation of arguments.\"\"\"\n        assert isinstance(block_string, str)\n        ops = block_string.split('_')\n        options = {}\n        for op in ops:\n            splits = re.split(r'(\\d.*)', op)\n            if len(splits) >= 2:\n                key, value = splits[:2]\n                options[key] = value\n\n        if 's' not in options or len(options['s']) != 2:\n            raise ValueError('Strides options should be a pair of integers.')\n\n        return BlockArgs(\n            kernel_size=int(options['k']),\n            num_repeat=int(options['r']),\n            input_filters=int(options['i']),\n            output_filters=int(options['o']),\n            expand_ratio=int(options['e']),\n            id_skip=('noskip' not in block_string),\n            se_ratio=float(options['se']) if 'se' in options else None,\n            strides=[int(options['s'][0]), int(options['s'][1])]\n        )\n\n    @staticmethod\n    def _encode_block_string(block):\n        \"\"\"Encodes a block to a string.\"\"\"\n        args = [\n            'r%d' % block.num_repeat,\n            'k%d' % block.kernel_size,\n            's%d%d' % (block.strides[0], block.strides[1]),\n            'e%s' % block.expand_ratio,\n            'i%d' % block.input_filters,\n            'o%d' % block.output_filters\n        ]\n        if 0 < block.se_ratio <= 1:\n            args.append('se%s' % block.se_ratio)\n        if block.id_skip is False:\n            args.append('noskip')\n        return '_'.join(args)\n\n    def decode(self, string_list):\n        \"\"\"Decodes a list of string notations to specify blocks inside the network.\n        Args:\n          string_list: a list of strings, each string is a notation of block.\n        Returns:\n          A list of namedtuples to represent blocks arguments.\n        \"\"\"\n        assert isinstance(string_list, list)\n        blocks_args = []\n        for block_string in string_list:\n            blocks_args.append(self._decode_block_string(block_string))\n        return blocks_args\n\n    def encode(self, blocks_args):\n        \"\"\"Encodes a list of Blocks to a list of strings.\n        Args:\n          blocks_args: A list of namedtuples to represent blocks arguments.\n        Returns:\n          a list of strings, each string is a notation of block.\n        \"\"\"\n        block_strings = []\n        for block in blocks_args:\n            block_strings.append(self._encode_block_string(block))\n        return block_strings\n\n\nclass Swish(layers.Layer):\n    def __init__(self, name=None, **kwargs):\n        super().__init__(name=name, **kwargs)\n\n    def call(self, inputs, **kwargs):\n        return tf.nn.swish(inputs)\n\n    def get_config(self):\n        config = super().get_config()\n        config['name'] = self.name\n        return config\n\n\ndef SEBlock(block_args, **kwargs):\n    num_reduced_filters = max(\n        1, int(block_args.input_filters * block_args.se_ratio))\n    filters = block_args.input_filters * block_args.expand_ratio\n\n    spatial_dims = [1, 2]\n\n    try:\n        block_name = kwargs['block_name']\n    except KeyError:\n        block_name = ''\n\n    def block(inputs):\n        x = inputs\n        x = layers.Lambda(lambda a: K.mean(a, axis=spatial_dims, keepdims=True))(x)\n        x = layers.Conv2D(\n            num_reduced_filters,\n            kernel_size=[1, 1],\n            strides=[1, 1],\n            kernel_initializer=conv_kernel_initializer,\n            padding='same',\n            name=block_name + 'se_reduce_conv2d',\n            use_bias=True\n        )(x)\n\n        x = Swish(name=block_name + 'se_swish')(x)\n\n        x = layers.Conv2D(\n            filters,\n            kernel_size=[1, 1],\n            strides=[1, 1],\n            kernel_initializer=conv_kernel_initializer,\n            padding='same',\n            name=block_name + 'se_expand_conv2d',\n            use_bias=True\n        )(x)\n\n        x = layers.Activation('sigmoid')(x)\n        out = layers.Multiply()([x, inputs])\n        return out\n\n    return block\n\n\nclass DropConnect(layers.Layer):\n\n    def __init__(self, drop_connect_rate, **kwargs):\n        super().__init__(**kwargs)\n        self.drop_connect_rate = drop_connect_rate\n\n    def call(self, inputs, **kwargs):\n        def drop_connect():\n            keep_prob = 1.0 - self.drop_connect_rate\n\n            # Compute drop_connect tensor\n            batch_size = tf.shape(inputs)[0]\n            random_tensor = keep_prob\n            random_tensor += tf.random.uniform([batch_size, 1, 1, 1], dtype=inputs.dtype)\n            binary_tensor = tf.floor(random_tensor)\n            output = tf.math.divide(inputs, keep_prob) * binary_tensor\n            return output\n\n        return K.in_train_phase(drop_connect(), inputs, training=None)\n\n    def get_config(self):\n        config = super().get_config()\n        config['drop_connect_rate'] = self.drop_connect_rate\n        return config\n\n\ndef conv_kernel_initializer(shape, dtype=K.floatx()):\n    \"\"\"Initialization for convolutional kernels.\n    The main difference with tf.variance_scaling_initializer is that\n    tf.variance_scaling_initializer uses a truncated normal with an uncorrected\n    standard deviation, whereas here we use a normal distribution. Similarly,\n    tf.contrib.layers.variance_scaling_initializer uses a truncated normal with\n    a corrected standard deviation.\n    Args:\n        shape: shape of variable\n        dtype: dtype of variable\n    Returns:\n        an initialization for the variable\n    \"\"\"\n    kernel_height, kernel_width, _, out_filters = shape\n    fan_out = int(kernel_height * kernel_width * out_filters)\n    return tf.random.normal(\n        shape, mean=0.0, stddev=np.sqrt(2.0 / fan_out), dtype=dtype)\n\n\ndef dense_kernel_initializer(shape, dtype=K.floatx()):\n    init_range = 1.0 / np.sqrt(shape[1])\n    return tf.random.uniform(shape, -init_range, init_range, dtype=dtype)\n\n\ndef MBConvBlock(block_args, global_params, idx, drop_connect_rate=None):\n    filters = block_args.input_filters * block_args.expand_ratio\n    batch_norm_momentum = global_params.batch_norm_momentum\n    batch_norm_epsilon = global_params.batch_norm_epsilon\n    has_se = (block_args.se_ratio is not None) and (0 < block_args.se_ratio <= 1)\n\n    block_name = 'blocks_' + str(idx) + '_'\n\n    def block(inputs):\n        x = inputs\n\n        # Expansion phase\n        if block_args.expand_ratio != 1:\n            expand_conv = layers.Conv2D(filters,\n                                        kernel_size=[1, 1],\n                                        strides=[1, 1],\n                                        kernel_initializer=conv_kernel_initializer,\n                                        padding='same',\n                                        use_bias=False,\n                                        name=block_name + 'expansion_conv2d'\n                                        )(x)\n            bn0 = layers.BatchNormalization(momentum=batch_norm_momentum,\n                                            epsilon=batch_norm_epsilon,\n                                            name=block_name + 'expansion_batch_norm')(expand_conv)\n\n            x = Swish(name=block_name + 'expansion_swish')(bn0)\n\n        # Depth-wise convolution phase\n        kernel_size = block_args.kernel_size\n        depthwise_conv = layers.DepthwiseConv2D(\n            [kernel_size, kernel_size],\n            strides=block_args.strides,\n            depthwise_initializer=conv_kernel_initializer,\n            padding='same',\n            use_bias=False,\n            name=block_name + 'depthwise_conv2d'\n        )(x)\n        bn1 = layers.BatchNormalization(momentum=batch_norm_momentum,\n                                        epsilon=batch_norm_epsilon,\n                                        name=block_name + 'depthwise_batch_norm'\n                                        )(depthwise_conv)\n        x = Swish(name=block_name + 'depthwise_swish')(bn1)\n\n        if has_se:\n            x = SEBlock(block_args, block_name=block_name)(x)\n\n        # Output phase\n        project_conv = layers.Conv2D(\n            block_args.output_filters,\n            kernel_size=[1, 1],\n            strides=[1, 1],\n            kernel_initializer=conv_kernel_initializer,\n            padding='same',\n            name=block_name + 'output_conv2d',\n            use_bias=False)(x)\n        x = layers.BatchNormalization(momentum=batch_norm_momentum,\n                                      epsilon=batch_norm_epsilon,\n                                      name=block_name + 'output_batch_norm'\n                                      )(project_conv)\n        if block_args.id_skip:\n            if all(\n                    s == 1 for s in block_args.strides\n            ) and block_args.input_filters == block_args.output_filters:\n                # only apply drop_connect if skip presents.\n                if drop_connect_rate:\n                    x = DropConnect(drop_connect_rate)(x)\n                x = layers.add([x, inputs])\n\n        return x\n\n    return block\n\n\ndef freeze_efficientunet_first_n_blocks(model, n):\n    mbblock_nr = 0\n    while True:\n        try:\n            model.get_layer('blocks_{}_output_batch_norm'.format(mbblock_nr))\n            mbblock_nr += 1\n        except ValueError:\n            break\n\n    all_block_names = ['blocks_{}_output_batch_norm'.format(i) for i in range(mbblock_nr)]\n    all_block_index = []\n    for idx, layer in enumerate(model.layers):\n        if layer.name == all_block_names[0]:\n            all_block_index.append(idx)\n            all_block_names.pop(0)\n            if len(all_block_names) == 0:\n                break\n    n_blocks = len(all_block_index)\n\n    if n <= 0:\n        print('n is less than or equal to 0, therefore no layer will be frozen.')\n        return\n    if n > n_blocks:\n        raise ValueError(\"There are {} blocks in total, n cannot be greater than {}.\".format(n_blocks, n_blocks))\n\n    idx_of_last_block_to_be_frozen = all_block_index[n - 1]\n    for layer in model.layers[:idx_of_last_block_to_be_frozen + 1]:\n        layer.trainable = False\n\n\ndef unfreeze_efficientunet(model):\n    for layer in model.layers:\n        layer.trainable = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from tensorflow.keras import models, layers\nfrom tensorflow.keras.utils import get_file\n\n__all__ = ['get_model_by_name', 'get_efficientnet_b0_encoder', 'get_efficientnet_b1_encoder',\n           'get_efficientnet_b2_encoder', 'get_efficientnet_b3_encoder', 'get_efficientnet_b4_encoder',\n           'get_efficientnet_b5_encoder', 'get_efficientnet_b6_encoder', 'get_efficientnet_b7_encoder']\n\n\ndef _efficientnet(input_shape, blocks_args_list, global_params):\n    batch_norm_momentum = global_params.batch_norm_momentum\n    batch_norm_epsilon = global_params.batch_norm_epsilon\n\n    # Stem part\n    model_input = layers.Input(shape=input_shape)\n    x = layers.Conv2D(\n        filters=round_filters(32, global_params),\n        kernel_size=[3, 3],\n        strides=[2, 2],\n        kernel_initializer=conv_kernel_initializer,\n        padding='same',\n        use_bias=False,\n        name='stem_conv2d'\n    )(model_input)\n\n    x = layers.BatchNormalization(\n        momentum=batch_norm_momentum,\n        epsilon=batch_norm_epsilon,\n        name='stem_batch_norm'\n    )(x)\n\n    x = Swish(name='stem_swish')(x)\n\n    # Blocks part\n    idx = 0\n    drop_rate = global_params.drop_connect_rate\n    n_blocks = sum([blocks_args.num_repeat for blocks_args in blocks_args_list])\n    drop_rate_dx = drop_rate / n_blocks\n\n    for blocks_args in blocks_args_list:\n        assert blocks_args.num_repeat > 0\n        # Update block input and output filters based on depth multiplier.\n        blocks_args = blocks_args._replace(\n            input_filters=round_filters(blocks_args.input_filters, global_params),\n            output_filters=round_filters(blocks_args.output_filters, global_params),\n            num_repeat=round_repeats(blocks_args.num_repeat, global_params)\n        )\n\n        # The first block needs to take care of stride and filter size increase.\n        x = MBConvBlock(blocks_args, global_params, idx, drop_connect_rate=drop_rate_dx * idx)(x)\n        idx += 1\n\n        if blocks_args.num_repeat > 1:\n            blocks_args = blocks_args._replace(input_filters=blocks_args.output_filters, strides=[1, 1])\n\n        for _ in range(blocks_args.num_repeat - 1):\n            x = MBConvBlock(blocks_args, global_params, idx, drop_connect_rate=drop_rate_dx * idx)(x)\n            idx += 1\n\n    # Head part\n    x = layers.Conv2D(\n        filters=round_filters(1280, global_params),\n        kernel_size=[1, 1],\n        strides=[1, 1],\n        kernel_initializer=conv_kernel_initializer,\n        padding='same',\n        use_bias=False,\n        name='head_conv2d'\n    )(x)\n\n    x = layers.BatchNormalization(\n        momentum=batch_norm_momentum,\n        epsilon=batch_norm_epsilon,\n        name='head_batch_norm'\n    )(x)\n\n    x = Swish(name='head_swish')(x)\n\n    x = layers.GlobalAveragePooling2D(name='global_average_pooling2d')(x)\n\n    if global_params.dropout_rate > 0:\n        x = layers.Dropout(global_params.dropout_rate)(x)\n\n    x = layers.Dense(\n        global_params.num_classes,\n        kernel_initializer=dense_kernel_initializer,\n        activation='softmax',\n        name='head_dense'\n    )(x)\n\n    model = models.Model(model_input, x)\n\n    return model\n\n\ndef get_model_by_name(model_name, input_shape, classes=1000, pretrained=False):\n    \"\"\"Get an EfficientNet model by its name.\n    \"\"\"\n    blocks_args, global_params = get_efficientnet_params(model_name, override_params={'num_classes': classes})\n    model = _efficientnet(input_shape, blocks_args, global_params)\n\n    try:\n        if pretrained==True:\n            weights = IMAGENET_WEIGHTS[model_name]\n            weights_path = get_file(\n                weights['name'],\n                weights['url'],\n                cache_subdir='models',\n                md5_hash=weights['md5'],\n            )\n            model.load_weights(weights_path)\n        elif type(pretrained) is str:\n            print(f\"loading {pretrained}...\")\n            model.load_weights(pretrained)\n    except KeyError as e:\n        print(\"NOTE: Currently model {} doesn't have pretrained weights, therefore a model with randomly initialized\"\n              \" weights is returned.\".format(e))\n\n    return model\n\n\ndef _get_efficientnet_encoder(model_name, input_shape, pretrained=False):\n    model = get_model_by_name(model_name, input_shape, pretrained=pretrained)\n    encoder = models.Model(model.input, model.get_layer('global_average_pooling2d').output)\n    encoder.layers.pop()  # remove GAP layer\n    return encoder\n\n\ndef get_efficientnet_b0_encoder(input_shape, pretrained=False):\n    return _get_efficientnet_encoder('efficientnet-b0', input_shape, pretrained=pretrained)\n\n\ndef get_efficientnet_b1_encoder(input_shape, pretrained=False):\n    return _get_efficientnet_encoder('efficientnet-b1', input_shape, pretrained=pretrained)\n\n\ndef get_efficientnet_b2_encoder(input_shape, pretrained=False):\n    return _get_efficientnet_encoder('efficientnet-b2', input_shape, pretrained=pretrained)\n\n\ndef get_efficientnet_b3_encoder(input_shape, pretrained=False):\n    return _get_efficientnet_encoder('efficientnet-b3', input_shape, pretrained=pretrained)\n\n\ndef get_efficientnet_b4_encoder(input_shape, pretrained=False):\n    return _get_efficientnet_encoder('efficientnet-b4', input_shape, pretrained=pretrained)\n\n\ndef get_efficientnet_b5_encoder(input_shape, pretrained=False):\n    return _get_efficientnet_encoder('efficientnet-b5', input_shape, pretrained=pretrained)\n\n\ndef get_efficientnet_b6_encoder(input_shape, pretrained=False):\n    return _get_efficientnet_encoder('efficientnet-b6', input_shape, pretrained=pretrained)\n\n\ndef get_efficientnet_b7_encoder(input_shape, pretrained=False):\n    return _get_efficientnet_encoder('efficientnet-b7', input_shape, pretrained=pretrained)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from tensorflow.keras.layers import *\nfrom tensorflow.keras import models\n\n\n__all__ = ['get_efficient_unet_b0', 'get_efficient_unet_b1', 'get_efficient_unet_b2', 'get_efficient_unet_b3',\n           'get_efficient_unet_b4', 'get_efficient_unet_b5', 'get_efficient_unet_b6', 'get_efficient_unet_b7',\n           'get_blocknr_of_skip_candidates']\n\n\ndef get_blocknr_of_skip_candidates(encoder, verbose=False):\n    \"\"\"\n    Get block numbers of the blocks which will be used for concatenation in the Unet.\n    :param encoder: the encoder\n    :param verbose: if set to True, the shape information of all blocks will be printed in the console\n    :return: a list of block numbers\n    \"\"\"\n    shapes = []\n    candidates = []\n    mbblock_nr = 0\n    while True:\n        try:\n            mbblock = encoder.get_layer('blocks_{}_output_batch_norm'.format(mbblock_nr)).output\n            shape = int(mbblock.shape[1]), int(mbblock.shape[2])\n            if shape not in shapes:\n                shapes.append(shape)\n                candidates.append(mbblock_nr)\n            if verbose:\n                print('blocks_{}_output_shape: {}'.format(mbblock_nr, shape))\n            mbblock_nr += 1\n        except ValueError:\n            break\n    return candidates\n\n\ndef DoubleConv(filters, kernel_size, initializer='glorot_uniform'):\n\n    def layer(x):\n\n        x = Conv2D(filters, kernel_size, padding='same', use_bias=False, kernel_initializer=initializer)(x)\n        x = BatchNormalization()(x)\n        x = Activation('relu')(x)\n        x = Conv2D(filters, kernel_size, padding='same', use_bias=False, kernel_initializer=initializer)(x)\n        x = BatchNormalization()(x)\n        x = Activation('relu')(x)\n\n        return x\n\n    return layer\n\n\ndef UpSampling2D_block(filters, kernel_size=(3, 3), upsample_rate=(2, 2), interpolation='bilinear',\n                       initializer='glorot_uniform', skip=None):\n    def layer(input_tensor):\n\n        x = UpSampling2D(size=upsample_rate, interpolation=interpolation)(input_tensor)\n\n        if skip is not None:\n            x = Concatenate()([x, skip])\n\n        x = DoubleConv(filters, kernel_size, initializer=initializer)(x)\n\n        return x\n    return layer\n\n\ndef Conv2DTranspose_block(filters, kernel_size=(3, 3), transpose_kernel_size=(2, 2), upsample_rate=(2, 2),\n                          initializer='glorot_uniform', skip=None):\n    def layer(input_tensor):\n\n        x = Conv2DTranspose(filters, transpose_kernel_size, strides=upsample_rate, padding='same')(input_tensor)\n\n        if skip is not None:\n            x = Concatenate()([x, skip])\n\n        x = DoubleConv(filters, kernel_size, initializer=initializer)(x)\n\n        return x\n\n    return layer\n\n\n# noinspection PyTypeChecker\ndef _get_efficient_unet(encoder, out_channels=2, block_type='upsampling', concat_input=True):\n    MBConvBlocks = []\n\n    skip_candidates = get_blocknr_of_skip_candidates(encoder)\n\n    for mbblock_nr in skip_candidates:\n        mbblock = encoder.get_layer('blocks_{}_output_batch_norm'.format(mbblock_nr)).output\n        MBConvBlocks.append(mbblock)\n\n    # delete the last block since it won't be used in the process of concatenation\n    MBConvBlocks.pop()\n\n    input_ = encoder.input\n    head = encoder.get_layer('head_swish').output\n    blocks = [input_] + MBConvBlocks + [head]\n\n    if block_type == 'upsampling':\n        UpBlock = UpSampling2D_block\n    else:\n        UpBlock = Conv2DTranspose_block\n\n    o = blocks.pop()\n    o = UpBlock(512, initializer=conv_kernel_initializer, skip=blocks.pop())(o)\n    o = UpBlock(256, initializer=conv_kernel_initializer, skip=blocks.pop())(o)\n    o = UpBlock(128, initializer=conv_kernel_initializer, skip=blocks.pop())(o)\n    o = UpBlock(64, initializer=conv_kernel_initializer, skip=blocks.pop())(o)\n    if concat_input:\n        o = UpBlock(32, initializer=conv_kernel_initializer, skip=blocks.pop())(o)\n    else:\n        o = UpBlock(32, initializer=conv_kernel_initializer, skip=None)(o)\n    o = Conv2D(out_channels, (1, 1), padding='same', kernel_initializer=conv_kernel_initializer, activation=\"sigmoid\")(o)\n\n    model = models.Model(encoder.input, o)\n\n    return model\n\n\ndef get_efficient_unet_b0(input_shape, out_channels=2, pretrained=False, block_type='transpose', concat_input=True):\n    \"\"\"Get a Unet model with Efficient-B0 encoder\n    :param input_shape: shape of input (cannot have None element)\n    :param out_channels: the number of output channels\n    :param pretrained: True for ImageNet pretrained weights\n    :param block_type: \"upsampling\" to use UpSampling layer, otherwise use Conv2DTranspose layer\n    :param concat_input: if True, input image will be concatenated with the last conv layer\n    :return: an EfficientUnet_B0 model\n    \"\"\"\n    encoder = get_efficientnet_b0_encoder(input_shape, pretrained=pretrained)\n    model = _get_efficient_unet(encoder, out_channels, block_type=block_type, concat_input=concat_input)\n    return model\n\n\ndef get_efficient_unet_b1(input_shape, out_channels=2, pretrained=False, block_type='transpose', concat_input=True):\n    \"\"\"Get a Unet model with Efficient-B1 encoder\n    :param input_shape: shape of input (cannot have None element)\n    :param out_channels: the number of output channels\n    :param pretrained: True for ImageNet pretrained weights\n    :param block_type: \"upsampling\" to use UpSampling layer, otherwise use Conv2DTranspose layer\n    :param concat_input: if True, input image will be concatenated with the last conv layer\n    :return: an EfficientUnet_B1 model\n    \"\"\"\n    encoder = get_efficientnet_b1_encoder(input_shape, pretrained=pretrained)\n    model = _get_efficient_unet(encoder, out_channels, block_type=block_type, concat_input=concat_input)\n    return model\n\n\ndef get_efficient_unet_b2(input_shape, out_channels=2, pretrained=False, block_type='transpose', concat_input=True):\n    \"\"\"Get a Unet model with Efficient-B2 encoder\n    :param input_shape: shape of input (cannot have None element)\n    :param out_channels: the number of output channels\n    :param pretrained: True for ImageNet pretrained weights\n    :param block_type: \"upsampling\" to use UpSampling layer, otherwise use Conv2DTranspose layer\n    :param concat_input: if True, input image will be concatenated with the last conv layer\n    :return: an EfficientUnet_B2 model\n    \"\"\"\n    encoder = get_efficientnet_b2_encoder(input_shape, pretrained=pretrained)\n    model = _get_efficient_unet(encoder, out_channels, block_type=block_type, concat_input=concat_input)\n    return model\n\n\ndef get_efficient_unet_b3(input_shape, out_channels=2, pretrained=False, block_type='transpose', concat_input=True):\n    \"\"\"Get a Unet model with Efficient-B3 encoder\n    :param input_shape: shape of input (cannot have None element)\n    :param out_channels: the number of output channels\n    :param pretrained: True for ImageNet pretrained weights\n    :param block_type: \"upsampling\" to use UpSampling layer, otherwise use Conv2DTranspose layer\n    :param concat_input: if True, input image will be concatenated with the last conv layer\n    :return: an EfficientUnet_B3 model\n    \"\"\"\n    encoder = get_efficientnet_b3_encoder(input_shape, pretrained=pretrained)\n    model = _get_efficient_unet(encoder, out_channels, block_type=block_type, concat_input=concat_input)\n    return model\n\n\ndef get_efficient_unet_b4(input_shape, out_channels=2, pretrained=False, block_type='transpose', concat_input=True):\n    \"\"\"Get a Unet model with Efficient-B4 encoder\n    :param input_shape: shape of input (cannot have None element)\n    :param out_channels: the number of output channels\n    :param pretrained: True for ImageNet pretrained weights\n    :param block_type: \"upsampling\" to use UpSampling layer, otherwise use Conv2DTranspose layer\n    :param concat_input: if True, input image will be concatenated with the last conv layer\n    :return: an EfficientUnet_B4 model\n    \"\"\"\n    encoder = get_efficientnet_b4_encoder(input_shape, pretrained=pretrained)\n    model = _get_efficient_unet(encoder, out_channels, block_type=block_type, concat_input=concat_input)\n    return model\n\n\ndef get_efficient_unet_b5(input_shape, out_channels=2, pretrained=False, block_type='transpose', concat_input=True):\n    \"\"\"Get a Unet model with Efficient-B5 encoder\n    :param input_shape: shape of input (cannot have None element)\n    :param out_channels: the number of output channels\n    :param pretrained: True for ImageNet pretrained weights\n    :param block_type: \"upsampling\" to use UpSampling layer, otherwise use Conv2DTranspose layer\n    :param concat_input: if True, input image will be concatenated with the last conv layer\n    :return: an EfficientUnet_B5 model\n    \"\"\"\n    encoder = get_efficientnet_b5_encoder(input_shape, pretrained=pretrained)\n    model = _get_efficient_unet(encoder, out_channels, block_type=block_type, concat_input=concat_input)\n    return model\n\n\ndef get_efficient_unet_b6(input_shape, out_channels=2, pretrained=False, block_type='transpose', concat_input=True):\n    \"\"\"Get a Unet model with Efficient-B6 encoder\n    :param input_shape: shape of input (cannot have None element)\n    :param out_channels: the number of output channels\n    :param pretrained: True for ImageNet pretrained weights\n    :param block_type: \"upsampling\" to use UpSampling layer, otherwise use Conv2DTranspose layer\n    :param concat_input: if True, input image will be concatenated with the last conv layer\n    :return: an EfficientUnet_B6 model\n    \"\"\"\n    encoder = get_efficientnet_b6_encoder(input_shape, pretrained=pretrained)\n    model = _get_efficient_unet(encoder, out_channels, block_type=block_type, concat_input=concat_input)\n    return model\n\n\ndef get_efficient_unet_b7(input_shape, out_channels=2, pretrained=False, block_type='transpose', concat_input=True):\n    \"\"\"Get a Unet model with Efficient-B7 encoder\n    :param input_shape: shape of input (cannot have None element)\n    :param out_channels: the number of output channels\n    :param pretrained: True for ImageNet pretrained weights\n    :param block_type: \"upsampling\" to use UpSampling layer, otherwise use Conv2DTranspose layer\n    :param concat_input: if True, input image will be concatenated with the last conv layer\n    :return: an EfficientUnet_B7 model\n    \"\"\"\n    encoder = get_efficientnet_b7_encoder(input_shape, pretrained=pretrained)\n    model = _get_efficient_unet(encoder, out_channels, block_type=block_type, concat_input=concat_input)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EFNS = [get_efficient_unet_b0, get_efficient_unet_b1, get_efficient_unet_b2, get_efficient_unet_b3, \n        get_efficient_unet_b4,get_efficient_unet_b5, get_efficient_unet_b6, get_efficient_unet_b7]\ndef EfficientUnet(efun=0):\n    # noisy-student\n    model = EFNS[efun]((DIM, DIM, 3),out_channels=1, pretrained=f'../input/qubvelefficientnetweights/efficientnet-b{efun}_noisy-student.h5', block_type='transpose', concat_input=True)\n    # if you want to use imagenet, fix comment-out\n    # model = EFNS[efun]((DIM, DIM, 3),out_channels=1, pretrained=True, block_type='transpose', concat_input=True)\n    \n    #model.summary()\n    return model    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define loss function"},{"metadata":{"trusted":true},"cell_type":"code","source":"# metrics and loss functions\nsmooth = 1.\n\ndef dice_coef(y_true, y_pred):\n    y_true_f = tf.reshape(y_true,[-1])\n    y_pred_f = tf.reshape(y_pred,[-1])\n    intersection = tf.reduce_sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n\n\ndef dice_coef_loss(y_true, y_pred):\n    return 1.-dice_coef(y_true, y_pred)\n\n\ndef iou(y_true, y_pred):\n    intersection = tf.reduce_sum(y_true * y_pred)\n    union = tf.reduce_sum(y_true)+tf.reduce_sum(y_pred)-intersection\n    x = intersection/(union+1e-15)\n    return x\n\ndef tversky(y_true, y_pred, smooth=1, alpha=0.7):\n    y_true_pos = tf.reshape(y_true,[-1])\n    y_pred_pos = tf.reshape(y_pred,[-1])\n    true_pos = tf.reduce_sum(y_true_pos * y_pred_pos)\n    false_neg = tf.reduce_sum(y_true_pos * (1 - y_pred_pos))\n    false_pos = tf.reduce_sum((1 - y_true_pos) * y_pred_pos)\n    return (true_pos + smooth) / (true_pos + alpha * false_neg + (1 - alpha) * false_pos + smooth)\n\n\ndef tversky_loss(y_true, y_pred):\n    return 1 - tversky(y_true, y_pred)\n\ndef focal_tversky_loss(y_true, y_pred, gamma=0.75):\n    tv = tversky(y_true, y_pred)\n    return K.pow((1 - tv), gamma)\n\nget_custom_objects().update({\"dice\": dice_coef_loss})\nget_custom_objects().update({\"focal_tversky_loss\": focal_tversky_loss})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build EfficientUNet"},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = 1e-3\n\ndef get_model(efun=0):\n    with strategy.scope(): \n        model = EfficientUnet(efun)\n        opt = tf.keras.optimizers.Adam(lr)\n        metrics = [\"acc\", iou, dice_coef, tversky]\n        model.compile(loss=\"focal_tversky_loss\", optimizer=opt, metrics=metrics)\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr_callback(batch_size=8):\n    lr_start   = 5e-4\n    lr_max     = 1e-3\n    lr_min     = 1e-5\n    lr_ramp_ep = 5\n    lr_sus_ep  = 10\n    lr_decay   = 0.9\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = KFold(n_splits=FOLDS,shuffle=True,random_state=12)\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(8))):\n    if fold==(FOLDS-1):\n        idxTT = idxT; idxVV = idxV\n        print('### Using fold',fold,'for experiments')\n    print('Fold',fold,'has TRAIN:',idxT,'VALID:',idxV)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold,(idxT,idxV) in enumerate(skf.split(np.arange(8))):\n    tf.keras.backend.clear_session()\n    print(); print('#'*25)\n    print('### FOLD',fold+1)\n    print('#'*25)\n  \n    files_train = tf.io.gfile.glob([GCS_PATH + '/train%.2i*.tfrec'%x for x in idxT])\n    files_valid = tf.io.gfile.glob([GCS_PATH + '/train%.2i*.tfrec'%x for x in idxV])\n\n    NUM_TRAINING_IMAGES = int( count_data_items(files_train))\n    NUM_VALIDATION_IMAGES = int( count_data_items(files_valid) )\n    STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n    print('Dataset: {} training images, {} validation images,'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))\n\n    train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': files_train}).loc[:]['TRAINING_FILENAMES']), labeled = True)\n    val_dataset = load_dataset(list(pd.DataFrame({'VALIDATION_FILENAMES': files_valid}).loc[:]['VALIDATION_FILENAMES']), labeled = True, ordered = True)\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor=\"val_loss\", verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n    rlr = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor='val_loss', factor=0.5, patience=3, verbose=1, mode='auto',\n        min_delta=0., cooldown=0, min_lr=1e-5,)\n    es = tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto')\n    model = get_model(efun=EFUN)\n    history = model.fit(\n        get_training_dataset(train_dataset), \n        steps_per_epoch = STEPS_PER_EPOCH,\n        epochs = EPOCHS,\n        callbacks = [get_lr_callback(), sv, es],\n        validation_data = get_validation_dataset(val_dataset),\n        verbose=2\n    )\n    model.save_weights('fold-%if.h5'%fold)\n\n    \n    del model; z = gc.collect()\n    \n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}