{"cells":[{"metadata":{},"cell_type":"markdown","source":"**birdcall_using_TPU_train**\nbirdcall identification Using TPU\n\nReferences:\n[CutMix and MixUp on GPU/TPU](https://www.kaggle.com/cdeotte/cutmix-and-mixup-on-gpu-tpu)\n\n[Getting started with 100+ flowers on TPU](https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu)\n\n[Triple Stratified KFold with TFRecords](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords)\n\n[Flower Classification focal loss [+0.98]](https://www.kaggle.com/afshiin/flower-classification-focal-loss-0-98)\n\n[1つの画像が複数のクラスに属する場合（Multi-label）の画像分類](https://qiita.com/koshian2/items/ab5e0c68a257585d7c6f)(Japanese language)\n\n[Kerasで評価関数にF1スコアを使う方法](https://blog.shikoan.com/keras-f1score/)(Japanese language)\n\n[tensorflow](https://github.com/tensorflow/tensorflow/blob/v2.2.0/tensorflow/python/keras/applications/efficientnet.py)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# **Libraries and Configurations**","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd, numpy as np, gc\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DEVICE = \"TPU\" # \"TPU\" or \"GPU\"\nIMG_SIZE = [128, 313]\nFOLDS = 5\nBATCH_SIZE = 512\nEPOCHS = [50]*FOLDS\nAUG_BATCH = BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Directories and Classes","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS; GCS_PATH3 = [None]*FOLDS; GCS_PATH4 = [None]*FOLDS; GCS_PATH5 = [None]*FOLDS\nfor i in range(FOLDS):\n    GCS_PATH[i] = KaggleDatasets().get_gcs_path('birdsongab')\n    GCS_PATH2[i] = KaggleDatasets().get_gcs_path('birdsongcf')\n    GCS_PATH3[i] = KaggleDatasets().get_gcs_path('birdsonggm')\n    GCS_PATH4[i] = KaggleDatasets().get_gcs_path('birdsongnr')\n    GCS_PATH5[i] = KaggleDatasets().get_gcs_path('birdsongsy')\n\nfiles_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/*.tfrec')))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset and Augmentation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [128, 313, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled = True, ordered = False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # use data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO) # returns a dataset of (image, label) pairs if labeled = True or (image, id) pair if labeld = False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    #image = tf.image.random_flip_left_right(image)\n    return image, label   \n\ndef get_training_dataset(dataset, do_aug=True):\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.batch(AUG_BATCH)\n    if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO) # note we put AFTER batching\n    dataset = dataset.unbatch()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(dataset, do_onehot=True):\n    dataset = dataset.batch(BATCH_SIZE)\n    if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTO) # we must use one hot like augmented train data\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = int( count_data_items(files_train) * (FOLDS-1.)/FOLDS )\nNUM_VALIDATION_IMAGES = int( count_data_items(files_train) * (1./FOLDS) )\n\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\n#print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def onehot(image,label):\n    CLASSES = 264\n    return image,tf.one_hot(label,CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mixup(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    #DIM = IMAGE_SIZE[0]\n    CLASSES = 264\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n        b = 1-a\n        #a = tf.sqrt(a)\n        #b = tf.sqrt(b)\n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append(b*img1 + a*img2)\n        # MAKE CUTMIX LABEL\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        \n        \n        labs.append(tf.math.minimum(b*lab1 + a*lab2,1))\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,128,313,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def augmentation(img):\n    image = tf.image.flip_left_right(img)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform(image,label):\n    # THIS FUNCTION APPLIES BOTH CUTMIX AND MIXUP\n    CLASSES = 264\n    SWITCH = 0.5\n    CUTMIX_PROB = 0.666\n    MIXUP_PROB = 0.\n    # FOR SWITCH PERCENT OF TIME WE DO CUTMIX AND (1-SWITCH) WE DO MIXUP\n    #image2, label2 = cutmix(image, label, CUTMIX_PROB)\n    image3, label3 = mixup(image, label, MIXUP_PROB)\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        imgs.append(augmentation(image[j,]))\n        labs.append(label3[j,])\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,128,313,3))\n    label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image4,label4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nrow = 2; col = 2;\nrow = min(row,AUG_BATCH//col)\nall_elements = get_training_dataset(load_dataset(files_train),do_aug=False).unbatch()\naugmented_element = all_elements.repeat().batch(AUG_BATCH).map(transform)\n\nfor (img,label) in augmented_element:\n    print(label)\n    plt.figure(figsize=(15,int(15*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build and Train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import Callback\ndef total_acc(y_true, y_pred):\n    pred = K.cast(K.greater_equal(y_pred, 0.5), \"float\")\n    flag = K.cast(K.equal(y_true, pred), \"float\")\n    return K.prod(flag, axis=-1)\n\ndef binary_acc(y_true, y_pred):\n    pred = K.cast(K.greater_equal(y_pred, 0.5), \"float\")\n    flag = K.cast(K.equal(y_true, pred), \"float\")\n    return K.mean(flag, axis=-1)\n\nclass F1Callback(Callback):\n    def __init__(self):\n        self.f1s = []\n\n    def on_epoch_end(self, epoch, logs):\n        eps = np.finfo(np.float32).eps\n        recall = logs[\"val_true_positives\"] / (logs[\"val_possible_positives\"] + eps)\n        precision = logs[\"val_true_positives\"] / (logs[\"val_predicted_positives\"] + eps)\n        f1 = 2*precision*recall / (precision+recall+eps)\n        print(\"f1_val (from log) =\", f1)\n        self.f1s.append(f1)\n\ndef true_positives(y_true, y_pred):\n    return K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n\ndef possible_positives(y_true, y_pred):\n    return K.sum(K.round(K.clip(y_true, 0, 1)))\n\ndef predicted_positives(y_true, y_pred):\n    return K.sum(K.round(K.clip(y_pred, 0, 1)))\n\ndef F1(y_true, y_pred):\n    TPFN = possible_positives(y_true, y_pred)\n    TPFP = predicted_positives(y_true, y_pred)\n    TP = true_positives(y_true, y_pred)\n    return  (TP * 2) / (TPFN + TPFP + K.epsilon())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f1cb = F1Callback()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def binary_loss(y_true, y_pred):\n    bce = K.binary_crossentropy(y_true, y_pred)\n    return K.sum(bce, axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend as K\n\nimport dill\n\n\ndef binary_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n\n        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\n    return binary_focal_loss_fixed\n\n\ndef categorical_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Softmax version of focal loss.\n           m\n      FL = ∑  -alpha * (1 - p_o,c)^gamma * y_o,c * log(p_o,c)\n          c=1\n      where m = number of classes, c = class and o = observation\n    Parameters:\n      alpha -- the same as weighing factor in balanced cross entropy\n      gamma -- focusing parameter for modulating factor (1-p)\n    Default value:\n      gamma -- 2.0 as mentioned in the paper\n      alpha -- 0.25 as mentioned in the paper\n    References:\n        Official paper: https://arxiv.org/pdf/1708.02002.pdf\n        https://www.tensorflow.org/api_docs/python/tf/keras/backend/categorical_crossentropy\n    Usage:\n     model.compile(loss=[categorical_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def categorical_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred: A tensor resulting from a softmax\n        :return: Output tensor.\n        \"\"\"\n        epsilon = K.epsilon()\n        # Scale predictions so that the class probas of each sample sum to 1\n        y_pred /= K.sum(y_pred+epsilon, axis=-1, keepdims=True)\n\n        # Clip the prediction value to prevent NaN's and Inf's\n        epsilon = K.epsilon()\n        y_pred = K.clip(y_pred, epsilon, 1. - epsilon)\n\n        # Calculate Cross Entropy\n        cross_entropy = -y_true * K.log(y_pred)\n\n        # Calculate Focal Loss\n        loss = alpha * K.pow(1 - y_pred, gamma) * cross_entropy\n\n        # Sum the losses in mini_batch\n        return K.sum(loss, axis=1)\n\n    return categorical_focal_loss_fixed\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Copyright 2019 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n# pylint: disable=invalid-name\n\"\"\"EfficientNet models for Keras.\n\nReference paper:\n  - [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks]\n    (https://arxiv.org/abs/1905.11946) (ICML 2019)\n\"\"\"\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport copy\nimport math\nimport os\n\nfrom tensorflow.python.keras import backend\nfrom tensorflow.python.keras import layers\nfrom tensorflow.python.keras.applications import imagenet_utils\nfrom tensorflow.python.keras.engine import training\nfrom tensorflow.python.keras.utils import data_utils\nfrom tensorflow.python.keras.utils import layer_utils\nfrom tensorflow.python.util.tf_export import keras_export\n\n\nBASE_WEIGHTS_PATH = 'https://storage.googleapis.com/keras-applications/'\n\nWEIGHTS_HASHES = {\n    'b0': ('902e53a9f72be733fc0bcb005b3ebbac',\n           '50bc09e76180e00e4465e1a485ddc09d'),\n    'b1': ('1d254153d4ab51201f1646940f018540',\n           '74c4e6b3e1f6a1eea24c589628592432'),\n    'b2': ('b15cce36ff4dcbd00b6dd88e7857a6ad',\n           '111f8e2ac8aa800a7a99e3239f7bfb39'),\n    'b3': ('ffd1fdc53d0ce67064dc6a9c7960ede0',\n           'af6d107764bb5b1abb91932881670226'),\n    'b4': ('18c95ad55216b8f92d7e70b3a046e2fc',\n           'ebc24e6d6c33eaebbd558eafbeedf1ba'),\n    'b5': ('ace28f2a6363774853a83a0b21b9421a',\n           '38879255a25d3c92d5e44e04ae6cec6f'),\n    'b6': ('165f6e37dce68623721b423839de8be5',\n           '9ecce42647a20130c1f39a5d4cb75743'),\n    'b7': ('8c03f828fec3ef71311cd463b6759d99',\n           'cbcfe4450ddf6f3ad90b1b398090fe4a'),\n}\n\nDEFAULT_BLOCKS_ARGS = [{\n    'kernel_size': 3,\n    'repeats': 1,\n    'filters_in': 32,\n    'filters_out': 16,\n    'expand_ratio': 1,\n    'id_skip': True,\n    'strides': 1,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 3,\n    'repeats': 2,\n    'filters_in': 16,\n    'filters_out': 24,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 5,\n    'repeats': 2,\n    'filters_in': 24,\n    'filters_out': 40,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 3,\n    'repeats': 3,\n    'filters_in': 40,\n    'filters_out': 80,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 5,\n    'repeats': 3,\n    'filters_in': 80,\n    'filters_out': 112,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 1,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 5,\n    'repeats': 4,\n    'filters_in': 112,\n    'filters_out': 192,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 3,\n    'repeats': 1,\n    'filters_in': 192,\n    'filters_out': 320,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 1,\n    'se_ratio': 0.25\n}]\n\nCONV_KERNEL_INITIALIZER = {\n    'class_name': 'VarianceScaling',\n    'config': {\n        'scale': 2.0,\n        'mode': 'fan_out',\n        'distribution': 'truncated_normal'\n    }\n}\n\nDENSE_KERNEL_INITIALIZER = {\n    'class_name': 'VarianceScaling',\n    'config': {\n        'scale': 1. / 3.,\n        'mode': 'fan_out',\n        'distribution': 'uniform'\n    }\n}\n\n\ndef EfficientNet(\n    width_coefficient,\n    depth_coefficient,\n    default_size,\n    dropout_rate=0.2,\n    drop_connect_rate=0.2,\n    depth_divisor=8,\n    activation='swish',\n    blocks_args='default',\n    model_name='efficientnet',\n    include_top=True,\n    weights='imagenet',\n    input_tensor=None,\n    input_shape=None,\n    pooling=None,\n    classes=1000,\n    classifier_activation='softmax',\n):\n  \"\"\"Instantiates the EfficientNet architecture using given scaling coefficients.\n\n  Optionally loads weights pre-trained on ImageNet.\n  Note that the data format convention used by the model is\n  the one specified in your Keras config at `~/.keras/keras.json`.\n\n  Arguments:\n    width_coefficient: float, scaling coefficient for network width.\n    depth_coefficient: float, scaling coefficient for network depth.\n    default_size: integer, default input image size.\n    dropout_rate: float, dropout rate before final classifier layer.\n    drop_connect_rate: float, dropout rate at skip connections.\n    depth_divisor: integer, a unit of network width.\n    activation: activation function.\n    blocks_args: list of dicts, parameters to construct block modules.\n    model_name: string, model name.\n    include_top: whether to include the fully-connected\n        layer at the top of the network.\n    weights: one of `None` (random initialization),\n          'imagenet' (pre-training on ImageNet),\n          or the path to the weights file to be loaded.\n    input_tensor: optional Keras tensor\n        (i.e. output of `layers.Input()`)\n        to use as image input for the model.\n    input_shape: optional shape tuple, only to be specified\n        if `include_top` is False.\n        It should have exactly 3 inputs channels.\n    pooling: optional pooling mode for feature extraction\n        when `include_top` is `False`.\n        - `None` means that the output of the model will be\n            the 4D tensor output of the\n            last convolutional layer.\n        - `avg` means that global average pooling\n            will be applied to the output of the\n            last convolutional layer, and thus\n            the output of the model will be a 2D tensor.\n        - `max` means that global max pooling will\n            be applied.\n    classes: optional number of classes to classify images\n        into, only to be specified if `include_top` is True, and\n        if no `weights` argument is specified.\n    classifier_activation: A `str` or callable. The activation function to use\n        on the \"top\" layer. Ignored unless `include_top=True`. Set\n        `classifier_activation=None` to return the logits of the \"top\" layer.\n\n  Returns:\n    A `keras.Model` instance.\n\n  Raises:\n    ValueError: in case of invalid argument for `weights`,\n      or invalid input shape.\n    ValueError: if `classifier_activation` is not `softmax` or `None` when\n      using a pretrained top layer.\n  \"\"\"\n  if blocks_args == 'default':\n    blocks_args = DEFAULT_BLOCKS_ARGS\n\n  if not (weights in {'imagenet', None} or os.path.exists(weights)):\n    raise ValueError('The `weights` argument should be either '\n                     '`None` (random initialization), `imagenet` '\n                     '(pre-training on ImageNet), '\n                     'or the path to the weights file to be loaded.')\n\n  if weights == 'imagenet' and include_top and classes != 1000:\n    raise ValueError('If using `weights` as `\"imagenet\"` with `include_top`'\n                     ' as true, `classes` should be 1000')\n\n  # Determine proper input shape\n  input_shape = imagenet_utils.obtain_input_shape(\n      input_shape,\n      default_size=default_size,\n      min_size=32,\n      data_format=backend.image_data_format(),\n      require_flatten=include_top,\n      weights=weights)\n\n  if input_tensor is None:\n    img_input = layers.Input(shape=input_shape)\n  else:\n    if not backend.is_keras_tensor(input_tensor):\n      img_input = layers.Input(tensor=input_tensor, shape=input_shape)\n    else:\n      img_input = input_tensor\n\n  bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1\n\n  def round_filters(filters, divisor=depth_divisor):\n    \"\"\"Round number of filters based on depth multiplier.\"\"\"\n    filters *= width_coefficient\n    new_filters = max(divisor, int(filters + divisor / 2) // divisor * divisor)\n    # Make sure that round down does not go down by more than 10%.\n    if new_filters < 0.9 * filters:\n      new_filters += divisor\n    return int(new_filters)\n\n  def round_repeats(repeats):\n    \"\"\"Round number of repeats based on depth multiplier.\"\"\"\n    return int(math.ceil(depth_coefficient * repeats))\n\n  # Build stem\n  x = img_input\n  #x = layers.Rescaling(1. / 255.)(x)\n  x = layers.Normalization(axis=bn_axis)(x)\n\n  x = layers.ZeroPadding2D(\n      padding=imagenet_utils.correct_pad(x, 3),\n      name='stem_conv_pad')(x)\n  x = layers.Conv2D(\n      round_filters(32),\n      3,\n      strides=2,\n      padding='valid',\n      use_bias=False,\n      kernel_initializer=CONV_KERNEL_INITIALIZER,\n      name='stem_conv')(x)\n  x = layers.BatchNormalization(axis=bn_axis, name='stem_bn')(x)\n  x = layers.Activation(activation, name='stem_activation')(x)\n\n  # Build blocks\n  blocks_args = copy.deepcopy(blocks_args)\n\n  b = 0\n  blocks = float(sum(args['repeats'] for args in blocks_args))\n  for (i, args) in enumerate(blocks_args):\n    assert args['repeats'] > 0\n    # Update block input and output filters based on depth multiplier.\n    args['filters_in'] = round_filters(args['filters_in'])\n    args['filters_out'] = round_filters(args['filters_out'])\n\n    for j in range(round_repeats(args.pop('repeats'))):\n      # The first block needs to take care of stride and filter size increase.\n      if j > 0:\n        args['strides'] = 1\n        args['filters_in'] = args['filters_out']\n      x = block(\n          x,\n          activation,\n          drop_connect_rate * b / blocks,\n          name='block{}{}_'.format(i + 1, chr(j + 97)),\n          **args)\n      b += 1\n\n  # Build top\n  x = layers.Conv2D(\n      round_filters(1280),\n      1,\n      padding='same',\n      use_bias=False,\n      kernel_initializer=CONV_KERNEL_INITIALIZER,\n      name='top_conv')(x)\n  x = layers.BatchNormalization(axis=bn_axis, name='top_bn')(x)\n  x = layers.Activation(activation, name='top_activation')(x)\n  if include_top:\n    x = layers.GlobalAveragePooling2D(name='avg_pool')(x)\n    if dropout_rate > 0:\n      x = layers.Dropout(dropout_rate, name='top_dropout')(x)\n    imagenet_utils.validate_activation(classifier_activation, weights)\n    x = layers.Dense(\n        classes,\n        activation=classifier_activation,\n        kernel_initializer=DENSE_KERNEL_INITIALIZER,\n        name='predictions')(x)\n  else:\n    if pooling == 'avg':\n      x = layers.GlobalAveragePooling2D(name='avg_pool')(x)\n    elif pooling == 'max':\n      x = layers.GlobalMaxPooling2D(name='max_pool')(x)\n\n  # Ensure that the model takes into account\n  # any potential predecessors of `input_tensor`.\n  if input_tensor is not None:\n    inputs = layer_utils.get_source_inputs(input_tensor)\n  else:\n    inputs = img_input\n\n  # Create model.\n  model = training.Model(inputs, x, name=model_name)\n\n  # Load weights.\n  if weights == 'imagenet':\n    if include_top:\n      file_suffix = '.h5'\n      file_hash = WEIGHTS_HASHES[model_name[-2:]][0]\n    else:\n      file_suffix = '_notop.h5'\n      file_hash = WEIGHTS_HASHES[model_name[-2:]][1]\n    file_name = model_name + file_suffix\n    weights_path = data_utils.get_file(\n        file_name,\n        BASE_WEIGHTS_PATH + file_name,\n        cache_subdir='models',\n        file_hash=file_hash)\n    model.load_weights(weights_path)\n  elif weights is not None:\n    model.load_weights(weights)\n  return model\n\n\ndef block(inputs,\n          activation='swish',\n          drop_rate=0.,\n          name='',\n          filters_in=32,\n          filters_out=16,\n          kernel_size=3,\n          strides=1,\n          expand_ratio=1,\n          se_ratio=0.,\n          id_skip=True):\n  \"\"\"An inverted residual block.\n\n  Arguments:\n      inputs: input tensor.\n      activation: activation function.\n      drop_rate: float between 0 and 1, fraction of the input units to drop.\n      name: string, block label.\n      filters_in: integer, the number of input filters.\n      filters_out: integer, the number of output filters.\n      kernel_size: integer, the dimension of the convolution window.\n      strides: integer, the stride of the convolution.\n      expand_ratio: integer, scaling coefficient for the input filters.\n      se_ratio: float between 0 and 1, fraction to squeeze the input filters.\n      id_skip: boolean.\n\n  Returns:\n      output tensor for the block.\n  \"\"\"\n  bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1\n\n  # Expansion phase\n  filters = filters_in * expand_ratio\n  if expand_ratio != 1:\n    x = layers.Conv2D(\n        filters,\n        1,\n        padding='same',\n        use_bias=False,\n        kernel_initializer=CONV_KERNEL_INITIALIZER,\n        name=name + 'expand_conv')(\n            inputs)\n    x = layers.BatchNormalization(axis=bn_axis, name=name + 'expand_bn')(x)\n    x = layers.Activation(activation, name=name + 'expand_activation')(x)\n  else:\n    x = inputs\n\n  # Depthwise Convolution\n  if strides == 2:\n    x = layers.ZeroPadding2D(\n        padding=imagenet_utils.correct_pad(x, kernel_size),\n        name=name + 'dwconv_pad')(x)\n    conv_pad = 'valid'\n  else:\n    conv_pad = 'same'\n  x = layers.DepthwiseConv2D(\n      kernel_size,\n      strides=strides,\n      padding=conv_pad,\n      use_bias=False,\n      depthwise_initializer=CONV_KERNEL_INITIALIZER,\n      name=name + 'dwconv')(x)\n  x = layers.BatchNormalization(axis=bn_axis, name=name + 'bn')(x)\n  x = layers.Activation(activation, name=name + 'activation')(x)\n\n  # Squeeze and Excitation phase\n  if 0 < se_ratio <= 1:\n    filters_se = max(1, int(filters_in * se_ratio))\n    se = layers.GlobalAveragePooling2D(name=name + 'se_squeeze')(x)\n    se = layers.Reshape((1, 1, filters), name=name + 'se_reshape')(se)\n    se = layers.Conv2D(\n        filters_se,\n        1,\n        padding='same',\n        activation=activation,\n        kernel_initializer=CONV_KERNEL_INITIALIZER,\n        name=name + 'se_reduce')(\n            se)\n    se = layers.Conv2D(\n        filters,\n        1,\n        padding='same',\n        activation='sigmoid',\n        kernel_initializer=CONV_KERNEL_INITIALIZER,\n        name=name + 'se_expand')(se)\n    x = layers.multiply([x, se], name=name + 'se_excite')\n\n  # Output phase\n  x = layers.Conv2D(\n      filters_out,\n      1,\n      padding='same',\n      use_bias=False,\n      kernel_initializer=CONV_KERNEL_INITIALIZER,\n      name=name + 'project_conv')(x)\n  x = layers.BatchNormalization(axis=bn_axis, name=name + 'project_bn')(x)\n  if id_skip and strides == 1 and filters_in == filters_out:\n    if drop_rate > 0:\n      x = layers.Dropout(\n          drop_rate, noise_shape=(None, 1, 1, 1), name=name + 'drop')(x)\n    x = layers.add([x, inputs], name=name + 'add')\n  return x\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB0',\n              'keras.applications.EfficientNetB0')\ndef EfficientNetB0(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.0,\n      1.0,\n      224,\n      0.2,\n      model_name='efficientnetb0',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB1',\n              'keras.applications.EfficientNetB1')\ndef EfficientNetB1(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.0,\n      1.1,\n      240,\n      0.2,\n      model_name='efficientnetb1',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB2',\n              'keras.applications.EfficientNetB2')\ndef EfficientNetB2(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.1,\n      1.2,\n      260,\n      0.3,\n      model_name='efficientnetb2',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB3',\n              'keras.applications.EfficientNetB3')\ndef EfficientNetB3(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.2,\n      1.4,\n      300,\n      0.3,\n      model_name='efficientnetb3',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB4',\n              'keras.applications.EfficientNetB4')\ndef EfficientNetB4(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.4,\n      1.8,\n      380,\n      0.4,\n      model_name='efficientnetb4',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB5',\n              'keras.applications.EfficientNetB5')\ndef EfficientNetB5(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.6,\n      2.2,\n      456,\n      0.4,\n      model_name='efficientnetb5',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB6',\n              'keras.applications.EfficientNetB6')\ndef EfficientNetB6(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.8,\n      2.6,\n      528,\n      0.5,\n      model_name='efficientnetb6',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB7',\n              'keras.applications.EfficientNetB7')\ndef EfficientNetB7(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      2.0,\n      3.1,\n      600,\n      0.5,\n      model_name='efficientnetb7',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.preprocess_input')\ndef preprocess_input(x, data_format=None):  # pylint: disable=unused-argument\n  return x\n\n\n@keras_export('keras.applications.efficientnet.decode_predictions')\ndef decode_predictions(preds, top=5):\n  \"\"\"Decodes the prediction result from the model.\n\n  Arguments\n    preds: Numpy tensor encoding a batch of predictions.\n    top: Integer, how many top-guesses to return.\n\n  Returns\n    A list of lists of top class prediction tuples\n    `(class_name, class_description, score)`.\n    One list of tuples per sample in batch input.\n\n  Raises\n    ValueError: In case of invalid shape of the `preds` array (must be 2D).\n  \"\"\"\n  return imagenet_utils.decode_predictions(preds, top=top)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model():\n    inp = tf.keras.layers.Input(shape=(128,313,3))\n    #base = tf.keras.applications.Xception(include_top=False, weights='imagenet')\n    base = EfficientNetB3(include_top=False,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=[128,313,3])\n    x = base(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(264,activation='sigmoid')(x)\n    model = tf.keras.Model(inputs=inp,outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    model.compile(optimizer=opt,loss=binary_focal_loss(),metrics=[total_acc, F1,true_positives,possible_positives,predicted_positives])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr_callback(batch_size=8):\n    lr_start   = 0.0005\n    lr_max     = 0.001\n    lr_min     = 0.00001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 5\n    lr_decay   = 0.9\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = KFold(n_splits=FOLDS,shuffle=True,random_state=12)\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(5))):\n    if fold==(FOLDS-1):\n        idxTT = idxT; idxVV = idxV\n        print('### Using fold',fold,'for experiments')\n    print('Fold',fold,'has TRAIN:',idxT,'VALID:',idxV)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold in range(FOLDS):\n    if fold>0:\n        break;\n    # REPEAT SAME FOLD OVER AND OVER\n    idxT = idxTT\n    idxV = idxVV\n    \n    # DISPLAY FOLD INFO\n    if DEVICE=='TPU':\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print('#'*25); print('#### EXPERIMENT',fold+1)\n\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train-*-%.2i*.tfrec'%x for x in idxT])\n    files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train-*-%.2i*.tfrec'%x for x in idxT])\n    files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train-*-%.2i*.tfrec'%x for x in idxT])\n    files_train += tf.io.gfile.glob([GCS_PATH4[fold] + '/train-*-%.2i*.tfrec'%x for x in idxT])\n    files_train += tf.io.gfile.glob([GCS_PATH5[fold] + '/train-*-%.2i*.tfrec'%x for x in idxT])\n    print('#### all trains',len(files_train))\n        \n    files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train-*-%.2i*.tfrec'%x for x in idxV])\n    files_valid += tf.io.gfile.glob([GCS_PATH2[fold] + '/train-*-%.2i*.tfrec'%x for x in idxV])\n    files_valid += tf.io.gfile.glob([GCS_PATH3[fold] + '/train-*-%.2i*.tfrec'%x for x in idxV])\n    files_valid += tf.io.gfile.glob([GCS_PATH4[fold] + '/train-*-%.2i*.tfrec'%x for x in idxV])\n    files_valid += tf.io.gfile.glob([GCS_PATH5[fold] + '/train-*-%.2i*.tfrec'%x for x in idxV])\n    print('#### all valids',len(files_valid))\n    \n    NUM_TRAINING_IMAGES = int( count_data_items(files_train))\n    NUM_VALIDATION_IMAGES = int( count_data_items(files_valid) )\n    STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n    print('Dataset: {} training images, {} validation images,'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))\n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():\n        model = build_model()\n        \n    # SAVE BEST MODEL EACH FOLD\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n    es = tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto',\n        baseline=None, restore_best_weights=False\n    )\n    # TRAIN\n    train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': files_train}).loc[:]['TRAINING_FILENAMES']), labeled = True)\n    val_dataset = load_dataset(list(pd.DataFrame({'VALIDATION_FILENAMES': files_valid}).loc[:]['VALIDATION_FILENAMES']), labeled = True, ordered = True)\n    print(model.summary())\n    print('Training...')\n    history = model.fit(\n            get_training_dataset(train_dataset), \n            steps_per_epoch = STEPS_PER_EPOCH,\n            epochs = EPOCHS[fold],\n            callbacks = [sv,get_lr_callback(BATCH_SIZE),f1cb],\n            validation_data = get_validation_dataset(val_dataset),\n            verbose=2\n        )\n    model.save_weights('fold-%if.h5'%fold)\n        \n    del model; z = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}