{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Set up environment"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import math, re, os, gc, glob\nfrom tqdm import tqdm\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nimport numpy as np, cv2\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow import keras\nfrom keras.utils import to_categorical, Sequence\nfrom functools import partial\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nprint(\"Tensorflow version \" + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Set up Variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_PATH = \"../input/cassava-leaf-disease-classification\"\nGCS_PATH_STRATIFICATED =\"../input/cassava-recreate-stratificated-tfrecords\"\nREPLICAS = strategy.num_replicas_in_sync\nBATCH_SIZE = 128 \nAUG_BATCH = BATCH_SIZE\nIMAGE_SIZE = [512, 512]\nDIM = IMAGE_SIZE[0]\nCLASSES = ['0', '1', '2', '3', '4']\nEPOCHS = 40\nFOLDS = 5\ndebug = True\nTTA = 2\nWGTS = [1/FOLDS]*FOLDS\ncalc_oof = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(GCS_PATH + '/sample_submission.csv')\ntrain_df = pd.read_csv(GCS_PATH + '/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files_test  = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test_tfrecords/*.tfrec')))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"ROT_ = 180.0\nSHR_ = 2.0\nHZOOM_ = 8.0\nWZOOM_ = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform_mat(image, DIM=IMAGE_SIZE[0]):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 \n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n    \n    return tf.reshape(d, [DIM, DIM,3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Decode the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# this code will convert our test image data to a float32 \ndef to_float32(image, label):\n    return tf.cast(image, tf.float32), label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'target'                       : tf.io.FixedLenFeature([], tf.int64)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_unlabeled_tfrecord(example, return_image_name):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['image_name'] if return_image_name else '0'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAINING_FILENAMES = tf.io.gfile.glob('../input/cassava-leaf-disease-classification/train_tfrecords/' + '*train*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob('../input/cassava-leaf-disease-classification/test_tfrecords/' + 'ld_test*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = int( count_data_items(TRAINING_FILENAMES) * (FOLDS-1.)/FOLDS )\nNUM_VALIDATION_IMAGES = int( count_data_items(TRAINING_FILENAMES) * (1./FOLDS) )\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    # Thanks to the dataset.prefetch(AUTO) statement in the following function this happens essentially for free on TPU. \n    # Data pipeline code is executed on the \"CPU\" part of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define data loading methods"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def onehot(image,label):\n    CLASSES = 5\n    return image,tf.one_hot(label,CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_image(img, augment=True, dim=DIM):    \n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.cast(img, tf.float32) / 255.0\n    \n    if augment:\n        img = transform_mat(img,DIM=DIM)\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_saturation(img, 0.7, 1.3)\n        img = tf.image.random_contrast(img, 0.8, 1.2)\n        img = tf.image.random_brightness(img, 0.1)\n                      \n    img = tf.reshape(img, [dim,dim, 3])\n            \n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_dataset(dataset, do_onehot=True):\n    dataset = dataset.batch(BATCH_SIZE)\n    if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTOTUNE) # we must use one hot like augmented train data\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTOTUNE) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(files, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_names=False, batch_size=16, dim=512):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTOTUNE)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTOTUNE)\n    else:\n        \n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n                    num_parallel_calls=AUTOTUNE)      \n    \n    ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment, dim=dim), \n                                               imgname_or_label), \n                num_parallel_calls=AUTOTUNE)\n    \n    ds = ds.batch(batch_size * REPLICAS)\n    ds = ds.prefetch(AUTOTUNE)\n    return ds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#  Define ImageGenerator"},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataGenerator(Sequence):\n    def __init__(self, path, list_IDs, labels, batch_size, img_size, img_channel):\n        self.path = path\n        self.list_IDs = list_IDs\n        self.labels = labels\n        self.batch_size = batch_size\n        self.img_size = img_size\n        self.img_channel = img_channel\n        self.indexes = np.arange(len(self.list_IDs))\n        \n    def __len__(self):\n        return int(np.floor(len(self.list_IDs)/self.batch_size))\n    \n    \n    def __getitem__(self, index):\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        list_IDs_temp = [self.list_IDs[k] for k in indexes]\n        X, y = self.__data_generation(list_IDs_temp)\n        return X, y\n\n    \n    def __data_generation(self, list_IDs_temp):\n        X = np.empty((self.batch_size, self.img_size, self.img_size, self.img_channel))\n        y = np.empty((self.batch_size, 5), dtype=int)\n        for i, ID in enumerate(list_IDs_temp):\n            data_file = cv2.imread(self.path+ID)\n            img = cv2.resize(data_file, (self.img_size, self.img_size))\n            X[i, ] = img\n            y[i, ] = self.labels[i]\n        X = X.astype('float32')\n        #X -= X.mean()\n        #X /= X.std()\n        X /= 255.\n        return X, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess_image(image_path, desired_size=DIM):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n    im = cv2.resize(img, (desired_size,desired_size), interpolation = cv2.INTER_AREA) \n    im = np.array(im)\n    return im","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nif os.path.exists('../input/cassava-leaf-disease-classification/test_images'):\n    \n    # do the same thing as the last cell but on the test\\holdout set\n    N = test_df.shape[0]\n    x_test = np.empty((N, IMAGE_SIZE[0], IMAGE_SIZE[1], 3), dtype=np.uint8)\n    for i, image_id in enumerate(tqdm(test_df['image_id'])):\n        #print(i,image_id)\n        x_test[i, :, :, :] = preprocess_image(\n            f'../input/cassava-leaf-disease-classification/test_images/{image_id}'\n        )\n       \nelse:\n    print(\"error: no image directory/files\")\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plt.imshow(x_test[0])\n#plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test_datagen.fit(x_test)\n#x_test_len = len(glob.glob('../input/cassava-leaf-disease-classification/test_images/*jpg'))\n#x_test_len","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test_datagen = tf.keras.preprocessing.image.ImageDataGenerator(\n#    featurewise_center=False, samplewise_center=False,\n#    featurewise_std_normalization=False, samplewise_std_normalization=False,\n#    zca_whitening=False, zca_epsilon=1e-06, rotation_range=180, width_shift_range=0.1,\n#    height_shift_range=0.1, brightness_range=None, shear_range=0.1, zoom_range=0.1,\n#    channel_shift_range=0.0, fill_mode='reflect', cval=0.0, horizontal_flip=True,\n#    vertical_flip=True, rescale=1/255., preprocessing_function=None,\n#    data_format=None, validation_split=0.0, dtype=None\n#)\n\n#test_generator = test_datagen.flow_from_dataframe(dataframe=test_df,\n#                                                  directory=\"../input/cassava-leaf-disease-classification/test_images/\",\n#                                                  x_col=\"image_id\",\n#                                                  y_col='label',\n#                                                  target_size=(DIM, DIM),\n#                                                  batch_size=1,\n#                                                  shuffle=False,\n#                                                  class_mode='raw'\n#                                                     )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_generator = DataGenerator('../input/cassava-leaf-disease-classification/'+'test_images/', test_df['image_id'], test_df['label'], 1, DIM, 3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building the model"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Copyright 2019 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n# pylint: disable=invalid-name\n\"\"\"EfficientNet models for Keras.\n\nReference paper:\n  - [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks]\n    (https://arxiv.org/abs/1905.11946) (ICML 2019)\n\"\"\"\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport copy\nimport math\nimport os\n\nfrom tensorflow.python.keras import backend\nfrom tensorflow.python.keras import layers\nfrom tensorflow.python.keras.applications import imagenet_utils\nfrom tensorflow.python.keras.engine import training\nfrom tensorflow.python.keras.utils import data_utils\nfrom tensorflow.python.keras.utils import layer_utils\nfrom tensorflow.python.util.tf_export import keras_export\n\n\nBASE_WEIGHTS_PATH = 'https://storage.googleapis.com/keras-applications/'\n\nWEIGHTS_HASHES = {\n    'b0': ('902e53a9f72be733fc0bcb005b3ebbac',\n           '50bc09e76180e00e4465e1a485ddc09d'),\n    'b1': ('1d254153d4ab51201f1646940f018540',\n           '74c4e6b3e1f6a1eea24c589628592432'),\n    'b2': ('b15cce36ff4dcbd00b6dd88e7857a6ad',\n           '111f8e2ac8aa800a7a99e3239f7bfb39'),\n    'b3': ('ffd1fdc53d0ce67064dc6a9c7960ede0',\n           'af6d107764bb5b1abb91932881670226'),\n    'b4': ('18c95ad55216b8f92d7e70b3a046e2fc',\n           'ebc24e6d6c33eaebbd558eafbeedf1ba'),\n    'b5': ('ace28f2a6363774853a83a0b21b9421a',\n           '38879255a25d3c92d5e44e04ae6cec6f'),\n    'b6': ('165f6e37dce68623721b423839de8be5',\n           '9ecce42647a20130c1f39a5d4cb75743'),\n    'b7': ('8c03f828fec3ef71311cd463b6759d99',\n           'cbcfe4450ddf6f3ad90b1b398090fe4a'),\n}\n\nDEFAULT_BLOCKS_ARGS = [{\n    'kernel_size': 3,\n    'repeats': 1,\n    'filters_in': 32,\n    'filters_out': 16,\n    'expand_ratio': 1,\n    'id_skip': True,\n    'strides': 1,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 3,\n    'repeats': 2,\n    'filters_in': 16,\n    'filters_out': 24,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 5,\n    'repeats': 2,\n    'filters_in': 24,\n    'filters_out': 40,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 3,\n    'repeats': 3,\n    'filters_in': 40,\n    'filters_out': 80,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 5,\n    'repeats': 3,\n    'filters_in': 80,\n    'filters_out': 112,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 1,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 5,\n    'repeats': 4,\n    'filters_in': 112,\n    'filters_out': 192,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 3,\n    'repeats': 1,\n    'filters_in': 192,\n    'filters_out': 320,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 1,\n    'se_ratio': 0.25\n}]\n\nCONV_KERNEL_INITIALIZER = {\n    'class_name': 'VarianceScaling',\n    'config': {\n        'scale': 2.0,\n        'mode': 'fan_out',\n        'distribution': 'truncated_normal'\n    }\n}\n\nDENSE_KERNEL_INITIALIZER = {\n    'class_name': 'VarianceScaling',\n    'config': {\n        'scale': 1. / 3.,\n        'mode': 'fan_out',\n        'distribution': 'uniform'\n    }\n}\n\n\ndef EfficientNet(\n    width_coefficient,\n    depth_coefficient,\n    default_size,\n    dropout_rate=0.2,\n    drop_connect_rate=0.2,\n    depth_divisor=8,\n    activation='swish',\n    blocks_args='default',\n    model_name='efficientnet',\n    include_top=True,\n    weights='imagenet',\n    input_tensor=None,\n    input_shape=None,\n    pooling=None,\n    classes=1000,\n    classifier_activation='softmax',\n):\n  \"\"\"Instantiates the EfficientNet architecture using given scaling coefficients.\n\n  Optionally loads weights pre-trained on ImageNet.\n  Note that the data format convention used by the model is\n  the one specified in your Keras config at `~/.keras/keras.json`.\n\n  Arguments:\n    width_coefficient: float, scaling coefficient for network width.\n    depth_coefficient: float, scaling coefficient for network depth.\n    default_size: integer, default input image size.\n    dropout_rate: float, dropout rate before final classifier layer.\n    drop_connect_rate: float, dropout rate at skip connections.\n    depth_divisor: integer, a unit of network width.\n    activation: activation function.\n    blocks_args: list of dicts, parameters to construct block modules.\n    model_name: string, model name.\n    include_top: whether to include the fully-connected\n        layer at the top of the network.\n    weights: one of `None` (random initialization),\n          'imagenet' (pre-training on ImageNet),\n          or the path to the weights file to be loaded.\n    input_tensor: optional Keras tensor\n        (i.e. output of `layers.Input()`)\n        to use as image input for the model.\n    input_shape: optional shape tuple, only to be specified\n        if `include_top` is False.\n        It should have exactly 3 inputs channels.\n    pooling: optional pooling mode for feature extraction\n        when `include_top` is `False`.\n        - `None` means that the output of the model will be\n            the 4D tensor output of the\n            last convolutional layer.\n        - `avg` means that global average pooling\n            will be applied to the output of the\n            last convolutional layer, and thus\n            the output of the model will be a 2D tensor.\n        - `max` means that global max pooling will\n            be applied.\n    classes: optional number of classes to classify images\n        into, only to be specified if `include_top` is True, and\n        if no `weights` argument is specified.\n    classifier_activation: A `str` or callable. The activation function to use\n        on the \"top\" layer. Ignored unless `include_top=True`. Set\n        `classifier_activation=None` to return the logits of the \"top\" layer.\n\n  Returns:\n    A `keras.Model` instance.\n\n  Raises:\n    ValueError: in case of invalid argument for `weights`,\n      or invalid input shape.\n    ValueError: if `classifier_activation` is not `softmax` or `None` when\n      using a pretrained top layer.\n  \"\"\"\n  if blocks_args == 'default':\n    blocks_args = DEFAULT_BLOCKS_ARGS\n\n  if not (weights in {'imagenet', None} or os.path.exists(weights)):\n    raise ValueError('The `weights` argument should be either '\n                     '`None` (random initialization), `imagenet` '\n                     '(pre-training on ImageNet), '\n                     'or the path to the weights file to be loaded.')\n\n  if weights == 'imagenet' and include_top and classes != 1000:\n    raise ValueError('If using `weights` as `\"imagenet\"` with `include_top`'\n                     ' as true, `classes` should be 1000')\n\n  # Determine proper input shape\n  input_shape = imagenet_utils.obtain_input_shape(\n      input_shape,\n      default_size=default_size,\n      min_size=32,\n      data_format=backend.image_data_format(),\n      require_flatten=include_top,\n      weights=weights)\n\n  if input_tensor is None:\n    img_input = layers.Input(shape=input_shape)\n  else:\n    if not backend.is_keras_tensor(input_tensor):\n      img_input = layers.Input(tensor=input_tensor, shape=input_shape)\n    else:\n      img_input = input_tensor\n\n  bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1\n\n  def round_filters(filters, divisor=depth_divisor):\n    \"\"\"Round number of filters based on depth multiplier.\"\"\"\n    filters *= width_coefficient\n    new_filters = max(divisor, int(filters + divisor / 2) // divisor * divisor)\n    # Make sure that round down does not go down by more than 10%.\n    if new_filters < 0.9 * filters:\n      new_filters += divisor\n    return int(new_filters)\n\n  def round_repeats(repeats):\n    \"\"\"Round number of repeats based on depth multiplier.\"\"\"\n    return int(math.ceil(depth_coefficient * repeats))\n\n  # Build stem\n  x = img_input\n  #x = layers.Rescaling(1. / 255.)(x)\n  x = layers.Normalization(axis=bn_axis)(x)\n\n  x = layers.ZeroPadding2D(\n      padding=imagenet_utils.correct_pad(x, 3),\n      name='stem_conv_pad')(x)\n  x = layers.Conv2D(\n      round_filters(32),\n      3,\n      strides=2,\n      padding='valid',\n      use_bias=False,\n      kernel_initializer=CONV_KERNEL_INITIALIZER,\n      name='stem_conv')(x)\n  x = layers.BatchNormalization(axis=bn_axis, name='stem_bn')(x)\n  x = layers.Activation(activation, name='stem_activation')(x)\n\n  # Build blocks\n  blocks_args = copy.deepcopy(blocks_args)\n\n  b = 0\n  blocks = float(sum(args['repeats'] for args in blocks_args))\n  for (i, args) in enumerate(blocks_args):\n    assert args['repeats'] > 0\n    # Update block input and output filters based on depth multiplier.\n    args['filters_in'] = round_filters(args['filters_in'])\n    args['filters_out'] = round_filters(args['filters_out'])\n\n    for j in range(round_repeats(args.pop('repeats'))):\n      # The first block needs to take care of stride and filter size increase.\n      if j > 0:\n        args['strides'] = 1\n        args['filters_in'] = args['filters_out']\n      x = block(\n          x,\n          activation,\n          drop_connect_rate * b / blocks,\n          name='block{}{}_'.format(i + 1, chr(j + 97)),\n          **args)\n      b += 1\n\n  # Build top\n  x = layers.Conv2D(\n      round_filters(1280),\n      1,\n      padding='same',\n      use_bias=False,\n      kernel_initializer=CONV_KERNEL_INITIALIZER,\n      name='top_conv')(x)\n  x = layers.BatchNormalization(axis=bn_axis, name='top_bn')(x)\n  x = layers.Activation(activation, name='top_activation')(x)\n  if include_top:\n    x = layers.GlobalAveragePooling2D(name='avg_pool')(x)\n    if dropout_rate > 0:\n      x = layers.Dropout(dropout_rate, name='top_dropout')(x)\n    imagenet_utils.validate_activation(classifier_activation, weights)\n    x = layers.Dense(\n        classes,\n        activation=classifier_activation,\n        kernel_initializer=DENSE_KERNEL_INITIALIZER,\n        name='predictions')(x)\n  else:\n    if pooling == 'avg':\n      x = layers.GlobalAveragePooling2D(name='avg_pool')(x)\n    elif pooling == 'max':\n      x = layers.GlobalMaxPooling2D(name='max_pool')(x)\n\n  # Ensure that the model takes into account\n  # any potential predecessors of `input_tensor`.\n  if input_tensor is not None:\n    inputs = layer_utils.get_source_inputs(input_tensor)\n  else:\n    inputs = img_input\n\n  # Create model.\n  model = training.Model(inputs, x, name=model_name)\n\n  # Load weights.\n  if weights == 'imagenet':\n    if include_top:\n      file_suffix = '.h5'\n      file_hash = WEIGHTS_HASHES[model_name[-2:]][0]\n    else:\n      file_suffix = '_notop.h5'\n      file_hash = WEIGHTS_HASHES[model_name[-2:]][1]\n    file_name = model_name + file_suffix\n    weights_path = data_utils.get_file(\n        file_name,\n        BASE_WEIGHTS_PATH + file_name,\n        cache_subdir='models',\n        file_hash=file_hash)\n    model.load_weights(weights_path)\n  elif weights is not None:\n    model.load_weights(weights)\n  return model\n\n\ndef block(inputs,\n          activation='swish',\n          drop_rate=0.,\n          name='',\n          filters_in=32,\n          filters_out=16,\n          kernel_size=3,\n          strides=1,\n          expand_ratio=1,\n          se_ratio=0.,\n          id_skip=True):\n  \"\"\"An inverted residual block.\n\n  Arguments:\n      inputs: input tensor.\n      activation: activation function.\n      drop_rate: float between 0 and 1, fraction of the input units to drop.\n      name: string, block label.\n      filters_in: integer, the number of input filters.\n      filters_out: integer, the number of output filters.\n      kernel_size: integer, the dimension of the convolution window.\n      strides: integer, the stride of the convolution.\n      expand_ratio: integer, scaling coefficient for the input filters.\n      se_ratio: float between 0 and 1, fraction to squeeze the input filters.\n      id_skip: boolean.\n\n  Returns:\n      output tensor for the block.\n  \"\"\"\n  bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1\n\n  # Expansion phase\n  filters = filters_in * expand_ratio\n  if expand_ratio != 1:\n    x = layers.Conv2D(\n        filters,\n        1,\n        padding='same',\n        use_bias=False,\n        kernel_initializer=CONV_KERNEL_INITIALIZER,\n        name=name + 'expand_conv')(\n            inputs)\n    x = layers.BatchNormalization(axis=bn_axis, name=name + 'expand_bn')(x)\n    x = layers.Activation(activation, name=name + 'expand_activation')(x)\n  else:\n    x = inputs\n\n  # Depthwise Convolution\n  if strides == 2:\n    x = layers.ZeroPadding2D(\n        padding=imagenet_utils.correct_pad(x, kernel_size),\n        name=name + 'dwconv_pad')(x)\n    conv_pad = 'valid'\n  else:\n    conv_pad = 'same'\n  x = layers.DepthwiseConv2D(\n      kernel_size,\n      strides=strides,\n      padding=conv_pad,\n      use_bias=False,\n      depthwise_initializer=CONV_KERNEL_INITIALIZER,\n      name=name + 'dwconv')(x)\n  x = layers.BatchNormalization(axis=bn_axis, name=name + 'bn')(x)\n  x = layers.Activation(activation, name=name + 'activation')(x)\n\n  # Squeeze and Excitation phase\n  if 0 < se_ratio <= 1:\n    filters_se = max(1, int(filters_in * se_ratio))\n    se = layers.GlobalAveragePooling2D(name=name + 'se_squeeze')(x)\n    se = layers.Reshape((1, 1, filters), name=name + 'se_reshape')(se)\n    se = layers.Conv2D(\n        filters_se,\n        1,\n        padding='same',\n        activation=activation,\n        kernel_initializer=CONV_KERNEL_INITIALIZER,\n        name=name + 'se_reduce')(\n            se)\n    se = layers.Conv2D(\n        filters,\n        1,\n        padding='same',\n        activation='sigmoid',\n        kernel_initializer=CONV_KERNEL_INITIALIZER,\n        name=name + 'se_expand')(se)\n    x = layers.multiply([x, se], name=name + 'se_excite')\n\n  # Output phase\n  x = layers.Conv2D(\n      filters_out,\n      1,\n      padding='same',\n      use_bias=False,\n      kernel_initializer=CONV_KERNEL_INITIALIZER,\n      name=name + 'project_conv')(x)\n  x = layers.BatchNormalization(axis=bn_axis, name=name + 'project_bn')(x)\n  if id_skip and strides == 1 and filters_in == filters_out:\n    if drop_rate > 0:\n      x = layers.Dropout(\n          drop_rate, noise_shape=(None, 1, 1, 1), name=name + 'drop')(x)\n    x = layers.add([x, inputs], name=name + 'add')\n  return x\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB0',\n              'keras.applications.EfficientNetB0')\ndef EfficientNetB0(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.0,\n      1.0,\n      224,\n      0.2,\n      model_name='efficientnetb0',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB1',\n              'keras.applications.EfficientNetB1')\ndef EfficientNetB1(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.0,\n      1.1,\n      240,\n      0.2,\n      model_name='efficientnetb1',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB2',\n              'keras.applications.EfficientNetB2')\ndef EfficientNetB2(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.1,\n      1.2,\n      260,\n      0.3,\n      model_name='efficientnetb2',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB3',\n              'keras.applications.EfficientNetB3')\ndef EfficientNetB3(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.2,\n      1.4,\n      300,\n      0.3,\n      model_name='efficientnetb3',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB4',\n              'keras.applications.EfficientNetB4')\ndef EfficientNetB4(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.4,\n      1.8,\n      380,\n      0.4,\n      model_name='efficientnetb4',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB5',\n              'keras.applications.EfficientNetB5')\ndef EfficientNetB5(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.6,\n      2.2,\n      456,\n      0.4,\n      model_name='efficientnetb5',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB6',\n              'keras.applications.EfficientNetB6')\ndef EfficientNetB6(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.8,\n      2.6,\n      528,\n      0.5,\n      model_name='efficientnetb6',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB7',\n              'keras.applications.EfficientNetB7')\ndef EfficientNetB7(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      2.0,\n      3.1,\n      600,\n      0.5,\n      model_name='efficientnetb7',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.preprocess_input')\ndef preprocess_input(x, data_format=None):  # pylint: disable=unused-argument\n  return x\n\n\n@keras_export('keras.applications.efficientnet.decode_predictions')\ndef decode_predictions(preds, top=5):\n  \"\"\"Decodes the prediction result from the model.\n\n  Arguments\n    preds: Numpy tensor encoding a batch of predictions.\n    top: Integer, how many top-guesses to return.\n\n  Returns\n    A list of lists of top class prediction tuples\n    `(class_name, class_description, score)`.\n    One list of tuples per sample in batch input.\n\n  Raises\n    ValueError: In case of invalid shape of the `preds` array (must be 2D).\n  \"\"\"\n  return imagenet_utils.decode_predictions(preds, top=top)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(weights='imagenet'):\n\n    inp = tf.keras.layers.Input(shape=(DIM,DIM,3))\n    base = EfficientNetB0(input_shape=(DIM,DIM,3),weights=None,include_top=False)\n    x = base(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x  = tf.keras.layers.Flatten()(x)\n    x = tf.keras.layers.Dense(5,activation='softmax')(x)\n    model = tf.keras.Model(inputs=inp, outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    loss = tf.keras.losses.CategoricalCrossentropy(\n    from_logits=False, label_smoothing=0.0001,\n    name='categorical_crossentropy'\n    )\n    model.compile(optimizer=opt,loss=loss,metrics=['categorical_accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = KFold(n_splits=FOLDS,shuffle=True,random_state=12)\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(5))):\n    if fold==(FOLDS-1):\n        idxTT = idxT; idxVV = idxV\n        print('### Using fold',fold,'for experiments')\n    print('Fold',fold,'has TRAIN:',idxT,'VALID:',idxV)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = np.zeros((test_df.shape[0],5))\n\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(5))):\n    print(); print('#'*25)\n    print('### FOLD',fold+1)\n    print('#'*25)\n\n    files_train = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train%.2i*.tfrec'%x for x in idxT])\n    files_valid = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train%.2i*.tfrec'%x for x in idxV])\n\n    NUM_TRAINING_IMAGES = int( count_data_items(files_train))\n    NUM_VALIDATION_IMAGES = int( count_data_items(files_valid) )\n    STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n    print('Dataset: {} training images, {} validation images,'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))\n\n    model = get_model()\n\n    print('Loading best model...')\n    model.load_weights('../input/cassavaefnetb0/fold-%i.h5'%fold)\n        \n    # PREDICT TEST USING TTA\n    print('Predicting Test with TTA...')\n    \n    tta_steps = TTA\n    predictions = []\n\n    for i in tqdm(range(tta_steps)):\n        preds = model.predict_generator(test_generator, verbose=1)\n        #preds = model.predict_generator(generator=test_generator, steps=test_df.shape[0], verbose=1)\n        predictions.append(preds)\n\n    pred += np.mean(predictions, axis=0)/FOLDS\n    \n    del model; z = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = get_dataset(files_test, augment=False, repeat=False, dim=IMAGE_SIZE[0],\n                 labeled=False, return_image_names=True)\n\nimage_names = np.array([img_name.numpy().decode(\"utf-8\") \n                        for img, img_name in iter(ds.unbatch())])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(pred)\nprediction = np.argmax(pred, axis=1)\n\n\ntest_df['label'] = prediction\ntest_df = test_df[[\"image_id\",\"label\"]]\ntest_df.to_csv('submission.csv',index=False)\ntest_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}