{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Objectives\n* V15 - score improved\n* V35 - [KFold](https://www.kaggle.com/ragnar123/4-kfold-densenet201) and [rotation augmentation](https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96)\n* V46 - TensorBoard logging and [GridMask augmentation](https://www.kaggle.com/xiejialun/gridmask-data-augmentation-with-tensorflow)\n* V53 - efficientnet model\n* V69 - confusion matrix\n* V77 - memory cleanup\n* V83 - class weights (balanced)\n\nThis notebook is based on [Getting started with 100+ flowers on TPU](https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu)<br>\nFine tuning for model is done with SGD and Adam + LR scheduler.<br>\nTensorBoard logs located in output folder (CPU and GPU)"},{"metadata":{},"cell_type":"markdown","source":"# 1. Setup"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\n\n# NOTE: internet should be ON\nfrom kaggle_datasets import KaggleDatasets\nGCS_DS_PATH = KaggleDatasets().get_gcs_path()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. TPU detection and log"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport datetime\nimport gc\n\nprint('TensorFlow version: %s' % tf.__version__)\nprint('Keras version: %s' % tf.keras.__version__)\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. \n    # No parameters necessary if TPU_NAME environment variable is set. \n    # On Kaggle this is always the case.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    print(\"TPU is not available\")\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # default distribution strategy in Tensorflow. \n    # Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy() \n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# helper log function\nimport os\n\ndt_start = datetime.datetime.utcnow()\ndef print_and_log(string):\n    seconds = (datetime.datetime.utcnow() - dt_start).seconds\n    time_diff = \"%d:%02d\" % (seconds / 60, seconds % 60)\n    print(time_diff, string)\n    os.system(f'echo \\\"{time_diff}\\\" \\\"{string}\\\"')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Configuration"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# at 512,512 size, a GPU will run out of memory. Use the TPU\n# 224, 224 is VGG16 input\n# 299, 299 is inception v3 input\nIMAGE_SIZE = [512, 512] \nIMAGE_CHANNELS = 3\n\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\nGCS_PATH_SELECT = { # available image sizes\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\n# predictions on this dataset should be submitted for the competition\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TENSOR_BOARD = False # CPU/GPU\nVERBOSE = 0\nIMAGE_AUGMENTATION = True\nGRID_MASK = False\nUSE_CLASS_WEIGHTS = False\n\n# 'EfficientNetB7', 'InceptionV3', 'DenseNet201', 'ResNet152V2', 'VGG16'\n# 'imagenet', EfficientNetB7 -> 'noisy-student'\n\nMODEL_NAME = 6 * ['InceptionV3']\nWEIGHTS_NAME = 6 * ['imagenet']\n\nselected_optimizer = 'adam'\nselected_loss = 'sparse_categorical_crossentropy' \nselected_metrics = 'sparse_categorical_accuracy' \n\nEPOCHS = 30\nFOLDS = len(MODEL_NAME)\nSEED = 92\n\nMODEL_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Classes"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"CLASSES = [\n    'pink primrose',    'hard-leaved pocket orchid',\n    'canterbury bells',                 'sweet pea',\n    'wild geranium',                   'tiger lily',           \n    'moon orchid',               'bird of paradise',\n    'monkshood',                    'globe thistle',# 00 - 09\n    'snapdragon',                     \"colt's foot\",\n    'king protea',                  'spear thistle',\n    'yellow iris',                   'globe-flower',         \n    'purple coneflower',            'peruvian lily',\n    'balloon flower',       'giant white arum lily',# 10 - 19\n    'fire lily',                'pincushion flower',\n    'fritillary',                      'red ginger',\n    'grape hyacinth',                  'corn poppy',           \n    'prince of wales feathers',  'stemless gentian',\n    'artichoke',                    'sweet william',# 20 - 29\n    'carnation',                     'garden phlox',\n    'love in the mist',                    'cosmos',\n    'alpine sea holly',      'ruby-lipped cattleya', \n    'cape flower',               'great masterwort',\n    'siam tulip',                     'lenten rose',# 30 - 39\n    'barberton daisy',                   'daffodil',\n    'sword lily',                      'poinsettia',\n    'bolero deep blue',                'wallflower',\n    'marigold',                         'buttercup',\n    'daisy',                     'common dandelion',# 40 - 49\n    'petunia',                         'wild pansy',\n    'primula',                          'sunflower',\n    'lilac hibiscus',          'bishop of llandaff',\n    'gaura',                             'geranium',\n    'orange dahlia',           'pink-yellow dahlia',# 50 - 59\n    'cautleya spicata',          'japanese anemone',\n    'black-eyed susan',                'silverbush',\n    'californian poppy',             'osteospermum',         \n    'spring crocus',                         'iris',\n    'windflower',                      'tree poppy',# 60 - 69\n    'gazania',                             'azalea',\n    'water lily',                            'rose',\n    'thorn apple',                  'morning glory',     \n    'passion flower',                       'lotus',\n    'toad lily',                        'anthurium',# 70 - 79\n    'frangipani',                        'clematis',\n    'hibiscus',                         'columbine',\n    'desert-rose',                    'tree mallow',      \n    'magnolia',                         'cyclamen ',\n    'watercress',                      'canna lily',# 80 - 89\n    'hippeastrum ',                      'bee balm',\n    'pink quill',                        'foxglove',\n    'bougainvillea',                     'camellia',        \n    'mallow',                     'mexican petunia',\n    'bromelia',                    'blanket flower',# 90 - 99\n    'trumpet creeper',            'blackberry lily',\n    'common tulip',                     'wild rose']# 100 - 102","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Image processing"},{"metadata":{},"cell_type":"markdown","source":"### Rotation, zoom"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\nimport math\nimport tensorflow.keras.backend as K\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    rotation_matrix = tf.reshape( \n        tf.concat([\n              c1,  s1, zero,\n             -s1,  c1, zero, \n            zero,zero, one],\n        axis=0),\n        [3,3] \n    )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( \n        tf.concat([\n            one,   s2, zero, \n            zero,  c2, zero, \n            zero,zero, one],\n        axis=0),\n        [3,3] \n    )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( \n        tf.concat([\n          one/height_zoom,  zero,  zero, \n          zero,   one/width_zoom,  zero,\n          zero,             zero,  one],\n        axis=0),\n        [3,3] \n    )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( \n        tf.concat([\n          one,  zero, height_shift, \n          zero,  one,  width_shift, \n          zero, zero,         one],\n        axis=0),\n        [3,3]\n    )\n    \n    return K.dot(\n            K.dot(rotation_matrix, shear_matrix), \n            K.dot(zoom_matrix, shift_matrix))\n\ndef data_rotate(image,label):\n    # input image - is one image of size [dim,dim,3] \n    #               not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    \n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 15. * tf.random.normal([1],dtype='float32')\n    shr = 5. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 16. * tf.random.normal([1],dtype='float32') \n    w_shift = 16. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3]),label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Grid mask"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# https://www.kaggle.com/xiejialun/gridmask-data-augmentation-with-tensorflow\n\n# todo: switch to pytorch\ndef GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):\n    #\n    def mask_transform(image, inv_mat, image_shape):\n        h, w, c = image_shape\n        cx, cy = w//2, h//2\n\n        new_xs = tf.repeat( tf.range(-cx, cx, 1), h)\n        new_ys = tf.tile( tf.range(-cy, cy, 1), [w])\n        new_zs = tf.ones([h*w], dtype=tf.int32)\n\n        old_coords = tf.matmul(inv_mat, \n                        tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))\n        \n        old_coords_x  = tf.round(old_coords[0, :] + w//2)\n        old_coords_y  = tf.round(old_coords[1, :] + h//2)\n\n        clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)\n        clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)\n        clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)\n\n        old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))\n        old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))\n        new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))\n        new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))\n\n        old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)\n        new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)\n        rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))\n        rotated_image_channel = list()\n        for i in range(c):\n            vals = rotated_image_values[:,i]\n            sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])\n            rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, \n                                            default_value=0, validate_indices=False))\n\n        return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])\n    #\n    def mask_random_rotate(image, angle, image_shape):\n        def get_rotation_mat_inv(angle):\n              #transform to radian\n            angle = math.pi * angle / 180\n\n            cos_val = tf.math.cos(angle)\n            sin_val = tf.math.sin(angle)\n            one = tf.constant([1], tf.float32)\n            zero = tf.constant([0], tf.float32)\n\n            rot_mat_inv = tf.concat([cos_val, sin_val, zero,\n                                         -sin_val, cos_val, zero,\n                                         zero, zero, one], axis=0)\n            rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])\n            return rot_mat_inv\n\n        angle = float(angle) * tf.random.normal([1],dtype='float32')\n        rot_mat_inv = get_rotation_mat_inv(angle)\n        return mask_transform(image, rot_mat_inv, image_shape)    \n    #\n    h, w = image_height, image_width\n    hh = int(np.ceil(np.sqrt(h*h+w*w)))\n    hh = hh+1 if hh%2==1 else hh\n    d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)\n    l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)\n\n    st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n    st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n\n    y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)\n    x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)\n\n    for i in range(0, hh//d+1):\n        s1 = i * d + st_h\n        s2 = i * d + st_w\n        y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)\n        x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)\n\n    x_clip_mask = tf.logical_or(x_ranges < 0 , x_ranges > hh-1)\n    y_clip_mask = tf.logical_or(y_ranges < 0 , y_ranges > hh-1)\n    clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)\n\n    x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))\n    y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))\n\n    hh_ranges = tf.tile(tf.range(0,hh), \n                        [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)), tf.int32)])\n    x_ranges = tf.repeat(x_ranges, hh)\n    y_ranges = tf.repeat(y_ranges, hh)\n\n    y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))\n    x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))\n\n    y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64), \n                                    tf.zeros_like(y_ranges), [hh, hh])\n    y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)\n\n    x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), \n                                    tf.zeros_like(x_ranges), [hh, hh])\n    x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)\n\n    mask = tf.expand_dims( tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)\n\n    mask = mask_random_rotate(mask, rotate_angle, [hh, hh, 1])\n    mask = tf.image.crop_to_bounding_box(mask, (hh-h)//2, (hh-w)//2, \n                                         image_height, image_width)\n\n    return mask\n\ndef data_gridmask(image,label):\n    AugParams = {\n        'd1' : 100,\n        'd2': 160,\n        'rotate' : 45,\n        'ratio' : 0.3\n    }\n    mask = GridMask(IMAGE_SIZE[0], IMAGE_SIZE[1], AugParams['d1'], \n                    AugParams['d2'], AugParams['rotate'], AugParams['ratio'])\n    if IMAGE_CHANNELS == 3:\n        mask = tf.concat([mask, mask, mask], axis=-1)\n    return image * tf.cast(mask,tf.float32), label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### TFRecord loader"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import re\n\nAUTO = tf.data.experimental.AUTOTUNE\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    # convert image to floats in [0, 1] range\n    image = tf.cast(image, tf.float32) / 255.0  \n    # explicit size needed for TPU\n    image = tf.reshape(image, [*IMAGE_SIZE, IMAGE_CHANNELS]) \n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        # tf.string means bytestring\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        # shape [] means single element\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  \n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    # returns a dataset of (image, label) pairs\n    return image, label\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        # tf.string means bytestring\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        # shape [] means single element\n        \"id\": tf.io.FixedLenFeature([], tf.string),  \n        # class is missing, this competitions's challenge \n        # is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum \n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, \n    # reading from multiple files at once and\n    # disregarding data order. Order does not matter \n    # since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        # disable order, increase speed\n        ignore_order.experimental_deterministic = False \n\n    # automatically interleaves reads from multiple files\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) \n    # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.with_options(ignore_order) \n    dataset = dataset.map(read_labeled_tfrecord if labeled else \n                          read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True \n    # or (image, id) pairs if labeled=False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in \n    # the next function (below), this happens essentially for free on TPU. \n    # Data pipeline code is executed on the \"CPU\" part of the TPU while \n    # the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    #image = tf.image.random_saturation(image, 0, 2)\n    return image, label   \n\ndef get_training_dataset(dataset, do_aug=False, do_grid=False):\n    # dataset = load_dataset(filenames, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    # Rotation Augmentation GPU/TPU\n    if do_aug: dataset = dataset.map(data_rotate, num_parallel_calls=AUTO)\n    # grid mask\n    if do_grid: dataset = dataset.map(data_gridmask, num_parallel_calls=AUTO)   \n    # the training dataset must repeat for several epochs\n    dataset = dataset.repeat() \n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    # prefetch next batch while training (autotune prefetch buffer size)\n    dataset = dataset.prefetch(AUTO) \n    return dataset\n\ndef get_validation_dataset(dataset):\n    # dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    # prefetch next batch while training (autotune prefetch buffer size)\n    dataset = dataset.prefetch(AUTO) \n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    # prefetch next batch while training (autotune prefetch buffer size)\n    dataset = dataset.prefetch(AUTO) \n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files,\n    # i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.\n      format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Class weights"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils import class_weight\n\ndef read_tfrecord_label(example):\n    LABELED_TFREC_FORMAT = {\n        # tf.string means bytestring\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        # shape [] means single element\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  \n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    #image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    # returns a dataset of (image, label) pairs\n    return label\n\ndef get_class_weights(filenames):\n    count = count_data_items(filenames)\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.map(read_tfrecord_label, num_parallel_calls=AUTO)\n    labels = next(iter(dataset.batch(count))).numpy()\n    return class_weight.compute_class_weight('balanced', [x for x in range(len(CLASSES))], labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_select(name, weights = 'imagenet'):\n    if name == 'EfficientNetB7':\n        try:\n            import efficientnet.tfkeras as efn\n        except ImportError:\n            !pip install -q efficientnet\n            import efficientnet.tfkeras as efn\n        pretrained_model = efn.EfficientNetB7(\n            weights=weights, \n            include_top=False,\n            input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])\n        # model fine tuning\n        pretrained_model.trainable = True\n        return tf.keras.Sequential([\n            pretrained_model,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            # tf.keras.layers.Dropout(0.2, name=\"dropout_out\"),\n            tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n        ])\n    elif name == 'InceptionV3':\n        pretrained_model = tf.keras.applications.InceptionV3(\n            weights=weights, \n            include_top=False,\n            input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])\n        # model fine tuning\n        pretrained_model.trainable = True\n        return tf.keras.Sequential([\n            pretrained_model,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(4096, activation='relu'), \n            tf.keras.layers.Dense(4096, activation='relu'), \n            tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n        ])       \n    elif name == 'DenseNet201':\n        pretrained_model = tf.keras.applications.DenseNet201(\n            weights=weights, \n            include_top=False,\n            input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])\n        # model fine tuning\n        pretrained_model.trainable = True\n        return tf.keras.Sequential([\n            pretrained_model,\n            tf.keras.layers.GlobalAveragePooling2D(),\n#             tf.keras.layers.Dense(4096, activation='relu'), \n#             tf.keras.layers.Dense(4096, activation='relu'), \n            tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n        ])\n    elif name == 'ResNet152V2':\n        pretrained_model = tf.keras.applications.ResNet152V2(\n            weights=weights, \n            include_top=False,\n            input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])\n        # model fine tuning\n        pretrained_model.trainable = True\n        return tf.keras.Sequential([\n            pretrained_model,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(4096, activation='relu'), \n            tf.keras.layers.Dense(4096, activation='relu'),            \n            tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n        ])  \n    elif name == 'VGG16':\n        pretrained_model = tf.keras.applications.VGG16(\n            weights=weights, \n            include_top=False,\n            input_shape=[*IMAGE_SIZE, IMAGE_CHANNELS])\n        # model fine tuning\n        pretrained_model.trainable = True\n        return tf.keras.Sequential([\n            pretrained_model,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(4096, activation='relu'), \n            tf.keras.layers.Dense(4096, activation='relu'), \n            tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n        ])\n    else:\n        print('Unknown model: %s' % name)\n        raise\n    \ndef create_model(name, weights = 'imagenet'):\n    # transfer learning\n    with strategy.scope():\n        # https://www.tensorflow.org/tutorials/keras/overfit_and_underfit\n        # add dropout, weight regularization to decrease overfitting\n        model = model_select(name, weights)\n    \n        # v25 => lr=0.01, decay=1e-6, momentum=0.9, nesterov=True \n        # v27 => lr=0.001, decay=1e-5, momentum=0.8, nesterov=True \n        #opt = tf.keras.optimizers.SGD(lr=0.01, \n        #                              decay=1e-6, \n        #                              momentum=0.9, \n        #                              nesterov=True)\n        #opt = 'adam' #tf.keras.optimizers.Adam(0.0001)\n        model.compile(\n            optimizer = selected_optimizer,\n            loss = selected_loss,\n            metrics=[selected_metrics]\n        )\n        return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# save configuration for post processing\nimport json\n\ndef save_config():\n    # Data to be written \n    config = { \n        \"IMAGE_SIZE\" : IMAGE_SIZE,\n        \"IMAGE_CHANNELS\" : IMAGE_CHANNELS,\n        \"BATCH_SIZE\" : BATCH_SIZE,\n        \"MODEL_NAME\" : MODEL_NAME,\n        \"WEIGHTS_NAME\" : WEIGHTS_NAME,\n        \"EPOCHS\" : EPOCHS,\n        \"FOLDS\" : FOLDS,\n        \"TENSOR_BOARD\" : TENSOR_BOARD, \n        \"VERBOSE\" : VERBOSE, \n        \"IMAGE_AUGMENTATION\" : IMAGE_AUGMENTATION, \n        \"GRID_MASK\" : GRID_MASK,\n        \"USE_CLASS_WEIGHTS\" : USE_CLASS_WEIGHTS,\n        \"SEED\" : SEED\n    } \n    # Serializing json  \n    # json_object = json.dumps(config, indent = 4) \n    with open('config.json', 'w') as outfile:\n        json.dump(config, outfile)\n        \nsave_config()        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\n\n# Learning rate schedule for TPU, GPU and CPU.\n# Using an LR ramp up because fine-tuning a pre-trained model.\n# Starting with a high LR would break the pre-trained weights.\n\nLR_START = 0.00001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 4\nLR_SUSTAIN_EPOCHS = 6\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n        #lr = (LR_MAX - LR_START) * (epoch/LR_RAMPUP_EPOCHS)**2 + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - \n                        LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n\n# Clear any logs from previous runs\n!rm -rf ./logs/\n\n# define tensorboard log root\nlog_dir = \"logs/fit/\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\") \n\ndef callbacks(ifold, board = False):\n    # prepare callbacks\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = False)\n    if board:\n        #writer = tf.summary.create_file_writer(log_dir + f\"/fold{ifold}\")\n        tensorboard_callback = tf.keras.callbacks.TensorBoard(\n            log_dir=log_dir + f\"/fold{ifold}\", histogram_freq=1)\n        return [lr_callback, tensorboard_callback]\n    else:\n        return [lr_callback]\n    \nkfold = KFold(FOLDS, shuffle = True, random_state = SEED)\n\n# https://www.kaggle.com/c/flower-classification-with-tpus/discussion/131876\n# array of weights works on TPU with the standard sparse_categorical_crossentropy loss\nCLASS_WEIGHTS = get_class_weights(MODEL_FILENAMES) if USE_CLASS_WEIGHTS else None\n\n# since we are splitting the dataset and iterating separately on \n# images and ids, order matters.\ntest_ds = get_test_dataset(ordered=True)\n\ndef train_cross_validate_predict(filenames):\n    print(f'Start training {FOLDS} folds {EPOCHS} epochs, img_aug {IMAGE_AUGMENTATION} grid {GRID_MASK} class weights {USE_CLASS_WEIGHTS}')\n    histories = []\n    for ifold, (trn_ind, val_ind) in enumerate(kfold.split(filenames)):\n        # select files\n        tdf = list(pd.DataFrame({'FILENAMES': filenames}).loc[trn_ind]['FILENAMES'])\n        vdf = list(pd.DataFrame({'FILENAMES': filenames}).loc[val_ind]['FILENAMES'])\n        steps_per_epoch = count_data_items(tdf) // BATCH_SIZE\n        train_dataset = load_dataset(tdf, labeled = True)\n        val_dataset = load_dataset(vdf, labeled = True, ordered = True)\n        validation_data = get_validation_dataset(val_dataset)\n        # run fit\n        print_and_log(f'# FOLD: {ifold+1} {steps_per_epoch}');\n        # Recreate the exact same model purely from the file\n        #model = tf.keras.models.load_model('my_model.h5')\n        model = create_model(MODEL_NAME[ifold], WEIGHTS_NAME[ifold])\n        history = model.fit(\n            get_training_dataset(train_dataset, IMAGE_AUGMENTATION, GRID_MASK), \n            steps_per_epoch = steps_per_epoch,\n            epochs = EPOCHS,\n            callbacks = callbacks(ifold, TENSOR_BOARD),\n            validation_data = validation_data,\n            class_weight = CLASS_WEIGHTS,\n            verbose = VERBOSE\n        )\n        test_images_ds = test_ds.map(lambda image, idnum: image)\n        predict = model.predict(test_images_ds)\n        np.save(f'fold{ifold+1}.npy', predict)\n        #\n        cm_predict = model.predict(validation_data)\n        np.save(f'cm_fold{ifold+1}.npy', cm_predict)\n        histories.append(history)\n        # to avoid memory issues with TPU\n        # https://www.kaggle.com/c/flower-classification-with-tpus/discussion/131045\n        del model, train_dataset, val_dataset, validation_data, predict, cm_predict\n        #K.clear_session()\n        gc.collect()\n        if tpu:\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n    return histories #, predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run train\nhistories = train_cross_validate_predict(MODEL_FILENAMES)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6. Graphs"},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib import pyplot as plt\n\ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,8), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    if(validation):\n        ax.plot(validation)\n    if len(title) > 0:\n        ax.set_title(title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])\n\nif not TENSOR_BOARD:  \n    # Learning rate schedule graph\n    lrfn_rng = [i for i in range(25 if EPOCHS<25 else EPOCHS)]\n    lrfn_y = [lrfn(x) for x in lrfn_rng]\n    a = plt.plot(lrfn_rng, lrfn_y)\n\n    print_and_log('show loss and accuracy')\n    for ifold, history in enumerate(histories):\n        display_training_curves(\n            history.history['loss'], \n            history.history['val_loss'], \n            f'kfold{ifold+1} {MODEL_NAME[ifold]} loss', 221)\n        display_training_curves(\n            history.history[selected_metrics], \n            history.history['val_' + selected_metrics], \n            f'kfold{ifold+1} {MODEL_NAME[ifold]} accuracy', 222)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Save tensorboard logs"},{"metadata":{"trusted":true},"cell_type":"code","source":"# logs\n#!tar -zcvf tensorboard.tar.gz logs\n\n# # Upload to colab \n# !tar -xvf tensorboard.tar.gz\n# # run tensor board\n# %tensorboard --logdir logs/fit","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 7. Confusion Matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nfrom matplotlib.colors import LinearSegmentedColormap\n\ndef custom_cmap():\n    # custom cmap, replace 0 with gray\n    cmap = plt.cm.Reds\n    cmaplist = [cmap(i) for i in range(cmap.N)]\n    cmaplist[0] = (0, 0, 0, 0.2)\n    return LinearSegmentedColormap.from_list('mcm' ,cmaplist, cmap.N)\n\ndef display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap=custom_cmap())\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, \n            'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()\n    \ndef plot_confusion_matrix(model_predictions):\n    all_labels = []; all_pred = []\n\n    for j, (trn_ind, val_ind) in enumerate( kfold.split(MODEL_FILENAMES) ):\n        vdf = list(pd.DataFrame({'FILENAMES': MODEL_FILENAMES}).loc[val_ind]['FILENAMES'])\n        NUM_VALIDATION_IMAGES = count_data_items(vdf)\n        val_dataset = load_dataset(vdf, labeled = True, ordered = True)\n        cmdataset = get_validation_dataset(val_dataset)\n        labels_ds = cmdataset.map(lambda image, label: label).unbatch()\n        all_labels.append(next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy())\n        prob = model_predictions[j]\n        all_pred.append( np.argmax(prob, axis=-1) )\n\n    cm_correct_labels = np.concatenate(all_labels)\n    cm_predictions = np.concatenate(all_pred)\n\n    cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))\n    score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\n    precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\n    recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\n    display_confusion_matrix(cmat, score, precision, recall)\n\n    print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#load predictions\nfold_predictions = []\ntry:\n    for ifold in range(FOLDS):\n        predict = np.load(f'cm_fold{ifold+1}.npy')\n        fold_predictions.append(predict)\nexcept:\n    print('KFold processing issue')\n\nprint_and_log('show confusion matrix')\nplot_confusion_matrix(fold_predictions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 8. Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_predictions(preds):\n    #test_images_ds = test_ds.map(lambda image, idnum: image)\n    print('Computing predictions...')\n    # get the mean probability of the folds models\n    probabilities = np.average([preds[i] for i in range(len(preds))], axis = 0)\n    predictions = np.argmax(probabilities, axis=-1)\n    print(predictions)\n    return predictions\n\ndef save_prediction(predictions):\n    print('Generating submission.csv file...')\n    test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n    # all in one batch\n    test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') \n    np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), \n               fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#load predictions\npredictions = []\ntry:\n    for ifold in range(FOLDS):\n        predict = np.load(f'fold{ifold+1}.npy')\n        predictions.append(predict)\nexcept:\n    print('KFold processing issue')\n    \nprint_and_log('save prediction')\nsave_prediction(get_predictions(predictions))\n\n# print head\n!head submission.csv","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}