{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Download Latest Version of TensorFlow and Tensorflow Addons"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":false,"_kg_hide-output":true},"cell_type":"code","source":"#!pip install tensorflow==2.2.0rc2\n!pip install tensorflow-addons==0.9.1\n!pip install -q efficientnet --upgrade","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Importing Dependencies"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import re, os, gc\nimport sys, time\nimport math\nimport pprint\ngc.enable()\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom sklearn.model_selection import (train_test_split, \n                                     KFold, \n                                     StratifiedKFold)\nfrom sklearn.utils.class_weight import compute_class_weight\n\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom tensorflow.keras import backend as K\nimport efficientnet.tfkeras as efn\n\nfrom kaggle_datasets import KaggleDatasets","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### TF - Code Adjustments"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check API version\nprint(f\"TensorFlow version: {tf.__version__}\")\nprint(f\"Keras version: {tf.keras.__version__}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"TensorFlow is executing eagerly: {tf.executing_eagerly()}\")\nprint(\"No of physical devices available: {}\".format(len(tf.config.experimental.list_physical_devices())))\nif not len(tf.config.experimental.list_physical_devices()) == 0:\n    print(\"GPU is available.\" if tf.test.is_gpu_available() else \"TPU is available.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Initializing radom seeds ..\")\ntf.random.set_seed(2020)\n\nprint(f\"Enabling TensorFlow Device Debugger ..\")\ntf.config.set_soft_device_placement(False)\ntf.debugging.set_log_device_placement(False)\n\n%reload_ext tensorboard\n\nprint(\"Done.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Flags for TPU, XLA Compiler & Mixed Precision"},{"metadata":{},"cell_type":"markdown","source":"#### TPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"if 'XRT_TPU_CONFIG' not in os.environ and \"TPU_NAME\" not in os.environ:\n    print('ERROR: Not connected to a TPU runtime; please see the first cell in this notebook for instructions!')\nelse:\n    tpu_address = os.environ['TPU_NAME']\n    print('TPU address is: ', tpu_address)\n    print('TPU Configuration: ',os.environ['XRT_TPU_CONFIG'])\n    with tf.compat.v1.Session(tpu_address) as session:\n        devices = session.list_devices()\n    print('TPU devices:')\n    pprint.pprint(devices)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n    print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\nexcept ValueError:\n    tpu = None\n    gpus = tf.config.experimental.list_logical_devices(\"GPU\")\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelif len(gpus) > 1: # multiple GPUs in one VM\n    strategy = tf.distribute.MirroredStrategy(gpus)\nelse: # default strategy that works on CPU and single GPU\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### XLA Compiler"},{"metadata":{"trusted":true},"cell_type":"code","source":"# This ensures that XLA and ptxas work well together, and helps with scaling.\nprint('Configuring XLA ..')\nif os.getenv(\"XLA_FLAGS\") is None:\n    print('XLA not found.')\n    tf.config.optimizer.set_jit(False) # Make XLA disabled.\nelse:\n    print(\"XLA_FLAGS='{}'\".format(os.getenv(\"XLA_FLAGS\")))\n    tf.config.optimizer.set_jit(True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Mixed Precision"},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.config.optimizer.set_jit(True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\nIMAGE_SIZE = [512, 512]\nEPOCHS = 25\nFOLDS = 2\nSEED = 2020\nBATCH_SIZE = 12 * strategy.num_replicas_in_sync\nAUG_BATCH = BATCH_SIZE\nFIRST_FOLD_ONLY = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path()\n\nGCS_PATH_SELECT = { \n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\n\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec') + tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### List of Labels "},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily',\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',          \n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             \n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',         \n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',     \n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Processing\n\nLearn here: \n\n[TensoFlow Tutorials - Images](https://www.tensorflow.org/tutorials/load_data/images)\n\n[TensoFlow Tutorials - TFRecord and tf.Example](https://www.tensorflow.org/tutorials/load_data/tfrecord)"},{"metadata":{},"cell_type":"markdown","source":"#### TF-Record to Image Preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  \n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"class\": tf.io.FixedLenFeature([], tf.int64), \n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"id\": tf.io.FixedLenFeature([], tf.string),  \n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum\n\ndef load_dataset(filenames, labeled = True, ordered = False):    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO) \n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Transform Images to Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset(dataset, do_aug=True):\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat()\n    dataset = dataset.batch(AUG_BATCH)\n    if do_aug: \n        dataset = dataset.unbatch().map(apply_rotation, num_parallel_calls=AUTO).batch(AUG_BATCH).map(transform, num_parallel_calls=AUTO)\n    dataset = dataset.unbatch()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(dataset, do_onehot=True):\n    dataset = dataset.batch(BATCH_SIZE)\n    if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTO)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Utitlity Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    image = tf.image.random_flip_left_right(image)\n    return image, label   \n\ndef onehot(image,label):\n    CLASSES = 104\n    return image,tf.one_hot(label,CLASSES)\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"NUM_TRAINING_IMAGES = int( count_data_items(TRAINING_FILENAMES) * (FOLDS-1.)/FOLDS )\nNUM_VALIDATION_IMAGES = int( count_data_items(TRAINING_FILENAMES) * (1./FOLDS) )\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = int(NUM_TRAINING_IMAGES/BATCH_SIZE)\n\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Augmentation"},{"metadata":{},"cell_type":"markdown","source":"##### CutMix Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def cutmix(image, label, PROBABILITY = 1.0):\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 104\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1) \n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"row = 6; col = 4;\nrow = min(row,AUG_BATCH//col)\nall_elements = get_training_dataset(load_dataset(TRAINING_FILENAMES),do_aug=False).unbatch()\naugmented_element = all_elements.repeat().batch(AUG_BATCH).map(cutmix)\n\nfor (img,label) in augmented_element:\n    plt.figure(figsize=(15,int(15*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### Mixup Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def mixup(image, label, PROBABILITY = 1.0):\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 104\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        a = tf.random.uniform([],0,1)*P\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":false,"_kg_hide-input":true},"cell_type":"code","source":"row = 6; col = 4;\nrow = min(row,AUG_BATCH//col)\nall_elements = get_training_dataset(load_dataset(TRAINING_FILENAMES),do_aug=False).unbatch()\naugmented_element = all_elements.repeat().batch(AUG_BATCH).map(mixup)\n\nfor (img,label) in augmented_element:\n    plt.figure(figsize=(15,int(15*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### Rotation Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def apply_rotation(image,label):\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2\n    \n    rot = 15. * tf.random.normal([1],dtype='float32')\n    shr = 5. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 16. * tf.random.normal([1],dtype='float32') \n    w_shift = 16. * tf.random.normal([1],dtype='float32') \n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n    return tf.reshape(d,[DIM,DIM,3]),label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CUTMIX = True\nMIXUP = True\nROTATION = True\n\ndef transform(image,label):\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 104\n    SWITCH = 0.5\n    CUTMIX_PROB = 0.666\n    MIXUP_PROB = 0.666\n    if CUTMIX and MIXUP:\n        image2, label2 = cutmix(image, label, CUTMIX_PROB)\n        image3, label3 = mixup(image, label, MIXUP_PROB)\n        imgs = []; labs = []\n        for j in range(AUG_BATCH):\n            P = tf.cast( tf.random.uniform([],0,1)<=SWITCH, tf.float32)\n            imgs.append(P*image2[j,]+(1-P)*image3[j,])\n            labs.append(P*label2[j,]+(1-P)*label3[j,])\n        image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n        label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n        return image4,label4\n    else:\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Build, Train and Infer"},{"metadata":{},"cell_type":"markdown","source":"#### Focal Loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"def FocalLoss(target, input):\n    gamma = 2.\n    input = tf.cast(input, tf.float32)\n    max_val = K.clip(-input, 0, 1)\n    loss = input - input * target + max_val + K.log(K.exp(-max_val) + K.exp(-input - max_val))\n    invprobs = tf.math.log_sigmoid(-input * (target * 2.0 - 1.0))\n    loss = K.exp(invprobs * gamma) * loss\n    return K.mean(K.sum(loss, axis=1))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Model - Efficient Net - B7"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(model_name):\n    with strategy.scope():\n        if model_name == 'efn-b7':\n            efnet = efn.EfficientNetB7(input_shape=[512, 512, 3],\n                                       weights='noisy-student',\n                                       include_top=False)\n        elif model_name == 'efn-b6':\n            efnet = efn.EfficientNetB6(input_shape=[512, 512, 3],\n                                       weights='noisy-student',\n                                       include_top=False)\n        elif model_name == 'efn-b5':\n            efnet = efn.EfficientNetB5(input_shape=[512, 512, 3],\n                                       weights='noisy-student',\n                                       include_top=False)\n        efnet.trainable = True\n        model = tf.keras.Sequential([\n            efnet,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(len(CLASSES)),\n            tf.keras.layers.Activation(\"softmax\", dtype='float32', name='predictions')\n        ])\n        model.compile(\n            optimizer='adam',\n            loss = \"categorical_crossentropy\",\n            metrics=['categorical_accuracy',tfa.metrics.f_scores.F1Score(num_classes=104,average=\"macro\")]\n        )\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Callbacks"},{"metadata":{"trusted":true},"cell_type":"code","source":"if strategy.num_replicas_in_sync == 8:\n    start_lr = 0.00001\n    max_lr = 0.00005 * strategy.num_replicas_in_sync\n    min_lr = 0.00001\n    rampup_epochs = 5\n    sustain_epochs = 0\n    exp_decay = .8\nelif strategy.num_replicas_in_sync == 1:\n    start_lr = 0.00001\n    min_lr = 0.00001\n    max_lr = 0.0002\n    rampup_epochs = 5\n    sustain_epochs = 0\n    exp_decay = .8\nelse: \n    start_lr = 0.00001\n    min_lr = 0.00001\n    max_lr = 0.00002 * strategy.num_replicas_in_sync\n    rampup_epochs = 7\n    sustain_epochs = 0\n    exp_decay = .8\n    \ndef lrfn(epoch):\n    def lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay):\n        if epoch < rampup_epochs:\n            lr = (max_lr - start_lr)/rampup_epochs * epoch + start_lr\n        elif epoch < rampup_epochs + sustain_epochs:\n            lr = max_lr\n        else:\n            lr = (max_lr - min_lr) * exp_decay**(epoch-rampup_epochs-sustain_epochs) + min_lr\n        return lr\n    return lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay)\n    \n    \nimport os\ndef get_callbacks(model_name):\n    if not os.path.exists(str(model_name)+\"_callback_output\"):\n        os.mkdir(str(model_name)+\"_callback_output\")\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lambda epoch: lrfn(epoch), verbose=True)\n    #tqdm_callback = tfa.callbacks.TQDMProgressBar()\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(str(model_name)+\"_callback_output\" + '/best.h5', \n                              monitor = 'val_loss', \n                              verbose = True, save_best_only=True, \n                              mode = 'min',\n                              save_weights_only = True)\n    csv_logger = tf.keras.callbacks.CSVLogger(str(model_name)+\"_callback_output\" + '/log.csv')\n    early = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', patience=2,verbose=True)\n    return [lr_callback, checkpoint, early, csv_logger]\n\nrng = [i for i in range(25)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, [lrfn(x) for x in rng])\nprint(y[0], y[-1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_cross_validate(folds, model_name):\n    histories = []\n    models = []\n    #model_name = 'efn-b7'\n    kfold = KFold(folds, shuffle = True, random_state = SEED)\n    for f, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n    \n        print(); print('#'*30); print('### FOLD',f+1); print('#'*30)\n        \n        train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)\n        val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)\n        \n        with strategy.scope():\n            model = get_model(model_name)\n            callbacks = get_callbacks(model_name)\n        \n        history = model.fit(\n            get_training_dataset(train_dataset), \n            steps_per_epoch = STEPS_PER_EPOCH,\n            epochs = EPOCHS,\n            callbacks = callbacks,\n            validation_data = get_validation_dataset(val_dataset),\n            verbose=True\n        )\n        \n        models.append(model)\n        histories.append(history)\n        \n        tf.compat.v1.reset_default_graph()\n        del model\n        gc.collect()\n        \n        if FIRST_FOLD_ONLY: break\n        \n    return histories, models\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_and_predict(folds, model_name):\n    \n    test_ds = get_test_dataset(ordered=True) \n    test_images_ds = test_ds.map(lambda image, idnum: image)\n    print('Start training %i folds'%folds)\n    histories, models = train_cross_validate(folds, model_name)\n    \n    print('Computing predictions...')\n    if FIRST_FOLD_ONLY: probabilities = np.average([models[i].predict(test_images_ds) for i in range(1)], axis = 0)\n    else: probabilities = np.average([models[i].predict(test_images_ds) for i in range(folds)], axis = 0)\n    \n    predictions = np.argmax(probabilities, axis=-1)\n    print('Generating submission.csv file...')\n    \n    test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n    test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n    np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\n    probs_df = pd.DataFrame(probabilities)\n    probs_df['ID'] = test_ids\n    probs_df.to_csv('probs.csv',index=False)    \n    return histories, models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run train and predict\nmodel_name = 'efn-b7'\nhistories, models = train_and_predict(folds = FOLDS, model_name=model_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}