{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install --quiet efficientnet\n\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport os, re, math, warnings, random, cv2\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import classification_report, confusion_matrix\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras import optimizers, applications, Sequential, losses\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\nimport efficientnet.tfkeras as efn\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TPU configuration"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16 * REPLICAS\nLEARNING_RATE = 3e-5 * REPLICAS\nEPOCHS = 20\nHEIGHT = 331\nWIDTH = 331\nCHANNELS = 3\nN_CLASSES = 104\nES_PATIENCE = 5\nN_FOLDS = 5\nFOLDS_USED = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path() + '/tfrecords-jpeg-%sx%s' % (HEIGHT, WIDTH)\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec') + tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')\n\nCLASSES = [\n    'pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', \n    'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', \n    'globe thistle', 'snapdragon', \"colt's foot\", 'king protea', 'spear thistle', \n    'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', \n    'balloon flower', 'giant white arum lily', 'fire lily', 'pincushion flower', \n    'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', \n    'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william', \n    'carnation', 'garden phlox', 'love in the mist', 'cosmos',  'alpine sea holly', \n    'ruby-lipped cattleya', 'cape flower', 'great masterwort',  'siam tulip', \n    'lenten rose', 'barberton daisy', 'daffodil',  'sword lily', 'poinsettia', \n    'bolero deep blue',  'wallflower', 'marigold', 'buttercup', 'daisy', \n    'common dandelion', 'petunia', 'wild pansy', 'primula',  'sunflower', \n    'lilac hibiscus', 'bishop of llandaff', 'gaura',  'geranium', 'orange dahlia', \n    'pink-yellow dahlia', 'cautleya spicata',  'japanese anemone', \n    'black-eyed susan', 'silverbush', 'californian poppy',  'osteospermum', \n    'spring crocus', 'iris', 'windflower',  'tree poppy', 'gazania', 'azalea', \n    'water lily',  'rose', 'thorn apple', 'morning glory', 'passion flower',  \n    'lotus', 'toad lily', 'anthurium', 'frangipani',  'clematis', 'hibiscus', \n    'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', \n    'watercress',  'canna lily', 'hippeastrum ', 'bee balm', 'pink quill',  \n    'foxglove', 'bougainvillea', 'camellia', 'mallow',  'mexican petunia',  \n    'bromelia', 'blanket flower', 'trumpet creeper',  'blackberry lily', \n    'common tulip', 'wild rose']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualization utility functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"np.set_printoptions(threshold=15, linewidth=80)\n\ndef plot_metrics(history, metric_list):\n    fig, axes = plt.subplots(len(metric_list), 1, sharex='col', figsize=(24, 12))\n    axes = axes.flatten()\n    \n    for index, metric in enumerate(metric_list):\n        axes[index].plot(history[metric], label='Train %s' % metric)\n        axes[index].plot(history['val_%s' % metric], label='Validation %s' % metric)\n        axes[index].legend(loc='best', fontsize=16)\n        axes[index].set_title(metric)\n\n    plt.xlabel('Epochs', fontsize=16)\n    sns.despine()\n    plt.show()\n\n    \ndef dataset_to_numpy_util(dataset, N):\n    dataset = dataset.unbatch().batch(N)\n    for images, labels in dataset:\n        numpy_images = images.numpy()\n        numpy_labels = labels.numpy()\n        break;  \n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    label = np.argmax(label, axis=-1)\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], str(correct), ', shoud be ' if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower_eval(image, title, subplot, red=False):\n    plt.subplot(subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    plt.title(title, fontsize=14, color='red' if red else 'black')\n    return subplot+1\n\ndef display_9_images_with_predictions(images, predictions, labels):\n    subplot=331\n    plt.figure(figsize=(10, 10))\n    for i, image in enumerate(images):\n        title, correct = title_from_label_and_target(predictions[i], labels[i])\n        subplot = display_one_flower_eval(image, title, subplot, not correct)\n        if i >= 8:\n            break;\n              \n    plt.tight_layout()\n    plt.subplots_adjust(wspace=0.1, hspace=0.1)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Methods to display images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target_(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n\ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target_(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Datasets utility functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=CHANNELS)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [HEIGHT, WIDTH, CHANNELS])\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    \n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n    }\n    \n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef get_dataset(filenames, labeled=True, ordered=True, repeated=False, shufled=False, augmented=False):\n    dataset = load_dataset(filenames, labeled=labeled, ordered=ordered)\n    if augmented:\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    if repeated:\n        dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    if shufled:\n        dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_training_dataset_preview(ordered=True):\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# About the datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files.\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n# Train data\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nprint(f'Number of training images {NUM_TRAINING_IMAGES}')\n\n# Test data\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nprint(f'Number of test images {NUM_TEST_IMAGES}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## ****train data****"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_dataset_preview(ordered=True)\ndisplay_batch_of_images(next(iter(train_dataset.unbatch().batch(20))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## ****test data****"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = get_test_dataset(ordered=True)\ndisplay_batch_of_images(next(iter(test_dataset.unbatch().batch(20))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dropout augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def random_dropout(img, sl=0.1, sh=0.2, rl=0.4):\n    p=random.random()\n    if p>=0.25:\n        w, h, c = HEIGHT, WIDTH, 3\n        origin_area = tf.cast(h*w, tf.float32)\n\n        e_size_l = tf.cast(tf.round(tf.sqrt(origin_area * sl * rl)), tf.int32)\n        e_size_h = tf.cast(tf.round(tf.sqrt(origin_area * sh / rl)), tf.int32)\n\n        e_height_h = tf.minimum(e_size_h, h)\n        e_width_h = tf.minimum(e_size_h, w)\n\n        erase_height = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_height_h, dtype=tf.int32)\n        erase_width = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_width_h, dtype=tf.int32)\n\n        erase_area = tf.zeros(shape=[erase_height, erase_width, c])\n        erase_area = tf.cast(erase_area, tf.uint8)\n\n        pad_h = h - erase_height\n        pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)\n        pad_bottom = pad_h - pad_top\n\n        pad_w = w - erase_width\n        pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)\n        pad_right = pad_w - pad_left\n\n        erase_mask = tf.pad([erase_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)\n        erase_mask = tf.squeeze(erase_mask, axis=0)\n        erased_img = tf.multiply(tf.cast(img,tf.float32), tf.cast(erase_mask, tf.float32))\n\n        return tf.cast(erased_img, img.dtype)\n    else:\n        return tf.cast(img, img.dtype)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Blockout augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def blockout(image, DIM = HEIGHT, PROBABILITY = 1, CT = 8, SZ = 0.2):\n    \n    prob = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (prob==0)|(CT==0)|(SZ==0): return image\n    \n    for k in range(CT):\n\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        WIDTH = tf.cast( SZ*DIM,tf.int32) * prob\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3]) \n        three = image[ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)\n            \n    image = tf.reshape(image,[DIM,DIM,3])\n    \n    return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Advanced augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel = tf.random.uniform([], 0, 1.0, dtype=tf.float32)    \n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shift = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    \n    # Flips\n    if p_spatial >= .2:\n        image = tf.image.random_flip_left_right(image)\n        image = tf.image.random_flip_up_down(image)\n        \n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n    \n    if p_rotation >= .3: # Rotation\n        image = transform_rotation(image, height=HEIGHT, rotation=45.)\n    if p_shift >= .3: # Shift\n        image = transform_shift(image, height=HEIGHT, h_shift=15., w_shift=15.)\n    if p_shear >= .3: # Shear\n        image = transform_shear(image, height=HEIGHT, shear=20.)\n        \n    # Crops\n    if p_crop > .4:\n        crop_size = tf.random.uniform([], int(HEIGHT*.7), HEIGHT, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n    elif p_crop > .7:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.8)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.9)\n            \n    image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n        \n    # Pixel-level transforms\n    if p_pixel >= .2:\n        if p_pixel >= .8:\n            image = tf.image.random_saturation(image, lower=0, upper=2)\n        elif p_pixel >= .6:\n            image = tf.image.random_contrast(image, lower=.8, upper=2)\n        elif p_pixel >= .4:\n            image = tf.image.random_brightness(image, max_delta=.2)\n        else:\n            image = tf.image.adjust_gamma(image, gamma=.6)\n            \n    # Dropout    \n    #image = dropout(image)\n    \n    # Blockout\n    image = blockout(image)\n    \n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shift(image, height, h_shift, w_shift):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly shifted\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    height_shift = h_shift * tf.random.uniform([1],dtype='float32') \n    width_shift = w_shift * tf.random.uniform([1],dtype='float32') \n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n        \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shift_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Look at some augmented samples"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset_aug = get_dataset(TRAINING_FILENAMES, labeled=True, ordered=False, repeated=True, shufled=True, augmented=True)\ndisplay_batch_of_images(next(iter(train_dataset_aug.unbatch().batch(20))))\ndisplay_batch_of_images(next(iter(train_dataset_aug.unbatch().batch(20))))\ndisplay_batch_of_images(next(iter(train_dataset_aug.unbatch().batch(20))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Learning rate schedule"},{"metadata":{"trusted":true},"cell_type":"code","source":"def exponential_schedule_with_warmup(epoch):\n    '''\n    Create a schedule with a learning rate that decreases exponentially after linearly increasing during a warmup period.\n    '''\n    \n    warmup_epochs=3\n    hold_max_epochs=0\n    lr_start=1e-6\n    lr_max=LEARNING_RATE\n    lr_min=1e-6\n    decay=0.8\n        \n        \n    if epoch < warmup_epochs:\n        lr = (lr_max - lr_start) / warmup_epochs * epoch + lr_start\n    elif epoch < warmup_epochs + hold_max_epochs:\n        lr = lr_max\n    else:\n        lr = lr_max * (decay ** (epoch - warmup_epochs - hold_max_epochs))\n        if lr_min is not None:\n            lr = tf.math.maximum(lr_min, lr)\n            \n    return lr\n\n    \nrng = [i for i in range(EPOCHS)]\ny = [exponential_schedule_with_warmup(x) for x in rng]\n\nsns.set(style='whitegrid')\nfig, ax = plt.subplots(figsize=(20, 6))\nplt.plot(rng, y)\n\nprint(f'{EPOCHS} total epochs and {NUM_TRAINING_IMAGES//BATCH_SIZE} steps per epoch')\nprint(f'Learning rate schedule: {y[0]:.3g} to { max(y):.3g} to { y[-1]:.3g}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model (EfficientNet)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(input_shape, N_CLASSES):\n    base_model = efn.EfficientNetB4(weights='noisy-student', \n                                    include_top=False,\n                                    input_shape=input_shape)\n\n    model = tf.keras.Sequential([\n                base_model,\n                L.GlobalAveragePooling2D(),\n                L.Dense(N_CLASSES, activation='softmax')\n            ])\n    \n    \n    optimizer = optimizers.Adam(lr=LEARNING_RATE)\n    model.compile(optimizer=optimizer, \n                  loss=losses.SparseCategoricalCrossentropy(), \n                  metrics=['sparse_categorical_accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"kfold = KFold(N_FOLDS, shuffle=True, random_state=seed)\nhistory_list = []\n\n# Datasets\ncomplete_dataset = get_dataset(TRAINING_FILENAMES, labeled=True, ordered=True)\ntest_dataset = get_dataset(TEST_FILENAMES, labeled=False, ordered=True)\nx_complete = complete_dataset.map(lambda image, label: image)\ny_complete = next(iter(complete_dataset.unbatch().map(lambda image, label: label).batch(NUM_TRAINING_IMAGES))).numpy()\nx_test = test_dataset.map(lambda image, idnum: image)\n# Predictions\ncomplete_preds = np.zeros((NUM_TRAINING_IMAGES, N_CLASSES))\ntest_preds = np.zeros((NUM_TEST_IMAGES, N_CLASSES))\n\n\nfor n_fold, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n    if n_fold >= FOLDS_USED:\n        break\n        \n    print(f'\\nFOLD: {n_fold+1}')\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    \n    ### Data\n    fold_train_filenames = np.asarray(TRAINING_FILENAMES)[trn_ind]\n    fold_valid_filenames = np.asarray(TRAINING_FILENAMES)[val_ind]\n    train_size = count_data_items(fold_train_filenames)\n    validation_size = count_data_items(fold_valid_filenames)\n    STEPS_PER_EPOCH = train_size // BATCH_SIZE\n\n    ### Train model\n    K.clear_session()\n    model_path = f'model_{HEIGHT}x{WIDTH}_fold_{n_fold+1}.h5'\n\n    with strategy.scope():\n        model = create_model((None, None, CHANNELS), N_CLASSES)\n\n    es = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE, restore_best_weights=True, verbose=1)\n    checkpoint = ModelCheckpoint(model_path, monitor='val_loss', mode='min', save_best_only=True)\n    lr_callback = LearningRateScheduler(exponential_schedule_with_warmup, verbose=0)\n\n\n    history = model.fit(x=get_dataset(fold_train_filenames, labeled=True, ordered=False, repeated=True, shufled=True, augmented=True), \n                        validation_data=get_dataset(fold_valid_filenames, labeled=True, ordered=True), \n                        callbacks=[checkpoint, es, lr_callback], \n                        steps_per_epoch=STEPS_PER_EPOCH, \n                        epochs=EPOCHS, \n                        verbose=2).history\n    \n    history_list.append(history)\n    complete_preds += model.predict(x_complete) / FOLDS_USED\n    test_preds += model.predict(x_test) / FOLDS_USED\n    \ncomplete_preds = np.argmax(complete_preds, axis=-1)\ntest_preds = np.argmax(test_preds, axis=-1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model loss graph"},{"metadata":{"trusted":true},"cell_type":"code","source":"for index, history in enumerate(history_list):\n    print(f'FOLD {index+1}')\n    plot_metrics(history, metric_list=['loss', 'sparse_categorical_accuracy'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model evaluation"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(classification_report(y_complete, complete_preds, target_names=CLASSES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Confusion matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(20, 45))\ncfn_matrix = confusion_matrix(y_complete, complete_preds, labels=range(len(CLASSES)))\ncfn_matrix = (cfn_matrix.T / cfn_matrix.sum(axis=1)).T\ndf_cm = pd.DataFrame(cfn_matrix, index=CLASSES, columns=CLASSES)\nax = sns.heatmap(df_cm, cmap='Blues').set_title('Labels', fontsize=30)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_samp, y_samp = dataset_to_numpy_util(complete_dataset, 9)\nsamp_preds = model.predict(x_samp, batch_size=9)\ndisplay_9_images_with_predictions(x_samp, samp_preds, y_samp)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test set predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ids_ds = test_dataset.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\nsubmission = pd.DataFrame(test_ids, columns=['id'])\nsubmission['label'] = test_preds\nsubmission.to_csv('submission.csv', index=False)\ndisplay(submission.head(10))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}