{"cells":[{"metadata":{},"cell_type":"markdown","source":"### About this kernel\n\nV4: 0.94786\n\nV5: Data Augmentation. Epoch:20\n\nV6: Fail\n\nV7: Fail\n\nV8: 0.96084\n\nV8: ----"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import math, re, os, random\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nimport efficientnet.tfkeras as efn\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.applications import DenseNet201\nprint(\"Tensorflow version \" + tf.__version__)\nfrom sklearn.model_selection import KFold","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configurations"},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Configuration\nIMAGE_SIZE = [512, 512]\nEPOCHS = 25\nFOLDS = 5\nSEED = 777\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Mixed Precision and/or XLA"},{"metadata":{},"cell_type":"markdown","source":"The following boolens can enable mixed precision and/or XLA on TPU/GPU. By default TPU already uses some mixed precision butwe can add more. These allow the GPU/TPU memory to handle larger batch sizes and can speed up the training process. The Nvidia V100 GPU has special Tensor cores which get utilized when mixed precision is enabled. Unfortunately Kaggle's Nnidia P100 GPU doesn't have that Tensor COres to receive speed up."},{"metadata":{"trusted":true},"cell_type":"code","source":"MIXED_PRECISION = True\nXLA_ACCELERATE = True\n\nif MIXED_PRECISION:\n    from tensorflow.keras.mixed_precision import experimental as mixed_precision\n    if tpu:\n        policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')\n    else:\n        policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n   \n    mixed_precision.set_policy(policy)\n    print('Mixed precision enabled')\n\n\nif XLA_ACCELERATE:\n    tf.config.optimizer.set_jit(True)\n    print('Accelerated Linear Algebra enabled')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Directories"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Data access\nGCS_DS_PATH = KaggleDatasets().get_gcs_path()\n\nGCS_PATH_SELECT = { # available image sizes\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec') + tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\n#VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec') # predictions on this dataset should be submitted for the competition\n\n# watch out for overfitting!\nSKIP_VALIDATION = False\nif SKIP_VALIDATION:\n    TRAINING_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Classes"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"CLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Custom LR scheduler"},{"metadata":{"trusted":true},"cell_type":"code","source":"LR_START = 0.00001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 10\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions\n## Visualization"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()\n    \ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Datasets Functions"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    #image = tf.image.random_saturation(image, 0, 2)\n    image = tf.image.random_flip_up_down(image, seed=SEED)\n    #image = tf.image.random_saturation(image, lower=0, upper=2, seed=None)\n    #image = tf.image.random_contrast(image, lower=.8, upper=2, seed=None)\n    #image = tf.image.random_brightness(image, max_delta=.2, seed=None)\n    \n    return image, label   \n\ndef get_training_dataset(dataset, do_aug=True):\n    #dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    if do_aug:\n        dataset = dataset.map(transform, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\n\ndef get_validation_dataset(dataset):#ordered=False):\n    #dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES) * (FOLDS - 1.0)/FOLDS)\nNUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES) * 1.0/FOLDS)\nNUM_TEST_IMAGES = int(count_data_items(TEST_FILENAMES))\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Augmnetation"},{"metadata":{},"cell_type":"markdown","source":"The following code does rotations, shear, shift, and zoom using the GPU/TPU. When an image gets moved away from an edge revealing blank space, the blank space is filled by stretching the colors on the original edge. Change the variables in function transform() below to control the desired amount of augmentation."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform(image,label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 15. * tf.random.normal([1],dtype='float32')\n    shr = 5. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 16. * tf.random.normal([1],dtype='float32') \n    w_shift = 16. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3]),label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Display Example Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# row = 3; col = 4;\n# all_elements = get_training_dataset(load_dataset(TRAINING_FILENAMES), do_aug=True).unbatch()\n# one_element = tf.data.Dataset.from_tensors(next(iter(all_elements)) )\n# augmented_element = one_element.repeat().map(transform).batch(row*col)\n\n# for (img,label) in augmented_element:\n#     plt.figure(figsize=(15,int(15*row/col)))\n#     for j in range(row*col):\n#         plt.subplot(row,col,j+1)\n#         plt.axis('off')\n#         plt.imshow(img[j,])\n#     plt.show()\n#     break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# row = 3; col = 4;\n# all_elements = get_training_dataset(load_dataset(TRAINING_FILENAMES), do_aug=True).unbatch()\n# one_element = tf.data.Dataset.from_tensors(next(iter(all_elements)) )\n# augmented_element = one_element.repeat().map(transform).batch(row*col)\n\n# for (img,label) in augmented_element:\n#     plt.figure(figsize=(15,int(15*row/col)))\n#     for j in range(row*col):\n#         plt.subplot(row,col,j+1)\n#         plt.axis('off')\n#         plt.imshow(img[j,])\n#     plt.show()\n#     break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# row = 3; col = 4;\n# all_elements = get_training_dataset(load_dataset(TRAINING_FILENAMES), do_aug=True).unbatch()\n# one_element = tf.data.Dataset.from_tensors(next(iter(all_elements)) )\n# augmented_element = one_element.repeat().map(transform).batch(row*col)\n\n# for (img,label) in augmented_element:\n#     plt.figure(figsize=(15,int(15*row/col)))\n#     for j in range(row*col):\n#         plt.subplot(row,col,j+1)\n#         plt.axis('off')\n#         plt.imshow(img[j,])\n#     plt.show()\n#     break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Model"},{"metadata":{},"cell_type":"markdown","source":"## Load Model into TPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Need this line so Google will recite some incantations\n# for Turing to magically load the model onto the TPU\ndef get_model():\n\n    with strategy.scope():\n        enet = efn.EfficientNetB7(\n            input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),\n            weights='imagenet',\n            include_top=False\n        )\n        enet.trainable = True\n\n        model = tf.keras.Sequential([\n            enet,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(len(CLASSES), activation='softmax', dtype='float32')\n        ])\n\n    model.compile(\n        optimizer='adam',\n        loss = 'sparse_categorical_crossentropy',\n        metrics=['sparse_categorical_accuracy']\n    )\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_cross_validate(test_images_ds, folds=5):\n    histories = []\n    probabilities = []\n    all_labels = []\n    all_prob = []\n    all_pred = []\n    \n    early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 5)\n    kfold = KFold(folds, shuffle=True, random_state=SEED)\n    \n    for f, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n        \n        # train part\n        print()\n        print('#'*25)\n        print('### FOLD', f+1)\n        print('#'*25)\n        \n        train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)\n        val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)\n        model = get_model()\n        history = model.fit(\n            get_training_dataset(train_dataset, do_aug=True), \n            steps_per_epoch = STEPS_PER_EPOCH,\n            epochs = EPOCHS,\n            callbacks = [lr_callback, early_stopping],\n            validation_data = get_validation_dataset(val_dataset),\n            verbose=2\n        )\n        probabilities.append(model.predict(test_images_ds))\n        histories.append(history)\n        \n        # inference part\n        print('Inferring fold',f+1,'validation images...')\n        VAL_FILES = list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES'])\n        NUM_VALIDATION_IMAGES = count_data_items(VAL_FILES)\n        cmdataset = get_validation_dataset(load_dataset(VAL_FILES, labeled = True, ordered = True))\n        images_ds = cmdataset.map(lambda image, label: image)\n        labels_ds = cmdataset.map(lambda image, label: label).unbatch()\n        all_labels.append( next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy() ) # get everything as one batch\n        prob = model.predict(images_ds)\n        all_prob.append( prob )\n        all_pred.append( np.argmax(prob, axis=-1))\n        \n        \n    cm_correct_labels = np.concatenate(all_labels)\n    cm_probabilities = np.concatenate(all_prob)\n    cm_predictions = np.concatenate(all_pred)\n    \n    \n    return histories, probabilities, cm_correct_labels, cm_predictions\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#def train_and_predict(folds = 5):\n\n\ntest_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\ntest_images_ds = test_ds.map(lambda image, idnum: image)\n\nprint('Start training %i folds'%FOLDS)\nhistories, probabilities, cm_correct_labels, cm_pre = train_cross_validate(test_images_ds, folds = FOLDS)\nprint('Computing predictions...')\n\n# get the mean probability of the folds models\nprobabilities = np.average(probabilities, axis = 0)\npredictions = np.argmax(probabilities, axis=-1)\n\nprint('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\n\n\n\n# run train and predict\n#histories, models = train_and_predict(FOLDS = FOLDS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Correct   labels: \", cm_correct_labels.shape, cm_correct_labels)\nprint(\"Predicted labels: \", cm_pre.shape, cm_pre)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cmat = confusion_matrix(cm_correct_labels, cm_pre, labels=range(len(CLASSES)))\nscore = f1_score(cm_correct_labels, cm_pre, labels=range(len(CLASSES)), average='macro')\nprecision = precision_score(cm_correct_labels, cm_pre, labels=range(len(CLASSES)), average='macro')\nrecall = recall_score(cm_correct_labels, cm_pre, labels=range(len(CLASSES)), average='macro')\ndisplay_confusion_matrix(cmat, score, precision, recall)\nprint('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Need this line so Google will recite some incantations\n# for Turing to magically load the model onto the TPU\ndef get_model_2():\n\n    with strategy.scope():\n        dnet = DenseNet201(\n            input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),\n            weights='imagenet',\n            include_top=False\n        )\n        dnet.trainable = True\n\n        model = tf.keras.Sequential([\n            dnet,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(len(CLASSES), activation='softmax', dtype='float32')\n        ])\n\n    model.compile(\n        optimizer='adam',\n        loss = 'sparse_categorical_crossentropy',\n        metrics=['sparse_categorical_accuracy']\n    )\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_cross_validate_2(test_images_ds, folds=5):\n    histories = []\n    probabilities = []\n    all_labels = []\n    all_prob = []\n    all_pred = []\n    \n    early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 5)\n    kfold = KFold(folds, shuffle=True, random_state=SEED)\n    \n    for f, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n        \n        # train part\n        print()\n        print('#'*25)\n        print('### FOLD', f+1)\n        print('#'*25)\n        \n        train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)\n        val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)\n        model = get_model()\n        history = model.fit(\n            get_training_dataset(train_dataset, do_aug=True), \n            steps_per_epoch = STEPS_PER_EPOCH,\n            epochs = EPOCHS,\n            callbacks = [lr_callback, early_stopping],\n            validation_data = get_validation_dataset(val_dataset),\n            verbose=2\n        )\n        probabilities.append(model.predict(test_images_ds))\n        histories.append(history)\n        \n        # inference part\n        print('Inferring fold',f+1,'validation images...')\n        VAL_FILES = list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES'])\n        NUM_VALIDATION_IMAGES = count_data_items(VAL_FILES)\n        cmdataset = get_validation_dataset(load_dataset(VAL_FILES, labeled = True, ordered = True))\n        images_ds = cmdataset.map(lambda image, label: image)\n        labels_ds = cmdataset.map(lambda image, label: label).unbatch()\n        all_labels.append( next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy() ) # get everything as one batch\n        prob = model.predict(images_ds)\n        all_prob.append( prob )\n        all_pred.append( np.argmax(prob, axis=-1))\n        \n        \n    cm_correct_labels = np.concatenate(all_labels)\n    cm_probabilities = np.concatenate(all_prob)\n    cm_predictions = np.concatenate(all_pred)\n    \n    \n    return histories, probabilities, cm_correct_labels, cm_predictions\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #def train_and_predict(folds = 5):\n# #model = get_model()\n\n# test_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n# test_images_ds = test_ds.map(lambda image, idnum: image)\n\n# print('Start training %i folds'%FOLDS)\n# histories_2, probabilities_2, cm_correct_labels_2, cm_predictions_2 = train_cross_validate_2(test_images_ds, folds = FOLDS)\n# print('Computing predictions...')\n\n# # get the mean probability of the folds models\n# probabilities_2 = np.average(probabilities_2, axis = 0)\n# predictions_2 = np.argmax(probabilities_2, axis=-1)\n\n# print('Generating submission.csv file...')\n# test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n# test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\n# #np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\n\n\n\n# # run train and predict\n# #histories, models = train_and_predict(FOLDS = FOLDS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# with strategy.scope():\n#     enet = efn.EfficientNetB7(\n#         input_shape=(512, 512, 3),\n#         weights='imagenet',\n#         include_top=False\n#     )\n\n#     model = tf.keras.Sequential([\n#         enet,\n#         tf.keras.layers.GlobalAveragePooling2D(),\n#         tf.keras.layers.Dense(len(CLASSES), activation='softmax', dtype='float32')\n#     ])\n        \n# model.compile(\n#     optimizer=tf.keras.optimizers.Adam(),\n#     loss = 'sparse_categorical_crossentropy',\n#     metrics=['sparse_categorical_accuracy']\n# )\n# model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# history = model.fit(\n#     get_training_dataset(do_aug=True), \n#     steps_per_epoch=STEPS_PER_EPOCH,\n#     epochs=EPOCHS,\n#     callbacks=[lr_callback],\n#     validation_data = None if SKIP_VALIDATION else get_validation_dataset()\n# )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# if not SKIP_VALIDATION:\n#     display_training_curves(history.history['loss'], history.history['val_loss'], 'loss', 211)\n#     display_training_curves(history.history['sparse_categorical_accuracy'], history.history['val_sparse_categorical_accuracy'], 'accuracy', 212)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# history = model.fit(\n#     get_training_dataset(do_aug=True), \n#     steps_per_epoch=STEPS_PER_EPOCH,\n#     epochs=5,\n#     #callbacks=[lr_callback],\n#     validation_data = None if SKIP_VALIDATION else get_validation_dataset()\n# )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training 2:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# with strategy.scope():\n#     rnet = DenseNet201(\n#         input_shape=(512, 512, 3),\n#         weights='imagenet',\n#         include_top=False\n#     )\n\n#     model2 = tf.keras.Sequential([\n#         rnet,\n#         tf.keras.layers.GlobalAveragePooling2D(),\n#         tf.keras.layers.Dense(len(CLASSES), activation='softmax', dtype='float32')\n#     ])\n        \n# model2.compile(\n#     optimizer=tf.keras.optimizers.Adam(),\n#     loss = 'sparse_categorical_crossentropy',\n#     metrics=['sparse_categorical_accuracy']\n# )\n# model2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# history2 = model2.fit(\n#     get_training_dataset(do_aug=True), \n#     steps_per_epoch=STEPS_PER_EPOCH,\n#     epochs=EPOCHS, \n#     callbacks=[lr_callback],\n#     validation_data=None if SKIP_VALIDATION else get_validation_dataset()\n# )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# if not SKIP_VALIDATION:\n#     display_training_curves(history2.history['loss'], history2.history['val_loss'], 'loss', 211)\n#     display_training_curves(history2.history['sparse_categorical_accuracy'], history2.history['val_sparse_categorical_accuracy'], 'accuracy', 212)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Finding best alpha\nOur final model is just mix of two presented above. In the first commit it was arithmetic mean (alpha = 0.5). Note that using validation data as training will fit your model with accuracy equal 1.0.\nThus formula presented below of linear combination of models will work only with validation data:\n\nprob = alpha  prob(model) + (1 - alpha)  prob(model2)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# if not SKIP_VALIDATION:\n#     cmdataset = get_validation_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and labels, order matters.\n#     images_ds = cmdataset.map(lambda image, label: image)\n#     labels_ds = cmdataset.map(lambda image, label: label).unbatch()\n#     cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy() # get everything as one batch\n#     m = model.predict(images_ds)\n#     m2 = model2.predict(images_ds)\n#     scores = []\n#     for alpha in np.linspace(0,1,100):\n#         cm_probabilities = alpha*m+(1-alpha)*m2\n#         cm_predictions = np.argmax(cm_probabilities, axis=-1)\n#         scores.append(f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro'))\n        \n#     print(\"Correct   labels: \", cm_correct_labels.shape, cm_correct_labels)\n#     print(\"Predicted labels: \", cm_predictions.shape, cm_predictions)\n#     plt.plot(scores)\n#     best_alpha = np.argmax(scores)/100\n#     cm_probabilities = best_alpha*m+(1-best_alpha)*m2\n#     cm_predictions = np.argmax(cm_probabilities, axis=-1)\n# else:\n#     best_alpha = 0.44","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Confusion matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"# if not SKIP_VALIDATION:\n#     cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))\n#     score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\n#     precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\n#     recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\n#     #cmat = (cmat.T / cmat.sum(axis=1)).T # normalized\n#     display_confusion_matrix(cmat, score, precision, recall)\n#     print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n\n# print('Computing predictions...')\n# test_images_ds = test_ds.map(lambda image, idnum: image)\n# probabilities = best_alpha*model.predict(test_images_ds) + (1-best_alpha)*model2.predict(test_images_ds)\n# predictions = np.argmax(probabilities, axis=-1)\n# print(predictions)\n\n# print('Generating submission.csv file...')\n# test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n# test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\n# np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}