{"cells":[{"metadata":{},"cell_type":"markdown","source":"Little info:\n1. Using 224*224px images gives 0.97090 on public LB (https://www.kaggle.com/serosh/accurate-efficientnet-no-ensembling)\n1. Using 512*512px images gives 0.97650 on public LB"},{"metadata":{},"cell_type":"markdown","source":"In this notebook:\n1. train model on all 5 datasets 512*512px (not including validation set)\n2. compute metrics on validation dataset\n3. retrain model on validation dataset\n4. make predictions for test"},{"metadata":{"trusted":true,"pycharm":{"is_executing":false}},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"pycharm":{"is_executing":false}},"cell_type":"code","source":"import math, re, gc\nimport numpy as np # linear algebra\nimport pickle\nfrom datetime import datetime, timedelta\nimport tensorflow as tf\nimport efficientnet.tfkeras as efficientnet\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nprint('TensorFlow version', tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Initial Setup"},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nprint('Replicas:', strategy.num_replicas_in_sync)\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('flower-classification-with-tpus')\nMORE_IMAGES_GCS_DS_PATH = KaggleDatasets().get_gcs_path('tf-flower-photo-tfrec')\nprint(GCS_DS_PATH, '\\n', MORE_IMAGES_GCS_DS_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = datetime.now()\nprint('Time now is', start_time)\nend_training_by_tdelta = timedelta(seconds=8400)\nthis_run_file_prefix = start_time.strftime('%Y%m%d_%H%M_')\nprint(this_run_file_prefix)\n\nIMAGE_SIZE = [512, 512] # [512, 512]\n\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\nGCS_PATH_SELECT = {\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')\n\nMOREIMAGES_PATH_SELECT = {\n    192: '/tfrecords-jpeg-192x192',\n    224: '/tfrecords-jpeg-224x224',\n    331: '/tfrecords-jpeg-331x331',\n    512: '/tfrecords-jpeg-512x512'\n}\nMOREIMAGES_PATH = MOREIMAGES_PATH_SELECT[IMAGE_SIZE[0]]\n\nIMAGENET_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/imagenet' + MOREIMAGES_PATH + '/*.tfrec')\nINATURELIST_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/inaturalist' + MOREIMAGES_PATH + '/*.tfrec')\nOPENIMAGE_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/openimage' + MOREIMAGES_PATH + '/*.tfrec')\nOXFORD_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/oxford_102' + MOREIMAGES_PATH + '/*.tfrec')\nTENSORFLOW_FILES = tf.io.gfile.glob(MORE_IMAGES_GCS_DS_PATH + '/tf_flowers' + MOREIMAGES_PATH + '/*.tfrec')\nADDITIONAL_TRAINING_FILENAMES = IMAGENET_FILES + INATURELIST_FILES + OPENIMAGE_FILES + OXFORD_FILES + TENSORFLOW_FILES\n#print(VALIDATION_FILENAMES)\nprint('----')\nTRAINING_FILENAMES = TRAINING_FILENAMES + ADDITIONAL_TRAINING_FILENAMES\n#print(TRAINING_FILENAMES)\n\n# This is so awkward. Everyone is doing this for an extra few points.\n# TRAINING_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES\n# VALIDATION_FILENAMES = TRAINING_FILENAMES\n\nCLASSES = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', 'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', 'globe thistle', # 00 - 09\n           'snapdragon', \"colt's foot\", 'king protea', 'spear thistle', 'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower', 'giant white arum lily', # 10 - 19\n           'fire lily', 'pincushion flower', 'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william', # 20 - 29\n           'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip', 'lenten rose', # 30 - 39\n           'barberton daisy', 'daffodil', 'sword lily', 'poinsettia', 'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion', # 40 - 49\n           'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia', # 50 - 59\n           'cautleya spicata', 'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy', # 60 - 69\n           'gazania', 'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily', 'anthurium', # 70 - 79\n           'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily', # 80 - 89\n           'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower', # 90 - 99\n           'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose'] # 100 - 102","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Out of 4 models used for ensembling, two of the models use LR_EXP_DECAY of 0.8 and two of the models use 0.75."},{"metadata":{"trusted":true},"cell_type":"code","source":"LR_START = 0.00001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = LR_START\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0 # 3\nLR_EXP_DECAY = 0.80\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = LR_START + (epoch * (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS)\n    elif epoch < (LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS):\n        lr = LR_MAX\n    else:\n        lr = LR_MIN + (LR_MAX - LR_MIN) * LR_EXP_DECAY ** (epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)\n#    print('For epoch', epoch, 'setting lr to', lr)\n    return lr\n\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n\nrng = [i for i in range(20)]\ny = [lrfn(x) for x in rng]\nprint(y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# numpy and matplotlib defaults\nnp.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n    \ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n\ndef display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()\n    \ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels = 3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n#\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'class': tf.io.FixedLenFeature([], tf.int64),\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label\n#\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'id': tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum\n#\n\ndef load_dataset(filenames, labeled = True, ordered = False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO)\n    return dataset\n#\n\ndef data_augment(image, label):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_saturation(image, 0, 2)\n    return image, label\n#\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled = True)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n#\n\ndef get_validation_dataset(ordered = False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled = True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n#\n\ndef get_test_dataset(ordered = False):\n    dataset = load_dataset(TEST_FILENAMES, labeled = False, ordered = ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n#\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n#\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Training data shapes')\nfor image, label in get_training_dataset().take(3):\n    print(image.numpy().shape, label.numpy().shape)\nprint('Training data label examples:', label.numpy())\n#\n\nprint('Validation data shapes')\nfor image, label in get_validation_dataset().take(3):\n    print(image.numpy().shape, label.numpy().shape)\nprint('Validation data label examples:', label.numpy())\n#\n\nprint('Test data shapes')\nfor image, idnum in get_test_dataset().take(3):\n    print(image.numpy().shape, idnum.numpy().shape)\nprint('Test data IDs:', idnum.numpy().astype('U'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_dataset = get_training_dataset()\ntraining_dataset = training_dataset.unbatch().batch(20)\ntrain_batch = iter(training_dataset)\n#\n#display_batch_of_images(next(train_batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = get_test_dataset()\ntest_dataset = test_dataset.unbatch().batch(20)\ntest_batch = iter(test_dataset)\n#\n#display_batch_of_images(next(test_batch))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Setup and Training\nA lot of code below are not used in this particular notebook. But code previously used in other notebooks were not deleted intentionally. It should give an indication on the different kinds of experiments one could run for this competition."},{"metadata":{"trusted":true},"cell_type":"code","source":"early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=20, restore_best_weights = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_VGG16_model():\n    pretrained_model = tf.keras.applications.VGG16(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])\n    pretrained_model.trainable = True # False\n\n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')\n    ])\n\n    model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_Xception_model():\n    pretrained_model = tf.keras.applications.Xception(include_top = False, input_shape = [*IMAGE_SIZE, 3])\n    pretrained_model.trainable = True\n\n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')\n    ])\n\n    model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_DenseNet_model():\n    pretrained_model = tf.keras.applications.DenseNet201(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])\n    pretrained_model.trainable = True\n\n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')\n    ])\n\n    model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_EfficientNet_model():\n    pretrained_model = efficientnet.EfficientNetB7(weights = 'noisy-student', include_top = False, input_shape = [*IMAGE_SIZE, 3])\n    pretrained_model.trainable = True\n\n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')\n    ])\n\n    model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_InceptionV3_model():\n    pretrained_model = tf.keras.applications.InceptionV3(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])\n    pretrained_model.trainable = True\n\n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')\n    ])\n\n    model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_ResNet152_model():\n    pretrained_model = tf.keras.applications.ResNet152V2(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])\n    pretrained_model.trainable = True\n\n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')\n    ])\n\n    model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_MobileNetV2_model():\n    pretrained_model = tf.keras.applications.MobileNetV2(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])\n    pretrained_model.trainable = True\n\n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')\n    ])\n\n    model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_InceptionResNetV2_model():\n    pretrained_model = tf.keras.applications.InceptionResNetV2(weights = 'imagenet', include_top = False, input_shape = [*IMAGE_SIZE, 3])\n    pretrained_model.trainable = True\n\n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')\n    ])\n\n    model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['sparse_categorical_accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# no_of_models = 1\n# models = [0] * no_of_models\n# start_model = 0\n# end_model = 1\n# model_indx_0 = start_model\n#model_indx_1 = start_model + 1\n#model_indx_2 = start_model + 2\n\n# val_probabilities = [0] * no_of_models\n# test_probabilities = [0] * no_of_models\n# all_probabilities = [0] * no_of_models","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Out of the four models used in ensembling, two of the models use EfficientNetB7 and two others use DenseNet201."},{"metadata":{"trusted":true},"cell_type":"code","source":"#with strategy.scope():\n#    models[0] = create_DenseNet_model()\n#    models[1] = create_EfficientNet_model()\n#print(models[0].summary())\n#print(models[1].summary())\n#\nwith strategy.scope():\n    # model = create_VGG16_model()\n    # model = create_Xception_model()\n    # model = create_DenseNet_model()\n    model = create_EfficientNet_model()\n    # model = create_InceptionV3_model()\n    # model = create_ResNet152_model()\n    # model = create_MobileNetV2_model()\n    # model = create_InceptionResNetV2_model()\n\nmodel_name = model.layers[0].name\nmodel.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TimeCallback(tf.keras.callbacks.Callback): \n    def on_epoch_end(self, epoch, logs={}):\n        if((datetime.now() - start_time).total_seconds() > 9500):\n            self.model.stop_training = True\n            \ntime_callback = TimeCallback()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nEPOCHS = 30 # 30 # 50 # 35 # 2 # 20\n\nprint('LR_EXP_DECAY:', LR_EXP_DECAY, '. LR_MAX:', LR_MAX)\nhistory = model.fit(get_training_dataset(), steps_per_epoch = STEPS_PER_EPOCH, epochs = EPOCHS, validation_data = get_validation_dataset(), callbacks = [lr_callback, early_stop, time_callback])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cmdataset = get_validation_dataset(ordered = True)\nimages_ds = cmdataset.map(lambda image, label: image)\nlabels_ds = cmdataset.map(lambda image, label: label).unbatch()\ncm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntest_ds = get_test_dataset(ordered = True)\n#print('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dataset = get_validation_dataset()\n# dataset = dataset.unbatch().batch(20)\n# batch = iter(dataset)\n# \n# images, labels = next(batch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(datetime.now())\n# \n# for j in range(start_model, end_model):\n#     val_probabilities[j] = models[j].predict(images_ds)\n#     test_probabilities[j] = models[j].predict(test_images_ds)\n#     all_probabilities[j] = models[j].predict(images)\n# \n# print(datetime.now())\n#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_training_curves(history.history['loss'], history.history['val_loss'], 'loss', 211)\ndisplay_training_curves(history.history['sparse_categorical_accuracy'], history.history['val_sparse_categorical_accuracy'], 'accuracy', 212)","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n"},"trusted":true},"cell_type":"code","source":"val_probabilities = model.predict(images_ds)\nnp.save(model_name + '_val_probabilities', val_probabilities)\n\ncm_predictions = np.argmax(val_probabilities, axis = -1)\nprint('Correct labels: ', cm_correct_labels.shape, cm_correct_labels)\nprint('Predicted labels: ', cm_predictions.shape, cm_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def getFitPrecisionRecall(correct_labels, predictions):\n    score = f1_score(correct_labels, predictions, labels = range(len(CLASSES)), average = 'macro')\n    precision = precision_score(correct_labels, predictions, labels = range(len(CLASSES)), average = 'macro')\n    recall = recall_score(correct_labels, predictions, labels = range(len(CLASSES)), average = 'macro')\n    return score, precision, recall\n#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels = range(len(CLASSES)))\nscore, precision, recall = getFitPrecisionRecall(cm_correct_labels, cm_predictions)\ncmat = (cmat.T / cmat.sum(axis = -1)).T\ndisplay_confusion_matrix(cmat, score, precision, recall)\nprint('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_submission_file(filename, probabilities):\n    predictions = np.argmax(probabilities, axis = -1)\n    print('Generating submission file...', filename)\n    test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n    test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\n    np.savetxt(filename, np.rec.fromarrays([test_ids, predictions]), fmt = ['%s', '%d'], delimiter = ',', header = 'id,label', comments = '')\n#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntest_probabilities_no_val = model.predict(test_images_ds)\ncreate_submission_file('submission_no_val.csv', test_probabilities_no_val)","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n"},"trusted":true},"cell_type":"code","source":"%%time\n\nprint('LR_EXP_DECAY:', LR_EXP_DECAY, '. LR_MAX:', LR_MAX)\nmodel.fit(get_validation_dataset(), steps_per_epoch = STEPS_PER_EPOCH, epochs = 25, callbacks = [lr_callback])","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n"},"trusted":true},"cell_type":"code","source":"%%time\n\ntest_probabilities = model.predict(test_images_ds)\ncreate_submission_file('submission.csv', test_probabilities)","execution_count":null,"outputs":[]},{"metadata":{"pycharm":{"name":"#%%\n"},"trusted":true},"cell_type":"code","source":"pklfile = open(model_name + '_test_probs_with_val.pkl', 'ab')\npickle.dump(dict(zip(test_ids, test_probabilities)), pklfile)\npklfile.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save_weights(model_name + '_weights.h5')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"pycharm":{"stem_cell":{"cell_type":"raw","source":[],"metadata":{"collapsed":false}}}},"nbformat":4,"nbformat_minor":4}