{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Petals to the Metal - DL project\n\nNames : Géraud FAYE, Lila SAINERO, Arnaud LOUYS\n\nThis notebook is inspired by the notebook provided by Kaggle to setup a TPU environment.","metadata":{}},{"cell_type":"markdown","source":"# Setting up a TPU environment","metadata":{}},{"cell_type":"code","source":"import os\nos.environ['TF_XLA_FLAGS'] = '--tf_xla_enable_xla_devices'\n\nimport tensorflow as tf\nfrom kaggle_datasets import KaggleDatasets\n\nprint(\"Tensorflow version \" + tf.__version__)\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-01-02T16:23:28.952352Z","iopub.execute_input":"2022-01-02T16:23:28.95271Z","iopub.status.idle":"2022-01-02T16:23:40.847995Z","shell.execute_reply.started":"2022-01-02T16:23:28.952618Z","shell.execute_reply":"2022-01-02T16:23:40.84712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Getting Kaggle data paths","metadata":{}},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path()","metadata":{"execution":{"iopub.status.busy":"2022-01-02T16:23:40.849382Z","iopub.execute_input":"2022-01-02T16:23:40.849612Z","iopub.status.idle":"2022-01-02T16:23:41.236428Z","shell.execute_reply.started":"2022-01-02T16:23:40.849585Z","shell.execute_reply":"2022-01-02T16:23:41.235615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Setting learning parameters","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE = [512, 512] # at this size, a GPU will run out of memory. We can use the TPU to solve this issue\nEPOCHS = 200\n# The batch size can be a multiple of the number of TPU cores to balance the computing load\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\n# Dataset characteristics\nNUM_TRAINING_IMAGES = 12753\nNUM_TEST_IMAGES = 7382\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE","metadata":{"execution":{"iopub.status.busy":"2022-01-02T16:23:56.286067Z","iopub.execute_input":"2022-01-02T16:23:56.286757Z","iopub.status.idle":"2022-01-02T16:23:56.291716Z","shell.execute_reply.started":"2022-01-02T16:23:56.286713Z","shell.execute_reply":"2022-01-02T16:23:56.291089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load my data\n\nThis data is loaded from Kaggle and automatically sharded to maximize parallelization.","metadata":{}},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n#     image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n#     For InceptionV3, the pictures must have pixels values between -1 and 1\n    image = (tf.cast(image, tf.float32) / 255.0) * 2.0 - 1.0\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\n# Data augmentation to make generalization better\n# Note that only functions in tf.keras.layers.experimental.preprocessing are supported by TPUs\naugmentations = tf.keras.Sequential([\n    # Flip the picture horizontally with a 50% chance\n    tf.keras.layers.experimental.preprocessing.RandomFlip(mode='horizontal'),\n    # Randomly change the picture contrast between [1 - factor, 1 + factor]\n    tf.keras.layers.experimental.preprocessing.RandomContrast(factor=0.2),\n    # Random rotation between [-factor * 2*pi, + factor * 2*pi]\n    tf.keras.layers.experimental.preprocessing.RandomRotation(factor=0.5, dtype=tf.float32),\n    # Randomly zoom or dezoom the picture. By default, the picture will be reflected at its ends if it is dezoomed\n    tf.keras.layers.experimental.preprocessing.RandomZoom((-0.5, 0.5))\n])\n\ndef data_augment(image, label):\n    image = augmentations(image, training=True)\n    return image, label\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)\n    # Data augmentation\n#     if labeled:\n#         dataset.map(data_augment, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef get_training_dataset():\n#     dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-192x192/train/*.tfrec'), labeled=True)\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-512x512/train/*.tfrec'), labeled=True)\n#     dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-331x331/train/*.tfrec'), labeled=True)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset():\n#     dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-192x192/val/*.tfrec'), labeled=True, ordered=False)\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-512x512/val/*.tfrec'), labeled=True, ordered=False)\n#     dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-331x331/val/*.tfrec'), labeled=True, ordered=False)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    return dataset\n\ndef get_test_dataset(ordered=False):\n#     dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-192x192/test/*.tfrec'), labeled=False, ordered=ordered)\n    dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-512x512/test/*.tfrec'), labeled=False, ordered=ordered)\n#     dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords-jpeg-331x331/test/*.tfrec'), labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset\n\ntraining_dataset = get_training_dataset()\nvalidation_dataset = get_validation_dataset()","metadata":{"execution":{"iopub.status.busy":"2022-01-02T16:26:38.472948Z","iopub.execute_input":"2022-01-02T16:26:38.473233Z","iopub.status.idle":"2022-01-02T16:26:39.151036Z","shell.execute_reply.started":"2022-01-02T16:26:38.4732Z","shell.execute_reply":"2022-01-02T16:26:39.150092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing data","metadata":{}},{"cell_type":"code","source":" CLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 103","metadata":{"execution":{"iopub.status.busy":"2022-01-02T16:26:49.919822Z","iopub.execute_input":"2022-01-02T16:26:49.920695Z","iopub.status.idle":"2022-01-02T16:26:49.929169Z","shell.execute_reply.started":"2022-01-02T16:26:49.92066Z","shell.execute_reply":"2022-01-02T16:26:49.928571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nimport matplotlib.pyplot as plt\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case,these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n        # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], \n                                'OK' if correct else 'NO', \n                                u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n\ndef display_batch_of_images(databatch, predictions=None, display_mismatches_only=False):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        if display_mismatches_only:\n            if predictions[i] != label:\n                subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n        else:        \n            subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-02T16:26:53.171615Z","iopub.execute_input":"2022-01-02T16:26:53.171908Z","iopub.status.idle":"2022-01-02T16:26:53.19114Z","shell.execute_reply.started":"2022-01-02T16:26:53.171877Z","shell.execute_reply":"2022-01-02T16:26:53.190519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We visualize a small batch of validation pictures.","metadata":{}},{"cell_type":"code","source":"ds_iter = iter(validation_dataset.unbatch().batch(20))\ndisplay_batch_of_images(next(ds_iter))","metadata":{"execution":{"iopub.status.busy":"2022-01-02T16:27:15.064285Z","iopub.execute_input":"2022-01-02T16:27:15.065118Z","iopub.status.idle":"2022-01-02T16:27:19.078075Z","shell.execute_reply.started":"2022-01-02T16:27:15.065082Z","shell.execute_reply":"2022-01-02T16:27:19.07742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And then a batch of augmented data used for training.","metadata":{}},{"cell_type":"code","source":"ds_iter = iter(training_dataset.unbatch().batch(20))\ndisplay_batch_of_images(next(ds_iter))","metadata":{"execution":{"iopub.status.busy":"2022-01-02T16:27:34.036223Z","iopub.execute_input":"2022-01-02T16:27:34.036544Z","iopub.status.idle":"2022-01-02T16:27:55.565773Z","shell.execute_reply.started":"2022-01-02T16:27:34.036514Z","shell.execute_reply":"2022-01-02T16:27:55.564588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build and fit our model","metadata":{}},{"cell_type":"markdown","source":"Experiments show that we obtain best accuracy while using a model pretrained with ImageNet and adding fully-connected layers after it.\nWe finetune this network and do not use it only as a feature extractor, as ImageNet is not a dataset specialized in flowers.","metadata":{}},{"cell_type":"code","source":"# # with strategy.scope() is necessary to build the model inside the TPU\n# with strategy.scope():    \n#     pretrained_model = tf.keras.applications.EfficientNetB7(\n#                                                     weights='imagenet',\n#                                                     include_top=False ,\n#                                                     input_shape=(*IMAGE_SIZE, 3),\n#                                                     pooling=None,\n#                                                     classes=1000)\n#     pretrained_model.trainable = True # tramsfer learning\n    \n#     model = tf.keras.Sequential([\n#         pretrained_model,\n#         tf.keras.layers.GlobalAveragePooling2D(),\n#         tf.keras.layers.Dense(500, activation=\"relu\"),\n#         tf.keras.layers.Dense(104, activation='softmax')\n#     ])\n        \n# model.compile(\n#     optimizer=tf.keras.optimizers.Adam(lr=0.0001),\n#     loss = 'sparse_categorical_crossentropy',\n#     metrics=['sparse_categorical_accuracy']\n# )\n\n# # We use early stopping to prevent overfitting\n# early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n\n# historical = model.fit(training_dataset, \n#           steps_per_epoch=STEPS_PER_EPOCH, \n#           epochs=EPOCHS, \n#           validation_data=validation_dataset,\n#           callbacks=[early_stopping])","metadata":{"execution":{"iopub.status.busy":"2021-12-24T13:09:44.094414Z","iopub.execute_input":"2021-12-24T13:09:44.094682Z","iopub.status.idle":"2021-12-24T13:09:47.231326Z","shell.execute_reply.started":"2021-12-24T13:09:44.094653Z","shell.execute_reply":"2021-12-24T13:09:47.2306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing training results","metadata":{}},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n\n# plt.plot(historical.history['sparse_categorical_accuracy'])\n# plt.plot(historical.history['val_sparse_categorical_accuracy'])\n# plt.title('model sparse categorical accuracy')\n# plt.ylabel('accuracy')\n# plt.xlabel('epoch')\n# plt.legend(['train', 'test'], loc='upper left')\n# plt.show()\n# plt.plot(historical.history['loss'])\n# plt.plot(historical.history['val_loss'])\n# plt.title('model loss')\n# plt.ylabel('loss')\n# plt.xlabel('epoch')\n# plt.legend(['train', 'test'], loc='upper left')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-24T13:10:14.388966Z","iopub.execute_input":"2021-12-24T13:10:14.389228Z","iopub.status.idle":"2021-12-24T13:10:14.393779Z","shell.execute_reply.started":"2021-12-24T13:10:14.389201Z","shell.execute_reply":"2021-12-24T13:10:14.393086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compute your predictions on the test set!\n\nThis will create a file that can be submitted to the competition.","metadata":{}},{"cell_type":"code","source":"# import numpy as np\n\n# test_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n\n# print('Computing predictions...')\n# test_images_ds = test_ds.map(lambda image, idnum: image)\n# probabilities = model.predict(test_images_ds)\n# predictions = np.argmax(probabilities, axis=-1)\n# print(predictions)\n\n# print('Generating submission.csv file...')\n# test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n# test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\n# np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\n# np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')","metadata":{"execution":{"iopub.status.busy":"2021-12-24T13:10:16.580203Z","iopub.execute_input":"2021-12-24T13:10:16.580916Z","iopub.status.idle":"2021-12-24T13:10:16.58592Z","shell.execute_reply.started":"2021-12-24T13:10:16.58088Z","shell.execute_reply":"2021-12-24T13:10:16.585063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble learning to improve accuracy","metadata":{}},{"cell_type":"markdown","source":"We train several models as trained above.\nFor each model, we compute the probabilities for each picture to be of each category.\nWe then sum these probabilities and choose the category with the best sum.\n\nIt is different of majority vote where each model takes a decision and the most frequent decision is adopted.\nWe choose the most probable category according to the ensemble of models.","metadata":{}},{"cell_type":"code","source":"import numpy as np\n\nprobabilities_k = []\n\nNUMBER_OF_MODELS = 1\nEPOCHS = 20\nfor k in range(NUMBER_OF_MODELS):\n    with strategy.scope():    \n        pretrained_model = tf.keras.applications.Xception(\n                                                        weights='imagenet',\n                                                        include_top=False ,\n                                                        input_shape=(*IMAGE_SIZE, 3),\n                                                        pooling=None,\n                                                        classes=1000)\n        pretrained_model.trainable = True # transfer learning\n        model = tf.keras.Sequential([\n            pretrained_model,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(500, activation=\"relu\"),\n            tf.keras.layers.Dense(104, activation='softmax')\n        ])\n\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(lr=0.0001),\n        loss = 'sparse_categorical_crossentropy',\n        metrics=['sparse_categorical_accuracy']\n    )\n    early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n    historical = model.fit(training_dataset, \n              steps_per_epoch=STEPS_PER_EPOCH, \n              epochs=EPOCHS, \n              validation_data=validation_dataset,\n              callbacks=[early_stopping])\n    test_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n    print(f'Computing predictions for model {k}')\n    test_images_ds = test_ds.map(lambda image, idnum: image)\n    probabilities_k.append(model.predict(test_images_ds))\n    \n    with strategy.scope():    \n        pretrained_model = tf.keras.applications.DenseNet201(\n                                                        weights='imagenet',\n                                                        include_top=False ,\n                                                        input_shape=(*IMAGE_SIZE, 3),\n                                                        pooling=None,\n                                                        classes=1000)\n        pretrained_model.trainable = True # transfer learning\n        model = tf.keras.Sequential([\n            pretrained_model,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(500, activation=\"relu\"),\n            tf.keras.layers.Dense(104, activation='softmax')\n        ])\n\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(lr=0.0001),\n        loss = 'sparse_categorical_crossentropy',\n        metrics=['sparse_categorical_accuracy']\n    )\n    early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n    historical = model.fit(training_dataset, \n              steps_per_epoch=STEPS_PER_EPOCH, \n              epochs=EPOCHS, \n              validation_data=validation_dataset,\n              callbacks=[early_stopping])\n    test_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n    print(f'Computing predictions for model {k}')\n    test_images_ds = test_ds.map(lambda image, idnum: image)\n    probabilities_k.append(model.predict(test_images_ds))\n    \n    with strategy.scope():    \n        pretrained_model = tf.keras.applications.InceptionResNetV2(\n                                                        weights='imagenet',\n                                                        include_top=False ,\n                                                        input_shape=(*IMAGE_SIZE, 3),\n                                                        pooling=None,\n                                                        classes=1000)\n        pretrained_model.trainable = True # transfer learning\n        model = tf.keras.Sequential([\n            pretrained_model,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(500, activation=\"relu\"),\n            tf.keras.layers.Dense(104, activation='softmax')\n        ])\n\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(lr=0.0001),\n        loss = 'sparse_categorical_crossentropy',\n        metrics=['sparse_categorical_accuracy']\n    )\n    early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n    historical = model.fit(training_dataset, \n              steps_per_epoch=STEPS_PER_EPOCH, \n              epochs=EPOCHS, \n              validation_data=validation_dataset,\n              callbacks=[early_stopping])\n    test_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n    print(f'Computing predictions for model {k}')\n    test_images_ds = test_ds.map(lambda image, idnum: image)\n    probabilities_k.append(model.predict(test_images_ds))\n    \n    with strategy.scope():    \n        pretrained_model = tf.keras.applications.EfficientNetB7(\n                                                        weights='imagenet',\n                                                        include_top=False ,\n                                                        input_shape=(*IMAGE_SIZE, 3),\n                                                        pooling=None,\n                                                        classes=1000)\n        pretrained_model.trainable = True # transfer learning\n        model = tf.keras.Sequential([\n            pretrained_model,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(500, activation=\"relu\"),\n            tf.keras.layers.Dense(104, activation='softmax')\n        ])\n\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(lr=0.0001),\n        loss = 'sparse_categorical_crossentropy',\n        metrics=['sparse_categorical_accuracy']\n    )\n    early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n    historical = model.fit(training_dataset, \n              steps_per_epoch=STEPS_PER_EPOCH, \n              epochs=EPOCHS, \n              validation_data=validation_dataset,\n              callbacks=[early_stopping])\n    test_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n    print(f'Computing predictions for model {k}')\n    test_images_ds = test_ds.map(lambda image, idnum: image)\n    probabilities_k.append(model.predict(test_images_ds))\n    \nprobabilities = np.zeros_like(probabilities_k[0])\nfor prob in probabilities_k:\n    probabilities += prob\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\n\nprint('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\n# np.savetxt('submission_ensemble.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')","metadata":{"execution":{"iopub.status.busy":"2021-12-24T13:10:21.811163Z","iopub.execute_input":"2021-12-24T13:10:21.81161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}