{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Libraries initialization"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import math, re, os\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\n\nfrom kaggle_datasets import KaggleDatasets\n\nfrom tensorflow.keras import datasets, layers, models\nimport matplotlib.pyplot as plt\nprint(tf.__version__)\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Coopied data loading funcionality"},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path('flower-classification-with-tpus')\nIMAGE_SIZE = [192, 192]\n\nGCS_PATH_SELECT = { # available image sizes\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192/'\n}\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + 'train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + 'val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + 'test/*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAINING_FILENAMES","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nBATCH_SIZE = 150\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int64)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    #image = tf.image.random_saturation(image, 0, 2)\n    return image, label   \n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_TESTING_IMAGES = count_data_items(TEST_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"No of training images: {NUM_TRAINING_IMAGES}\")\nprint(f\"No of validation images: {NUM_VALIDATION_IMAGES}\")\nprint(f\"No of test images: {NUM_TESTING_IMAGES}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Example data set images"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,10))\nprefetch_ds = get_validation_dataset().take(25)\niterator = prefetch_ds.enumerate().as_numpy_iterator()\n\nfor i, data in iterator:\n    plt.subplot(5,5,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(data[0][0], cmap=plt.cm.binary)\n    plt.xlabel(CLASSES[data[1][0]])\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Models"},{"metadata":{"trusted":true},"cell_type":"code","source":"MODELS_MAP = {}\nMODELS_HISTORY = {}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Base model"},{"metadata":{"trusted":true},"cell_type":"code","source":"basic_model = models.Sequential()\nbasic_model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(192, 192, 3)))\nbasic_model.add(layers.MaxPooling2D((2, 2)))\nbasic_model.add(layers.Conv2D(16, (3, 3), activation='relu'))\nbasic_model.add(layers.MaxPooling2D((2, 2)))\nbasic_model.add(layers.Conv2D(8, (3, 3), activation='relu'))\nbasic_model.add(layers.Flatten())\nbasic_model.add(layers.Dense(512, activation='relu'))\nbasic_model.add(layers.Dense(len(CLASSES), activation='relu'))\n\nbasic_model.summary()\n\nMODELS_MAP[\"basic\"] = basic_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Base with dropout"},{"metadata":{"trusted":true},"cell_type":"code","source":"dropout_model = models.Sequential()\ndropout_model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(192, 192, 3)))\ndropout_model.add(layers.MaxPooling2D((2, 2)))\ndropout_model.add(layers.Conv2D(16, (3, 3), activation='relu'))\ndropout_model.add(layers.MaxPooling2D((2, 2)))\ndropout_model.add(layers.Conv2D(8, (3, 3), activation='relu'))\ndropout_model.add(layers.Dropout(0.2))\ndropout_model.add(layers.Flatten())\ndropout_model.add(layers.Dense(512, activation='relu'))\ndropout_model.add(layers.Dense(len(CLASSES), activation='relu'))\n\ndropout_model.summary()\n\nMODELS_MAP[\"dropout\"] = dropout_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Base dropout with different filters"},{"metadata":{"trusted":true},"cell_type":"code","source":"filtered_model = models.Sequential()\nfiltered_model.add(layers.Conv2D(32, (5, 5), activation='relu', input_shape=(192, 192, 3)))\nfiltered_model.add(layers.MaxPooling2D((2, 2)))\nfiltered_model.add(layers.Conv2D(16, (3, 3), activation='relu'))\nfiltered_model.add(layers.MaxPooling2D((2, 2)))\nfiltered_model.add(layers.Conv2D(8, (1, 1), activation='relu'))\nfiltered_model.add(layers.Dropout(0.2))\nfiltered_model.add(layers.Flatten())\nfiltered_model.add(layers.Dense(512, activation='relu'))\nfiltered_model.add(layers.Dense(len(CLASSES), activation='relu'))\n\nfiltered_model.summary()\n\nMODELS_MAP[\"filtered\"] = filtered_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# VGG Like"},{"metadata":{"trusted":true},"cell_type":"code","source":"vgg_like = models.Sequential()\nvgg_like.add(layers.Conv2D(filters=8, kernel_size=3, padding='same', activation='relu', input_shape=(192, 192, 3)))\nvgg_like.add(layers.Conv2D(filters=8, kernel_size=3, padding='same', activation='relu'))\nvgg_like.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like.add(layers.Conv2D(filters=16, kernel_size=3, padding='same', activation='relu'))\nvgg_like.add(layers.Conv2D(filters=16, kernel_size=3, padding='same', activation='relu'))\nvgg_like.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like.add(layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))\nvgg_like.add(layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))\nvgg_like.add(layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))\nvgg_like.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like.add(layers.GlobalAveragePooling2D())\nvgg_like.add(layers.Dense(512, activation='relu'))\nvgg_like.add(layers.Dense(512, activation='relu'))\nvgg_like.add(layers.Dense(len(CLASSES), activation='softmax'))\n\n\nvgg_like.summary()\n\nMODELS_MAP[\"vgg-like\"] = vgg_like","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Base With 1x1"},{"metadata":{"trusted":true},"cell_type":"code","source":"x1_model = models.Sequential()\nx1_model.add(layers.Conv2D(32, (1, 1), activation='relu', input_shape=(192, 192, 3)))\nx1_model.add(layers.Conv2D(32, (7, 7), activation='relu'))\nx1_model.add(layers.MaxPooling2D((2, 2)))\nx1_model.add(layers.Conv2D(32, (1, 1), activation='relu'))\nx1_model.add(layers.Conv2D(32, (5, 5), activation='relu'))\nx1_model.add(layers.MaxPooling2D((2, 2)))\nx1_model.add(layers.Conv2D(32, (1, 1), activation='relu'))\nx1_model.add(layers.Conv2D(32, (3, 3), activation='relu'))\nx1_model.add(layers.Dropout(0.2))\nx1_model.add(layers.Flatten())\nx1_model.add(layers.Dense(512, activation='relu'))\nx1_model.add(layers.Dense(len(CLASSES), activation='relu'))\n\nx1_model.summary()\n\nMODELS_MAP[\"1x1\"] = x1_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# VGG-Like with 1x1"},{"metadata":{"trusted":true},"cell_type":"code","source":"vgg_like_nxn = models.Sequential()\nvgg_like_nxn.add(layers.Conv2D(filters=8, kernel_size=3, padding='same', activation='relu', input_shape=(192, 192, 3)))\nvgg_like_nxn.add(layers.Conv2D(filters=8, kernel_size=3, padding='same', activation='relu'))\nvgg_like_nxn.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_nxn.add(layers.Conv2D(16, (1,1), padding='same', activation='relu'))\nvgg_like_nxn.add(layers.Conv2D(filters=16, kernel_size=3, padding='same', activation='relu'))\nvgg_like_nxn.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_nxn.add(layers.Conv2D(32, (1,1), padding='same', activation='relu'))\nvgg_like_nxn.add(layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))\nvgg_like_nxn.add(layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))\nvgg_like_nxn.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_nxn.add(layers.Conv2D(64, (1,1), padding='same', activation='relu'))\nvgg_like_nxn.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_nxn.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_nxn.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_nxn.add(layers.Conv2D(64, (1,1), padding='same', activation='relu'))\nvgg_like_nxn.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_nxn.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_nxn.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_nxn.add(layers.GlobalAveragePooling2D())\nvgg_like_nxn.add(layers.Dense(512, activation='relu'))\nvgg_like_nxn.add(layers.Dense(512, activation='relu'))\nvgg_like_nxn.add(layers.Dense(len(CLASSES), activation='softmax'))\n\n\nvgg_like_nxn.summary()\n\nMODELS_MAP[\"vgg-like-1x1\"] = vgg_like_nxn","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# VGG Like 1x1 with nxn"},{"metadata":{"trusted":true},"cell_type":"code","source":"vgg_like_nxn_1x1 = models.Sequential()\nvgg_like_nxn_1x1.add(layers.Conv2D(filters=8, kernel_size=3, padding='same', activation='relu', input_shape=(192, 192, 3)))\nvgg_like_nxn_1x1.add(layers.Conv2D(filters=8, kernel_size=3, padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_nxn_1x1.add(layers.Conv2D(16, (1,1), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.Conv2D(16, (3, 1), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.Conv2D(16, (1, 3), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_nxn_1x1.add(layers.Conv2D(32, (1,1), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.Conv2D(32, (3, 1), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.Conv2D(32, (1, 3), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.Conv2D(32, (3, 1), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.Conv2D(32, (1, 3), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_nxn_1x1.add(layers.Conv2D(64, (1,1), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.Conv2D(64, (3, 1), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.Conv2D(64, (1, 3), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.Conv2D(64, (3, 1), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.Conv2D(64, (1, 3), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_nxn_1x1.add(layers.Conv2D(64, (1,1), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.Conv2D(64, (3, 1), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.Conv2D(64, (1, 3), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.Conv2D(64, (3, 1), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.Conv2D(64, (1, 3), padding='same', activation='relu'))\nvgg_like_nxn_1x1.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_nxn_1x1.add(layers.GlobalAveragePooling2D())\nvgg_like_nxn_1x1.add(layers.Dense(512, activation='relu'))\nvgg_like_nxn_1x1.add(layers.Dense(512, activation='relu'))\nvgg_like_nxn_1x1.add(layers.Dense(len(CLASSES), activation='softmax'))\n\n\nvgg_like_nxn_1x1.summary()\n\nMODELS_MAP[\"vgg-like-nxn\"] = vgg_like_nxn_1x1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pseudo xception"},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv_bn(x, filters, kernel_size, strides=1):\n    \n    x = layers.Conv2D(filters=filters, \n               kernel_size = kernel_size, \n               strides=strides, \n               padding = 'same', \n               use_bias = False)(x)\n    x = layers.BatchNormalization()(x)\n    return x\n\ndef sep_bn(x, filters, kernel_size, strides=1):\n    \n    x = layers.SeparableConv2D(filters=filters, \n                        kernel_size = kernel_size, \n                        strides=strides, \n                        padding = 'same', \n                        use_bias = False)(x)\n    x = layers.BatchNormalization()(x)\n    return x\n\ndef entry_flow(x):\n    \n    x = conv_bn(x, filters =32, kernel_size =3, strides=2)\n    x = layers.ReLU()(x)\n    x = conv_bn(x, filters =64, kernel_size =3, strides=1)\n    tensor = layers.ReLU()(x)\n    \n    x = sep_bn(tensor, filters = 128, kernel_size =3)\n    x = layers.ReLU()(x)\n    x = sep_bn(x, filters = 128, kernel_size =3)\n    x = layers.MaxPool2D(pool_size=3, strides=2, padding = 'same')(x)\n    \n    tensor = conv_bn(tensor, filters=128, kernel_size = 1,strides=2)\n    x = layers.Add()([tensor,x])\n    \n    x = layers.ReLU()(x)\n    x = sep_bn(x, filters =256, kernel_size=3)\n    x = layers.ReLU()(x)\n    x = sep_bn(x, filters =256, kernel_size=3)\n    x = layers.MaxPool2D(pool_size=3, strides=2, padding = 'same')(x)\n    \n    tensor = conv_bn(tensor, filters=256, kernel_size = 1,strides=2)\n    x = layers.Add()([tensor,x])\n    \n    x = layers.ReLU()(x)\n    x = sep_bn(x, filters =728, kernel_size=3)\n    x = layers.ReLU()(x)\n    x = sep_bn(x, filters =728, kernel_size=3)\n    x = layers.MaxPool2D(pool_size=3, strides=2, padding = 'same')(x)\n    \n    tensor = conv_bn(tensor, filters=728, kernel_size = 1,strides=2)\n    x = layers.Add()([tensor,x])\n    return x\n\ndef middle_flow(tensor, nums):\n    \n    for _ in range(nums):\n        x = layers.ReLU()(tensor)\n        x = sep_bn(x, filters = 728, kernel_size = 3)\n        x = layers.ReLU()(x)\n        x = sep_bn(x, filters = 728, kernel_size = 3)\n        x = layers.ReLU()(x)\n        x = sep_bn(x, filters = 728, kernel_size = 3)\n        x = layers.ReLU()(x)\n        tensor = layers.Add()([tensor,x])\n        \n    return tensor\n\ndef exit_flow(tensor):\n    \n    x = layers.ReLU()(tensor)\n    x = sep_bn(x, filters = 728,  kernel_size=3)\n    x = layers.ReLU()(x)\n    x = sep_bn(x, filters = 1024,  kernel_size=3)\n    x = layers.MaxPool2D(pool_size = 3, strides = 2, padding ='same')(x)\n    \n    tensor = conv_bn(tensor, filters =1024, kernel_size=1, strides =2)\n    x = layers.Add()([tensor,x])\n    \n    x = sep_bn(x, filters = 1536,  kernel_size=3)\n    x = layers.ReLU()(x)\n    x = sep_bn(x, filters = 2048,  kernel_size=3)\n    x = layers.GlobalAvgPool2D()(x)\n    \n    x = layers.Dense(units = 1000, activation = 'softmax')(x)\n    \n    return x\n\ninput = layers.Input(shape = (192,192,3))\nx = entry_flow(input)\nx = middle_flow(x, 4)\noutput = exit_flow(x)\n\nmodel_xce = models.Model(inputs=input, outputs=output)\nmodel_xce.summary()\n\nMODELS_MAP[\"xce\"] = model_xce","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Base with nx1 1xn"},{"metadata":{"trusted":true},"cell_type":"code","source":"xn_model = models.Sequential()\nxn_model.add(layers.Conv2D(32, (7, 1), activation='relu', input_shape=(192, 192, 3)))\nxn_model.add(layers.Conv2D(32, (1, 7), activation='relu'))\nxn_model.add(layers.MaxPooling2D((2, 2)))\nxn_model.add(layers.Conv2D(32, (5, 1), activation='relu'))\nxn_model.add(layers.Conv2D(32, (1, 5), activation='relu'))\nxn_model.add(layers.MaxPooling2D((2, 2)))\nxn_model.add(layers.Conv2D(32, (3, 1), activation='relu'))\nxn_model.add(layers.Conv2D(32, (1, 3), activation='relu'))\nxn_model.add(layers.Dropout(0.2))\nxn_model.add(layers.Flatten())\nxn_model.add(layers.Dense(512, activation='relu'))\nxn_model.add(layers.Dense(len(CLASSES), activation='relu'))\n\nxn_model.summary()\n\nMODELS_MAP[\"nx1\"] = xn_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# VGG Like with dropout and normalization"},{"metadata":{"trusted":true},"cell_type":"code","source":"vgg_like_drop = models.Sequential()\nvgg_like_drop.add(layers.Conv2D(filters=8, kernel_size=3, padding='same', activation='relu', input_shape=(192, 192, 3)))\nvgg_like_drop.add(layers.Conv2D(filters=8, kernel_size=3, padding='same', activation='relu'))\nvgg_like_drop.add(layers.BatchNormalization())\nvgg_like_drop.add(layers.Dropout(0.1))\nvgg_like_drop.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_drop.add(layers.Conv2D(filters=16, kernel_size=3, padding='same', activation='relu'))\nvgg_like_drop.add(layers.Conv2D(filters=16, kernel_size=3, padding='same', activation='relu'))\nvgg_like_drop.add(layers.BatchNormalization())\nvgg_like_drop.add(layers.Dropout(0.1))\nvgg_like_drop.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_drop.add(layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))\nvgg_like_drop.add(layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))\nvgg_like_drop.add(layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))\nvgg_like_drop.add(layers.BatchNormalization())\nvgg_like_drop.add(layers.Dropout(0.1))\nvgg_like_drop.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_drop.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_drop.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_drop.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_drop.add(layers.BatchNormalization())\nvgg_like_drop.add(layers.Dropout(0.1))\nvgg_like_drop.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_drop.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_drop.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_drop.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_drop.add(layers.BatchNormalization())\nvgg_like_drop.add(layers.Dropout(0.1))\nvgg_like_drop.add(layers.MaxPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_drop.add(layers.GlobalAveragePooling2D())\nvgg_like_drop.add(layers.Dense(512, activation='relu'))\nvgg_like_drop.add(layers.Dense(512, activation='relu'))\nvgg_like_drop.add(layers.Dense(len(CLASSES), activation='softmax'))\n\n\nvgg_like_drop.summary()\n\nMODELS_MAP[\"vgg-like-drop\"] = vgg_like_drop","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Previous VGG with avg pooling"},{"metadata":{"trusted":true},"cell_type":"code","source":"vgg_like_avg = models.Sequential()\nvgg_like_avg.add(layers.Conv2D(filters=8, kernel_size=3, padding='same', activation='relu', input_shape=(192, 192, 3)))\nvgg_like_avg.add(layers.Conv2D(filters=8, kernel_size=3, padding='same', activation='relu'))\nvgg_like_avg.add(layers.BatchNormalization())\nvgg_like_avg.add(layers.Dropout(0.1))\nvgg_like_avg.add(layers.AvgPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_avg.add(layers.Conv2D(filters=16, kernel_size=3, padding='same', activation='relu'))\nvgg_like_avg.add(layers.Conv2D(filters=16, kernel_size=3, padding='same', activation='relu'))\nvgg_like_avg.add(layers.BatchNormalization())\nvgg_like_avg.add(layers.Dropout(0.1))\nvgg_like_avg.add(layers.AvgPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_avg.add(layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))\nvgg_like_avg.add(layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))\nvgg_like_avg.add(layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))\nvgg_like_avg.add(layers.BatchNormalization())\nvgg_like_avg.add(layers.Dropout(0.1))\nvgg_like_avg.add(layers.AvgPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_avg.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_avg.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_avg.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_avg.add(layers.BatchNormalization())\nvgg_like_avg.add(layers.Dropout(0.1))\nvgg_like_avg.add(layers.AvgPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_avg.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_avg.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_avg.add(layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\nvgg_like_avg.add(layers.BatchNormalization())\nvgg_like_avg.add(layers.Dropout(0.1))\nvgg_like_avg.add(layers.AvgPool2D(pool_size=2, strides=2, padding ='same'))\n\nvgg_like_avg.add(layers.GlobalAveragePooling2D())\nvgg_like_avg.add(layers.Dense(512, activation='relu'))\nvgg_like_avg.add(layers.Dense(512, activation='relu'))\nvgg_like_avg.add(layers.Dense(len(CLASSES), activation='softmax'))\n\n\nvgg_like_avg.summary()\n\nMODELS_MAP[\"vgg-like-avg\"] = vgg_like_avg","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Xception with different filters"},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv_bn(x, filters, kernel_size, strides=1):\n    x = layers.Conv2D(filters=filters, \n               kernel_size = kernel_size, \n               strides=strides, \n               padding = 'same', \n               use_bias = False)(x)\n    x = layers.BatchNormalization()(x)\n    return x\n\ndef sep_bn(x, filters, kernel_size, strides=1):\n    x = layers.SeparableConv2D(filters=filters, \n                        kernel_size = kernel_size, \n                        strides=strides, \n                        padding = 'same', \n                        use_bias = False)(x)\n    x = layers.BatchNormalization()(x)\n    return x\n\ndef entry_flow(x):\n    \n    x = conv_bn(x, filters =32, kernel_size =7, strides=2)\n    x = layers.ReLU()(x)\n    x = conv_bn(x, filters =64, kernel_size =7, strides=1)\n    tensor = layers.ReLU()(x)\n    \n    x = sep_bn(tensor, filters = 128, kernel_size =7)\n    x = layers.ReLU()(x)\n    x = sep_bn(x, filters = 128, kernel_size =7)\n    x = layers.MaxPool2D(pool_size=3, strides=2, padding = 'same')(x)\n    \n    tensor = conv_bn(tensor, filters=128, kernel_size = 1,strides=2)\n    x = layers.Add()([tensor,x])\n    \n    x = layers.ReLU()(x)\n    x = sep_bn(x, filters =256, kernel_size=7)\n    x = layers.ReLU()(x)\n    x = sep_bn(x, filters =256, kernel_size=7)\n    x = layers.MaxPool2D(pool_size=3, strides=2, padding = 'same')(x)\n    \n    tensor = conv_bn(tensor, filters=256, kernel_size = 1,strides=2)\n    x = layers.Add()([tensor,x])\n    \n    x = layers.ReLU()(x)\n    x = sep_bn(x, filters =728, kernel_size=7)\n    x = layers.ReLU()(x)\n    x = sep_bn(x, filters =728, kernel_size=7)\n    x = layers.MaxPool2D(pool_size=3, strides=2, padding = 'same')(x)\n    \n    tensor = conv_bn(tensor, filters=728, kernel_size = 1,strides=2)\n    x = layers.Add()([tensor,x])\n    return x\n\ndef middle_flow(tensor, nums):\n    \n    for _ in range(nums):\n        x = layers.ReLU()(tensor)\n        x = sep_bn(x, filters = 728, kernel_size = 5)\n        x = layers.ReLU()(x)\n        x = sep_bn(x, filters = 728, kernel_size = 5)\n        x = layers.ReLU()(x)\n        x = sep_bn(x, filters = 728, kernel_size = 5)\n        x = layers.ReLU()(x)\n        tensor = layers.Add()([tensor,x])\n        \n    return tensor\n\ndef exit_flow(tensor):\n    \n    x = layers.ReLU()(tensor)\n    x = sep_bn(x, filters = 728,  kernel_size=3)\n    x = layers.ReLU()(x)\n    x = sep_bn(x, filters = 1024,  kernel_size=3)\n    x = layers.MaxPool2D(pool_size = 3, strides = 2, padding ='same')(x)\n    \n    tensor = conv_bn(tensor, filters =1024, kernel_size=1, strides =2)\n    x = layers.Add()([tensor,x])\n    \n    x = sep_bn(x, filters = 1536,  kernel_size=3)\n    x = layers.ReLU()(x)\n    x = sep_bn(x, filters = 2048,  kernel_size=3)\n    x = layers.GlobalAvgPool2D()(x)\n    \n    x = layers.Dense(units = 1000, activation = 'softmax')(x)\n    \n    return x\n\ninput = layers.Input(shape = (192,192,3))\nx = entry_flow(input)\nx = middle_flow(x, 4)\noutput = exit_flow(x)\n\nmodel_xce_nxn = models.Model(inputs=input, outputs=output)\nmodel_xce_nxn.summary()\n\nMODELS_MAP[\"xce_nxn\"] = model_xce_nxn","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Models training"},{"metadata":{"trusted":true},"cell_type":"code","source":"nns = MODELS_MAP.keys()\n#nns = [\"xce_nxn\"]\n\nfor name in nns:\n    print(f\"training: {name}\")\n    MODELS_MAP[name].compile(optimizer='adam',\n                  loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n                  metrics=['accuracy'])\n\n    MODELS_HISTORY[name] = MODELS_MAP[name].fit(get_training_dataset(), epochs=25, \n                        validation_data=get_validation_dataset(), steps_per_epoch=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_model_summary(key):\n    history, model = MODELS_HISTORY[key], MODELS_MAP[key]\n    plt.plot(history.history['accuracy'], label='accuracy')\n    plt.plot(history.history['val_accuracy'], label = 'val_accuracy')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.ylim([0, 1])\n    plt.legend(loc='lower right')\n\n\n    return model.evaluate(get_validation_dataset(), verbose=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Results"},{"metadata":{},"cell_type":"markdown","source":"## Basic Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_model_summary(\"basic\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Basic with dropout"},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_model_summary(\"dropout\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Basic filtered"},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_model_summary(\"filtered\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## VGG Like"},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_model_summary(\"vgg-like\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1x1 Convs"},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_model_summary(\"1x1\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## xception"},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_model_summary(\"xce\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## nx1 1xn"},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_model_summary(\"nx1\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## VGG Like 1x1"},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_model_summary(\"vgg-like-1x1\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## VGG like with 1x1 nxn"},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_model_summary(\"vgg-like-nxn\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## VGG like with drop and normalization"},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_model_summary(\"vgg-like-drop\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# VGG like with avg"},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_model_summary(\"vgg-like-avg\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Exception with different filters"},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_model_summary(\"xce_nxn\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}