{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport math","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.train import BytesList, FloatList, Int64List\nfrom tensorflow.train import Example, Features, Feature\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    tpu_strategy = tf.distribute.get_strategy() \n    \nAUTO = tf.data.experimental.AUTOTUNE\nprint(\"REPLICAS: \", tpu_strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE=[331,331]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS=34","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here we set the batch size to 128. 16 for each core"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE=16*tpu_strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH=KaggleDatasets().get_gcs_path('tpu-getting-started')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EXT_GCS = KaggleDatasets().get_gcs_path('tf-flower-photo-tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH_SELECT={\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512',\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"here we select the path for 224* 224 data"},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH=GCS_PATH_SELECT[IMAGE_SIZE[0]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, we form consecutive lists for training filenames, validation filename and test filenames respectively. *** Asterik in the code denotes for all the files in the train path.****"},{"metadata":{"trusted":true},"cell_type":"code","source":"training_filenames=tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec' )\nvalidation_filenames=tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\ntest_filenames=tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#imagenet_files = tf.io.gfile.glob(EXT_GCS + '/imagenet/tfrecords-jpeg-224*224/*.tfrec')\n#inaturalist_files=tf.io.gfile.glob(EXT_GCS + '/inaturalist/tfrecords-jpeg-224*224/*.tfrec')\n#openimage_files=tf.io.gfile.glob(EXT_GCS + '/openimage/tfrecords-jpeg-224*224/*.tfrec')\n#tf_flowers_files=tf.io.gfile.glob(EXT_GCS + '/tf_flowers/tfrecords-jpeg-224*224/*.tfrec')\n#training_filenames=training_filenames + imagenet_files + inaturalist_files + openimage_files + tf_flowers_files","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are 104 classes in the dataset. So, we are going to specify the classes now."},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, we move on to learning rate scheduler. We have to increase the lr with batch size. but additional tuning may be necessary to find the optimal lr schedule for a given model and accelerator. so we move from linear to exponential learning rate scheduler."},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_start=0.0022\nlr_max=0.0023* tpu_strategy.num_replicas_in_sync\nlr_min=0.0022\nlr_ramp_up_epoch=9\nlr_sustain_epoch=0\nlr_exp_decay=.8","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we create a function for learning rate where interpolation is done for different epoch values. for the first case, the lr is assumed to be linear with epochs. where as lr is optimum value for maximum rampup epoch. and lr varies exponentially with epochs in the third case"},{"metadata":{"trusted":true},"cell_type":"code","source":"def lrfn(epoch):\n    if epoch < lr_ramp_up_epoch:\n        lr = (lr_max - lr_start) / lr_ramp_up_epoch * epoch + lr_start\n    elif epoch < lr_ramp_up_epoch + lr_sustain_epoch:\n        lr = lr_max\n    else:\n        lr = (lr_max-lr_min) * lr_exp_decay ** (epoch - lr_ramp_up_epoch - lr_sustain_epoch) + lr_min\n    return lr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"tf.keras.callbacks.LearningRateScheduler(schedule,verbose=0){{{documentaion}}}\nschedule: a function that takes an epoch index(integer, indexed from 0) and current learning rate lr(float) and returns a new learning rate(lr) as output.\n\njastai aghi ko mathi ko lrfn chai hamro schedule vayo hai ta.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rng = [i for i in range(34 if EPOCHS<34 else EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we create different functions for simplicity."},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image=tf.image.decode_jpeg(image_data,channels=3)\n    image=tf.cast(image,tf.float32)/255.0 #This converts image to floats in [0,1] range.\n    image=tf.reshape(image,[*IMAGE_SIZE,3]) #It is the explicit size needed for TPU\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.io import FixedLenFeature, VarLenFeature","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT={\n        'image':tf.io.FixedLenFeature([],tf.string),\n        'class':tf.io.FixedLenFeature([],tf.int64),}\n    \n    example=tf.io.parse_single_example(example,LABELED_TFREC_FORMAT)\n    image=decode_image(example['image'])\n    label=tf.cast(example['class'],tf.int32)\n    return image,label\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT={\n        'image':tf.io.FixedLenFeature([],tf.string),\n        'id':tf.io.FixedLenFeature([],tf.string),\n        \n    }\n    example=tf.io.parse_single_example(example,UNLABELED_TFREC_FORMAT)\n    image=decode_image(example['image'])\n    ids=example['id']\n    return image,ids\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames,labeled=True,ordered=False):\n    ignore_order=tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic=False\n    dataset=tf.data.TFRecordDataset(filenames,num_parallel_reads=AUTO)\n    dataset=dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord,num_parallel_calls=AUTO)\n    return dataset\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    #image = tf.image.random_saturation(image, 0, 2)\n    return image, label   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset():\n    dataset=load_dataset(training_filenames,labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset=dataset.repeat()\n    dataset=dataset.shuffle(100)\n    dataset=dataset.batch(BATCH_SIZE)\n    dataset=dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_dataset():\n    dataset=load_dataset(validation_filenames,labeled=True,ordered=False)\n    dataset=dataset.batch(BATCH_SIZE)\n    dataset=dataset.cache()\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(ordered=False):\n    dataset=load_dataset(test_filenames,labeled=False,ordered=ordered)\n    dataset=dataset.batch(BATCH_SIZE)\n    dataset=dataset.cache()\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset=get_training_dataset()\nvalidation_dataset=get_validation_dataset()\ntest_dataset=get_test_dataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(CLASSES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.set_printoptions(threshold=15,linewidth=80)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def batch_to_numpy_images_and_labels(data):\n    images,labels=data\n    numpy_images=images.numpy()\n    numpy_labels=labels.numpy()\n    \n    if numpy_labels.dtype==object:\n        numpy_labels=[None for i in enumerate(numpy_images)]\n    return numpy_images,numpy_labels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here, in function batch_to_numpy_images_and_labels, the None case for numpy_labels is for test data. numpy_labels.dtype is object for test data as it is the dtype for id strings."},{"metadata":{},"cell_type":"markdown","source":"Yo wala chai majale bujheko chaina hai"},{"metadata":{"trusted":true},"cell_type":"code","source":"def title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(training_filenames)\nNUM_VALIDATION_IMAGES = count_data_items(validation_filenames)\nNUM_TEST_IMAGES = count_data_items(test_filenames)\nSTEPS_FOR_EPOCH=NUM_TRAINING_IMAGES// BATCH_SIZE\nprint('Training_size=',NUM_TRAINING_IMAGES  ,'Validation size=',NUM_VALIDATION_IMAGES , 'Test size=',NUM_TEST_IMAGES )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for image, label in get_training_dataset().take(3):\n    print(image.numpy().shape, label.numpy().shape)\nprint('Training data label examples:', label.numpy())\n#\n\nprint('Validation data shapes')\nfor image, label in get_validation_dataset().take(3):\n    print(image.numpy().shape, label.numpy().shape)\nprint('Validation data label examples:', label.numpy())\n#\n\nprint('Test data shapes')\nfor image, ids in get_test_dataset().take(3):\n    print(image.numpy().shape, ids.numpy().shape)\nprint('Test data IDs:', ids.numpy().astype('U'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntraining_dataset = train_dataset.unbatch().batch(20)\ntrain_batch = iter(training_dataset)\n#\ndisplay_batch_of_images(next(train_batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntest_dataset = test_dataset.unbatch().batch(20)\ntest_batch = iter(test_dataset)\n#\ndisplay_batch_of_images(next(test_batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=15, restore_best_weights = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install efficientnet\nimport efficientnet.tfkeras","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Building and training the model \nwith tpu_strategy.scope():    \n\n    pretrained_model =efficientnet.tfkeras.EfficientNetB7(\n        include_top=False, weights='imagenet', input_shape=[*IMAGE_SIZE,3])\n    pretrained_model.trainable = False # tramsfer learning\n    model=tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(104, activation='softmax')\n    ])\n            \nmodel.compile(\n    optimizer='adam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"historical = model.fit(train_dataset, \n          steps_per_epoch=STEPS_FOR_EPOCH, \n          epochs=EPOCHS, \n          validation_data=validation_dataset,callbacks=[lr_callback,early_stop])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with tpu_strategy.scope():    \n\n    pretrained_model =tf.keras.applications.DenseNet201(\n        include_top=False, weights='imagenet', input_shape=[*IMAGE_SIZE,3])\n    pretrained_model.trainable = False # tramsfer learning\n    model2=tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(104, activation='softmax')\n    ])\n            \nmodel2.compile(\n    optimizer='adam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)\n\nmodel2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"historical = model2.fit(train_dataset, \n          steps_per_epoch=STEPS_FOR_EPOCH, \n          epochs=EPOCHS, callbacks=[lr_callback],\n          validation_data=validation_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_start1=0.0004\nlr_max1=0.001* tpu_strategy.num_replicas_in_sync\nlr_min1=0.0006\nlr_ramp_up_epoch1=8\nlr_sustain_epoch1=0\nlr_exp_decay1=.8\ndef lrfn1(epoch):\n    if epoch < lr_ramp_up_epoch1:\n        lr = (lr_max1 - lr_start1) / lr_ramp_up_epoch1 * epoch + lr_start1\n    elif epoch < lr_ramp_up_epoch1 + lr_sustain_epoch1:\n        lr = lr_max1\n    else:\n        lr = (lr_max1-lr_min1) * lr_exp_decay1 ** (epoch - lr_ramp_up_epoch1 - lr_sustain_epoch1) + lr_min1\n    return lr\nlr_callback1 = tf.keras.callbacks.LearningRateScheduler(lrfn1, verbose = True)\n\nrng1 = [i for i in range(12 if EPOCHS<12 else EPOCHS)]\ny = [lrfn1(x) for x in rng1]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with tpu_strategy.scope():    \n\n    pretrained_model =tf.keras.applications.ResNet50V2(\n        include_top=False, weights='imagenet', input_shape=[*IMAGE_SIZE,3])\n    pretrained_model.trainable = False # tramsfer learning\n    model3=tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(104, activation='softmax')\n    ])\n            \nmodel3.compile(\n    optimizer='adam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)\n\nmodel3.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"historical3 = model3.fit(train_dataset, \n          steps_per_epoch=STEPS_FOR_EPOCH, \n          epochs=EPOCHS,callbacks=[lr_callback1],\n          validation_data=validation_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True)\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities1 = model.predict(test_images_ds)\npredictions1= np.argmax(probabilities1, axis=-1)\nprint(predictions1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities3 = model3.predict(test_images_ds)\npredictions3= np.argmax(probabilities3, axis=-1)\nprint(predictions3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"probabilities=(probabilities+probabilities3+probabilities1)/3\npredictions=np.argmax(probabilities,axis=-1)\nprint('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#\n\n#print('Generating submission.csv file...')\n#test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n#test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\n#np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}