{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nfrom matplotlib import pyplot as plt\nfrom tensorflow.keras import layers\nimport os,math,re","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-01T03:36:28.936684Z","iopub.execute_input":"2021-07-01T03:36:28.937059Z","iopub.status.idle":"2021-07-01T03:36:28.943956Z","shell.execute_reply.started":"2021-07-01T03:36:28.937021Z","shell.execute_reply":"2021-07-01T03:36:28.942669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Outline\n1. TPU Setting\n2. Data Import and Visualization\n3. Training: VCG Transfer Learning\n4. Validation\n5. Predicting Test Data\n6. References","metadata":{}},{"cell_type":"markdown","source":"# TPU Setting","metadata":{}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() \n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:28.945939Z","iopub.execute_input":"2021-07-01T03:36:28.946725Z","iopub.status.idle":"2021-07-01T03:36:34.28113Z","shell.execute_reply.started":"2021-07-01T03:36:28.946666Z","shell.execute_reply":"2021-07-01T03:36:34.280012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Import and Visualization","metadata":{}},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')\nprint(GCS_DS_PATH) # what do gcs paths look like?","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:34.282767Z","iopub.execute_input":"2021-07-01T03:36:34.283091Z","iopub.status.idle":"2021-07-01T03:36:34.590535Z","shell.execute_reply.started":"2021-07-01T03:36:34.283062Z","shell.execute_reply":"2021-07-01T03:36:34.58954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = [299, 299] \n# VGG16 : Input size 224 x 224\n# Xception: 299 x 299\nGCS_PATH = GCS_DS_PATH + '/tfrecords-jpeg-512x512'\nAUTO = tf.data.experimental.AUTOTUNE\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec') ","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:34.592309Z","iopub.execute_input":"2021-07-01T03:36:34.592726Z","iopub.status.idle":"2021-07-01T03:36:34.798619Z","shell.execute_reply.started":"2021-07-01T03:36:34.592683Z","shell.execute_reply":"2021-07-01T03:36:34.797628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Flower classes encoding\nCLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:34.800128Z","iopub.execute_input":"2021-07-01T03:36:34.800425Z","iopub.status.idle":"2021-07-01T03:36:34.809093Z","shell.execute_reply.started":"2021-07-01T03:36:34.800397Z","shell.execute_reply":"2021-07-01T03:36:34.808119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image_data):\n    image =tf.image.decode_jpeg(image_data, channels=3)\n    image =tf.image.resize(image,[*IMAGE_SIZE])  # resize image to the dimension needed for the pretrained model\n    image =tf.cast(image, tf.float32) /255.0\n    image = tf.reshape(image,[*IMAGE_SIZE, 3])\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([],tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([],tf.int64),\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example[\"image\"])\n    label = tf.cast(example[\"class\"], tf.int32)\n    return image,label\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([],tf.string),\n        \"id\": tf.io.FixedLenFeature([],tf.string),\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example[\"image\"])\n    idnum = example[\"id\"]\n    return image,idnum\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:34.810327Z","iopub.execute_input":"2021-07-01T03:36:34.810604Z","iopub.status.idle":"2021-07-01T03:36:34.832278Z","shell.execute_reply.started":"2021-07-01T03:36:34.810577Z","shell.execute_reply":"2021-07-01T03:36:34.831243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_augment(image, label):\n    image = tf.image.random_flip_left_right(image)\n    return image,label\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # repeats for several epochs\n    dataset = dataset.shuffle(buffer_size=2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(ordered = False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items in the name of the .tfrec \n    n  = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nfTrainImages = count_data_items(TRAINING_FILENAMES)\nfValidationImages = count_data_items(VALIDATION_FILENAMES)\nfTestImages = count_data_items(TEST_FILENAMES)\nprint(f\"{fTrainImages} training images, {fValidationImages} validation images, {fTestImages} test images \")","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:34.835324Z","iopub.execute_input":"2021-07-01T03:36:34.835771Z","iopub.status.idle":"2021-07-01T03:36:34.853448Z","shell.execute_reply.started":"2021-07-01T03:36:34.835723Z","shell.execute_reply":"2021-07-01T03:36:34.852322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 16* strategy.num_replicas_in_sync\nds_train = get_training_dataset()\nds_valid = get_validation_dataset()\nds_test = get_test_dataset()\n\nprint(\"Training: \", ds_train)\nprint(\"Validation: \", ds_valid)\nprint(\"Test: \", ds_test)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:34.855435Z","iopub.execute_input":"2021-07-01T03:36:34.85584Z","iopub.status.idle":"2021-07-01T03:36:35.123568Z","shell.execute_reply.started":"2021-07-01T03:36:34.855778Z","shell.execute_reply":"2021-07-01T03:36:35.122625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image, label in ds_train.take(3):\n    print(image.numpy().shape, label.numpy().shape)\n    \nprint(\"Training data label examples:\", label.numpy())\n","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:35.124941Z","iopub.execute_input":"2021-07-01T03:36:35.125318Z","iopub.status.idle":"2021-07-01T03:36:36.850337Z","shell.execute_reply.started":"2021-07-01T03:36:35.125285Z","shell.execute_reply":"2021-07-01T03:36:36.849208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image, idnum in ds_test.take(3):\n    print(image.numpy().shape, idnum.numpy().shape)\nprint(\"Test data IDs: \", idnum.numpy().astype('U')) # U = unicode string","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:36.853661Z","iopub.execute_input":"2021-07-01T03:36:36.853987Z","iopub.status.idle":"2021-07-01T03:36:38.089137Z","shell.execute_reply.started":"2021-07-01T03:36:36.853958Z","shell.execute_reply":"2021-07-01T03:36:38.08791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image Visualization ","metadata":{}},{"cell_type":"code","source":"def batch_to_numpy_images_and_labels(data):\n    images,labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object:\n        numpy_labels = [ None for _ in enumerate(numpy_images)]\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label==correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], \n                                'OK' if correct else 'NO',\n                                u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title)>0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2),\n                  color= 'red' if red else 'black',\n                fontdict={'verticalalignment':'center'}, \n                  pad=int(titlesize/1.5))\n    return (subplot[0],subplot[1],subplot[2]+1)\ndef display_batch_of_images(databatch,predictions=None):\n    images,labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels  = [None for _ in enumerate(images)]\n        \n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n    \n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot = (rows,cols,1)\n    if(rows < cols):\n        plt.figure(figsize=(FIGSIZE, FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols, FIGSIZE))\n    \n    #display \n    for i, (image,label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i],label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3\n        subplot = display_one_flower(image,title,subplot, not correct,\n                                    titlesize = dynamic_titlesize)\n    \n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0,hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n               ","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:38.090666Z","iopub.execute_input":"2021-07-01T03:36:38.091051Z","iopub.status.idle":"2021-07-01T03:36:38.110297Z","shell.execute_reply.started":"2021-07-01T03:36:38.091018Z","shell.execute_reply":"2021-07-01T03:36:38.109087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_iter = iter(ds_train.unbatch().batch(20))","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:38.112309Z","iopub.execute_input":"2021-07-01T03:36:38.112943Z","iopub.status.idle":"2021-07-01T03:36:38.140729Z","shell.execute_reply.started":"2021-07-01T03:36:38.112891Z","shell.execute_reply":"2021-07-01T03:36:38.139401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_batch = next(ds_iter)\ndisplay_batch_of_images(one_batch)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:38.142294Z","iopub.execute_input":"2021-07-01T03:36:38.1426Z","iopub.status.idle":"2021-07-01T03:36:42.456113Z","shell.execute_reply.started":"2021-07-01T03:36:38.142571Z","shell.execute_reply":"2021-07-01T03:36:42.455142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Model","metadata":{}},{"cell_type":"code","source":"def display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:42.45741Z","iopub.execute_input":"2021-07-01T03:36:42.458049Z","iopub.status.idle":"2021-07-01T03:36:42.464841Z","shell.execute_reply.started":"2021-07-01T03:36:42.458013Z","shell.execute_reply":"2021-07-01T03:36:42.463691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 10","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:42.46631Z","iopub.execute_input":"2021-07-01T03:36:42.46668Z","iopub.status.idle":"2021-07-01T03:36:42.479427Z","shell.execute_reply.started":"2021-07-01T03:36:42.466642Z","shell.execute_reply":"2021-07-01T03:36:42.478276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Learning Rate Schedule\n        \ndef exponential_lr(epoch, start_lr = 1e-5, min_lr = 1e-5, max_lr = 5e-5, \n                   rampup_epochs=5, sustain_epochs=0, exp_decay=0.8):\n    if epoch < rampup_epochs:\n        lr = (max_lr-start_lr)/rampup_epochs*epoch + start_lr\n    elif epoch < rampup_epochs + sustain_epochs:\n        lr = max_lr\n    else:\n        lr = (max_lr - min_lr)*exp_decay**(epoch- rampup_epochs - sustain_epochs) + min_lr\n    return lr\n\nlr_callback = tf.keras.callbacks.LearningRateScheduler(exponential_lr, verbose=False)\n\nrng = [i for i in range(EPOCHS)]\ny = [exponential_lr(x) for x in rng]\nplt.plot(rng,y)\nprint(\"Learning Rate Schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0],max(y),y[-1]))\n    ","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:42.481083Z","iopub.execute_input":"2021-07-01T03:36:42.481466Z","iopub.status.idle":"2021-07-01T03:36:42.65668Z","shell.execute_reply.started":"2021-07-01T03:36:42.481428Z","shell.execute_reply":"2021-07-01T03:36:42.655448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pretrained Xception Model","metadata":{}},{"cell_type":"code","source":"with strategy.scope():\n    img_adjust_layer = tf.keras.layers.Lambda(lambda data: tf.keras.applications.xception.preproces_input(tf.cast(data,tf.float32)), input_shape=[*IMAGE_SIZE,3])\n    xce_pretrained_model = tf.keras.applications.Xception(weights='imagenet',include_top=False)    \n    xce_pretrained_model.trainable = True\n    \n    xce_model = tf.keras.Sequential();\n    xce_model.add(xce_pretrained_model)\n    xce_model.add(layers.GlobalAveragePooling2D())\n    xce_model.add(layers.Dense(len(CLASSES), activation='softmax'))\n    \nxce_model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])\nxce_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:42.660448Z","iopub.execute_input":"2021-07-01T03:36:42.660815Z","iopub.status.idle":"2021-07-01T03:36:50.625011Z","shell.execute_reply.started":"2021-07-01T03:36:42.660783Z","shell.execute_reply":"2021-07-01T03:36:50.623851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STEPS_PER_EPOCH = fTrainImages // BATCH_SIZE\nxce_history = xce_model.fit(ds_train, validation_data=ds_valid, \n                            epochs=EPOCHS, steps_per_epoch = STEPS_PER_EPOCH, \n                            callbacks=[lr_callback], verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:36:50.626549Z","iopub.execute_input":"2021-07-01T03:36:50.627005Z","iopub.status.idle":"2021-07-01T03:40:53.690802Z","shell.execute_reply.started":"2021-07-01T03:36:50.626953Z","shell.execute_reply":"2021-07-01T03:40:53.68985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_training_curves(xce_history.history['loss'], xce_history.history['val_loss'], 'loss', 211)\ndisplay_training_curves(xce_history.history['sparse_categorical_accuracy'], xce_history.history['val_sparse_categorical_accuracy'],'accuracy',212)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:40:53.69401Z","iopub.execute_input":"2021-07-01T03:40:53.694321Z","iopub.status.idle":"2021-07-01T03:40:54.127939Z","shell.execute_reply.started":"2021-07-01T03:40:53.694293Z","shell.execute_reply":"2021-07-01T03:40:54.126981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pretrained ResNet Model","metadata":{}},{"cell_type":"code","source":"with strategy.scope():\n    img_adjust_layer = tf.keras.layers.Lambda(lambda data: tf.keras.applications.resnet_v2.preproces_input(tf.cast(data,tf.float32)), input_shape=[*IMAGE_SIZE,3])\n    resnet_pretrained_model = tf.keras.applications.ResNet50V2(weights='imagenet',include_top=False)    \n    resnet_pretrained_model.trainable = True\n    \n    resnet_model = tf.keras.Sequential();\n    resnet_model.add(resnet_pretrained_model)\n    resnet_model.add(layers.GlobalAveragePooling2D())\n    resnet_model.add(layers.Dense(len(CLASSES), activation='softmax'))\n    \nresnet_model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])\nresnet_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:40:54.129656Z","iopub.execute_input":"2021-07-01T03:40:54.129959Z","iopub.status.idle":"2021-07-01T03:41:02.892569Z","shell.execute_reply.started":"2021-07-01T03:40:54.12993Z","shell.execute_reply":"2021-07-01T03:41:02.890926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resnet_history = resnet_model.fit(ds_train, validation_data=ds_valid, \n                            epochs=EPOCHS, steps_per_epoch = STEPS_PER_EPOCH, \n                            callbacks=[lr_callback], verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:41:02.893844Z","iopub.execute_input":"2021-07-01T03:41:02.894312Z","iopub.status.idle":"2021-07-01T03:44:52.307362Z","shell.execute_reply.started":"2021-07-01T03:41:02.894281Z","shell.execute_reply":"2021-07-01T03:44:52.3063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_training_curves(resnet_history.history['loss'], resnet_history.history['val_loss'], 'loss', 211)\ndisplay_training_curves(resnet_history.history['sparse_categorical_accuracy'], resnet_history.history['val_sparse_categorical_accuracy'],'accuracy',212)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:44:52.308623Z","iopub.execute_input":"2021-07-01T03:44:52.308938Z","iopub.status.idle":"2021-07-01T03:44:52.790787Z","shell.execute_reply.started":"2021-07-01T03:44:52.30891Z","shell.execute_reply":"2021-07-01T03:44:52.789649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation","metadata":{}},{"cell_type":"markdown","source":"## Macro F1 Scores","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import f1_score\n\nvalidation_ds = get_validation_dataset(ordered=True)\nvalidation_images_ds = validation_ds.map(lambda image, label: image)\nval_label_ds = validation_ds.map(lambda image, label: label).unbatch()\nval_labels = next(iter(val_label_ds.batch(fValidationImages))).numpy()","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:44:52.792538Z","iopub.execute_input":"2021-07-01T03:44:52.793263Z","iopub.status.idle":"2021-07-01T03:44:53.833687Z","shell.execute_reply.started":"2021-07-01T03:44:52.793214Z","shell.execute_reply":"2021-07-01T03:44:53.832948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_proba = xce_model.predict(validation_images_ds)\npredictions = np.argmax(validation_proba, axis=-1)\nxce_f1 = f1_score(val_labels, predictions,average='macro')\nprint(f\"Macro F1 scores: {xce_f1}\")\n\nxce_model.evaluate(validation_ds)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:44:53.835205Z","iopub.execute_input":"2021-07-01T03:44:53.835866Z","iopub.status.idle":"2021-07-01T03:45:03.183345Z","shell.execute_reply.started":"2021-07-01T03:44:53.835803Z","shell.execute_reply":"2021-07-01T03:45:03.182478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_proba = resnet_model.predict(validation_images_ds)\npredictions = np.argmax(validation_proba, axis=-1)\nresnet_f1 = f1_score(val_labels, predictions,average='macro')\nprint(f\"Macro F1 scores: {resnet_f1}\")\n\nresnet_model.evaluate(validation_ds)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:45:03.184607Z","iopub.execute_input":"2021-07-01T03:45:03.184911Z","iopub.status.idle":"2021-07-01T03:45:14.204985Z","shell.execute_reply.started":"2021-07-01T03:45:03.184883Z","shell.execute_reply":"2021-07-01T03:45:14.203999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = get_validation_dataset()\ndataset = dataset.unbatch().batch(20)\nbatch = iter(dataset)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:45:14.206265Z","iopub.execute_input":"2021-07-01T03:45:14.206587Z","iopub.status.idle":"2021-07-01T03:45:14.252977Z","shell.execute_reply.started":"2021-07-01T03:45:14.206556Z","shell.execute_reply":"2021-07-01T03:45:14.252061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, labels = next(batch)\nproba  = xce_model.predict(images)\npredictions = np.argmax(proba, axis=-1)\ndisplay_batch_of_images((images,labels), predictions)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:45:14.254992Z","iopub.execute_input":"2021-07-01T03:45:14.255505Z","iopub.status.idle":"2021-07-01T03:45:21.64134Z","shell.execute_reply.started":"2021-07-01T03:45:14.255471Z","shell.execute_reply":"2021-07-01T03:45:21.639989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predicting Test Data","metadata":{}},{"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True)\ntest_images_ds = test_ds.map(lambda image, idnum: image)\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(fTestImages))).numpy().astype('U')","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:45:21.643305Z","iopub.execute_input":"2021-07-01T03:45:21.643697Z","iopub.status.idle":"2021-07-01T03:45:23.482341Z","shell.execute_reply.started":"2021-07-01T03:45:21.643665Z","shell.execute_reply":"2021-07-01T03:45:23.481126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"proba = resnet_model.predict(test_images_ds)\npredictions = np.argmax(proba, axis=-1)\n\nnp.savetxt('submission_resnet50.csv',\n          np.rec.fromarrays([test_ids,predictions]),\n           fmt=['%s', '%d'],\n           delimiter=',',\n           header='id,label',\n           comments='',)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:47:54.172764Z","iopub.execute_input":"2021-07-01T03:47:54.173398Z","iopub.status.idle":"2021-07-01T03:47:58.090501Z","shell.execute_reply.started":"2021-07-01T03:47:54.173363Z","shell.execute_reply":"2021-07-01T03:47:58.089676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"proba = xce_model.predict(test_images_ds)\npredictions = np.argmax(proba, axis=-1)\n\nnp.savetxt('submission_xce.csv',\n          np.rec.fromarrays([test_ids,predictions]),\n           fmt=['%s', '%d'],\n           delimiter=',',\n           header='id,label',\n           comments='',)\n\nnp.savetxt('submission.csv',\n          np.rec.fromarrays([test_ids,predictions]),\n           fmt=['%s', '%d'],\n           delimiter=',',\n           header='id,label',\n           comments='',)","metadata":{"execution":{"iopub.status.busy":"2021-07-01T03:46:56.343722Z","iopub.execute_input":"2021-07-01T03:46:56.344117Z","iopub.status.idle":"2021-07-01T03:47:04.518404Z","shell.execute_reply.started":"2021-07-01T03:46:56.344085Z","shell.execute_reply":"2021-07-01T03:47:04.517484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# References\n1. https://www.kaggle.com/ryanholbrook/create-your-first-submission\n2. https://www.kaggle.com/mgornergoogle/five-flowers-with-keras-and-xception-on-tpu\n3. https://www.kaggle.com/philculliton/a-simple-petals-tf-2-2-notebook","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}