{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nfrom kaggle_datasets import KaggleDatasets\nimport numpy as np\nimport math\nimport time\nfrom matplotlib import pyplot as plt\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout\nfrom keras import regularizers\nimport re\n\nAUTO = tf.data.experimental.AUTOTUNE\n\nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:36:56.211720Z","iopub.execute_input":"2023-11-12T20:36:56.212116Z","iopub.status.idle":"2023-11-12T20:37:12.213713Z","shell.execute_reply.started":"2023-11-12T20:36:56.212081Z","shell.execute_reply":"2023-11-12T20:37:12.212978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# code in this cell is referenced from https://www.kaggle.com/code/philculliton/a-simple-petals-tf-2-2-notebook/notebook\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:37:12.215242Z","iopub.execute_input":"2023-11-12T20:37:12.216672Z","iopub.status.idle":"2023-11-12T20:37:21.461033Z","shell.execute_reply.started":"2023-11-12T20:37:12.216642Z","shell.execute_reply":"2023-11-12T20:37:21.460325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:37:21.461976Z","iopub.execute_input":"2023-11-12T20:37:21.462223Z","iopub.status.idle":"2023-11-12T20:37:21.466484Z","shell.execute_reply.started":"2023-11-12T20:37:21.462196Z","shell.execute_reply":"2023-11-12T20:37:21.465807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# code in this cell is referenced from https://www.kaggle.com/code/philculliton/a-simple-petals-tf-2-2-notebook/notebook\nIMAGE_SIZE = [192, 192] # at this size, a GPU will run out of memory. Use the TPU\nEPOCHS = 25\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nHEIGHT = 192\nWIDTH = 192\nCHANNELS = 3\nGCS_PATH = GCS_DS_PATH + '/tfrecords-jpeg-192x192'\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec') \n\nCLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']  \n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:37:21.467492Z","iopub.execute_input":"2023-11-12T20:37:21.467832Z","iopub.status.idle":"2023-11-12T20:37:21.502582Z","shell.execute_reply.started":"2023-11-12T20:37:21.467789Z","shell.execute_reply":"2023-11-12T20:37:21.501807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# code in this cell is referenced from https://www.kaggle.com/code/philculliton/a-simple-petals-tf-2-2-notebook/notebook\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:37:21.504575Z","iopub.execute_input":"2023-11-12T20:37:21.504927Z","iopub.status.idle":"2023-11-12T20:37:21.511529Z","shell.execute_reply.started":"2023-11-12T20:37:21.504898Z","shell.execute_reply":"2023-11-12T20:37:21.510796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# code in this cell is referenced from https://www.kaggle.com/code/philculliton/a-simple-petals-tf-2-2-notebook/notebook\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames,num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:37:21.512494Z","iopub.execute_input":"2023-11-12T20:37:21.512745Z","iopub.status.idle":"2023-11-12T20:37:21.526413Z","shell.execute_reply.started":"2023-11-12T20:37:21.512719Z","shell.execute_reply":"2023-11-12T20:37:21.525730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# code in this cell is referenced from https://www.kaggle.com/code/philculliton/a-simple-petals-tf-2-2-notebook/notebook\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    #dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset\n\ndef get_validation_dataset():\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=False)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:37:21.527241Z","iopub.execute_input":"2023-11-12T20:37:21.527454Z","iopub.status.idle":"2023-11-12T20:37:21.541315Z","shell.execute_reply.started":"2023-11-12T20:37:21.527431Z","shell.execute_reply":"2023-11-12T20:37:21.540653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# code in this cell is referenced from https://www.kaggle.com/code/philculliton/a-simple-petals-tf-2-2-notebook/notebook\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec\n    # files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\ntraining_dataset = get_training_dataset()\nvalidation_dataset = get_validation_dataset()\ntest_dataset = get_test_dataset(ordered=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:37:21.542484Z","iopub.execute_input":"2023-11-12T20:37:21.542739Z","iopub.status.idle":"2023-11-12T20:37:21.774581Z","shell.execute_reply.started":"2023-11-12T20:37:21.542711Z","shell.execute_reply":"2023-11-12T20:37:21.773651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# code in this cell is referenced from https://www.kaggle.com/code/philculliton/a-simple-petals-tf-2-2-notebook/notebook\nnp.set_printoptions(threshold=15, linewidth=80)\n\nprint(\"Training data shapes:\")\nfor image, label in training_dataset.take(3):\n    print(image.numpy().shape, label.numpy().shape)\nprint(\"Training data label examples:\", label.numpy())\n\nprint(\"Test data shapes:\")\nfor image, idnum in test_dataset.take(3):\n    print(image.numpy().shape, idnum.numpy().shape)\nprint(\"Test data IDs:\", idnum.numpy().astype('U')) # U=unicode string","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:37:21.775648Z","iopub.execute_input":"2023-11-12T20:37:21.775957Z","iopub.status.idle":"2023-11-12T20:37:22.502850Z","shell.execute_reply.started":"2023-11-12T20:37:21.775928Z","shell.execute_reply":"2023-11-12T20:37:22.501988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# code in this cell is referenced from https://www.kaggle.com/code/philculliton/a-simple-petals-tf-2-2-notebook/notebook\ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])\n    \n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:37:22.503915Z","iopub.execute_input":"2023-11-12T20:37:22.504289Z","iopub.status.idle":"2023-11-12T20:37:22.509575Z","shell.execute_reply.started":"2023-11-12T20:37:22.504260Z","shell.execute_reply":"2023-11-12T20:37:22.508848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#my code\nmodel_names = []\nmodel_training_time = []\ndef display_training_time_plot(model_names, model_training_time):\n    plt.bar(model_names, model_training_time)\n    plt.xlabel('models')\n    plt.ylabel('training time')\n    plt.xticks(rotation=90)\n    plt.title(\"Training Time of  different models\")\n    plt.show()\n\ntrain_accuracy = []\nval_accuracy = []\n\n\ndef display_accuracy_plot(model_names, train_accuracy, val_accuracy):\n    fig, ax = plt.subplots()\n    ax.plot(model_names, train_accuracy)\n    ax.plot(model_names, val_accuracy)\n    plt.xticks(rotation=90)\n    ax.legend(['train', 'valid.'])\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:37:22.510485Z","iopub.execute_input":"2023-11-12T20:37:22.510909Z","iopub.status.idle":"2023-11-12T20:37:22.521978Z","shell.execute_reply.started":"2023-11-12T20:37:22.510877Z","shell.execute_reply":"2023-11-12T20:37:22.521314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#my code; the model defination is original\nwith strategy.scope():    \n    \n    # Define the model as a sequential sequence of layers\n    model1 = Sequential()\n\n    # Define convolutional layers\n    model1.add(Conv2D(64, (3, 3), activation='relu', padding='same', input_shape=(*IMAGE_SIZE, 3)))\n    model1.add(MaxPooling2D((2, 2)))\n\n    model1.add(Conv2D(256, (3, 3), activation='relu', padding='same'))\n    model1.add(MaxPooling2D((2, 2)))\n    \n    model1.add(Conv2D(512, (3, 3), activation='relu', padding='same'))\n    model1.add(MaxPooling2D((2, 2)))\n\n\n    # Define classification layers\n    model1.add(Flatten())\n    model1.add(Dense(1024, activation='relu'))\n    model1.add(Dropout(0.5))\n    model1.add(Dense(len(CLASSES), activation='softmax'))\n\n    # Print a summary of the model architecture\n    print(model1.summary())\n\n      \nmodel1.compile(\n    optimizer='adam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)\n\n\nstart_time = time.time()\nmodel1_history = model1.fit(training_dataset, \n          steps_per_epoch=STEPS_PER_EPOCH, \n          epochs=EPOCHS, \n          validation_data=validation_dataset)\nend_time = time.time()\nmodel_training_time.append(end_time-start_time)\nmodel_names.append('model1')\n\ndisplay_training_curves(\n    model1_history.history['loss'],\n    model1_history.history['val_loss'],\n    'loss',\n    211,\n)\ndisplay_training_curves(\n    model1_history.history['sparse_categorical_accuracy'],\n    model1_history.history['val_sparse_categorical_accuracy'],\n    'accuracy',\n    212,\n)\n\ntrain_accuracy.append(model1_history.history['sparse_categorical_accuracy'][-1])\nval_accuracy.append(model1_history.history['val_sparse_categorical_accuracy'][-1])","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:37:22.522883Z","iopub.execute_input":"2023-11-12T20:37:22.523297Z","iopub.status.idle":"2023-11-12T20:43:04.606221Z","shell.execute_reply.started":"2023-11-12T20:37:22.523268Z","shell.execute_reply":"2023-11-12T20:43:04.605197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The training accuracy is 97.43% and validation accuracy is 33.35%, which is a strong indication that the model is overfitted. \nhttps://www.analyticsvidhya.com/blog/2020/09/overfitting-in-cnn-show-to-treat-overfitting-in-convolutional-neural-networks/\nOverfitting can be resolved using following \n* Regularization\n* Weight Initialization\n* Dropout Regularization\n* Weight Constraints\n\nBelow I have added L2 Regularization to the hidden layers 2 and 3","metadata":{}},{"cell_type":"code","source":"#my code; the model defination is original\nwith strategy.scope():    \n    \n    # Define the model as a sequential sequence of layers\n    model1_regularization = Sequential()\n\n    # Define convolutional layers\n    model1_regularization.add(Conv2D(64, (3, 3), activation='relu', padding='same', input_shape=(*IMAGE_SIZE, 3)))\n    model1_regularization.add(MaxPooling2D((2, 2)))\n\n    model1_regularization.add(Conv2D(256, (3, 3), activation='relu',kernel_regularizer=regularizers.l2(l=0.01), padding='same'))\n    model1_regularization.add(MaxPooling2D((2, 2)))\n    \n    model1_regularization.add(Conv2D(512, (3, 3), activation='relu', kernel_regularizer=regularizers.l2(l=0.01),padding='same'))\n    model1_regularization.add(MaxPooling2D((2, 2)))\n\n\n    # Define classification layers\n    model1_regularization.add(Flatten())\n    model1_regularization.add(Dense(1024, activation='relu'))\n    model1_regularization.add(Dropout(0.5))\n    model1_regularization.add(Dense(len(CLASSES), activation='softmax'))\n\n    # Print a summary of the model architecture\n    print(model1_regularization.summary())\n\n      \nmodel1_regularization.compile(\n    optimizer='adam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)\n\n\nstart_time = time.time()\nmodel1_regularization_history = model1_regularization.fit(training_dataset, \n          steps_per_epoch=STEPS_PER_EPOCH, \n          epochs=EPOCHS, \n          validation_data=validation_dataset)\nend_time = time.time()\nmodel_training_time.append(end_time-start_time)\nmodel_names.append('model1_regularization')\n\ndisplay_training_curves(\n    model1_regularization_history.history['loss'],\n    model1_regularization_history.history['val_loss'],\n    'loss',\n    211,\n)\ndisplay_training_curves(\n    model1_regularization_history.history['sparse_categorical_accuracy'],\n    model1_regularization_history.history['val_sparse_categorical_accuracy'],\n    'accuracy',\n    212,\n)\n\ntrain_accuracy.append(model1_regularization_history.history['sparse_categorical_accuracy'][-1])\nval_accuracy.append(model1_regularization_history.history['val_sparse_categorical_accuracy'][-1])","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:43:04.607491Z","iopub.execute_input":"2023-11-12T20:43:04.607867Z","iopub.status.idle":"2023-11-12T20:48:43.672997Z","shell.execute_reply.started":"2023-11-12T20:43:04.607839Z","shell.execute_reply":"2023-11-12T20:48:43.672004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Addition of L2 Regularizer to layer 2 and layer 3 led to the increase in the validation accuracy. Althought there is an increase in accuracy for validation data set. It still appears that the model is overfitting to the training data.\n","metadata":{}},{"cell_type":"markdown","source":"Next we build a more complex CNN model with more convolution layers.","metadata":{}},{"cell_type":"code","source":"#my code; the model defination is original\nwith strategy.scope():    \n    \n    # Define the model as a sequential sequence of layers\n    model2 = Sequential()\n\n    # Define convolutional layers\n    model2.add(Conv2D(64, (3, 3), activation='relu', padding='same', input_shape=(*IMAGE_SIZE, 3)))\n    model2.add(MaxPooling2D((2, 2)))\n\n    model2.add(Conv2D(256, (3, 3), activation='relu', padding='same'))\n    model2.add(Conv2D(256, (3, 3), activation='relu', padding='same'))\n    model2.add(MaxPooling2D((2, 2)))\n    \n    model2.add(Conv2D(512, (3, 3), activation='relu', padding='same'))\n    model2.add(Conv2D(512, (3, 3), activation='relu', padding='same'))\n    model2.add(MaxPooling2D((2, 2)))\n    \n    model2.add(Conv2D(1024, (3, 3), activation='relu', padding='same'))\n    model2.add(Conv2D(1024, (3, 3), activation='relu', padding='same'))\n    model2.add(MaxPooling2D((2, 2)))\n    \n    # Define classification layers\n    model2.add(Flatten())\n    model2.add(Dense(1024, activation='relu'))\n    model2.add(Dropout(0.4))\n    model2.add(Dense(512, activation='relu'))\n    model2.add(Dropout(0.2))\n    model2.add(Dense(len(CLASSES), activation='softmax'))\n\n    # Print a summary of the model architecture\n    print(model2.summary())\n\n      \nmodel2.compile(\n    optimizer='adam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)\n\n\nstart_time = time.time()\nmodel2_history = model2.fit(training_dataset, \n          steps_per_epoch=STEPS_PER_EPOCH, \n          epochs=EPOCHS, \n          validation_data=validation_dataset)\nend_time = time.time()\nmodel_training_time.append(end_time-start_time)\nmodel_names.append('model2')\n\ndisplay_training_curves(\n    model2_history.history['loss'],\n    model2_history.history['val_loss'],\n    'loss',\n    211,\n)\ndisplay_training_curves(\n    model2_history.history['sparse_categorical_accuracy'],\n    model2_history.history['val_sparse_categorical_accuracy'],\n    'accuracy',\n    212,\n)\n\ntrain_accuracy.append(model2_history.history['sparse_categorical_accuracy'][-1])\nval_accuracy.append(model2_history.history['val_sparse_categorical_accuracy'][-1])","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:48:43.675639Z","iopub.execute_input":"2023-11-12T20:48:43.675928Z","iopub.status.idle":"2023-11-12T20:54:54.275697Z","shell.execute_reply.started":"2023-11-12T20:48:43.675901Z","shell.execute_reply":"2023-11-12T20:54:54.274699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#my code; the model defination is original\nwith strategy.scope():    \n    \n    # Define the model as a sequential sequence of layers\n    model2_regularization = Sequential()\n\n    # Define convolutional layers\n    model2_regularization.add(Conv2D(64, (3, 3), activation='relu', padding='same', input_shape=(*IMAGE_SIZE, 3)))\n    model2_regularization.add(MaxPooling2D((2, 2)))\n\n    model2_regularization.add(Conv2D(256, (3, 3), activation='relu', kernel_regularizer=regularizers.l2(l=0.01), padding='same'))\n    model2_regularization.add(Conv2D(256, (3, 3), activation='relu', kernel_regularizer=regularizers.l2(l=0.01), padding='same'))\n    model2_regularization.add(MaxPooling2D((2, 2)))\n    \n    model2_regularization.add(Conv2D(512, (3, 3), activation='relu', padding='same'))\n    model2_regularization.add(Conv2D(512, (3, 3), activation='relu', padding='same'))\n    model2_regularization.add(MaxPooling2D((2, 2)))\n    \n    model2_regularization.add(Conv2D(1024, (3, 3), activation='relu', kernel_regularizer=regularizers.l2(l=0.01),padding='same'))\n    model2_regularization.add(Conv2D(1024, (3, 3), activation='relu', kernel_regularizer=regularizers.l2(l=0.01),padding='same'))\n    model2_regularization.add(MaxPooling2D((2, 2)))\n    \n\n\n    # Define classification layers\n    model2_regularization.add(Flatten())\n    model2_regularization.add(Dense(1024, activation='relu'))\n    model2_regularization.add(Dropout(0.4))\n    model2_regularization.add(Dense(512, activation='relu'))\n    model2_regularization.add(Dropout(0.2))\n    model2_regularization.add(Dense(len(CLASSES), activation='softmax'))\n\n    # Print a summary of the model architecture\n    print(model2_regularization.summary())\n\n      \nmodel2_regularization.compile(\n    optimizer='adam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)\n\n\nstart_time = time.time()\nmodel2_regularization_history = model2_regularization.fit(training_dataset, \n          steps_per_epoch=STEPS_PER_EPOCH, \n          epochs=EPOCHS, \n          validation_data=validation_dataset)\nend_time = time.time()\nmodel_training_time.append(end_time-start_time)\nmodel_names.append('model2_regularization')\n\ndisplay_training_curves(\n    model2_regularization_history.history['loss'],\n    model2_regularization_history.history['val_loss'],\n    'loss',\n    211,\n)\ndisplay_training_curves(\n    model2_regularization_history.history['sparse_categorical_accuracy'],\n    model2_regularization_history.history['val_sparse_categorical_accuracy'],\n    'accuracy',\n    212,\n)\n\ntrain_accuracy.append(model2_regularization_history.history['sparse_categorical_accuracy'][-1])\nval_accuracy.append(model2_regularization_history.history['val_sparse_categorical_accuracy'][-1])","metadata":{"execution":{"iopub.status.busy":"2023-11-12T20:54:54.276811Z","iopub.execute_input":"2023-11-12T20:54:54.277087Z","iopub.status.idle":"2023-11-12T21:01:01.887350Z","shell.execute_reply.started":"2023-11-12T20:54:54.277057Z","shell.execute_reply":"2023-11-12T21:01:01.886457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next up, we try to fit VGG16 model for flower classification with and without the pretraining.\nThe below cell's code is referenced from :\nhttps://medium.com/geekculture/boost-your-image-classification-model-with-pretrained-vgg-16-ec185f763104#:~:text=The%20VGG16%20model%20is%20a,and%203%20fully%20connected%20layers.&text=The%20pretrained%20VGG16%20model%20is,a%20wide%20range%20of%20features.\nhttps://datagen.tech/guides/computer-vision/vgg16/","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-12T05:44:05.830391Z","iopub.execute_input":"2023-11-12T05:44:05.830646Z","iopub.status.idle":"2023-11-12T05:44:22.279632Z","shell.execute_reply.started":"2023-11-12T05:44:05.830619Z","shell.execute_reply":"2023-11-12T05:44:22.278273Z"}}},{"cell_type":"code","source":"with strategy.scope():    \n    \n    #VGG16 \n    # Define the model as a sequential sequence of layers\n    vgg16_model = Sequential()\n\n    # Define convolutional layers\n    vgg16_model.add(Conv2D(64, (3, 3), activation='relu', padding='same', input_shape=(*IMAGE_SIZE, 3)))\n    vgg16_model.add(Conv2D(64, (3, 3), activation='relu', padding='same'))\n    vgg16_model.add(MaxPooling2D((2, 2)))\n\n    vgg16_model.add(Conv2D(128, (3, 3), activation='relu', padding='same'))\n    vgg16_model.add(Conv2D(128, (3, 3), activation='relu', padding='same'))\n    vgg16_model.add(MaxPooling2D((2, 2)))\n\n    vgg16_model.add(Conv2D(256, (3, 3), activation='relu', padding='same'))\n    vgg16_model.add(Conv2D(256, (3, 3), activation='relu', padding='same'))\n    vgg16_model.add(Conv2D(256, (3, 3), activation='relu', padding='same'))\n    vgg16_model.add(MaxPooling2D((2, 2)))\n\n    vgg16_model.add(Conv2D(512, (3, 3), activation='relu', padding='same'))\n    vgg16_model.add(Conv2D(512, (3, 3), activation='relu', padding='same'))\n    vgg16_model.add(Conv2D(512, (3, 3), activation='relu', padding='same'))\n    vgg16_model.add(MaxPooling2D((2, 2)))\n\n    vgg16_model.add(Conv2D(512, (3, 3), activation='relu', padding='same'))\n    vgg16_model.add(Conv2D(512, (3, 3), activation='relu', padding='same'))\n    vgg16_model.add(Conv2D(512, (3, 3), activation='relu', padding='same'))\n    vgg16_model.add(MaxPooling2D((2, 2)))\n\n    # Define classification layers\n    vgg16_model.add(Flatten())\n    vgg16_model.add(Dense(4096, activation='relu'))\n    vgg16_model.add(Dropout(0.5))\n    vgg16_model.add(Dense(4096, activation='relu'))\n    vgg16_model.add(Dropout(0.5))\n    vgg16_model.add(Dense(len(CLASSES), activation='softmax'))\n\n    # Print a summary of the model architecture\n    print(vgg16_model.summary())\n\n      \nvgg16_model.compile(\n    optimizer='adam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)\n\n\nstart_time = time.time()\nvgg16_model_history = vgg16_model.fit(training_dataset, \n          steps_per_epoch=STEPS_PER_EPOCH, \n          epochs=EPOCHS, \n          validation_data=validation_dataset)\nend_time = time.time()\nmodel_training_time.append(end_time-start_time)\nmodel_names.append('vgg16_model')\n\ndisplay_training_curves(\n    vgg16_model_history.history['loss'],\n    vgg16_model_history.history['val_loss'],\n    'loss',\n    211,\n)\ndisplay_training_curves(\n    vgg16_model_history.history['sparse_categorical_accuracy'],\n    vgg16_model_history.history['val_sparse_categorical_accuracy'],\n    'accuracy',\n    212,\n)\n\ntrain_accuracy.append(vgg16_model_history.history['sparse_categorical_accuracy'][-1])\nval_accuracy.append(vgg16_model_history.history['val_sparse_categorical_accuracy'][-1])","metadata":{"execution":{"iopub.status.busy":"2023-11-12T21:01:01.888461Z","iopub.execute_input":"2023-11-12T21:01:01.888763Z","iopub.status.idle":"2023-11-12T21:06:03.772808Z","shell.execute_reply.started":"2023-11-12T21:01:01.888734Z","shell.execute_reply":"2023-11-12T21:06:03.771811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():    \n    pretrained_VGG16 = tf.keras.applications.VGG16(\n        weights='imagenet',\n        include_top=False ,\n        input_shape=[*IMAGE_SIZE, 3]\n    )\n    pretrained_VGG16.trainable = False\n    \n    pretrained_VGG16_model = tf.keras.Sequential([\n        # To a base pretrained on ImageNet to extract features from images...\n        pretrained_VGG16,\n        # ... attach a new head to act as a classifier.\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n    ])\n    print(pretrained_VGG16_model.summary())\n\npretrained_VGG16_model.compile(\n    optimizer='adam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)\n\nstart_time = time.time()\npretrained_VGG16_model_history = pretrained_VGG16_model.fit(training_dataset, \n          steps_per_epoch=STEPS_PER_EPOCH, \n          epochs=EPOCHS, \n          validation_data=validation_dataset)\nend_time = time.time()\nmodel_training_time.append(end_time-start_time)\nmodel_names.append('pretrained_VGG16_model')\n\ndisplay_training_curves(\n    pretrained_VGG16_model_history.history['loss'],\n    pretrained_VGG16_model_history.history['val_loss'],\n    'loss',\n    211,\n)\ndisplay_training_curves(\n    pretrained_VGG16_model_history.history['sparse_categorical_accuracy'],\n    pretrained_VGG16_model_history.history['val_sparse_categorical_accuracy'],\n    'accuracy',\n    212,\n)\ntrain_accuracy.append(pretrained_VGG16_model_history.history['sparse_categorical_accuracy'][-1])\nval_accuracy.append(pretrained_VGG16_model_history.history['val_sparse_categorical_accuracy'][-1])","metadata":{"execution":{"iopub.status.busy":"2023-11-12T21:06:03.773929Z","iopub.execute_input":"2023-11-12T21:06:03.774211Z","iopub.status.idle":"2023-11-12T21:09:01.655036Z","shell.execute_reply.started":"2023-11-12T21:06:03.774177Z","shell.execute_reply":"2023-11-12T21:09:01.654193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_training_time_plot(model_names, model_training_time)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T21:09:01.656135Z","iopub.execute_input":"2023-11-12T21:09:01.656427Z","iopub.status.idle":"2023-11-12T21:09:01.816719Z","shell.execute_reply.started":"2023-11-12T21:09:01.656396Z","shell.execute_reply":"2023-11-12T21:09:01.815798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_accuracy_plot(model_names, train_accuracy, val_accuracy)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T21:09:01.817695Z","iopub.execute_input":"2023-11-12T21:09:01.817995Z","iopub.status.idle":"2023-11-12T21:09:01.952612Z","shell.execute_reply.started":"2023-11-12T21:09:01.817965Z","shell.execute_reply":"2023-11-12T21:09:01.951879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#saving predictions from model2\ntest_images_ds = test_dataset.map(lambda image, idnum: image)\nprobabilities = model1_regularization.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ids_ds = test_dataset.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\nprint('submission.csv file saved')","metadata":{"execution":{"iopub.status.busy":"2023-11-12T21:09:17.725373Z","iopub.execute_input":"2023-11-12T21:09:17.725823Z","iopub.status.idle":"2023-11-12T21:09:18.438515Z","shell.execute_reply.started":"2023-11-12T21:09:17.725753Z","shell.execute_reply":"2023-11-12T21:09:18.437591Z"},"trusted":true},"execution_count":null,"outputs":[]}]}