{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Notebooks To Check Out For Beginners.\nThis notebook has several lines lines of code from [https://www.kaggle.com/ryanholbrook/create-your-first-submission](http://).\nThat notebook was really helpful for beginners like me who might have gotten a bit overwhelmed with all of this.   \nPlease go check out that notebook before attempting this competition as it is really helpful for beginners.\n\nFor Data Augmentation i referred to this notebook [https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96](http://).\nIt is an excellent notebook that explains how the data augmentation is exactly done. Refer to it if you want to gain some intution.","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet --quiet\n#Importing Stuff\nimport math, re, os\nimport numpy as np\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efficientnet","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-06-21T14:12:19.432473Z","iopub.execute_input":"2021-06-21T14:12:19.432906Z","iopub.status.idle":"2021-06-21T14:12:34.524101Z","shell.execute_reply.started":"2021-06-21T14:12:19.432820Z","shell.execute_reply":"2021-06-21T14:12:34.523051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Detect TPU, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() \n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T14:12:35.679244Z","iopub.execute_input":"2021-06-21T14:12:35.679658Z","iopub.status.idle":"2021-06-21T14:12:41.795411Z","shell.execute_reply.started":"2021-06-21T14:12:35.679611Z","shell.execute_reply":"2021-06-21T14:12:41.794267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare To Load The Data\n\nData for this competition is taken from Google Cloud Storage. Since TPUs process data very fast we need the inputs in an equally fast fashion.\n","metadata":{}},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')\nprint(GCS_DS_PATH)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T14:12:41.797197Z","iopub.execute_input":"2021-06-21T14:12:41.797623Z","iopub.status.idle":"2021-06-21T14:12:42.180417Z","shell.execute_reply.started":"2021-06-21T14:12:41.797577Z","shell.execute_reply":"2021-06-21T14:12:42.179393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Here we define which image sizes we are going to use. \n#We also load the training,validation and test filenames.\nIMAGE_SIZE = [512, 512]\nGCS_PATH = GCS_DS_PATH + '/tfrecords-jpeg-512x512'\nAUTO = tf.data.experimental.AUTOTUNE\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec') \n\n#Name of the different types of Flowers.\nCLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         \n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', \n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         \n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           \n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      \n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    \n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            \n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             \n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            \n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        \n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                              ","metadata":{"execution":{"iopub.status.busy":"2021-06-21T14:16:02.384790Z","iopub.execute_input":"2021-06-21T14:16:02.385263Z","iopub.status.idle":"2021-06-21T14:16:02.580664Z","shell.execute_reply.started":"2021-06-21T14:16:02.385218Z","shell.execute_reply":"2021-06-21T14:16:02.579762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#This function rescales and reshapes the images so that they could be trained by a neural net\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  \n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) \n    return image\n\n#This function creates image,label pairs for training. \ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        \"class\": tf.io.FixedLenFeature([], tf.int64),  \n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label\n\n#This creates image,id pairs for unlabeled data. Useful for test set\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"id\": tf.io.FixedLenFeature([], tf.string),  \n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum \n\n#This function helps in loading the datasets.\n#It can return both labeled and unlabaled data because of the two functions we specified above.\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2021-06-21T14:16:02.582266Z","iopub.execute_input":"2021-06-21T14:16:02.582560Z","iopub.status.idle":"2021-06-21T14:16:02.593513Z","shell.execute_reply.started":"2021-06-21T14:16:02.582533Z","shell.execute_reply":"2021-06-21T14:16:02.592321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Augmentation Using Custom Functions","metadata":{}},{"cell_type":"code","source":"#This function will be used to augment the data. Augmentation almost always helps in improving the performance of the model.\n#It is always recommended to augment your data.\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transform matrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))\n\ndef transform(image,label):\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 \n    \n    rot = 15. * tf.random.normal([1],dtype='float32')\n    shr = 5. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 16. * tf.random.normal([1],dtype='float32') \n    w_shift = 16. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3]),label\n\n#Function to load the Training dataset\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(transform, num_parallel_calls=AUTO)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) \n    return dataset\n\n#Function to load the Validation dataset\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n#Function to load the Test dataset\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n#Here we specify the number of images in each categories as they are fixed. You can also write a custom function using regex \n#to find the same\nNUM_TRAINING_IMAGES = 12753\nNUM_VALIDATION_IMAGES = 3712\nNUM_TEST_IMAGES = 7382","metadata":{"execution":{"iopub.status.busy":"2021-06-21T14:16:02.595062Z","iopub.execute_input":"2021-06-21T14:16:02.595434Z","iopub.status.idle":"2021-06-21T14:16:02.623697Z","shell.execute_reply.started":"2021-06-21T14:16:02.595406Z","shell.execute_reply":"2021-06-21T14:16:02.622423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load The Data","metadata":{}},{"cell_type":"code","source":"#Batch size is dependent on whether we are using TPU or not\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\n\n#The functions mentioned above are called\nds_train = get_training_dataset()\nds_valid = get_validation_dataset()\nds_test = get_test_dataset()\n\nprint(\"Training:\", ds_train)\nprint (\"Validation:\", ds_valid)\nprint(\"Test:\", ds_test)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T14:16:02.625269Z","iopub.execute_input":"2021-06-21T14:16:02.625770Z","iopub.status.idle":"2021-06-21T14:16:03.072050Z","shell.execute_reply.started":"2021-06-21T14:16:02.625736Z","shell.execute_reply":"2021-06-21T14:16:03.070721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Defining The Models","metadata":{}},{"cell_type":"code","source":"#Remember while using TPUs it is necessary to define your model in  the strategy.scope() block.\nwith strategy.scope():\n    \n    pretrained_model = efficientnet.EfficientNetB7(\n        weights='noisy-student',\n        include_top=False ,\n        input_shape=[*IMAGE_SIZE, 3])\n    \n    pretrained_model.trainable = True\n    \n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dropout(0.3),\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n    ])","metadata":{"execution":{"iopub.status.busy":"2021-06-21T14:24:26.522548Z","iopub.execute_input":"2021-06-21T14:24:26.523187Z","iopub.status.idle":"2021-06-21T14:25:10.688234Z","shell.execute_reply.started":"2021-06-21T14:24:26.523144Z","shell.execute_reply":"2021-06-21T14:25:10.687324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Here we compile our model\n\nmodel.compile(\n    optimizer='adam',\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy'])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T14:25:10.689516Z","iopub.execute_input":"2021-06-21T14:25:10.690078Z","iopub.status.idle":"2021-06-21T14:25:10.821801Z","shell.execute_reply.started":"2021-06-21T14:25:10.690041Z","shell.execute_reply":"2021-06-21T14:25:10.820440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Callbacks","metadata":{}},{"cell_type":"code","source":"#A custom learning rate function is defined. If the learning rate was too high in the start our pre trained weights would become useless\ndef exponential_lr(epoch,\n                   start_lr = 0.00001, min_lr = 0.00001, max_lr = 0.00005,\n                   rampup_epochs = 5, sustain_epochs = 0,\n                   exp_decay = 0.8):\n\n    def lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay):\n        # linear increase from start to rampup_epochs\n        if epoch < rampup_epochs:\n            lr = ((max_lr - start_lr) / rampup_epochs * epoch + start_lr)\n        # constant max_lr during sustain_epochs\n        elif epoch < rampup_epochs + sustain_epochs:\n            lr = max_lr\n        # exponential decay towards min_lr\n        else:\n            lr = ((max_lr - min_lr) * exp_decay**(epoch - rampup_epochs - sustain_epochs) + min_lr)\n        return lr\n    return lr(epoch,start_lr,min_lr,max_lr,rampup_epochs,sustain_epochs,exp_decay)\n\n#It is used by the callbacks class. verbose is set to true to see the current learning rate at the start of the epoch\nlr_callback = tf.keras.callbacks.LearningRateScheduler(exponential_lr, verbose=True)\n\n\ncheck = tf.keras.callbacks.ModelCheckpoint(\"model.hdf5\",save_best_only=True,mode='max',monitor='val_sparse_categorical_accuracy')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T14:40:13.571720Z","iopub.execute_input":"2021-06-21T14:40:13.572151Z","iopub.status.idle":"2021-06-21T14:40:13.581732Z","shell.execute_reply.started":"2021-06-21T14:40:13.572117Z","shell.execute_reply":"2021-06-21T14:40:13.580559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training All The Models","metadata":{}},{"cell_type":"code","source":"#Here we finally put together everything we have done and start training our models\nEPOCHS = 10\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\nhistory = model.fit(\n    ds_train,\n    validation_data=ds_valid,\n    epochs=EPOCHS,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    callbacks=[lr_callback,check],\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T14:40:21.808764Z","iopub.execute_input":"2021-06-21T14:40:21.809275Z","iopub.status.idle":"2021-06-21T14:42:42.200929Z","shell.execute_reply.started":"2021-06-21T14:40:21.809242Z","shell.execute_reply":"2021-06-21T14:42:42.198524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n#This function is used to display the traing curves and loss.\ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1:\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])\n\n\n\n#Here we display the loss and accuracy curves\n#By changing the names we can view this data for any model\ndisplay_training_curves(\n    history.history['loss'],\n    history.history['val_loss'],\n    'loss',\n    211,\n)\ndisplay_training_curves(\n    history.history['sparse_categorical_accuracy'],\n    history.history['val_sparse_categorical_accuracy'],\n    'accuracy',\n    212,\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T14:42:42.739803Z","iopub.execute_input":"2021-06-21T14:42:42.740171Z","iopub.status.idle":"2021-06-21T14:42:42.774268Z","shell.execute_reply.started":"2021-06-21T14:42:42.740143Z","shell.execute_reply":"2021-06-21T14:42:42.772753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Here we load the test dataset but this in order as it is important to have the same order as the sample submission.\ntest_ds = get_test_dataset(ordered=True)\nmodel.load_weights(\"./model.hdf5\")\n\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T14:42:55.769105Z","iopub.execute_input":"2021-06-21T14:42:55.769465Z","iopub.status.idle":"2021-06-21T14:43:45.979505Z","shell.execute_reply.started":"2021-06-21T14:42:55.769436Z","shell.execute_reply":"2021-06-21T14:43:45.978501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Generating submission.csv file...')\n\n# Get image ids from test set and convert to unicode\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\n# Write the submission file\nnp.savetxt(\n    'submission.csv',\n    np.rec.fromarrays([test_ids, predictions]),\n    fmt=['%s', '%d'],\n    delimiter=',',\n    header='id,label',\n    comments='',\n)\n\n# Look at the first few predictions\n!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2021-06-21T14:43:50.437411Z","iopub.execute_input":"2021-06-21T14:43:50.437752Z","iopub.status.idle":"2021-06-21T14:43:53.894268Z","shell.execute_reply.started":"2021-06-21T14:43:50.437724Z","shell.execute_reply":"2021-06-21T14:43:53.893052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}