{"cells":[{"metadata":{"_cell_guid":"df6eea78-2fce-4b89-b147-6d173c47fb66","_uuid":"68be60ad-f3cd-4218-ac16-9da53654b573","papermill":{"duration":0.041621,"end_time":"2021-01-10T22:40:48.92247","exception":false,"start_time":"2021-01-10T22:40:48.880849","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Cassava Leaf Disease Classification\n\nInstantiates the Xception architecture."},{"metadata":{"_cell_guid":"6e64eeb0-fe1d-482f-b5d1-a362dcc96f8d","_uuid":"d7bf6224-fcb9-417d-9e97-2a79d329c8fd","execution":{"iopub.execute_input":"2021-01-10T22:40:49.031099Z","iopub.status.busy":"2021-01-10T22:40:49.028386Z","iopub.status.idle":"2021-01-10T22:40:54.41711Z","shell.execute_reply":"2021-01-10T22:40:54.416406Z"},"papermill":{"duration":5.44153,"end_time":"2021-01-10T22:40:54.417239","exception":false,"start_time":"2021-01-10T22:40:48.975709","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom tensorflow.keras import layers\nfrom tensorflow import keras\nfrom keras.layers import Dense, Flatten, Activation, Dropout\n\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.optimizers import Adamax\n\nimport matplotlib.pyplot as plt\nfrom keras.models import Sequential\n\nprint(\"Tensorflow version \" + tf.__version__)\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"bea56b1d-b944-427c-980d-af485ecae1c7","_uuid":"ab1d3ccf-ab32-46f8-9336-c339120e7feb","papermill":{"duration":0.027254,"end_time":"2021-01-10T22:40:54.539372","exception":false,"start_time":"2021-01-10T22:40:54.512118","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Check GPU state"},{"metadata":{"_cell_guid":"b9f25aef-c249-4221-badc-eb6a8dd0d242","_uuid":"7865a464-d3d5-4b69-9e0e-b191db7760c4","execution":{"iopub.execute_input":"2021-01-10T22:40:55.531072Z","iopub.status.busy":"2021-01-10T22:40:55.530111Z","iopub.status.idle":"2021-01-10T22:40:57.295221Z","shell.execute_reply":"2021-01-10T22:40:57.294545Z"},"papermill":{"duration":2.727741,"end_time":"2021-01-10T22:40:57.29534","exception":false,"start_time":"2021-01-10T22:40:54.567599","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from tensorflow.python.framework.config import set_memory_growth\ngpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n    try:\n        # Currently, memory growth needs to be the same across GPUs\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n    except RuntimeError as e:\n        # Memory growth must be set before GPUs have been initialized\n        print(e)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"147b82a7-721f-4f07-9be1-7b255c093366","_uuid":"f36a29f1-45d5-4a0f-aaa4-7023604bb3c2","execution":{"iopub.execute_input":"2021-01-10T22:40:54.480533Z","iopub.status.busy":"2021-01-10T22:40:54.479801Z","iopub.status.idle":"2021-01-10T22:40:54.483328Z","shell.execute_reply":"2021-01-10T22:40:54.483813Z"},"papermill":{"duration":0.038236,"end_time":"2021-01-10T22:40:54.483957","exception":false,"start_time":"2021-01-10T22:40:54.445721","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"img_width = 300\nimg_height = 300\nepochs =20\nEPOCHS =epochs\nlast_epoch=0\nBATCH_SIZE = 32\nDROPOUT_RATE = 0.3\nvalidation_split = 0.15\n\n\nInceptionV3_TOP = '../input/inceptionv3/inception_v3_weights_tf_dim_ordering_tf_kernels.h5'\nInceptionV3_NOTOP = '../input/inceptionv3/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5'\n\nXception_TOP = '../input/xception/xception_weights_tf_dim_ordering_tf_kernels.h5'\nXception_NOTOP = '../input/xception/xception_weights_tf_dim_ordering_tf_kernels_notop.h5'\n\ngeneral_path = '../input/cassava-leaf-disease-classification/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.mixed_precision import experimental as mixed_precision\npolicy = mixed_precision.Policy('mixed_float16')\nmixed_precision.set_policy(policy)\nprint('Compute dtype: %s' % policy.compute_dtype)\nprint('Variable dtype: %s' % policy.variable_dtype)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.mixed_precision import experimental as mixed_precision\n\npolicy = mixed_precision.Policy('mixed_float16')\nmixed_precision.set_policy(policy)\n\nprint('Compute dtype: %s' % policy.compute_dtype)\nprint('Variable dtype: %s' % policy.variable_dtype)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ffbe0e84-2d53-4448-be45-b56abcd1ae47","_uuid":"33579ec8-0120-49b5-af34-2146a84f1870","papermill":{"duration":0.029063,"end_time":"2021-01-10T22:40:57.353228","exception":false,"start_time":"2021-01-10T22:40:57.324165","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Read train data"},{"metadata":{"_cell_guid":"1acb40cc-a2e3-4dbc-b022-897012a9eaec","_uuid":"ce398461-7206-40be-9963-7a727ed0c285","execution":{"iopub.execute_input":"2021-01-10T22:40:57.450022Z","iopub.status.busy":"2021-01-10T22:40:57.449408Z","iopub.status.idle":"2021-01-10T22:40:57.508707Z","shell.execute_reply":"2021-01-10T22:40:57.507863Z"},"papermill":{"duration":0.127766,"end_time":"2021-01-10T22:40:57.508813","exception":false,"start_time":"2021-01-10T22:40:57.381047","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = pd.read_csv(general_path + 'train.csv')\ntrain['label'] = train['label'].astype('string')\ndisplay(train.head())","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"174674b7-e4fb-428b-b6e8-3ca59bdcf89f","_uuid":"6e9db000-1675-4b5b-b924-61ac19cfc7ee","papermill":{"duration":0.028167,"end_time":"2021-01-10T22:40:57.566215","exception":false,"start_time":"2021-01-10T22:40:57.538048","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Image augmentation using Keras Image Data Generator\nImage augmentation is a technique of applying different transformations to original images which results in multiple transformed copies of the same image. Each copy, however, is different from the other in certain aspects depending on the augmentation techniques you apply like shifting, rotating, flipping, etc.\n\nApplying these small amounts of variations on the original image does not change its target class but only provides a new perspective of capturing the object in real life. And so, we use it is quite often for building deep learning models. Keras ImageDataGenerator is a gem! It lets you augment your images in real-time while your model is still training! You can apply any random transformations on each training image as it is passed to the model. This will not only make your model robust but will also save up on the overhead memory.\n\nAdvantages of using Keras Image Data Generator:\nThe main benefit of using the Keras ImageDataGenerator class is that it is designed to provide real-time data augmentation. Meaning it is generating augmented images on the fly while your model is still in the training stage. But it only returns the transformed images and does not add it to the original corpus of images. If it was, in fact, the case, then the model would be seeing the original images multiple times which would definitely overfit our model.\n\nAnother advantage of ImageDataGenerator is that it requires lower memory usage. This is so because without using this class, we load all the images at once. But on using it, we are loading the images in batches which saves a lot of memory."},{"metadata":{"_cell_guid":"7a7a6159-295e-4edb-8c96-bc26600af2ff","_uuid":"7399fe37-e015-48e6-b358-58d2249bb919","execution":{"iopub.execute_input":"2021-01-10T22:40:57.66519Z","iopub.status.busy":"2021-01-10T22:40:57.634972Z","iopub.status.idle":"2021-01-10T22:40:57.886977Z","shell.execute_reply":"2021-01-10T22:40:57.887688Z"},"papermill":{"duration":0.292974,"end_time":"2021-01-10T22:40:57.887868","exception":false,"start_time":"2021-01-10T22:40:57.594894","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"\ntrain_datagen = ImageDataGenerator(\n                            horizontal_flip=True,\n                            vertical_flip=True,\n                            rotation_range=75,\n                            shear_range=20,\n                            zoom_range=0.2,\n                            height_shift_range=0.1,\n                            width_shift_range=0.1,\n                            validation_split=validation_split)\n\ntrain_datagen_flow = train_datagen.flow_from_dataframe(\n    dataframe=train,\n    directory=general_path + 'train_images',\n    x_col='image_id',\n    y_col='label',\n    target_size=(img_width, img_height),\n    batch_size=32,\n    seed=128,\n    subset='training',\n    validate_filenames=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"db84ee87-a11e-4535-b967-914a3c9cb898","_uuid":"3f5c6245-711b-4409-958a-91ca4c4a6afa","papermill":{"duration":0.030018,"end_time":"2021-01-10T22:40:57.948912","exception":false,"start_time":"2021-01-10T22:40:57.918894","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Since we have provided validation_split=0.2 as a parameter, out of 21397 images our train data generator has generated 17118 images for train data.\n\nHere class_mode='categorical' means that for eg. label=4 will be encoded as [0, 0, 0, 0, 1].\n\nFor each image it's size will be (300, 300, 3)."},{"metadata":{"_cell_guid":"96b26ba3-e7f5-4716-9392-f367797123c7","_uuid":"61abace5-ef07-4877-a72e-45316ba6b5ba","execution":{"iopub.execute_input":"2021-01-10T22:40:58.01498Z","iopub.status.busy":"2021-01-10T22:40:58.014384Z","iopub.status.idle":"2021-01-10T22:40:58.155989Z","shell.execute_reply":"2021-01-10T22:40:58.154984Z"},"papermill":{"duration":0.177772,"end_time":"2021-01-10T22:40:58.156138","exception":false,"start_time":"2021-01-10T22:40:57.978366","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"valid_datagen = keras.preprocessing.image.ImageDataGenerator(validation_split=validation_split)\n\nvalid_datagen_flow = valid_datagen.flow_from_dataframe(\n    dataframe=train,\n    directory=general_path + 'train_images',\n    x_col='image_id',\n    y_col='label',\n    target_size=(img_width, img_height),\n    batch_size=32,\n    seed=128,\n    subset='validation',\n    validate_filenames=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3f16d3a5-d280-4d52-b502-0287f2da57f5","_uuid":"ea146fc3-b503-4155-aa6e-ae992c42e183","papermill":{"duration":0.028869,"end_time":"2021-01-10T22:40:58.21652","exception":false,"start_time":"2021-01-10T22:40:58.187651","status":"completed"},"tags":[]},"cell_type":"markdown","source":"A Sequential model is appropriate for a plain stack of layers where each layer has exactly one input tensor and one output tensor."},{"metadata":{"_cell_guid":"d08baf05-a2ce-4caa-af2e-525f0f4b33b0","_uuid":"b618c636-3d6c-4ebe-85af-83179d1dacae","execution":{"iopub.execute_input":"2021-01-10T22:40:58.27897Z","iopub.status.busy":"2021-01-10T22:40:58.278393Z","iopub.status.idle":"2021-01-10T22:40:58.317871Z","shell.execute_reply":"2021-01-10T22:40:58.318376Z"},"papermill":{"duration":0.072806,"end_time":"2021-01-10T22:40:58.318521","exception":false,"start_time":"2021-01-10T22:40:58.245715","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"model = keras.models.Sequential()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a0357535-cda5-4d2c-affa-4211988fe301","_uuid":"a143cf84-bab5-4c40-bc9a-43c362df7dd3","papermill":{"duration":0.028407,"end_time":"2021-01-10T22:40:58.376103","exception":false,"start_time":"2021-01-10T22:40:58.347696","status":"completed"},"tags":[]},"cell_type":"markdown","source":"The structure of our deep learning model is as follows:\n\n1. Xception model as a part of transfer learning application by Keras.\n1. Global Average Pooling technique to reduce the image shape and apply pooling on spatial dimensions.\n1. Dense layer to provide the probability of predictions for all 5 classes, acting as a output layer.\n1. If you are new to Deep Learning and want to understand the functionality of Global Average Pooling layer: click here"},{"metadata":{"_cell_guid":"7520e984-2557-49a6-9a70-b78316e41dfd","_uuid":"0bfaadf6-91eb-43d1-a404-e43edc1b52fe","execution":{"iopub.execute_input":"2021-01-10T22:40:58.439097Z","iopub.status.busy":"2021-01-10T22:40:58.438427Z","iopub.status.idle":"2021-01-10T22:41:03.444416Z","shell.execute_reply":"2021-01-10T22:41:03.443461Z"},"papermill":{"duration":5.039603,"end_time":"2021-01-10T22:41:03.444541","exception":false,"start_time":"2021-01-10T22:40:58.404938","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"model.add(keras.applications.Xception(\n    input_shape=(img_height, img_width, 3), \n    weights=Xception_NOTOP, \n    include_top=False)\n)\n\nmodel.add(keras.layers.GlobalAveragePooling2D())\n\nmodel.add(layers.Dropout(DROPOUT_RATE))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"90c80a3e-0149-46ed-9dc9-f6f4074d3f74","_uuid":"7bdd966b-440d-43e4-90b8-c7015ae4c851","papermill":{"duration":0.028419,"end_time":"2021-01-10T22:41:03.57121","exception":false,"start_time":"2021-01-10T22:41:03.542791","status":"completed"},"tags":[]},"cell_type":"markdown","source":"The dense layer is a neural network layer that is connected deeply, which means each neuron in the dense layer receives input from all neurons of its previous layer. T"},{"metadata":{"_cell_guid":"1a44524f-66fe-4b69-99a0-477dbf0da31d","_uuid":"a80a5007-e781-43ea-bb04-e3eff2d9bcbb","execution":{"iopub.execute_input":"2021-01-10T22:41:03.638206Z","iopub.status.busy":"2021-01-10T22:41:03.637249Z","iopub.status.idle":"2021-01-10T22:41:03.648212Z","shell.execute_reply":"2021-01-10T22:41:03.647599Z"},"papermill":{"duration":0.04814,"end_time":"2021-01-10T22:41:03.648326","exception":false,"start_time":"2021-01-10T22:41:03.600186","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"model.add(keras.layers.Dense(5, activation='softmax', dtype='float32', name='predictions'))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b41472f4-477c-45de-ac9f-c27fc010c937","_uuid":"54db7f32-61cb-4be2-832e-c75b9537904a","papermill":{"duration":0.030084,"end_time":"2021-01-10T22:41:03.707924","exception":false,"start_time":"2021-01-10T22:41:03.67784","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Print model summary"},{"metadata":{"_cell_guid":"d5347740-8e48-48c6-af5d-21d6cb99de78","_uuid":"c8cad55e-b740-49c0-83d6-be8cf85290e9","execution":{"iopub.execute_input":"2021-01-10T22:41:03.79268Z","iopub.status.busy":"2021-01-10T22:41:03.79177Z","iopub.status.idle":"2021-01-10T22:41:03.808795Z","shell.execute_reply":"2021-01-10T22:41:03.809803Z"},"papermill":{"duration":0.073139,"end_time":"2021-01-10T22:41:03.809994","exception":false,"start_time":"2021-01-10T22:41:03.736855","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"10e6a7b0-3ae1-4255-9820-6dbc7952b49e","_uuid":"ccd4807b-ec7d-4c9b-a2dd-62d3111a7111","papermill":{"duration":0.049095,"end_time":"2021-01-10T22:41:03.910003","exception":false,"start_time":"2021-01-10T22:41:03.860908","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Callback"},{"metadata":{"_cell_guid":"a9e26a9d-6012-48d6-9ecb-c087bb6624f4","_uuid":"b4410953-f82f-4ba9-b9ec-e10037c6dfa4","papermill":{"duration":0.049106,"end_time":"2021-01-10T22:41:04.004763","exception":false,"start_time":"2021-01-10T22:41:03.955657","status":"completed"},"tags":[]},"cell_type":"markdown","source":"A callback is an object that can perform actions at various stages of training (e.g. at the start or end of an epoch, before or after a single batch, etc).  In this model we will be using 3 callbacks as below:"},{"metadata":{"_cell_guid":"ad14bc0f-e620-41bc-ba4d-b9dbd3333143","_uuid":"64df7129-e066-488c-bbd9-2ca3194bb766","papermill":{"duration":0.047169,"end_time":"2021-01-10T22:41:04.105896","exception":false,"start_time":"2021-01-10T22:41:04.058727","status":"completed"},"tags":[]},"cell_type":"markdown","source":"1. ModelCheckpoint : Callback to save the Keras model or model weights at some frequency."},{"metadata":{"_cell_guid":"bd4ed412-eb08-44d3-b647-6460d9a3ae7a","_uuid":"4ce3b9cf-49cf-4df9-bc8a-1c2198596ea3","execution":{"iopub.execute_input":"2021-01-10T22:41:04.204162Z","iopub.status.busy":"2021-01-10T22:41:04.202729Z","iopub.status.idle":"2021-01-10T22:41:04.206908Z","shell.execute_reply":"2021-01-10T22:41:04.207752Z"},"papermill":{"duration":0.056319,"end_time":"2021-01-10T22:41:04.207925","exception":false,"start_time":"2021-01-10T22:41:04.151606","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"model_checkpoint = keras.callbacks.ModelCheckpoint(\n    './best_weights.h5',\n    monitor=\"val_loss\",\n    verbose=1,\n    save_best_only=True,\n    save_weights_only=True,\n    mode=\"min\"\n)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"62320687-4e5e-4f3f-adf0-110a9b77ef19","_uuid":"8d5d1c63-b098-4381-8f44-ab55dffba03e","papermill":{"duration":0.043815,"end_time":"2021-01-10T22:41:04.29594","exception":false,"start_time":"2021-01-10T22:41:04.252125","status":"completed"},"tags":[]},"cell_type":"markdown","source":"2. EarlyStopping : Stop training when a monitored metric has stopped improving."},{"metadata":{"_cell_guid":"dd7cdb3b-1e58-4206-8492-af498b45273c","_uuid":"66087bb7-3b9b-47a9-a91f-939f6f39e3ec","execution":{"iopub.execute_input":"2021-01-10T22:41:04.392415Z","iopub.status.busy":"2021-01-10T22:41:04.389202Z","iopub.status.idle":"2021-01-10T22:41:04.393455Z","shell.execute_reply":"2021-01-10T22:41:04.394237Z"},"papermill":{"duration":0.054669,"end_time":"2021-01-10T22:41:04.394373","exception":false,"start_time":"2021-01-10T22:41:04.339704","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"early_stopping = keras.callbacks.EarlyStopping(\n    monitor=\"val_loss\",\n    min_delta=0.001,\n    patience=5,\n    verbose=1,\n    mode=\"min\",\n    restore_best_weights=True,\n)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a4d36849-6eea-4063-8df7-ac017fc8aa40","_uuid":"483d584c-b1d2-4513-a527-d74ff652af82","papermill":{"duration":0.030037,"end_time":"2021-01-10T22:41:04.470245","exception":false,"start_time":"2021-01-10T22:41:04.440208","status":"completed"},"tags":[]},"cell_type":"markdown","source":"3. ReduceLROnPlateau : Reduce learning rate when a metric has stopped improving."},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\n# Using an LR ramp up because fine-tuning a pre-trained model.\n# Starting with a high LR would break the pre-trained weights.\n\nLR_START = 0.00001\nLR_MAX = 0.0001\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 3\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = 0.85\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        #cosine decay\n        progress = (epoch - LR_RAMPUP_EPOCHS) / (EPOCHS - LR_RAMPUP_EPOCHS)\n        lr = LR_MAX * (0.5 * (1.0 + tf.math.cos(np.pi * ((1.0 * progress) % 1.0))))\n        \n        #exponential decay\n        #lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \n#setting verbose=True allows us to see LR in model training\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n\n#visualizing the learning rate schedule\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\n\nsns.set(style='whitegrid')\nplt.figure(figsize=(13, 5))\nplt.xlabel('Epoch')\nplt.ylabel('Learning Rate')\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6d1ff257-d223-4e77-af7e-21b0e6484fa2","_uuid":"0bb260a9-e918-4de1-a89b-735e782011c0","execution":{"iopub.execute_input":"2021-01-10T22:41:04.538637Z","iopub.status.busy":"2021-01-10T22:41:04.536825Z","iopub.status.idle":"2021-01-10T22:41:04.539343Z","shell.execute_reply":"2021-01-10T22:41:04.539829Z"},"papermill":{"duration":0.038489,"end_time":"2021-01-10T22:41:04.539944","exception":false,"start_time":"2021-01-10T22:41:04.501455","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"reduce_lr = keras.callbacks.ReduceLROnPlateau(\n    monitor=\"val_loss\",\n    factor=0.1,\n    patience=2,\n    verbose=1,\n    mode=\"min\",\n    min_delta=0.001,\n)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4753c165-788c-43e5-a8b6-947d95bccffb","_uuid":"a613c83f-c419-4b77-898b-b668fd51fbe7","papermill":{"duration":0.030241,"end_time":"2021-01-10T22:41:04.598992","exception":false,"start_time":"2021-01-10T22:41:04.568751","status":"completed"},"tags":[]},"cell_type":"markdown","source":"4. Print the batch number at the beginning of every batch."},{"metadata":{"_cell_guid":"9f3fd2ff-ea9e-41f9-abcc-4484a4a19e40","_uuid":"ff0566de-0bf1-4fec-b941-b82e3021d215","execution":{"iopub.execute_input":"2021-01-10T22:41:04.665194Z","iopub.status.busy":"2021-01-10T22:41:04.664568Z","iopub.status.idle":"2021-01-10T22:41:04.668939Z","shell.execute_reply":"2021-01-10T22:41:04.668461Z"},"papermill":{"duration":0.037915,"end_time":"2021-01-10T22:41:04.669035","exception":false,"start_time":"2021-01-10T22:41:04.63112","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"\nbatch_print_callback = keras.callbacks.LambdaCallback(\n    #on_batch_begin=lambda batch,logs: print(batch),\n    on_epoch_end=lambda epoch,logs: last_epoch+1,\n),","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0515343c-8634-4c7c-a2cf-b443443013bc","_uuid":"650ed513-234d-4fd2-ad1d-6dd0821a4d71","papermill":{"duration":0.030288,"end_time":"2021-01-10T22:41:04.728984","exception":false,"start_time":"2021-01-10T22:41:04.698696","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Stream the epoch loss to a file in JSON format. The file content is not well-formed JSON but rather has a JSON object per line."},{"metadata":{"_cell_guid":"dac0fcc7-aef4-4d3d-a6ea-87b88f4700c5","_uuid":"8057f3e2-21aa-47f3-976b-c7b9ee4cf3aa","execution":{"iopub.execute_input":"2021-01-10T22:41:04.796891Z","iopub.status.busy":"2021-01-10T22:41:04.795886Z","iopub.status.idle":"2021-01-10T22:41:04.80019Z","shell.execute_reply":"2021-01-10T22:41:04.799645Z"},"papermill":{"duration":0.04154,"end_time":"2021-01-10T22:41:04.80029","exception":false,"start_time":"2021-01-10T22:41:04.75875","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import json\njson_log = open('/kaggle/working//loss_log.json', mode='w', buffering=1)\njson_logging_callback = keras.callbacks.LambdaCallback(\n    on_train_begin=lambda logs: json_log.write('{ \"train\": [ \\n'),\n    on_epoch_end=lambda epoch,logs: json_log.write(json.dumps({'epoch': epoch, 'loss': logs['loss'], 'acc':logs['accuracy']}) + ',\\n'),\n    on_train_end=lambda logs: json_log.write(']\\n}'),\n)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b3723ca2-6ec9-42f2-ae9c-846f90405973","_uuid":"bfacb3e7-a9a6-4758-a6e3-3b40cba89f4d","papermill":{"duration":0.030755,"end_time":"2021-01-10T22:41:04.861642","exception":false,"start_time":"2021-01-10T22:41:04.830887","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Model Compile"},{"metadata":{"_cell_guid":"e921012d-a45c-467b-8f4f-df67831e19a1","_uuid":"fe30ed50-690c-46ac-a745-3a1bfd993fac","papermill":{"duration":0.030004,"end_time":"2021-01-10T22:41:04.921589","exception":false,"start_time":"2021-01-10T22:41:04.891585","status":"completed"},"tags":[]},"cell_type":"markdown","source":"For this model we will be using Adam optimizer and since our output labels are categorical we will be using categorical_crossentropy as a loss function."},{"metadata":{"_cell_guid":"dee3eb2a-476a-45d0-a9ea-5ef84e8c4aea","_uuid":"f44522c1-5526-4910-b376-bf33a1398ca9","execution":{"iopub.execute_input":"2021-01-10T22:41:05.000737Z","iopub.status.busy":"2021-01-10T22:41:04.99978Z","iopub.status.idle":"2021-01-10T22:41:05.007658Z","shell.execute_reply":"2021-01-10T22:41:05.007123Z"},"papermill":{"duration":0.056631,"end_time":"2021-01-10T22:41:05.007749","exception":false,"start_time":"2021-01-10T22:41:04.951118","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"optimizer = Adamax(lr=0.0001)\nloss = tf.keras.losses.BinaryCrossentropy(reduction=tf.keras.losses.Reduction.AUTO)\nmodel.compile(\n            loss=loss,\n            optimizer=optimizer,\n            metrics=['binary_crossentropy','accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"cb2e0459-f3b5-4542-93bd-e495d0a04d3d","_uuid":"b916bf95-3cca-4bf2-a2b7-e5ba2e7627f1","papermill":{"duration":0.029446,"end_time":"2021-01-10T22:41:05.066779","exception":false,"start_time":"2021-01-10T22:41:05.037333","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Start Training"},{"metadata":{"_cell_guid":"78cd2d1b-9e8d-417b-b578-9be046196177","_uuid":"7a3a196e-e6ca-4fca-ac14-6123d2dfe7a6","papermill":{"duration":0.029377,"end_time":"2021-01-10T22:41:05.126209","exception":false,"start_time":"2021-01-10T22:41:05.096832","status":"completed"},"tags":[]},"cell_type":"markdown","source":"We have defined everything we need, it's time to train the model..."},{"metadata":{"_cell_guid":"99335f3e-af22-42c6-a12b-5f63a35460fc","_uuid":"c303b6ea-e0fa-4e29-a1d7-7d6c16124b54","execution":{"iopub.execute_input":"2021-01-10T22:41:05.19486Z","iopub.status.busy":"2021-01-10T22:41:05.194225Z","iopub.status.idle":"2021-01-11T01:18:53.40774Z","shell.execute_reply":"2021-01-11T01:18:53.406654Z"},"papermill":{"duration":9468.252081,"end_time":"2021-01-11T01:18:53.407904","exception":false,"start_time":"2021-01-10T22:41:05.155823","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(\"Started Training...\")\n\n# Trains the model for a fixed number of epochs (iterations on a dataset).\nhistory = model.fit(\n    train_datagen_flow,\n    epochs=epochs,\n    steps_per_epoch=(len(train)*(1-validation_split)) // BATCH_SIZE,\n    validation_data=valid_datagen_flow,\n    validation_steps=(len(train)*validation_split) // BATCH_SIZE,\n    callbacks = [\n        model_checkpoint, \n        early_stopping, \n        lr_callback,\n        batch_print_callback,\n        json_logging_callback\n    ],\n    use_multiprocessing=True,\n    verbose=1\n)\nprint(\"Training completed\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5865f7ab-398c-46d2-9f26-a47fc2b5dc18","_uuid":"2c550cb7-6eb9-4fbf-b380-fa609bca1187","papermill":{"duration":2.400759,"end_time":"2021-01-11T01:19:13.491012","exception":false,"start_time":"2021-01-11T01:19:11.090253","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Submission"},{"metadata":{"_cell_guid":"0294271f-c240-495e-bea3-47f99a2bcbf2","_uuid":"59a869ec-d8ff-4b11-b346-4eeac1484a3e","execution":{"iopub.execute_input":"2021-01-11T01:19:18.230752Z","iopub.status.busy":"2021-01-11T01:19:18.230139Z","iopub.status.idle":"2021-01-11T01:19:19.33891Z","shell.execute_reply":"2021-01-11T01:19:19.337759Z"},"papermill":{"duration":3.6139,"end_time":"2021-01-11T01:19:19.339035","exception":false,"start_time":"2021-01-11T01:19:15.725135","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame(columns=['image_id','label'])\nfor image_name in os.listdir(general_path + 'test_images'):\n    image_path = os.path.join(general_path + 'test_images', image_name)\n    image = tf.keras.preprocessing.image.load_img(image_path)\n    resized_image = image.resize((img_width, img_height))\n    numpied_image = np.expand_dims(resized_image, 0)\n    tensored_image = tf.cast(numpied_image, tf.float32)\n    submission = submission.append(pd.DataFrame({'image_id': image_name,\n                                                 'label': model.predict_classes(tensored_image)}))\n\nsubmission.to_csv('/kaggle/working/submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"64168242-2406-4314-9416-a1b51ce874dc","_uuid":"17acc35d-27ce-41bf-8d12-93343de52dd2","papermill":{"duration":2.217112,"end_time":"2021-01-11T01:18:58.084566","exception":false,"start_time":"2021-01-11T01:18:55.867454","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Performance"},{"metadata":{"_cell_guid":"5f0da6da-1166-473d-a984-e739d2c24f8d","_uuid":"6810d763-16ed-4ea8-8f10-c4d720d388c7","papermill":{"duration":2.19449,"end_time":"2021-01-11T01:19:02.824092","exception":false,"start_time":"2021-01-11T01:19:00.629602","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Since the performance metric for this competition is accuracy, let us plot the train and validation accuracy to monitor our model performance."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(13, 5))\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title(\"Model Loss\")\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend(['Train', 'Test'])\nplt.ylim(ymax = 2, ymin = 0)\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(13, 5))\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('Model Accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend(['Train','Test'])\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(13, 5))\nplt.plot(history.history['sparse_categorical_accuracy'])\nplt.plot(history.history['val_sparse_categorical_accuracy'])\nplt.title('Sparse Categorical Accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Sparse Categorical Accuracy')\nplt.legend(['Train','Test'])\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nfile = open('/kaggle/working//loss_log.json', 'r')\ncountriesStr = file.read()\ncountriesStr = countriesStr[::-1].replace(',', '', 1)[::-1]\nfile.close()\n\nwith open('/kaggle/working//loss_log.json', 'w') as file:\n    file.write(countriesStr)\n    file.close()\n\njsonFile = open('/kaggle/working//loss_log.json', 'r')\njson_array = json.load(jsonFile)\n\nfor item in json_array['train']:\n    last_epoch = item['epoch']\n    \nlast_epoch+=1\n\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\nepochs_range = range(last_epoch)\n\nplt.figure(figsize=(8, 8))\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Training Accuracy')\nplt.plot(epochs_range, val_acc, label='Validation Accuracy')\nplt.legend(loc='lower right')\nplt.title('Training and Validation Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Training Loss')\nplt.plot(epochs_range, val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.title('Training and Validation Loss')\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}