{"cells":[{"metadata":{},"cell_type":"markdown","source":"**In this project, I will build a model to classify 5 different types of Cassava Plants.<br>\nThe Data is from the `cassava-classification-competition` live at Kaggle.<br>\nI have pre-processed a sample of the data and saved it to Google Cloud Storage.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math, re, os\nimport tensorflow as tf\nimport numpy as np\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\n\n\n#@title Enable Eager Execution and Print Versions\nif tf.__version__ < \"2.0.0\":\n    tf.compat.v1.enable_eager_execution()\n    print(\"Eager execution enabled.\")\nelse:\n    print(\"Eager execution enabled by default.\")\n\nprint(\"TensorFlow \" + tf.__version__)\n\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check for TPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"try: # detect TPUs\n    tpu = None\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept Exception as e: # detect GPUs\n    #strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --quiet gs-wrap\nimport time\nimport gswrap\nclient = gswrap.Client('vibrant-reach-282320')\nprint('gswrap ready for use!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing import image_dataset_from_directory\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom numpy import asarray\nfrom sklearn.model_selection import train_test_split\nfrom skimage.transform import resize\nfrom tqdm import tqdm\nimport imageio\nimport keras\nfrom keras.utils import np_utils\nimport json\nimport seaborn as sns\nfrom google.cloud import storage\nimport shutil\nfrom collections import Counter\nimport glob\nfrom PIL import Image\nimport time\nimport tensorflow_hub as hub\n\nprint('All imported!')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Downloading the Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    st=time.time()\n    print('Copying files...')\n    client.cp(src=\"gs://kaggle1980/Kaggle/images\",\n              dst=\"./\",\n              recursive=True, multithreaded=True)\n    ed=time.time()\n    tot_files = !ls images/*/*.jpg | wc -l\n    print(f'{tot_files} files copied in {ed-st} seconds!')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EDA\n\nLet's understand the data...\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Label Distribution**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let's see the different labels of cassava leaves as listed in the `label_num_to_disease_map.json` file\nlabels_json = '../input/cassava-leaf-disease-classification/label_num_to_disease_map.json'\n\n# extracting the json file as a dataframe\nlabels_dict_df = pd.read_json(labels_json, typ='series').to_frame()\nlabels_dict_df.columns = ['label']\nlabels_dict_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Class Distribution**\n\nLets see class distribution of original data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv = '../input/cassava-leaf-disease-classification/train.csv'\ntrain_df = pd.read_csv(train_csv)\nprint(f'shape is {train_df.shape}')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count = Counter(train_df.label)\n\ndef plot_labels(count):\n    # make each val a list\n    count = {key:[val] for key, val in count.items()}\n    xAxis = [0.02, 0.12, 0.22, 0.32, 0.42]\n    yAxis = list(count.values())\n    yAxis = [int(i[0]) for i in yAxis]\n    \n    # create a dataframe\n    count_df = pd.DataFrame(count)\n\n    # plot the dataframe\n    sns.set_style('ticks')\n    count_df.plot(kind='bar', edgecolor='black', linewidth=1.2, align='edge', figsize=(8,5))\n    plt.title('% Label Distribution of Cassava Leaves')\n    plt.xlabel('Labels')\n    plt.ylabel('Count')\n    \n    for x, y in zip(xAxis, yAxis):\n        val = round(y/sum(yAxis),2)\n        plt.annotate(str(val), (x,y-1000),fontweight='bold')\n        \n    plt.show()\n\nprint(f'Label count distribution is\\n',count)\nplot_labels(count)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualizing The Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = '../input/cassava-leaf-disease-classification/train_images/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot(df, source_path, nrows=1, ncols=5):\n    fig = plt.gcf()\n    fig.set_size_inches(ncols * 4.5, nrows * 8)\n\n    pic_index = np.random.randint(0, len(df)-(ncols+1), 1)[0]\n    pic_index += ncols\n    _pix = [os.path.join(source_path, fname) \n                    for fname in df.image_id[pic_index-ncols:pic_index]]\n    \n    \n    for i, img_path in enumerate(_pix):\n    # Set up subplot; subplot indices start at 1\n        sp = plt.subplot(nrows, ncols, i + 1)\n        sp.axis('Off') # Don't show axes (or gridlines)\n\n        img = imageio.imread(img_path)\n        plt.imshow(img)\n\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cbb_df = train_df[train_df.label==0]  # Cassava Bacterial Blight leaves \ncbsd_df = train_df[train_df.label==1]  # Cassava Brown Streak Disease leaves\ncgm_df = train_df[train_df.label==2]  # Cassava Green Mottle leaves\ncmd_df = train_df[train_df.label==3]  # Cassava Healthy Disease leaves\nhealthy_df = train_df[train_df.label==4]\n\n# Let's see one of em\nhealthy_df.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# healthy leaves\n\nplot(healthy_df, train_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Cassava Bacterial Blight leaves\n\nplot(cbb_df, train_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Cassava Brown Streak Disease leaves\n\nplot(cbsd_df, train_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Cassava Green Mottle leaves\n\nplot(cgm_df, train_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Cassava Mosaic Disease leaves\n\nplot(cmd_df, train_path)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Working with Balanced Data\n\nWe selected just 6500 images from the dominant CMD leaves class and augmented the rest minor classes to produce a roughly balanced dataset...\n\nLet's read it"},{"metadata":{"trusted":true},"cell_type":"code","source":"dirs = ['zero','one', 'two', 'three', 'four']\nlabel = [0, 1, 2, 3, 4]\n\ndf_img = []\ndf_lab = []\n\nfor name, label in tqdm(zip(dirs, label)):\n    x = os.listdir('./images/'+name)\n    y = [label]*len(x)\n    df_img.extend(x)\n    df_lab.extend(y)\n\n\nnew_train_df = pd.DataFrame([df_img, df_lab]).T\nnew_train_df.columns = ['image_id', 'label']\n\nprint(f'new df shape is {new_train_df.shape}')\nnew_train_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Let's see the new distribution of Data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"count = Counter(new_train_df.label)\n\nprint(f'New Label Count Distribution is\\n',count)\nplot_labels(count)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate a Dataset\n\nHere. we'd create a dataset of flattened images. Then we'd separate via train and val and use for the prediction."},{"metadata":{},"cell_type":"markdown","source":"**Let's define some parameters...**"},{"metadata":{"trusted":true},"cell_type":"code","source":"SIZE= 299  # ideal for exception model\nIMAGE_SIZE = (SIZE, SIZE)\nBATCH_SIZE = 16\nNUM_CLASS = 5\nVAL_SPLIT = 0.15\n\nif tpu:\n    BATCH_SIZE = 16*strategy.num_replicas_in_sync  # A TPU has 8 cores so this will be 128\nelse:\n    BATCH_SIZE = BATCH_SIZE  # On Colab/GPU, a higher batch size does not help and sometimes does not fit on the GPU (OOM)\n    \nSTEP_SIZE_TRAIN = int(np.ceil(len(new_train_df)*(1-VAL_SPLIT) / BATCH_SIZE))\nSTEP_SIZE_VALID = int(np.ceil(len(new_train_df)*(VAL_SPLIT) / BATCH_SIZE))\n\nprint('All set!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = image_dataset_from_directory(directory='./images',\n                                label_mode='categorical',\n                                batch_size=BATCH_SIZE,\n                                image_size=IMAGE_SIZE,\n                                seed=0,\n                                validation_split=VAL_SPLIT,\n                                subset='training',\n                                interpolation=\"nearest\")\n\nval_ds = image_dataset_from_directory(\n                                directory='./images',\n                                label_mode='categorical',\n                                batch_size=BATCH_SIZE,\n                                image_size=IMAGE_SIZE,\n                                seed=0,\n                                validation_split=VAL_SPLIT,\n                                subset='validation',\n                                interpolation=\"nearest\"\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(val_ds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"visualize a few of the images and their labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nfor images, labels in train_ds.take(1):\n    for i in range(9):\n        ax = plt.subplot(3, 3, i + 1)\n        plt.imshow(images[i].numpy().astype(\"uint8\"))\n        plt.title(str(labels[i][-1]))\n        plt.axis(\"off\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h3>Using image data augmentation</h3>\n\nWhen you don't have a large image dataset, it's a good practice to artificially introduce sample diversity by applying random yet realistic transformations to the training images, such as random horizontal flipping or small random rotations. This helps expose the model to different aspects of the training data while slowing down overfitting."},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import layers\nfrom keras.layers.experimental.preprocessing import RandomCrop \nfrom keras.layers.experimental.preprocessing import RandomFlip\nfrom keras.layers.experimental.preprocessing import RandomRotation\nfrom keras.layers.experimental.preprocessing import RandomZoom\nfrom keras.layers.experimental.preprocessing import RandomHeight\nfrom keras.layers.experimental.preprocessing import RandomWidth\nfrom keras.layers.experimental.preprocessing import Rescaling\nprint('Done!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_augmentation = keras.Sequential(\n    [\n        RandomFlip(\"horizontal\"),\n        RandomFlip('vertical'),\n        RandomRotation(0.1),\n        RandomZoom(0.2, 0.2, seed=0),\n        RandomHeight(factor=0.2, interpolation='nearest'),\n        RandomWidth(factor=0.2, interpolation='nearest')\n    ]\n)\nprint('Data-augmentation Set!')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Let's see an image being randomly augmented**"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nfor images, _ in train_ds.take(1):\n    for i in range(9):\n        augmented_images = data_augmentation(images)\n        ax = plt.subplot(3, 3, i + 1)\n        plt.imshow(augmented_images[0].numpy().astype(\"uint8\"))\n        plt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Configure the dataset for performance\n\nLet's make sure to use buffered prefetching so we can yield data from disk without having I/O becoming blocking:"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = train_ds.prefetch(buffer_size=BATCH_SIZE)\nval_ds = val_ds.prefetch(buffer_size=BATCH_SIZE)\nprint('Done')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build a model"},{"metadata":{},"cell_type":"markdown","source":"First, we train a **base-model** with all layers frozen and we train with only the added layer to the top that we provide. We do this for about 30 Epochs.</br>Then next, we unfreeze about 20 layers and train our model with these layers, but we ensure not to unfreeze the batch-norm layers, otherwise we mess up the model's original training."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Next, some imports\n\nfrom keras.models import Sequential\nfrom keras.layers import GlobalAveragePooling2D, Flatten\nfrom keras.layers import Dense, Dropout, BatchNormalization\nfrom keras.optimizers import RMSprop, Adam\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers.experimental import preprocessing\n\nprint('Done!')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h3> The Base Model</h3>"},{"metadata":{"trusted":true},"cell_type":"code","source":"base_model = keras.applications.Xception(\n    weights=\"imagenet\",  # Load weights pre-trained on ImageNet.\n    input_shape=(SIZE, SIZE, 3),\n    include_top=False\n)\n\n# Freeze the base_model\nbase_model.trainable = False\n\n# Create new model on top\ninputs = keras.Input(shape=(SIZE, SIZE, 3))\nx = data_augmentation(inputs)  # Apply random data augmentation\nx = Rescaling(scale=1./255)(x)  # Rescale the data\n\n# Pre-trained Xception weights requires that input be normalized\n# from (0, 255) to a range (-1., +1.), the normalization layer\n# does the following, outputs = (inputs - mean) / sqrt(var)\nnorm_layer = keras.layers.experimental.preprocessing.Normalization()\nmean = np.array([127.5] * 3)\nvar = mean ** 2\n# Scale inputs to [-1, +1]\nx = norm_layer(x)\nnorm_layer.set_weights([mean, var])\n\n# The base model contains batchnorm layers. We want to keep them in inference mode\n# when we unfreeze the base model for fine-tuning, so we make sure that the\n# base_model is running in inference mode here.\n\nx = base_model(x, training=False)\nx = keras.layers.GlobalAveragePooling2D()(x)\nx = keras.layers.Dropout(0.4)(x)  # Regularize with dropout\nx = keras.layers.Dense(256, activation='relu')(x)\nx = keras.layers.Dropout(0.2)(x)\nx = keras.layers.Dense(512, activation='relu')(x)\nx = keras.layers.Dropout(0.3)(x)\nx = keras.layers.Dense(1024, activation='relu')(x)\nx = keras.layers.Dropout(0.4)(x)\noutputs = keras.layers.Dense(NUM_CLASS, activation='softmax')(x)\n\nprint('Done!')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Fit the model with the necessary callBacks and params..."},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit_(model, REDUCE_LR=True, FINE_TUNE=False):\n    '''Compiling the model'''\n    loss = tf.keras.losses.CategoricalCrossentropy(from_logits = False,\n                                                   label_smoothing=0.0001,\n                                                   name='categorical_crossentropy' )\n    \n    model.compile(optimizer = Adam(learning_rate=LEARNING_RATE),\n                        loss = loss, #'categorical_crossentropy'\n                        metrics = METRIC) #'acc'\n    \n    # Stop training when the val_loss has stopped decreasing for 6 epochs.\n    es = EarlyStopping(monitor='val_loss', \n                       mode='min', \n                       patience=EARLY_PATIENCE,\n                       restore_best_weights=True, \n                       verbose=1)\n    \n    # Save the model with the minimum validation loss\n    checkpoint_cb = ModelCheckpoint(CHECK_Pt_NAME,\n                                    save_best_only=True,\n                                    monitor = 'val_loss',\n                                    mode='min')\n    \n    # reduce learning rate\n    reduce_lr = ReduceLROnPlateau(monitor = 'val_loss',\n                                  factor = REDUCE_FACTOR,\n                                  patience = REDUCE_PATIENCE,\n                                  min_lr = MIN_LEARNING_RATE,\n                                  mode = 'min',\n                                  verbose = 1)\n    \n    \n    def increase_lr_exp(epoch, lr, wait=EARLY_PATIENCE//2):\n        \"\"\"This method exponentially increases LR \n            by a fixed Pct.ideal for Fine-Tuning models\n        \"\"\"\n        if epoch < wait+1:\n            return lr\n        else:\n            return lr * np.exp(0.1)\n        \n    \n    increase_lr = tf.keras.callbacks.LearningRateScheduler(increase_lr_exp)\n    \n    if REDUCE_LR:\n        CALLBACKS=[es, checkpoint_cb, reduce_lr]\n    else:\n        if FINE_TUNE:\n            CALLBACKS=[es, checkpoint_cb, increase_lr]\n        else:\n            CALLBACKS=[es, checkpoint_cb]\n        \n    history = model.fit( train_ds,\n                         validation_data = val_ds,\n                         epochs= EPOCHS,\n                         batch_size = BATCH_SIZE,\n                         steps_per_epoch = STEP_SIZE_TRAIN,\n                         validation_steps = STEP_SIZE_VALID,\n                         callbacks=CALLBACKS)\n    \n    model.save(SAVE_NAME)  \n    \n    return history\nprint('Fit defined!')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Part 1: `FEATURE-EXTRACTION`"},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 40\nMETRIC = 'categorical_accuracy'\nLEARNING_RATE = 0.05\nMIN_LEARNING_RATE = 1e-5\nEARLY_PATIENCE = 6\nREDUCE_PATIENCE = 3\nREDUCE_FACTOR = 0.8\nCHECK_Pt_NAME = \"Cassava_best_model.h5\"\nSAVE_NAME = 'Cassava_model.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = keras.Model(inputs, outputs)\n    print(model.summary())\n    results = fit_(model)\n    \n    start_time= time.time()\n    print('Starting Training...')\n\n    last5_mean_val_accuracy = results.history[\"val_categorical_accuracy\"][-5:]\n    print(\"LAST 5 MEAN VAL-ACCURACY:\", np.mean(last5_mean_val_accuracy))\n    print(\"TRAINING TIME: \", time.time() - start_time, \" secs!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#%% CHECKING THE METRIC\n\nprint('Train_Cat-Acc: ', max(results.history['categorical_accuracy']))\nprint('Val_Cat-Acc: ', max(results.history['val_categorical_accuracy']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#%% PLOTTING RESULTS (Train vs Validation FOLDER 1)\n\ndef Train_Val_Plot(acc,val_acc,loss,val_loss):\n    \n    fig, (ax1, ax2) = plt.subplots(1,2, figsize= (15,10))\n    fig.suptitle(\" MODEL'S METRICS VISUALIZATION \", fontsize=20)\n\n    ax1.plot(range(1, len(acc) + 1), acc)\n    ax1.plot(range(1, len(val_acc) + 1), val_acc)\n    ax1.set_title('History of Accuracy', fontsize=15)\n    ax1.set_xlabel('Epochs', fontsize=15)\n    ax1.set_ylabel('Accuracy', fontsize=15)\n    ax1.legend(['training', 'validation'])\n\n\n    ax2.plot(range(1, len(loss) + 1), loss)\n    ax2.plot(range(1, len(val_loss) + 1), val_loss)\n    ax2.set_title('History of Loss', fontsize=15)\n    ax2.set_xlabel('Epochs', fontsize=15)\n    ax2.set_ylabel('Loss', fontsize=15)\n    ax2.legend(['training', 'validation'])\n    plt.show()\n    \n\nTrain_Val_Plot(results.history['categorical_accuracy'],results.history['val_categorical_accuracy'],\n               results.history['loss'],results.history['val_loss'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Loading The Best Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model = keras.models.load_model(CHECK_Pt_NAME)\ntry:\n    best_model.summary()\nexcept Exception as e:\n    print(e)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Part 2: `Model FINE-TUNING`\n\nWe can fine-tune the model by unfreezing a few of it's later layers and then retraining these on our data.\nWe must ensure not to touch the batch-nrem layers if any, so as not to destroy the model's previous learning."},{"metadata":{},"cell_type":"markdown","source":"Let's first confirm howmany layers the model has..."},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(model.layers))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Next we set some new params for fine tuning"},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 70\nMETRIC = 'categorical_accuracy'\nLEARNING_RATE = 1e-4  # fine-tuning should start with very small LR\nEARLY_PATIENCE = 10\nCHECK_Pt_NAME = \"Cassava_best_finetuned_model.h5\"\nSAVE_NAME = 'Cassava_model.h5'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So we define a method that unfreezes a given number of the last hidden layers of the model... <br>Without unfreezing the batch-norm layers, else we mess up the entire model's learning.<br>Then we pass a really small learning rate and retrain the model with our data. <br>This function also recompiles the model after unfreezing the weights."},{"metadata":{"trusted":true},"cell_type":"code","source":"def unfreeze_model(model, num_layers):\n    # We unfreeze the top layers while leaving BatchNorm layers frozen\n    ind = num_layers\n    for layer in model.layers[-num_layers:]:\n        if not isinstance(layer, layers.BatchNormalization):\n            model.layers[-ind].trainable = True\n        ind-=1\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let's unfreeze the last 10 layers and train these with our pretrained model.\n\nnum_layers = 10\n\nmodel = unfreeze_model(model, num_layers)\nprint('model unfrozen!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Retrain with more hidden layers of the base model, "},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    print('Starting Training...')\n    start_time = time.time()\n    results = fit_(model, REDUCE_LR=False, FINE_TUNE=True)\n\n    last5_mean_val_accuracy = results.history[\"val_categorical_accuracy\"][-5:]\n    print(\"LAST 5 MEAN VAL-ACCURACY:\", np.mean(last5_mean_val_accuracy))\n    print(\"TRAINING TIME: \", time.time() - start_time, \" secs\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Evaluation"},{"metadata":{"trusted":true},"cell_type":"code","source":"#%% CHECKING THE METRIC\n\nprint('Train_Cat-Acc: ', max(results.history['categorical_accuracy']))\nprint('Val_Cat-Acc: ', max(results.history['val_categorical_accuracy']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#%% PLOTTING RESULTS (Train vs Validation FOLDER 1)\n\ndef Train_Val_Plot(acc,val_acc,loss,val_loss):\n    \n    fig, (ax1, ax2) = plt.subplots(1,2, figsize= (15,10))\n    fig.suptitle(\" MODEL'S METRICS VISUALIZATION \", fontsize=20)\n\n    ax1.plot(range(1, len(acc) + 1), acc)\n    ax1.plot(range(1, len(val_acc) + 1), val_acc)\n    ax1.set_title('History of Accuracy', fontsize=15)\n    ax1.set_xlabel('Epochs', fontsize=15)\n    ax1.set_ylabel('Accuracy', fontsize=15)\n    ax1.legend(['training', 'validation'])\n\n\n    ax2.plot(range(1, len(loss) + 1), loss)\n    ax2.plot(range(1, len(val_loss) + 1), val_loss)\n    ax2.set_title('History of Loss', fontsize=15)\n    ax2.set_xlabel('Epochs', fontsize=15)\n    ax2.set_ylabel('Loss', fontsize=15)\n    ax2.legend(['training', 'validation'])\n    plt.show()\n    \n\nTrain_Val_Plot(results.history['categorical_accuracy'],results.history['val_categorical_accuracy'],\n               results.history['loss'],results.history['val_loss'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Pass the Best Fine-Tuned-Model to the Submission Notebook and Submit..."},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}