{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Cassava Plant Disease Classification\n![](https://cff2.earth.com/uploads/2020/11/11065615/shutterstock_12823926312-scaled.jpg)","metadata":{}},{"cell_type":"markdown","source":"\nThe following topics will be covered:\n1. Exploratory Data Analysis\n2. Image Augmentation\n3. Model Building and Selection\n4. Stratified K-Fold Cross Validation\n5. Model Ensembling and Inference","metadata":{}},{"cell_type":"markdown","source":"# Importing Libraries and Data","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nimport random\nimport os\nimport cv2\nimport sys\nfrom pylab import rcParams\nfrom PIL import Image\nwarnings.filterwarnings('ignore')\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import Input\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.layers import Dense, Flatten, Dropout, Activation, Input, GlobalAveragePooling2D\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\nfrom tensorflow.keras.applications import InceptionV3, Xception\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nfrom sklearn.model_selection import StratifiedShuffleSplit","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-12-25T18:13:57.840413Z","iopub.execute_input":"2021-12-25T18:13:57.840801Z","iopub.status.idle":"2021-12-25T18:14:03.965339Z","shell.execute_reply.started":"2021-12-25T18:13:57.840719Z","shell.execute_reply":"2021-12-25T18:14:03.964593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"policy = mixed_precision.Policy('mixed_float16')\nmixed_precision.set_policy(policy) #shortens training time by 2x","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:03.966958Z","iopub.execute_input":"2021-12-25T18:14:03.967228Z","iopub.status.idle":"2021-12-25T18:14:06.479625Z","shell.execute_reply.started":"2021-12-25T18:14:03.967193Z","shell.execute_reply":"2021-12-25T18:14:06.478135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:06.481091Z","iopub.execute_input":"2021-12-25T18:14:06.48143Z","iopub.status.idle":"2021-12-25T18:14:06.521789Z","shell.execute_reply.started":"2021-12-25T18:14:06.481392Z","shell.execute_reply":"2021-12-25T18:14:06.520981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"label\"] = df_train[\"label\"].astype(str) #convert to str as we want to use Categorical Cross Entropy (CCE) later on\ndf_train.info()","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:06.523948Z","iopub.execute_input":"2021-12-25T18:14:06.524219Z","iopub.status.idle":"2021-12-25T18:14:06.571097Z","shell.execute_reply.started":"2021-12-25T18:14:06.524181Z","shell.execute_reply":"2021-12-25T18:14:06.57029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"sns.set_style(\"whitegrid\")\nplt.figure(figsize=(10,8))\nsns.countplot(df_train[\"label\"], edgecolor=\"black\", palette=\"mako\")","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:06.572356Z","iopub.execute_input":"2021-12-25T18:14:06.572673Z","iopub.status.idle":"2021-12-25T18:14:06.831649Z","shell.execute_reply.started":"2021-12-25T18:14:06.572636Z","shell.execute_reply":"2021-12-25T18:14:06.830849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From here, we can see that we have a highly unbalanced dataset. We have the largest number of samples for label 3, Cassava Mosiac Disease (CMD), and the fewest number of samples for label 0, Cassava Bacterial Blight (CBB). Let's visualize how some of these images look like next.","metadata":{}},{"cell_type":"markdown","source":"### Label 0: Cassava Bacterial Blight (CBB)","metadata":{}},{"cell_type":"code","source":"path = \"../input/cassava-leaf-disease-classification/train_images/\"\ndf0 = df_train[df_train[\"label\"] == \"0\"]\nfiles = df0[\"image_id\"].sample(3).tolist()\n\nplt.figure(figsize=(15,5))\nindex = 0\nfor file in files:\n    image = Image.open(path + file)\n    plt.subplot(1, 3, index + 1)\n    plt.imshow(image)\n    plt.axis(\"off\")\n    index += 1\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:06.832982Z","iopub.execute_input":"2021-12-25T18:14:06.833253Z","iopub.status.idle":"2021-12-25T18:14:07.404644Z","shell.execute_reply.started":"2021-12-25T18:14:06.833217Z","shell.execute_reply":"2021-12-25T18:14:07.403849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Label 1: Cassava Brown Streak Disease (CBSD)","metadata":{}},{"cell_type":"code","source":"df1 = df_train[df_train[\"label\"] == \"1\"]\nfiles = df1[\"image_id\"].sample(3).tolist()\n\nplt.figure(figsize=(15,5))\nindex = 0\nfor file in files:\n    image = Image.open(path + file)\n    plt.subplot(1, 3, index + 1)\n    plt.imshow(image)\n    plt.axis(\"off\")\n    index += 1\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:07.405903Z","iopub.execute_input":"2021-12-25T18:14:07.40676Z","iopub.status.idle":"2021-12-25T18:14:07.969668Z","shell.execute_reply.started":"2021-12-25T18:14:07.406725Z","shell.execute_reply":"2021-12-25T18:14:07.968864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Label 2: Cassava Green Mottle (CGM)","metadata":{}},{"cell_type":"code","source":"df2 = df_train[df_train[\"label\"] == \"2\"]\nfiles = df2[\"image_id\"].sample(3).tolist()\n\nplt.figure(figsize=(15,5))\nindex = 0\nfor file in files:\n    image = Image.open(path + file)\n    plt.subplot(1, 3, index + 1)\n    plt.imshow(image)\n    plt.axis(\"off\")\n    index += 1\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:07.970982Z","iopub.execute_input":"2021-12-25T18:14:07.971695Z","iopub.status.idle":"2021-12-25T18:14:08.540583Z","shell.execute_reply.started":"2021-12-25T18:14:07.971655Z","shell.execute_reply":"2021-12-25T18:14:08.536534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Label 3: Cassava Mosiac Disease (CMD)","metadata":{}},{"cell_type":"code","source":"df3 = df_train[df_train[\"label\"] == \"3\"]\nfiles = df3[\"image_id\"].sample(3).tolist()\n\nplt.figure(figsize=(15,5))\nindex = 0\nfor file in files:\n    image = Image.open(path + file)\n    plt.subplot(1, 3, index + 1)\n    plt.imshow(image)\n    plt.axis(\"off\")\n    index += 1\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:08.541784Z","iopub.execute_input":"2021-12-25T18:14:08.542445Z","iopub.status.idle":"2021-12-25T18:14:09.095511Z","shell.execute_reply.started":"2021-12-25T18:14:08.542408Z","shell.execute_reply":"2021-12-25T18:14:09.094143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Label 4: Healthy","metadata":{}},{"cell_type":"code","source":"df4 = df_train[df_train[\"label\"] == \"4\"]\nfiles = df4[\"image_id\"].sample(3).tolist()\n\nplt.figure(figsize=(15,5))\nindex = 0\nfor file in files:\n    image = Image.open(path + file)\n    plt.subplot(1, 3, index + 1)\n    plt.imshow(image)\n    plt.axis(\"off\")\n    index += 1\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:09.098472Z","iopub.execute_input":"2021-12-25T18:14:09.099281Z","iopub.status.idle":"2021-12-25T18:14:09.840561Z","shell.execute_reply.started":"2021-12-25T18:14:09.099242Z","shell.execute_reply":"2021-12-25T18:14:09.839976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note that the images in this training set are \"noisy\". This means that there are some images that are mislabelled and hence will affect predictive performance. One of the basic ways to mitigate this is to use label smoothing on our loss metric, as you will see later on.","metadata":{}},{"cell_type":"markdown","source":"# Image Augmentation (Tensorflow)\nImage augmentation is important as, with all Machine Learning problems, we do not want the model to \"memorise\" the training set. In this case, we do not want our algorithm to memorise features specific to the training images such as the orientation, position or shade of the cassava plant. Therefore, we need to augment the images so that the model is able to generalise well and not overfit to the training set.\n\nOne of the ways we can perform image augmentation is through Tensorflow preprocessing layers. They provide basic augmentations such as cropping, flipping and rotating.","metadata":{}},{"cell_type":"code","source":"batch_size=16\nimage_size=300\n\ninput_shape = (image_size, image_size, 3)\ntarget_size = (image_size, image_size)","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:09.841903Z","iopub.execute_input":"2021-12-25T18:14:09.842314Z","iopub.status.idle":"2021-12-25T18:14:09.846329Z","shell.execute_reply.started":"2021-12-25T18:14:09.84228Z","shell.execute_reply":"2021-12-25T18:14:09.845715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_augmentation = tf.keras.Sequential(\n    [\n        tf.keras.layers.experimental.preprocessing.RandomCrop(image_size, image_size),\n        tf.keras.layers.experimental.preprocessing.RandomFlip(\"horizontal_and_vertical\"),\n        tf.keras.layers.experimental.preprocessing.RandomRotation(0.25),\n        tf.keras.layers.experimental.preprocessing.RandomZoom((-0.25, 0.25), (-0.25, 0.25)),\n    ])","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:09.847464Z","iopub.execute_input":"2021-12-25T18:14:09.8479Z","iopub.status.idle":"2021-12-25T18:14:09.888398Z","shell.execute_reply.started":"2021-12-25T18:14:09.847867Z","shell.execute_reply":"2021-12-25T18:14:09.887777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let's see how an image looks like pre-augmentation and post-augmentation.","metadata":{}},{"cell_type":"code","source":"path = \"../input/cassava-leaf-disease-classification/train_images/\"\nfiles = df_train[\"image_id\"].tolist()\nfile = random.choice(files)\nimage = Image.open(path + file)\nplt.imshow(image)\nplt.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:09.890997Z","iopub.execute_input":"2021-12-25T18:14:09.8912Z","iopub.status.idle":"2021-12-25T18:14:10.141142Z","shell.execute_reply.started":"2021-12-25T18:14:09.891156Z","shell.execute_reply":"2021-12-25T18:14:10.140416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here are 9 possible outcomes after augmentation.","metadata":{}},{"cell_type":"code","source":"image = tf.expand_dims(np.array(image), 0)\n\nplt.figure(figsize=(14, 14))\nfor i in range(9):\n    augmented_image = img_augmentation(image)\n    ax = plt.subplot(3, 3, i + 1)\n    plt.imshow(augmented_image[0])\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:10.142447Z","iopub.execute_input":"2021-12-25T18:14:10.142779Z","iopub.status.idle":"2021-12-25T18:14:11.798003Z","shell.execute_reply.started":"2021-12-25T18:14:10.142738Z","shell.execute_reply":"2021-12-25T18:14:11.797377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image Augmentation (Albumentations)\nI will now show you how to perform image augmentations through Albumentation (an external image augmentation library with much more functionality) through both ImageDataGenerator. We will use a tool called ImageDataAugmentor (big thanks to mjkvaak at github) that allows us to do this.","metadata":{}},{"cell_type":"code","source":"!pip install git+https://github.com/mjkvaak/ImageDataAugmentor","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-25T18:14:11.799052Z","iopub.execute_input":"2021-12-25T18:14:11.799592Z","iopub.status.idle":"2021-12-25T18:14:27.100065Z","shell.execute_reply.started":"2021-12-25T18:14:11.799552Z","shell.execute_reply":"2021-12-25T18:14:27.099239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ImageDataAugmentor.image_data_augmentor import *\nimport albumentations as A\n\ntrain_augmentations = A.Compose([\n            A.RandomCrop(image_size, image_size, p=1),\n            A.CoarseDropout(p=0.5),\n            A.Cutout(p=0.5),\n            A.Flip(p=0.5),\n            A.ShiftScaleRotate(p=0.5),\n            A.HueSaturationValue(p=0.5, hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2),\n            A.RandomBrightnessContrast(p=0.5, brightness_limit=(-0.2,0.2), contrast_limit=(-0.2, 0.2)),\n            A.ToFloat()\n            ], p=1)\n\nval_augmentations = A.Compose([\n                A.CenterCrop(image_size, image_size, p=1),\n                A.ToFloat()\n                ], p=1)","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:27.101714Z","iopub.execute_input":"2021-12-25T18:14:27.101976Z","iopub.status.idle":"2021-12-25T18:14:28.16573Z","shell.execute_reply.started":"2021-12-25T18:14:27.101938Z","shell.execute_reply":"2021-12-25T18:14:28.165032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def TFDataGenerator(train_set, val_set):\n    \n    train_generator = ImageDataAugmentor(augment=train_augmentations)\n    val_generator = ImageDataAugmentor(augment=val_augmentations)\n    \n    train_datagen = train_generator.flow_from_dataframe(\n                  dataframe = train_set,\n                  directory='../input/cassava-leaf-disease-classification/train_images',\n                  x_col='image_id',\n                  y_col='label',\n                  target_size=target_size,\n                  batch_size=batch_size,\n                  shuffle=True,\n                  class_mode='categorical',\n                  seed=2020)\n\n    val_datagen = val_generator.flow_from_dataframe(\n                dataframe = val_set,\n                directory='../input/cassava-leaf-disease-classification/train_images',\n                x_col='image_id',\n                y_col='label',\n                target_size=target_size,\n                batch_size=batch_size,\n                shuffle=False,\n                class_mode='categorical',\n                seed=2020)\n    \n    return train_datagen, val_datagen","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:28.167151Z","iopub.execute_input":"2021-12-25T18:14:28.167447Z","iopub.status.idle":"2021-12-25T18:14:28.175138Z","shell.execute_reply.started":"2021-12-25T18:14:28.167411Z","shell.execute_reply":"2021-12-25T18:14:28.17353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let's see how the generated images look like post-augmentation for our training set.","metadata":{}},{"cell_type":"code","source":"train_set = df_train.iloc[:10]\nval_set = df_train.iloc[-10:]\n\ntrain_datagen, val_datagen = TFDataGenerator(train_set, val_set)","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:28.176597Z","iopub.execute_input":"2021-12-25T18:14:28.176869Z","iopub.status.idle":"2021-12-25T18:14:28.241185Z","shell.execute_reply.started":"2021-12-25T18:14:28.176831Z","shell.execute_reply":"2021-12-25T18:14:28.240449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images, _ = next(train_datagen)\n\nplt.figure(figsize=(14, 14))\nfor i in range(9):\n    image = train_images[i]\n    ax = plt.subplot(3, 3, i + 1)\n    plt.imshow(image)\n    plt.axis(\"off\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:28.242336Z","iopub.execute_input":"2021-12-25T18:14:28.242569Z","iopub.status.idle":"2021-12-25T18:14:29.540796Z","shell.execute_reply.started":"2021-12-25T18:14:28.242537Z","shell.execute_reply":"2021-12-25T18:14:29.540038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And now our validation set:","metadata":{}},{"cell_type":"code","source":"val_images, _ = next(val_datagen)\n\nplt.figure(figsize=(14, 14))\nfor i in range(9):\n    image = val_images[i]\n    ax = plt.subplot(3, 3, i + 1)\n    plt.imshow(image)\n    plt.axis(\"off\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:29.542249Z","iopub.execute_input":"2021-12-25T18:14:29.542643Z","iopub.status.idle":"2021-12-25T18:14:30.789063Z","shell.execute_reply.started":"2021-12-25T18:14:29.54261Z","shell.execute_reply":"2021-12-25T18:14:30.788414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Building and Selection\nEarlier on, I mentioned that to counter mislabelling, we would implement label smoothing on our loss metric (in this case, categorical cross-entropy). Here's how it works:\n\nWhen we apply the cross-entropy loss to a classification task, we’re expecting true labels to be 1, while the others to be 0. In other words, we are sure that the labels are 100% correct. However, we already know that ***they are not!***\n\nOne way to circumvent this is to lower the confidence level of our labels. So instead of being 100% confident (or sure) that our labels are correct, we can be 90% or even 80% confident instead. This allows the model to \"learn\" that some the labels are not correct and make better predictions.\n\nOriginally, if an image is labelled \"0\" (Cassava Bacterial Blight Disease), the resulting one-hot label would be this: [1, 0, 0, 0, 0]. However, after applying label smoothing, then we get new one-hot labels according to this formula:\n\nnew_onehot_labels = onehot_labels * (1 - label_smoothing) + label_smoothing / num_classes\n\nIn our case, with a label smoothing value of 0.2, \n\nnew_onehot_labels \n\n= [1, 0, 0, 0, 0] * (1 - 0.2) + 0.2 / 5\n\n= [1, 0, 0, 0, 0] * 0.8 + 0.04\n\n= [0.84, 0.04, 0.04, 0.04, 0.04]\n                  \nThis means that we are ~84% confident in our labels!","metadata":{}},{"cell_type":"code","source":"def create_Inception():\n    base_model = InceptionV3(include_top=False, weights=\"imagenet\", input_shape=input_shape)\n\n    # Rebuild top\n    inputs = Input(shape=input_shape)\n\n    model = base_model(inputs)\n    pooling = GlobalAveragePooling2D()(model)\n    dropout = Dropout(0.2)(pooling)\n\n    outputs = Dense(5, activation=\"softmax\", name=\"dense\", dtype='float32')(dropout)\n\n    # Compile\n    inception = Model(inputs=inputs, outputs=outputs)\n    optimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9, nesterov=True)\n    loss = tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.2, from_logits=True)\n\n    inception.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])\n    return inception","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:30.790178Z","iopub.execute_input":"2021-12-25T18:14:30.790718Z","iopub.status.idle":"2021-12-25T18:14:30.801035Z","shell.execute_reply.started":"2021-12-25T18:14:30.790677Z","shell.execute_reply":"2021-12-25T18:14:30.800235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_Xception():\n    base_model = Xception(include_top=False, weights=\"imagenet\", input_shape=input_shape)\n\n    # Rebuild top\n    inputs = Input(shape=input_shape)\n\n    model = base_model(inputs)\n    pooling = GlobalAveragePooling2D()(model)\n    dropout = Dropout(0.2)(pooling)\n\n    outputs = Dense(5, activation=\"softmax\", name=\"dense\", dtype='float32')(dropout)\n\n    # Compile\n    xception = Model(inputs=inputs, outputs=outputs)\n    optimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9, nesterov=True)\n    loss = tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.2, from_logits=True)\n\n    xception.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])\n    return xception","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:30.802992Z","iopub.execute_input":"2021-12-25T18:14:30.80342Z","iopub.status.idle":"2021-12-25T18:14:30.81316Z","shell.execute_reply.started":"2021-12-25T18:14:30.803376Z","shell.execute_reply":"2021-12-25T18:14:30.812304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will be using two model architectures, InceptionV3 and Xception, for our model ensembling. I chose Stochastic Gradient Descent (SGD) with momentum as it generalizes better. Additionally, I enabled Nesterov momentum as it (theoretically) converges to a minimum more quickly. To understand the math behind the two implementations of SGD with momentum, refer to this: https://stats.stackexchange.com/questions/179915/whats-the-difference-between-momentum-based-gradient-descent-and-nesterovs-acc\n\n![](https://i.stack.imgur.com/YrpGA.png)","metadata":{}},{"cell_type":"markdown","source":"# Training with Stratified K-Fold Cross Validation\nWe need to train our models with k-fold cross validation to ensure that our model isn't overfitting on the training data (or rather, specific aspects of the training data). Stratified k-fold cv ensures that we have the same class representation in every fold. Here's a brief intuition:\n\n![](https://scikit-learn.org/stable/_images/sphx_glr_plot_cv_indices_0041.png)\n\nIn our case, we take 90% of the train images for training and 10% for testing. However, this split can be arbitary. Moreover, we saw earlier on that the data is imbalanced. Typically, the class with the most images will be the easiest to predict as the model is able to train much better with more data. Hence, splitting the dataset randomly can cause huge variance in the model's performance. For example, on some splits, the model might get a validation subset that is easy to predict.. On others, it might get a difficult subset to predict.\n\nBy using stratified cross validation, we ensure that a different subset of the data is split into training and test sets every time. Additionally, we ensure that class representation remains the same for all subsets, thereby allocating equal difficulty in all subsets.\n\nWe will only do a stratified 3-fold cv this time due to time constraints.","metadata":{}},{"cell_type":"code","source":"fold_number = 0\nn_splits = 3\nepochs = 8\n\ntf.keras.backend.clear_session()\nsss = StratifiedShuffleSplit(n_splits=n_splits, test_size=0.1, random_state=2020)\nfor train_index, val_index in sss.split(df_train[\"image_id\"], df_train[\"label\"]):\n    train_set = df_train.loc[train_index]\n    val_set = df_train.loc[val_index]\n    train_datagen, val_datagen = TFDataGenerator(train_set, val_set)\n    model = create_Inception()\n    print(\"Training fold no.: \" + str(fold_number+1))\n\n    model_name = \"inception \"\n    fold_name = \"fold.h5\"\n    filepath = model_name + str(fold_number+1) + fold_name\n    callbacks = [ReduceLROnPlateau(monitor='val_loss', patience=1, verbose=1, factor=0.2),\n                 EarlyStopping(monitor='val_loss', patience=3),\n                 ModelCheckpoint(filepath=filepath, monitor='val_loss', save_best_only=True)]\n\n    history = model.fit(train_datagen, epochs=epochs, validation_data=val_datagen, callbacks=callbacks)\n    fold_number += 1\n    if fold_number == n_splits:\n        print(\"Training finished!\")","metadata":{"execution":{"iopub.status.busy":"2021-12-25T18:14:30.81486Z","iopub.execute_input":"2021-12-25T18:14:30.815118Z","iopub.status.idle":"2021-12-25T20:11:07.542063Z","shell.execute_reply.started":"2021-12-25T18:14:30.815085Z","shell.execute_reply":"2021-12-25T20:11:07.541322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold_number = 0\nn_splits = 3\nepochs = 8\n\ntf.keras.backend.clear_session()\nsss = StratifiedShuffleSplit(n_splits=n_splits, test_size=0.1, random_state=2020)\nfor train_index, val_index in sss.split(df_train[\"image_id\"], df_train[\"label\"]):\n    train_set = df_train.loc[train_index]\n    val_set = df_train.loc[val_index]\n    train_datagen, val_datagen = TFDataGenerator(train_set, val_set)\n    model = create_Xception()\n    print(\"Training fold no.: \" + str(fold_number+1))\n\n    model_name = \"xception \"\n    fold_name = \"fold.h5\"\n    filepath = model_name + str(fold_number+1) + fold_name\n    callbacks = [ReduceLROnPlateau(monitor='val_loss', patience=1, verbose=1, factor=0.2),\n                 EarlyStopping(monitor='val_loss', patience=3),\n                 ModelCheckpoint(filepath=filepath, monitor='val_loss', save_best_only=True)]\n\n    history = model.fit(train_datagen, epochs=epochs, validation_data=val_datagen, callbacks=callbacks)\n    fold_number += 1\n    if fold_number == n_splits:\n        print(\"Training finished!\")","metadata":{"execution":{"iopub.status.busy":"2021-12-25T20:11:07.545252Z","iopub.execute_input":"2021-12-25T20:11:07.545473Z","iopub.status.idle":"2021-12-25T22:25:34.027719Z","shell.execute_reply.started":"2021-12-25T20:11:07.545448Z","shell.execute_reply":"2021-12-25T22:25:34.027033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Ensembling and Inference\nThis is the part where you should take the code below into a separate inference notebook with minor code changes to the filepaths. Reason being that we needed the internet enabled to install ImageDataAugmentor but internet access is not allowed in this competition. ","metadata":{}},{"cell_type":"code","source":"models = []\nfor i in range(n_splits):\n    inception = load_model(\"./inception \" + str(i+1) + \"fold.h5\")\n    models.append(inception)\n    \nfor i in range(n_splits):\n    xception = load_model(\"./xception \" + str(i+1) + \"fold.h5\")\n    models.append(xception)","metadata":{"execution":{"iopub.status.busy":"2021-12-25T22:25:34.032115Z","iopub.execute_input":"2021-12-25T22:25:34.034352Z","iopub.status.idle":"2021-12-25T22:25:48.314377Z","shell.execute_reply.started":"2021-12-25T22:25:34.034306Z","shell.execute_reply":"2021-12-25T22:25:48.313638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss = pd.read_csv(os.path.join('../input/cassava-leaf-disease-classification', \"sample_submission.csv\"))\npreds = []\nresults = []\n\nfor image_id in ss.image_id:\n    image = Image.open(os.path.join('../input/cassava-leaf-disease-classification', \"test_images\", image_id))\n    image = image.resize((image_size, image_size))\n    image = np.expand_dims(image, axis = 0)\n    for model in models:\n        preds.append(np.argmax(model.predict(image)))\n    res = max(set(preds), key = preds.count)\n    results.append(res)\n\nss['label'] = results\nss.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-12-25T22:25:48.31619Z","iopub.execute_input":"2021-12-25T22:25:48.316441Z","iopub.status.idle":"2021-12-25T22:25:55.396473Z","shell.execute_reply.started":"2021-12-25T22:25:48.316408Z","shell.execute_reply":"2021-12-25T22:25:55.395735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}