{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Project Overview\n\nThis is my first code competition with Kaggle. I learned a significant amount of information reinforcing code methods, noisy label managment, data augmentation (specifically tta), and with more time, then I would have added in cut mix methods for even better data augmentation. The problem statement of determining the diseases within Cassava plants is intriguing and at this point I think I can probabaly identify Cassava Mosaic disease within a Cassava leaf. Let us have a look at the data. "},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# Import the needed paskages for this notebook\n# These are the general data management tools:\nimport numpy as np\nimport pandas as pd\nimport os\nimport json\n\n# Tools for visualization of the data:\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom keras.preprocessing.image import load_img,img_to_array\n\n# For data cleaning\nimport sys\nsys.path.append(\"../input/cleanlab/\")\nimport cleanlab\n\n# For data preparation:\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\n\n# For data models:\nimport tensorflow as tf\nimport keras\nfrom tensorflow.keras import backend, models\nfrom tensorflow.keras.applications import EfficientNetB4\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint \nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Flatten, BatchNormalization, GlobalAveragePooling2D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.utils import plot_model\n\n# For model performance review:\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import classification_report","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"#Import the dataset framework:\n# Define working directories\ndirectory = '../input/cassava-leaf-disease-classification/'\ntraindir = '/kaggle/input/cassava-leaf-disease-classification/train_images'\ntestdir = '/kaggle/input/cassava-leaf-disease-classification/test_images'\n\nwith open(directory +\"label_num_to_disease_map.json\") as file:\n    map_classes = json.loads(file.read())\n    map_classes = {int(k) : v for k, v in map_classes.items()}\n\nimage_map = pd.read_csv(directory+'train.csv')\nimage_map['names'] = image_map['label'].map(map_classes)\n\n# Copy for use in final CV tests \nimage_copy = image_map.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"#General image parameters:\nimg_size = 224\nsize = (img_size, img_size)\nshape = (img_size, img_size, 3)\n\n#Model Parameters:\nclasses = 5\nbatch_size = 32","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"#Functions for the Notebook:\n\ndef image_print(images):\n    i=0\n    for image in images:\n        plt.rcParams['figure.figsize'] = (25,15)\n        ax = plt.subplot(1, 4, i+1)\n        img = load_img(traindir +'/'+ image)\n        title = str(image_map.loc[image_map['image_id']==image]['names'])[5:-27]\n        plt.title(title) \n        plt.imshow(img)\n        plt.axis('off')\n        i+=1\n        if i == 4:\n            break\n            \ndef visualize_noisy(ids, labels, guesses, target_class):\n    c_ids, c_guess, c_labels = ids[guesses == target_class], \\\n                               guesses[guesses == target_class], \\\n                               labels[guesses == target_class]\n    fig, axes = plt.subplots(1,4, sharex=True, sharey=True, figsize=(15, 12))\n    i = 0\n    for ids in c_ids:\n        plt.rcParams['figure.figsize'] = (25,20)\n        ax = plt.subplot(2, 4, i+1)\n        img = load_img(traindir +'/'+ ids)\n        title = f'{map_classes[c_labels[i]]}\\n{ids}'\n        subtitle = f'Guess: {map_classes[c_guess[i]]}'\n        plt.title(title, color = 'r')\n        plt.suptitle(subtitle, y=.9, color='b', fontsize=20, fontweight='bold') , \n        plt.tight_layout()\n        plt.imshow(img)\n        plt.axis('off')\n        i+=1\n        if i == 4:\n            break\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Exploratory Data Analysis on Cassava\n\nThe Cassava dataset consists of mostly the Cassava Mosaic Disease (CMD) and a few Cassava Bacterial Blight (CBB). This is a very unbalanced dataset. "},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"disease_count = image_map[['names','label']].groupby('names').count()\ndisease_count.columns = ['count']\n\nfig, axes = plt.subplots(figsize=(7,5), dpi=100)\nplt.barh(disease_count.index,width = disease_count['count'])\nplt.title('Frequency of each leaf disease in images')\nplt.xlabel('Frequency');","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"label = 0\ncount = disease_count.iloc[label]['count']\ntotal = disease_count['count'].sum()\n#total.sum() \n#CBB_count\n\nprint(f\"The {disease_count.index[label]} classification has {count}. {(count*100/total).round(2)}% of the images. Here are a few images of the CBB diseased Cassava:\")\n\nimages = image_map.loc[image_map['label']==label]['image_id']\n\nimage_print(images)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"label = 1\ncount = disease_count.iloc[label]['count']\ntotal = disease_count['count'].sum()\n#total.sum() \n#CBB_count\n\nprint(f\"The {disease_count.index[label]} classification has {count}. {(count*100/total).round(2)}% of the images. Here are a few images of the CBB diseased Cassava:\")\n\n\nimages = image_map.loc[image_map['label']==label]['image_id']\n\nimage_print(images)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"label = 2\ncount = disease_count.iloc[label]['count']\ntotal = disease_count['count'].sum()\n#total.sum() \n#CBB_count\n\nprint(f\"The {disease_count.index[label]} classification has {count}. {(count*100/total).round(2)}% of the images. Here are a few images of the CBB diseased Cassava:\")\n\n\nimages = image_map.loc[image_map['label']==label]['image_id']\n\nimage_print(images)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"label = 3\ncount = disease_count.iloc[label]['count']\ntotal = disease_count['count'].sum()\n#total.sum() \n#CBB_count\n\nprint(f\"The {disease_count.index[label]} classification has {count}. {(count*100/total).round(2)}% of the images. Here are a few images of the CBB diseased Cassava:\")\n\n\nimages = image_map.loc[image_map['label']==label]['image_id']\n\nimage_print(images)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"label = 4\ncount = disease_count.iloc[label]['count']\ntotal = disease_count['count'].sum()\n#total.sum() \n#CBB_count\n\nprint(f\"The {disease_count.index[label]} classification has {count}. {(count*100/total).round(2)}% of the images. Here are a few images of the CBB diseased Cassava:\")\n\n\nimages = image_map.loc[image_map['label']==label]['image_id']\n\nimage_print(images)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There do appear to be some images that do not go with the label. For example only one of the healthy images shown looks healthy. "},{"metadata":{},"cell_type":"markdown","source":"# Data Cleaning\n\nHow do we deal with noisy labels?  One notebook that I read started this process by doing an image comparison and finding that there are duplicates in the dataset. [Duplicate images](https://www.kaggle.com/nakajima/duplicate-train-images) where identified through image hashing and comparison of hash codes. That notebook identifies 2 sets of duplicate images:"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"images = ['1562043567.jpg', '3551135685.jpg', '2252529694.jpg', '911861181.jpg']\nimage_print(images)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"That is pretty impressive. The 2 duplicate Healthy images are obviously slightly different if you look at the edges. Possibly the image is just slightly cropped but other nuaces lead me to believe that these are likely back to back photos taken in both cases. The Healthy and CBB labeled photo furthers my question of the data labels integrity. \n\nFurther reading on noisy labels leads to looking at [CleanLab and specifically this notebook](https://www.kaggle.com/telljoy/noisy-label-eda-with-cleanlab). I created my own code from this notebook to identify labels based off my models feature sets with these results. As ussual machine learning is iterative so the model I use is one I trained before cleaning out the noisy labels. I uploaded it here so you can see the results.  "},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class FBETA(tf.keras.metrics.Metric):\n\n    def __init__(self, beta = 1, \n                 name=\"Fbeta\", **kwargs):\n        super(FBETA, self).__init__(name=name, **kwargs)\n\n        \n        \n        self.beta = beta\n\n        self.true_poss = self.add_weight(name=\"ctp\", initializer='zeros')\n        self.false_neg = self.add_weight(name=\"cfn\", initializer='zeros')\n        self.false_poss = self.add_weight(name=\"cfp\", initializer='zeros')\n       \n\n    def update_state(self, y_true, y_pred, sample_weight=None):   \n        \n        threshold_shift = 0\n        y_pred = backend.clip(y_pred, 0, 1)\n        y_pred_bin = backend.round(y_pred + threshold_shift)\n\n        tp = backend.sum(backend.round(y_true * y_pred_bin)) + backend.epsilon()\n        fp = backend.sum(backend.round(backend.clip(y_pred_bin - y_true, 0, 1)))\n        fn = backend.sum(backend.round(backend.clip(y_true - y_pred, 0, 1)))   \n        \n        self.true_poss.assign_add(tp)\n        self.false_poss.assign_add(fp)\n        self.false_neg.assign_add(fn)\n        \n    def result(self):\n        \n        bb = self.beta**2\n        \n        p = self.true_poss/(self.true_poss + self.false_poss)\n        r = self.true_poss/(self.true_poss + self.false_neg)                     \n        \n        result = backend.mean(((1 + bb**2) * p * r) / (bb**2 * p + r + backend.epsilon()))\n        return result","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"datagen_general = ImageDataGenerator(preprocessing_function = tf.keras.applications.efficientnet.preprocess_input,\n                                    )\n\ndata_set = datagen_general.flow_from_dataframe(image_map,\n                             directory = traindir,\n                             seed=42,\n                             x_col = 'image_id',\n                             y_col = 'names',\n                             target_size = size,\n                             class_mode = 'categorical',\n                             interpolation = 'nearest',\n                             shuffle = False,\n                             batch_size = 64)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"model1 = keras.models.load_model('../input/models/Cassava_rev14.h5',custom_objects={'FBETA':FBETA()})\n    \npreds = model1.predict(data_set, verbose=1)\npred_dict = {'image_id': image_map['image_id'], 'labels': preds.tolist()}\npredictions = pd.DataFrame(pred_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"pred_df = predictions.sort_values(by='image_id', ascending=1)\nlabel_df = image_map.drop(['names'], axis = 1)\nlabel_df = label_df.sort_values(by='image_id', ascending=1)\n\nids, labels = label_df.image_id.values, label_df.label.values\npreds = np.array([pred for pred in pred_df.labels.values])\n\nprint(f'total {len(ids)} images')\nprint(f'prediction shape: {preds.shape}, label shape: {labels.shape}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"(Note: This code is taken from [cleanlab](https://github.com/cgnorthcutt/cleanlab) tutorial.)"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# STEP 1 - Compute confident joint\n\n# Verify inputs\ns = labels\npsx = preds\n\n# Find the number of unique classes if K is not given\nK = len(np.unique(s))\n\n# Estimate the probability thresholds for confident counting\n# You can specify these thresholds yourself if you want\n# as you may want to optimize them using a validation set.\n# By default they are set to the average class prob.\nthresholds = [np.mean(psx[:,k][s == k]) for k in range(K)] # P(s^=k|s=k)\nthresholds = np.asarray(thresholds)\n\n# Compute confident joint\nconfident_joint = np.zeros((K, K), dtype = int)\nfor i, row in enumerate(psx):\n    s_label = s[i]\n    # Find out how many classes each example is confidently labeled as\n    confident_bins = row >= thresholds - 1e-6\n    num_confident_bins = sum(confident_bins)\n    # If more than one conf class, inc the count of the max prob class\n    if num_confident_bins == 1:\n        confident_joint[s_label][np.argmax(confident_bins)] += 1\n    elif num_confident_bins > 1:\n        confident_joint[s_label][np.argmax(row)] += 1\n\n# Normalize confident joint (use cleanlab, trust me on this)\nconfident_joint = cleanlab.latent_estimation.calibrate_confident_joint(\n    confident_joint, s)\n\ncleanlab.util.print_joint_matrix(confident_joint)\n\n# STEP 2 - Find label errors\n\n# We arbitrarily choose at least 5 examples left in every class.\n# Regardless of whether some of them might be label errors.\nMIN_NUM_PER_CLASS = 5\n# Leave at least MIN_NUM_PER_CLASS examples per class.\n# NOTE prune_count_matrix is transposed (relative to confident_joint)\nprune_count_matrix = cleanlab.pruning.keep_at_least_n_per_class(\n    prune_count_matrix=confident_joint.T,\n    n=MIN_NUM_PER_CLASS,\n)\n\ns_counts = np.bincount(s)\nnoise_masks_per_class = []\n# For each row in the transposed confident joint\nfor k in range(K):\n    noise_mask = np.zeros(len(psx), dtype=bool)\n    psx_k = psx[:, k]\n    if s_counts[k] > MIN_NUM_PER_CLASS:  # Don't prune if not MIN_NUM_PER_CLASS\n        for j in range(K):  # noisy label index (k is the true label index)\n            if k != j:  # Only prune for noise rates, not diagonal entries\n                num2prune = prune_count_matrix[k][j]\n                if num2prune > 0:\n                    # num2prune'th largest p(classk) - p(class j)\n                    # for x with noisy label j\n                    margin = psx_k - psx[:, j]\n                    s_filter = s == j\n                    threshold = -np.partition(\n                        -margin[s_filter], num2prune - 1\n                    )[num2prune - 1]\n                    noise_mask = noise_mask | (s_filter & (margin >= threshold))\n        noise_masks_per_class.append(noise_mask)\n    else:\n        noise_masks_per_class.append(np.zeros(len(s), dtype=bool))\n\n# Boolean label error mask\nlabel_errors_bool = np.stack(noise_masks_per_class).any(axis=0)\n\n # Remove label errors if given label == model prediction\nfor i, pred_label in enumerate(psx.argmax(axis=1)):\n    # np.all let's this work for multi_label and single label\n    if label_errors_bool[i] and np.all(pred_label == s[i]):\n        label_errors_bool[i] = False\n\n# Convert boolean mask to an ordered list of indices for label errors\nlabel_errors_idx = np.arange(len(s))[label_errors_bool]\n# self confidence is the holdout probability that an example\n# belongs to its given class label\nself_confidence = np.array(\n    [np.mean(psx[i][s[i]]) for i in label_errors_idx]\n)\nmargin = self_confidence - psx[label_errors_bool].max(axis=1)\nlabel_errors_idx = label_errors_idx[np.argsort(margin)]\n\n\n#Reference: @misc{northcutt2019confidentlearning, title={Confident Learning: \n#Estimating Uncertainty in Dataset Labels}, author={Curtis G. Northcutt and Lu \n#Jiang and Isaac L. Chuang}, year={2019}, eprint={1911.00068}, \n#archivePrefix={arXiv}, primaryClass={stat.ML} }\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"total_idx = np.arange(len(ids))\nclean_idx = np.array([idx for idx in total_idx if idx not in label_errors_idx])\n\nguesses = np.stack(noise_masks_per_class).argmax(axis=0)\nguesses[clean_idx] = labels[clean_idx]\n\nclean_ids = ids[clean_idx]\nclean_labels = labels[clean_idx]\nclean_guesses = guesses[clean_idx]\n\nnoisy_ids = ids[label_errors_idx]\nnoisy_labels = labels[label_errors_idx]\nnoisy_guesses = guesses[label_errors_idx]\n\nprint(f'[clean ratio] \\t {len(clean_idx) / len(total_idx) * 100:.2f}%')\nprint(f'[noise ratio] \\t {len(noisy_ids) / len(total_idx) * 100:.2f}%')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(f\"The {len(clean_idx)} clean classifications consisted:\")\nvisualize_noisy(clean_ids, clean_labels, clean_guesses, target_class=0)\nvisualize_noisy(clean_ids, clean_labels, clean_guesses, target_class=1)\nvisualize_noisy(clean_ids, clean_labels, clean_guesses, target_class=2)\nvisualize_noisy(clean_ids, clean_labels, clean_guesses, target_class=3)\nvisualize_noisy(clean_ids, clean_labels, clean_guesses, target_class=4)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(f\"The {len(noisy_ids)} noisy -- lower prediction probability classifications consisted:\")\nvisualize_noisy(noisy_ids, noisy_labels, noisy_guesses, target_class=0)\nvisualize_noisy(noisy_ids, noisy_labels, noisy_guesses, target_class=1)\nvisualize_noisy(noisy_ids, noisy_labels, noisy_guesses, target_class=2)\nvisualize_noisy(noisy_ids, noisy_labels, noisy_guesses, target_class=3)\nvisualize_noisy(noisy_ids, noisy_labels, noisy_guesses, target_class=4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Using the noisy label data, I was able to do 2 options: 1) I can remove the noisy labels from the dataset or 2) replace the noisy labels with the new inferences. I evaluated both options and found my best performing model after training to be the one where noisy labels are replaced. This makes a lot of sense given that throwing away data means throwing away the potential for this model to learn. Looking at the noisy labels the new predictions appear correct to my non-expert botanist's eyes."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Drop the noisy labels and replace with the infered value.\nimage_copy = image_map.copy()\nfor id in range(len(noisy_ids)):    \n    i = image_map.index[image_map['image_id'] == noisy_ids[id]]\n    image_map.loc[i] = (noisy_ids[id], noisy_labels[id], map_classes[noisy_guesses[id]])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"After replacement of the noisy labels I want to re-evaluate the balance of the input labels and find that it is still an unbalanced problem."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"disease_count = image_map[['names','label']].groupby('names').count()\ndisease_count.columns = ['count']\n\nfig, axes = plt.subplots(figsize=(7,5), dpi=100)\nplt.barh(disease_count.index,width = disease_count['count'])\nplt.title('Frequency of each leaf disease in images')\nplt.xlabel('Frequency');","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"label = 0\ncount = disease_count.iloc[label]['count']\ntotal = disease_count['count'].sum()\n\nprint(f\"The {disease_count.index[label]} classification has {count}. {(count*100/total).round(2)}% of the images.\")\n\nlabel = 1\ncount = disease_count.iloc[label]['count']\ntotal = disease_count['count'].sum()\n\nprint(f\"The {disease_count.index[label]} classification has {count}. {(count*100/total).round(2)}% of the images.\")\nlabel = 2\ncount = disease_count.iloc[label]['count']\ntotal = disease_count['count'].sum()\n\nprint(f\"The {disease_count.index[label]} classification has {count}. {(count*100/total).round(2)}% of the images.\")\nlabel = 3\ncount = disease_count.iloc[label]['count']\ntotal = disease_count['count'].sum()\n\nprint(f\"The {disease_count.index[label]} classification has {count}. {(count*100/total).round(2)}% of the images.\")\nlabel = 4\ncount = disease_count.iloc[label]['count']\ntotal = disease_count['count'].sum()\n\nprint(f\"The {disease_count.index[label]} classification has {count}. {(count*100/total).round(2)}% of the images.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Most of the change is in the healthy group which has gained ~2% while all but CBB dropped slightly."},{"metadata":{},"cell_type":"markdown","source":"# Data Preparation\n\nFor the models, I will use train test split to create a training and validation set. Early in the competition I had a Test split as well that I used for finding the optimal model parameters but at the end I used all of the data for training/validation and used the LB score to guide my final model tweaks. "},{"metadata":{"trusted":true},"cell_type":"code","source":"train,val = train_test_split(image_map, test_size = 0.05, random_state = 42, stratify = image_map['names']) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I used ImageDataGenerator for both the augmentation and image processing. Remembering to set the val set to shuffle false so I can use it for predictions after the model is fit. A significant amount of this code was obtained from this [notebook](https://www.kaggle.com/marto24/keras-model-89-tta) which continues to evolve and change names since my first reading at version 6, so I hope this link still works.  "},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"datagen_general = ImageDataGenerator(preprocessing_function = tf.keras.applications.efficientnet.preprocess_input,\n                                    )\ndatagen_train = ImageDataGenerator(preprocessing_function = tf.keras.applications.efficientnet.preprocess_input, \n                            rotation_range = 90,\n                            width_shift_range = [0,0.2],\n                            height_shift_range = [0,0.2],\n                            shear_range = 0.2,\n                            zoom_range = 0.2,\n                            channel_shift_range=25,\n                            fill_mode = 'constant',\n                            cval = 175, \n                            horizontal_flip = True,\n                            vertical_flip = True\n                            )\n\ntrain_set = datagen_train.flow_from_dataframe(train,\n                             directory = traindir,\n                             seed=42,\n                             x_col = 'image_id',\n                             y_col = 'names',\n                             target_size = size,\n                             class_mode = 'categorical',\n                             interpolation = 'nearest',\n                             shuffle = True,\n                             batch_size = batch_size)\nval_set = datagen_general.flow_from_dataframe(val,\n                             directory = traindir,\n                             seed=42,\n                             x_col = 'image_id',\n                             y_col = 'names',\n                             target_size = size,\n                             class_mode = 'categorical',\n                             interpolation = 'nearest',\n                             shuffle = False,\n                             batch_size = batch_size)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I am going to use the class_weights option during modeling this first revision, but have not had that much success due to the domanence of the CMD class in the hidden test set and training data. Here are the weights: (Note: I used this for version 2 but removed it for version 3 do to poor training shown in the loss per epoch chart. This was consistent in the competition as well.)"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"total = disease_count['count'].sum()\n\nwt_0 = (1 / disease_count['count'][0] )*(total)\nwt_1 = (1 / disease_count['count'][1] )*(total) \nwt_2 = (1 / disease_count['count'][2] )*(total) \nwt_3 = (1 / disease_count['count'][3] )*(total)  \nwt_4 = (1 / disease_count['count'][4] )*(total) \n\nclass_weight = {0: wt_0, 1: wt_1, 2: wt_2, 3: wt_3, 4: wt_4}\nclass_weight","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modeling using EfficientNetB4\n\nI show the model architecture below. \n"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"\n\n########################################  MODELING PART #############################################\n\n#%% MODEL CREATION \nbackend.clear_session()\n\nmodel = Sequential()\n\nmodel.add(EfficientNetB4(input_shape = shape, include_top = False,\n                             weights = 'imagenet',\n                             drop_connect_rate=0.5))\nmodel.add(GlobalAveragePooling2D())\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = 'relu', bias_regularizer=tf.keras.regularizers.L1L2(l1=0.01, l2=0.001)))\nmodel.add(BatchNormalization())\nmodel.add(Dense(classes, activation = 'softmax'))\n\n   \nleaf_model = model\nleaf_model.summary()\n\nfrom tensorflow.keras.layers import Dense, Flatten, BatchNormalization, GlobalAveragePooling2D","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"keras.utils.plot_model(leaf_model)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"step_size_train = train_set.n//train_set.batch_size\nstep_size_test = val_set.n//val_set.batch_size\n\n#Compile\nloss = [tf.keras.losses.CategoricalCrossentropy(\n    name='categorical_crossentropy'),\n            \n       ]\nmetrics = [tf.keras.metrics.CategoricalAccuracy(name=\"categorical_accuracy\"),\n           FBETA(beta = 2),\n           tf.keras.metrics.Precision(name='precision'),\n           tf.keras.metrics.Recall(name='recall'),           \n        ]\ncheckpoint_cb = ModelCheckpoint('CassavaModelloss_Rev0.h5',\n                                    save_best_only=True,\n                                    monitor = 'val_loss',\n                                    mode='min')\nes = EarlyStopping(monitor='val_categorical_accuracy', patience = 7, restore_best_weights = True)\n\nplateau = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.6, patience = 2, verbose = 1)\n    \nleaf_model.compile(optimizer = Adam(learning_rate = .001),\n                        loss = 'categorical_crossentropy',\n                        metrics = metrics) \n#fit model\nhistory = leaf_model.fit( \n    train_set, \n    steps_per_epoch = step_size_train, \n    epochs = 50,\n    validation_data = val_set,\n    validation_steps = step_size_test,\n    #class_weight = class_weight,\n    verbose = 1,\n    callbacks=[es, plateau, checkpoint_cb] \n)\nleaf_model.save('CassavaModel_Rev0.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Performance"},{"metadata":{},"cell_type":"markdown","source":"After a maximum of 50 epochs with some early stopping if the model starts to plateau I review how the model learned. Ideally< I would like to see a steady increase in accuracy for both training a validation steps.  Based of the LB stuck at 91% or less a 90% on my accuracy is expected. The loass should show a steady drop with both training and validation tracking together to a plateau at the minimum loss. "},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# plot accuracy vs epoch\nplt.plot(history.history['categorical_accuracy'])\nplt.plot(history.history['val_categorical_accuracy'])\nplt.title('Model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Test'], loc='upper left')\nplt.show()\n\n# Plot loss values vs epoch\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.ylim([-1, 3])\nplt.legend(['Train', 'Test'], loc='upper left')\nplt.show()\n\n# Evaluate against test data.\nscores = leaf_model.evaluate(val_set, verbose=1)\nprint('Test loss:', scores[0])\nprint('Test accuracy:', scores[1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I like to review my models against a confusion matrix so that I can see where the model is not performing at its best. In these models CBB and Healthy are the consistent problem classifications. This is the entire original dataset."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nfrom sklearn.preprocessing import LabelEncoder\n\n#plot the random images.\n\ntrue_label, true_image = image_copy['names'], image_copy['image_id']\n\nval_images = os.listdir(traindir + '/')\n\npred = leaf_model.predict(data_set)\npred_classes=np.argmax(pred,axis=1)\n\ncode = LabelEncoder()\nlabel = code.fit_transform(true_label)\n\n#Code obtained from: https://scikit-learn.org/0.18/auto_examples/model_selection/plot_confusion_matrix.html\nimport itertools\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.cool):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        cm = np.round_(cm,2)\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    #print(cm)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n\n#confusion matrix\nconf_mat=confusion_matrix(label,pred_classes)\n\nplt.figure()\nplot_confusion_matrix(conf_mat, classes = map_classes, normalize = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"The accuracy of the model is fully reviewed using the sklearn metrics classification report.\")\nprint(classification_report(label,pred_classes))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Inference on Testset"},{"metadata":{},"cell_type":"markdown","source":"This is one area that I learned a significant amount during this code competition. I had never thought about creating an ensemble of multiple models or using data augmentation to fix the noise on the test dataset. So here I show the code of an ensemble (with models that I have previously uploaded) and multi-iteration augmentation of the images for testing. Since I am  not submitting this notebook, I will use my validation set for presentation purposes. "},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"model1 = keras.models.load_model('../input/models/Cassava_rev15.h5',custom_objects={'FBETA':FBETA()})\nmodel2 = keras.models.load_model('../input/models/Cassava_rev18.h5',custom_objects={'FBETA':FBETA()})\nmodel3 = keras.models.load_model('../input/models/Cassava_rev19loss.h5',custom_objects={'FBETA':FBETA()})\nmodel4 = keras.models.load_model('../input/models/Cassava_rev22.h5',custom_objects={'FBETA':FBETA()})\nmodel5 = keras.models.load_model('../input/models/Cassava_rev14.h5',custom_objects={'FBETA':FBETA()})","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"val_set_aug = datagen_train.flow_from_dataframe(image_copy,\n                             directory = traindir,\n                             x_col = 'image_id',\n                             y_col = 'names',\n                             target_size = size,\n                             class_mode = 'categorical',\n                             interpolation = 'nearest',\n                             shuffle = False,\n                             batch_size = 1)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"preds = []\n\npreds.append(model1.predict(data_set, verbose=1))\npreds.append(model2.predict(data_set, verbose=1))    \npreds.append(model3.predict(data_set, verbose=1))    \npreds.append(model4.predict(data_set, verbose=1))    \npreds.append(model5.predict(data_set, verbose=1)) \n\ni=0\nfor i in range(0,5):\n    preds.append(model1.predict(val_set_aug, verbose=1))\n    preds.append(model2.predict(val_set_aug, verbose=1))    \n    preds.append(model3.predict(val_set_aug, verbose=1))    \n    preds.append(model4.predict(val_set_aug, verbose=1))    \n    preds.append(model5.predict(val_set_aug, verbose=1))\n    print(f\"This is run{i+1} through augmentation.\")\n    i=+1 \n\np_ave = np.mean(preds, axis=0)\npred_classes = np.argmax(p_ave,axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nfrom sklearn.preprocessing import LabelEncoder\n\n#plot the random images.\n\ntrue_label, true_image = image_copy['names'], image_copy['image_id']\nval_images = os.listdir(traindir + '/')\ncode = LabelEncoder()\nlabel = code.fit_transform(true_label)\n\n#Code obtained from: https://scikit-learn.org/0.18/auto_examples/model_selection/plot_confusion_matrix.html\nimport itertools\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.cool):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        cm = np.round_(cm,2)\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n\n#confusion matrix\nconf_mat=confusion_matrix(label,pred_classes)\n\nplt.figure()\nplot_confusion_matrix(conf_mat, classes = map_classes, normalize = True)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(\"The accuracy of the model is fully reviewed using the sklearn metrics classification report.\")\nprint(classification_report(label,pred_classes))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Conclusion\n\nOverall, my models were able to achieve a 88.6% on the 30% hidden test set. Eagerly awaiting the count down to see the Private testset board open up."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}