{"cells":[{"metadata":{},"cell_type":"markdown","source":"# This Notebook Summarizes Results for the Cassava Leaf Disease Classification Competition \n\nThis notebook is part of my learning journey to deepen my knowledge in deep learning using Keras. \n\n**Main points of this work:**\n* Implements transfer learning using EfficientNetB4 and fine tuning the weights of all layers.\n* Using stratified K-fold cross-validation.\n* Implementing custom image generator since the dataset is extremely large [1].\n* Image augmentation using imgaug library [2].\n* Label smoothing was found to improve the performance - Most likely due to noisy training data labeling.\n* The custom image generator is taking random 400x400x3 crops of the during training. After each epoch, a custom validation step is applied where 6 random crops are applied per image with a less aggressive augmentation compared to test time. The final prediction is based on the sum over all 6 random crops.\n* Custom function implemented to reduce learning rate on plataue based on the custom validation step.\n* Custom function for keeping the best weights based on the custom validation step, very similar to early stopping.\n\n\n**Other things that was tried but didn't show any improvements:**\n* Using max instead of sum during validation.\n* Taking the N highest values per feature during validation instead of sum.\n* Fixed weights of EfficientNetB4 and adding trainable dense layers at the output.\n\n\n**Other comments:**\n* This notebook shows an example of training one fold. Folds are run in different notebooks due to Kaggle runtime limitation.\n\n[1] https://medium.com/@mrgarg.rajat/training-on-large-datasets-that-dont-fit-in-memory-in-keras-60a974785d71 \n\n[2] https://imgaug.readthedocs.io/en/latest/source/overview_of_augmenters.html"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport time as time\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport matplotlib.lines as mlines\nimport keras\nfrom keras import backend as K\nfrom keras.utils import to_categorical\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import confusion_matrix, classification_report\nfrom sklearn.preprocessing import MinMaxScaler\n\n\nfrom tensorflow.keras.applications import MobileNetV2, EfficientNetB3, InceptionResNetV2, EfficientNetB4\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras import Input\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Flatten, Dropout, GlobalAveragePooling2D, Flatten, BatchNormalization\n\nfrom skimage.transform import rescale, resize\nimport seaborn as sn\n\nimport imgaug as ia\nimport imgaug.augmenters as iaa\nfrom imgaug import parameters as iap\n\nfrom sklearn.model_selection import StratifiedKFold\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nimport pickle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Settings \nclass settings:\n    Nfolds = 5\n    run_folds = [4]\n    imsize = 400 #assuming square images\n    nsubcuts = 6\n    initial_learning_rate = 0.25e-4\n    reduceLR = 0.5\n    patience = 2\n    tolerance = 1\n    label_smoothing = 0.1\n    debug = False\n    input_shape = (400,400,3)\n    R_dropout_finalLayer = 0.4\n    batchSize = 16\n    numEpochs = 12\n    batchSize_predict = 1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"path = '../input/cassava-leaf-disease-classification/'\ndf = pd.read_csv(path+\"train.csv\")\ndf = shuffle(df, random_state=42)\n\n#If we need to debug we can use a subset of all images:\nif settings.debug: \n    df = df[1:150] #use only 150 items","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Examples of images from the training set\n* Image size is (600,800,3)"},{"metadata":{"trusted":true},"cell_type":"code","source":"imgPath_train = path+'train_images/'\nfileList = os.listdir(path+'train_images/')\n\nplt.figure(figsize = (40,10))\ngs = gridspec.GridSpec(2, 6)\ngs.update(wspace=0, hspace=0.005) # set the spacing between axes. \nfor k in range(12):\n    img = Image.open(path+\"train_images/\" + fileList[k+7])\n    plt.subplot(gs[k])\n    plt.imshow(img)\n    plt.axis('off')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Investigating label statistics"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = np.unique(df['label'])\nlabels_text = [\"Cassava Bacterial Blight (CBB)\", \"Cassava Brown Streak Disease (CBSD)\",\"Cassava Green Mottle (CGM)\",\"Cassava Mosaic Disease (CMD)\",\"Healthy\"]\n# Calculating the prior probabilities\nclassProb =np.zeros(len(labels))\nidx = 0\nfor k in labels:\n    print(f\"{k} contains {(df['label'] == k).sum()} samples\")\n    classProb[idx] = (df['label'] == k).sum()\n    idx+=1\n\n# Visualizing the results in a pie-chart:\nprint() #Empty line before figure\ncolor = ['#58508d','#bc5090','#ff6361', '#ffa600','#55AF21'] \nplt.figure(figsize=(15,7))\nplt.pie(classProb, shadow=True, explode=[0.4,0, 0, 0,0.0],labels=labels_text,\n        autopct='%1.2f%%', colors=color, startangle=0,\n        textprops={'fontsize': 14})\n\nclass_weight_vect =np.square(1 / (classProb/classProb.sum()) )# Calculate the weight per classbased on the prior probability dervied from the training data.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Functions to return randomly cropped images\n* We will use random cuts of size (400,400,3)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Note: Values for cropping hardcoded for now, should be updated! \n\n# Images are of shape (600, 800,3) we crop them into (400,400,3)\n    \n#Return one random crop:\ndef returnOneCropped(img): \n    img_height, img_width,_ = img.shape\n    y = np.random.randint(0,img_width-400)\n    x = np.random.randint(0,img_height-400)\n    imgCut = img[x:x+400 , y:y+400,:]    \n    return imgCut\n\n#Return batch of randomly cropped images with test time augmentation applied:\ndef returnCroppedBatch(img, nBatches): \n    img_height, img_width,_ = img.shape\n    imgCut = np.zeros((nBatches,400,400,3))\n    for k in range(nBatches):      \n        y = np.random.randint(0,img_width-400)\n        x = np.random.randint(0,img_height-400)\n\n        imgCut[k,:,:,:] = aug_TTA(image = img[x:x+400 , y:y+400,:])\n    \n    return imgCut.astype(int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Set up model for transfer learning\nHere we define the model using pre-trained **EfficientNetB4**. All layers are set to be trainable. Further, the output uses a Global Average Pooling layer and an output layer with softmax activation and dropout on the weights."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Construct the model for transfer learning \ndef return_model():\n    pretrained_layers = EfficientNetB4(weights='imagenet', include_top=False, input_shape=settings.input_shape)\n\n    #Set all layers to be trainable\n    for layer in pretrained_layers.layers:\n        layer.trainable = True\n\n    R_dropout = settings.R_dropout_finalLayer\n    model = Sequential()\n    model.add(pretrained_layers)\n    model.add(GlobalAveragePooling2D())\n    model.add(Dropout(R_dropout))\n    model.add(Dense(5, activation = 'softmax'))\n    return model\n\n#Custom loss used to be able to set label smoothing\ndef custom_loss(y_true, y_pred):\n    return keras.losses.categorical_crossentropy(y_true, y_pred, label_smoothing=settings.label_smoothing)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentation \n* Different augmentation for training and validation/test"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train time augmentation:\naug = iaa.Sequential([\n    iaa.Fliplr(0.5),\n    iaa.Flipud(0.25),\n    \n    iaa.Sometimes(0.5,\n                  iaa.LinearContrast((0.5, 1.25))),\n\n    iaa.Sometimes(0.95,\n                  iaa.CoarseDropout((0.03, 0.15),\n                  size_percent=(0.01, 0.02),\n                  per_channel=0.0 )),\n                        \n    iaa.Sometimes(0.4,\n                  iaa.Crop(percent=(0, 0.25))),\n        \n    iaa.Sometimes(0.95,\n            iaa.Affine(\n                rotate=(-45, 45),\n                shear=(-18, 18),        \n                mode=ia.ALL)),\n    \n], random_order=True)\n\n\n#Validation and test time augmentation: \naug_TTA = iaa.Sequential([    \n        iaa.Fliplr(0.5),\n        iaa.Flipud(0.25),    \n        iaa.Sometimes(0.25,\n                      iaa.Crop(percent=(0, 0.1))),\n        iaa.Sometimes(0.75,\n            iaa.Affine(\n                translate_percent={\"x\": (-0.1, 0.1), \"y\": (-0.1, 0.1)},\n                rotate=(-45, 45),\n                shear=(-18, 18),\n                cval=(0, 255),\n                mode=ia.ALL)),    \n], random_order=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Original image that will be used as an example to show training and validation/test augmentation****:"},{"metadata":{"trusted":true},"cell_type":"code","source":"img = Image.open(path+'train_images/' + fileList[7])\nplt.figure(figsize = (10,10))\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train time augmentation example:"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (40,10))\ngs = gridspec.GridSpec(2, 6)\ngs.update(wspace=0, hspace=0.000) # set the spacing between axes. \nimg = np.array(img)\nfor k in range(12):\n    img_plot = returnOneCropped(aug(image=img))\n    plt.subplot(gs[k])\n    plt.imshow(img_plot)\n    plt.axis('off')    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validation/test time augmentation example:"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (40,10))\ngs = gridspec.GridSpec(2, 6)\ngs.update(wspace=0, hspace=0.005) # set the spacing between axes. \nimg = np.array(img)\nimg_plot = returnCroppedBatch(img,12)\nfor k in range(12):\n    plt.subplot(gs[k])\n    plt.imshow( img_plot[k,:,:,:])\n    plt.axis('off')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# The custom image generator \n\nSee https://medium.com/@mrgarg.rajat/training-on-large-datasets-that-dont-fit-in-memory-in-keras-60a974785d71 for info on basic structure "},{"metadata":{"trusted":true},"cell_type":"code","source":"class customImageGenerator(keras.utils.Sequence):\n     \n  def __init__(self, image_filenames, labels, batch_size,imsize, nsubcuts, training) :\n    self.image_filenames = image_filenames\n    self.labels = labels\n    self.batch_size = batch_size\n    self.imsize = imsize\n    self.nsubcuts = nsubcuts\n    self.training = training\n    \n    \n  def __len__(self) :\n    return (np.floor(len(self.image_filenames) / float(self.batch_size))).astype(np.int)\n  \n  \n  def __getitem__(self, idx) :\n    batch_x = self.image_filenames[idx * self.batch_size : (idx+1) * self.batch_size]\n    batch_y = self.labels[idx * self.batch_size : (idx+1) * self.batch_size]\n    nsubcuts=self.nsubcuts\n    \n\n    batch_x_split = np.empty(shape=(self.nsubcuts*self.batch_size,self.imsize,self.imsize,3))\n    batch_y_split = []\n\n    for b in range(len(batch_x)):             \n        img = np.array(Image.open(batch_x.iloc[b]))\n        if  self.training==True:\n            img = aug(image=img)\n        if nsubcuts == 1:\n            splitBatch = returnOneCropped(img)\n        else:\n            splitBatch = returnCroppedBatch(img, settings.nsubcuts)\n\n        batch_x_split[self.nsubcuts*(b):self.nsubcuts*(b+1),:,:,:] = splitBatch\n\n        if nsubcuts == 1:\n            batch_y_split = np.append(batch_y_split, np.array(batch_y.iloc[b]))                \n        else:\n            batch_y_split = np.append(batch_y_split, (np.ones(len(splitBatch))*np.array(batch_y.iloc[b])))\n\n\n    return batch_x_split, to_categorical(batch_y_split, 5)\n\n\nclass customImageGenerator_predict(keras.utils.Sequence):  \n  def __init__(self, image_filenames, batch_size,imsize, nsubcuts) :\n    self.image_filenames = image_filenames\n    self.batch_size = batch_size\n    self.imsize = imsize\n    self.nsubcuts = nsubcuts\n        \n  def __len__(self) :\n    return self.nsubcuts\n    \n  def __getitem__(self, idx) :\n    batch_x = self.image_filenames\n    nsubcuts=self.nsubcuts\n    batch_x_split = np.empty(shape=(self.nsubcuts*self.batch_size,self.imsize,self.imsize,3))       \n    img = np.array(Image.open(batch_x))\n    splitBatch = returnCroppedBatch(img)\n    return splitBatch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training of model"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define the stratified KFold split:\nskf = StratifiedKFold(n_splits=settings.Nfolds, random_state=42, shuffle=True)\n\n#Prepare the data for training and pre-allocate variables: \nimage_id=df.drop('label', axis=1)\nlabel=df.label\nval_acc_results     = np.zeros((skf.n_splits,settings.numEpochs))\nval_acc_results_best = np.zeros((skf.n_splits,settings.numEpochs))\nctr = 0\n\n# Start training for one or several folds:\nfor train_index, val_index in skf.split(image_id,label):\n    if ctr in settings.run_folds:\n        print('##### Starting split: ' + str(ctr+1) +'/' +str(skf.n_splits) +' #####')\n\n        model = return_model()\n        model.compile(loss=custom_loss, optimizer=keras.optimizers.Adam(learning_rate=settings.initial_learning_rate), metrics=[\"accuracy\"])\n        image_id_train, image_id_val = image_id.iloc[train_index], image_id.iloc[val_index]\n        label_train, label_val       = label.iloc[train_index],    label.iloc[val_index]\n\n        #Loop for epochs:\n        currentAccuracy = 0\n        patience = 1\n        for k_epochs in range(settings.numEpochs):\n            t = time.time()\n            history = model.fit(customImageGenerator(imgPath_train+image_id_train['image_id'], label_train, settings.batchSize,  settings.imsize, 1, True),\n                                          steps_per_epoch = int(len(image_id_train)/settings.batchSize), epochs=1)    \n            elapsed = time.time() - t\n            print('Elapsed time: ' + str(np.round(elapsed)) +'s')\n            \n            ##########################################\n            # Custom validation step:\n            ##########################################\n            t = time.time()\n            #Predict on the validation data, using settings.nsubcuts number of random crops per image:\n            yhat = model.predict(customImageGenerator(imgPath_train+image_id_val['image_id'], label_val, settings.batchSize_predict,  settings.imsize, settings.nsubcuts, False))\n            \n            #Sum up the predictions and make prediction based on the max of the sum: \n            yhat_comb_sum = np.zeros((np.int(yhat.shape[0]/settings.nsubcuts),5))\n            for k in range(np.int(yhat.shape[0]/settings.nsubcuts)):\n                yhat_comb_sum[k,:] = np.sum(yhat[k*settings.nsubcuts:(k+1)*settings.nsubcuts],0)               \n            y_hat_labels_sum = np.argmax(yhat_comb_sum,1)\n            y_true = np.array(label_val)\n            val_accuracy_sum = np.sum((y_hat_labels_sum-y_true)==0)/len(y_hat_labels_sum)\n            print('Accuracy sum:  ' +str(val_accuracy_sum))\n\n            elapsed = time.time() - t\n            print('Elapsed time: ' + str(np.round(elapsed)) +'s')\n\n            # Check if the validation accuracy has improved, if so save the model. Also if the settings.patience is reached, reduce the learning rate if no improvment in validation accuracy is seen:\n            val_acc_results[ctr,k_epochs] = val_accuracy_sum\n            if val_accuracy_sum > currentAccuracy:\n                currentAccuracy = val_accuracy_sum\n                model.save('./split400x400_model_5fold_' +str(ctr) +'.h5')\n            elif val_accuracy_sum <= currentAccuracy:\n                if patience >= settings.patience: \n                    reduceLR = settings.reduceLR\n                    K.set_value(model.optimizer.lr, K.get_value(model.optimizer.lr) * reduceLR )\n                    print(str(val_accuracy_sum) +' <= ' +str(currentAccuracy) +' --- Lowering the LR to: ' +str( K.get_value(model.optimizer.lr)))\n                    patience = 1\n                else:\n                    patience+=1\n            #Store the best validation accuracy result:    \n            val_acc_results_best[ctr,k_epochs] = currentAccuracy\n\n        #Save results for this fold:\n        with open('results_fold_' +str(ctr) +'.pkl', 'wb') as f:  # Python 3: open(..., 'wb')\n            pickle.dump([currentAccuracy, val_acc_results_best,val_acc_results, settings], f)    \n    ctr += 1\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plotting the training results for all folds\n* Note that this is the results for slightly different (minor) settings in terms of augmentation compared to this notebook.\n* We store the model at the point where we have the best validation accuracy"},{"metadata":{"trusted":true},"cell_type":"code","source":"color = ['#58508d','#bc5090','#ff6361', '#ffa600','#0fa6f0', '#b05fff' ] \nfs = 15\nNfolds = 5\nplt.figure(figsize = (10,8))\nfor k in range(Nfolds):    \n    with open('../input/new-cassava-trans-learn-stratkfold-fold-'+str(k) +'/results_fold_' +str(k) +'.pkl', 'rb') as f:  # Python 3: open(..., 'rb')\n        currentAccuracy_old, val_acc_results_best_old,val_acc_results_old, settings_old = pickle.load(f)\n        plt.plot(val_acc_results_old[k,:],     'o--',color=color[k])\n        plt.plot(val_acc_results_best_old[k,:],'o-', color=color[k])\n\nplt.plot(val_acc_results[settings.run_folds,:][0],      'x--',color='k')\nplt.plot(val_acc_results_best[settings.run_folds,:][0], 'x-',color='k')\n        \nplt.grid('both', linestyle='--')\n\n\n#Ugly way to make a decent looking legend:\nl1 = mlines.Line2D([], [], color=color[0], marker='o' ,label='Fold 1', linewidth=0)\nl2 = mlines.Line2D([], [], color=color[1], marker='o' ,label='Fold 2', linewidth=0)\nl3 = mlines.Line2D([], [], color=color[2], marker='o' ,label='Fold 3', linewidth=0)\nl4 = mlines.Line2D([], [], color=color[3], marker='o' ,label='Fold 4', linewidth=0)\nl5 = mlines.Line2D([], [], color=color[4], marker='o' ,label='Fold 5', linewidth=0)\nl6 = mlines.Line2D([], [], color='k',      marker='x' ,label='This Notebook\\'s fold', linewidth=0)\nl7 = mlines.Line2D([], [], color='k', marker=None,label='Accuracy'            , linestyle= '--' )\nl8 = mlines.Line2D([], [], color='k', marker=None,label='Accuracy saved model', linestyle= '-')\nplt.legend(handles=[l1, l2, l3, l4, l5, l6, l7, l8],loc='center left', bbox_to_anchor=(1, 0.5),fontsize=fs )\n\nplt.xlabel('Epoch #')\nplt.ylabel('Accuracy')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. # Investigate the confusion matrix for the best model in this fold"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the classification performance in a confusion matrix\n#Define the stratified KFold split:\nskf = StratifiedKFold(n_splits=settings.Nfolds, random_state=42, shuffle=True)\nimage_id=df.drop('label', axis=1)\nlabel=df.label\nctr = 0\nfor train_index, val_index in skf.split(image_id,label):\n    if ctr in settings.run_folds:\n        image_id_train, image_id_val = image_id.iloc[train_index], image_id.iloc[val_index]\n        label_train, label_val       = label.iloc[train_index],    label.iloc[val_index]\n    ctr += 1\n\n#Using a previous fold here to get predictable results without having to re-run the whole training:\nmodel = return_model()\nmodel.load_weights('../input/new-cassava-trans-learn-stratkfold-fold-' +str(settings.run_folds[0]) +'/split400x400_model_5fold_' +str(settings.run_folds[0]) +'.h5')\nyhat = model.predict(customImageGenerator(imgPath_train+image_id_val['image_id'], label_val, settings.batchSize_predict,  settings.imsize, settings.nsubcuts, False))\nyhat_comb_sum = np.zeros((np.int(yhat.shape[0]/settings.nsubcuts),5))\nfor k in range(np.int(yhat.shape[0]/settings.nsubcuts)):\n    yhat_comb_sum[k,:] = np.sum(yhat[k*settings.nsubcuts:(k+1)*settings.nsubcuts],0)               \ny_hat_labels_sum = np.argmax(yhat_comb_sum,1)\ny_true = np.array(label_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels_num = [0,1,2,3]\ncm = confusion_matrix( y_true, y_hat_labels_sum, normalize='true')\nsn.set(font_scale=1.4) # for label size\nsn.heatmap(cm, annot=True, annot_kws={\"size\": 14}, cmap=\"rocket_r\", xticklabels = labels_text, yticklabels = labels_text)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Illustrate the probability evolution when taking the sum of several random crops with augmentation \n\n* Note that the plots are showing the probability calculated from the cumulative sum\n\n* Showing example of the accuracy on three different images\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"gs = gridspec.GridSpec(3, 1)\nctr = 0\nfor k in [8, 6, 1]: \n    plt.figure(figsize = (10,8))\n    #plt.subplot(gs[ctr])\n    yhat_this = yhat[k*settings.nsubcuts:(k+1)*settings.nsubcuts]\n    plt.plot(np.cumsum(yhat_this,0)/ (1 + np.array(range(len(yhat_this))).reshape([len(yhat_this),1])),'o-',mec='gray')    \n    plt.xlabel('Subcut #')\n    plt.ylabel('Probability')\n    plt.title('Image #' +str(k))\n    ctr = ctr+1\n    \n    plt.legend(labels_text,loc='center left', bbox_to_anchor=(1, 0.5),fontsize=fs )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Finally we investigate the gain in validation accuracy for one fold from using N number of random crops with augmentation at test time. Note that for the full K-folds, gains were observed up to 12 random crops and similar for the public leaderboard. However, for a single fold the results may look a bit noisy and for this specific case we seem to hit the maximum validation accuracy with 4 random crops. Note that it is dependent on the random crop and the random augmentations that are applied and that fluctuations in accuracy decreases with the number of subcuts. However, naturally the processing time also increases with the number of subcuts that are applied. "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading data that was run in a separate notebook due to memory and limited run-time on Kaggle:\nrun_subcutSweep = False\nif run_subcutSweep:\n    Nsubcut_sweep =[1, 2, 4, 6, 8, 10, 12]\n    val_accuracy_sum_sweep = np.zeros(len(Nsubcut_sweep))\n    #Using a previous fold here to get predictable results without having to re-run the whole training:\n    skf = StratifiedKFold(n_splits=settings.Nfolds, random_state=42, shuffle=True)\n    image_id=df.drop('label', axis=1)\n    label=df.label\n    ctr = 0\n    for train_index, val_index in skf.split(image_id,label):\n        if ctr in settings.run_folds:\n            image_id_train, image_id_val = image_id.iloc[train_index], image_id.iloc[val_index]\n            label_train, label_val       = label.iloc[train_index],    label.iloc[val_index]\n        ctr += 1\n\n    K.clear_session()\n    model = return_model()\n    model.load_weights('../input/new-cassava-trans-learn-stratkfold-fold-' +str(settings.run_folds[0]) +'/split400x400_model_5fold_' +str(settings.run_folds[0]) +'.h5')\n\n\n    val_accuracy_sum_sweep = np.zeros(len(Nsubcut_sweep))\n    for n in range(len(Nsubcut_sweep)):\n        print(\"Number of subcuts: \" + str(Nsubcut_sweep[n]))\n        #yhat_comb_sum = np.zeros((Nsubcut_sweep[n],5))\n        ctr = 0\n        for idx in image_id_val.index:  \n            img = np.array(Image.open(path+\"train_images/\"+image_id_val.loc[idx][0]))\n            splitBatch = returnCroppedBatch(img,Nsubcut_sweep[n])\n            yhat = model.predict_generator(np.array(splitBatch))\n            yhat_comb_sum = np.sum(yhat,0)\n            y_hat_labels_sum[ctr] = np.argmax(yhat_comb_sum,0)\n            y_true[ctr] = np.array(label_val[idx])\n            ctr = ctr+1\n        val_accuracy_sum_sweep[n] = np.sum((y_hat_labels_sum-y_true)==0)/len(y_hat_labels_sum)\n        \nelse:\n    with open('../input/cassava-trans-learn-stratified-k-only-last-part/subcut_sweep4279.pkl', 'rb') as f:  # Python 3: open(..., 'rb')\n        Nsubcut_sweep, val_accuracy_sum_sweep, settings_old = pickle.load(f)\n   \n    \nplt.figure(figsize = (10,5))\nplt.plot(Nsubcut_sweep,val_accuracy_sum_sweep,'ko-')\nval_accuracy_sum_sweep\nplt.grid('both', linestyle='--')\nplt.xlabel('# of random crops with augmentations per image')\nplt.ylabel('Accuracy')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}