{"cells":[{"metadata":{"id":"tIVg2bi73ClR","outputId":"6f24f9b0-20f6-4740-ecbd-997d778d5ae3","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"#from google.colab import drive\n#drive.mount('/gdrive')","execution_count":null,"outputs":[]},{"metadata":{"id":"qXsy78k3HEhl","outputId":"d5119f7b-84c1-4518-ff0d-e610188b4d10","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"#%cd ../gdrive/MyDrive/Kaggle/cassavaleaf\n#!ls","execution_count":null,"outputs":[]},{"metadata":{"id":"KDL_RTwxLVd_","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"#! pip install -q kaggle\n#! mkdir ~/.kaggle\n#### ES necesario que tengas el token de kaggle on your gdrive directory\n#! cp kaggle.json ~/.kaggle/\n#! chmod 600 ~/.kaggle/kaggle.json\n#!kaggle datasets download -d tahsin/cassava-leaf-disease-merged","execution_count":null,"outputs":[]},{"metadata":{"id":"4ZB9MiGDLWFH","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"#!mkdir kaggle_data\n#!unzip \"cassava-leaf-disease-merged.zip\" -d kaggle_data","execution_count":null,"outputs":[]},{"metadata":{"id":"05cIjBHmG2sU"},"cell_type":"markdown","source":"
\n \n
\n"},{"metadata":{"id":"hgdQU8-WG2sa"},"cell_type":"markdown","source":"
\n \n
\n

Geting Started | Transfer Learning and Data Aumentatin šŸ“ššŸ’¬

\n
"},{"metadata":{"id":"nSkIIdTKG2sb"},"cell_type":"markdown","source":"

ABOUT:

\n\n

Hi everyone, the aims of this notebooks is show you how to achive a classication problem going trowgh data augmentation and pretrained models for transfer learning.

\n\n

I'm going to start exploring the data and the type of data augmentation techniques that i'll use in the model. Then design the class for handle the data generator, and finaly implement the model with all the good stuffs.

\n\n

Walking throw the competition notebooks, i've learn a lot from others and found that many folks made his own dataset for cassava challenge, they did merge dataset from 2019 and 2020, many just image and other using tfrecord. So, i've taken one of those, this dataset , i really recomended.

\n\n

If you have any sugestion or observation, please let me know in the coments bellow.

\n\n\n

Let's do it!

"},{"metadata":{"id":"S0tm-LhWG2sc"},"cell_type":"markdown","source":"# Table of Contents\n1. [Chapter 1 - What is the problem?](#ch1)\n1. [Chapter 2 - About our tools](#ch2)\n1. [Chapter 3 - Overview data](#ch3)\n1. [Chapter 4 - Exploring Image Augmentation](#ch4)\n1. [Chapter 5 - Data Generator](#ch5)\n1. [Chapter 6 - Model](#ch6)\n1. [Chapter 7 - Training the base Model](#ch7)\n1. [Chapter 8 - Training Kfold Model](#ch8)\n1. [Chapter 9 - Submission - Test](#ch9)\n1. [Submission File](#ch99)"},{"metadata":{"id":"GPQc_p_sSk2L","outputId":"a8316fd9-24b3-45ce-f94a-2f8b5adbebc6","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"\"\"\"\nimport os\ncont = 0\nfor dirname, _, filenames in os.walk('kaggle_data/train'):\n for filename in filenames:\n cont= cont +1\n #print(os.path.join(dirname, filename))\nassert 26337 <= cont ,\"There are not enough images files, only {} files\".format(cont)\nprint(\"Number of images : {}\".format(cont))\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --quiet ../input/kerasapplications\n!pip install --quiet ../input/efficientnet-source-code","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"#!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"id":"2Lt85R05G2sd"},"cell_type":"code","source":"# import libraries\nimport numpy as np \nimport pandas as pd \nfrom os.path import join\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport plotly.express as px\nimport plotly.figure_factory as ff\nimport os\n\nimport IPython.display as display\n\nfrom PIL import Image\nimport warnings\n#warnings.filterwarnings(\"ignore\")\n\nimport random as rd\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import Input,optimizers\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.layers import Dense, Flatten, Dropout,BatchNormalization, Activation, Input, GlobalAveragePooling2D, concatenate,Lambda\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping,TensorBoard\nfrom tensorflow.keras.applications import EfficientNetB0,EfficientNetB4,EfficientNetB3, Xception\nimport efficientnet.tfkeras as efn\n\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import confusion_matrix,roc_curve,multilabel_confusion_matrix\nfrom sklearn.metrics import classification_report\nfrom tqdm import tqdm\nimport itertools\n\ncolors = plt.rcParams['axes.prop_cycle'].by_key()['color']\ndef plot_confusion_matrix(y_true, y_pred, class_names,title=\"Confusion matrix\",normalize=False,onehot = False):\n \"\"\"\n Returns a matplotlib figure containing the plotted confusion matrix.\n Args:\n cm (array, shape = [n, n]): a confusion matrix of integer classes\n class_names (array, shape = [n]): String names of the integer classes\n \"\"\"\n if onehot :\n cm = confusion_matrix([y_i.argmax() for y_i in y_true], [y_ip.argmax() for y_ip in y_pred])\n else:\n cm = confusion_matrix(y_true, y_pred)\n figure = plt.figure(figsize=(8, 8))\n plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)\n plt.title(title)\n plt.colorbar()\n tick_marks = np.arange(len(class_names))\n plt.xticks(tick_marks, class_names, rotation=45)\n plt.yticks(tick_marks, class_names)\n\n # Normalize the confusion matrix.\n cm = np.around(cm.astype('float') / cm.sum(axis=1)[:, np.newaxis], decimals=2) if normalize else cm\n\n # Use white text if squares are dark; otherwise black.\n threshold = cm.max() / 2.\n for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n color = \"white\" if cm[i, j] > threshold else \"black\"\n plt.text(j, i, cm[i, j], horizontalalignment=\"center\", color=color)\n\n plt.tight_layout()\n plt.ylabel('True label')\n plt.xlabel('Predicted label')\n plt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"mVgFAUQIG2se"},"cell_type":"markdown","source":"\n# CHAPTER 1. What is the problem?\n\nCassava leaf, like any other plant is vulnerable to diseases and if you have a sick leaf, then this disease will be going to propagate to others leaf and potentially spread the disease for the whole plantation. So, we need a cassava leaf desease expert at the plantation (that will be expensive) or create an image classifier to detect automatically if there are any of the 4 diseases or if the leaf is healthy.\n\nCan you identify a problem with a cassava plant using a photo from a relatively inexpensive camera? This competition will challenge you to distinguish between several diseases that cause material harm to the food supply of many African countries. In some cases the main remedy is to burn the infected plants to prevent further spread, which can make a rapid automated turnaround quite useful to the farmers."},{"metadata":{"id":"rcCd6onyG2se"},"cell_type":"markdown","source":"\n# CHAPTER 2. About our tools?\n\n1. [EfficientNET](https://github.com/qubvel/efficientnet)\n1. [Image aumentation](https://www.tensorflow.org/tutorials/images/data_augmentation)\n1. [TensorFlow Hub - Efficientnet](https://tfhub.dev/google/collections/efficientnet/1)\n1. [Noisy student paper - Efficienet improve ](https://arxiv.org/pdf/1911.04252.pdf)\n1. [Merge Dataset Cassava challenge 2019 and 2020](https://www.kaggle.com/tahsin/cassava-leaf-disease-merged)\n\n
\n \n
"},{"metadata":{"id":"JBYYWIjyG2sf"},"cell_type":"markdown","source":"\n# CHAPTER 3. Overview Data"},{"metadata":{"id":"1HKtHe-RG2sf"},"cell_type":"markdown","source":"### Paths šŸ“š"},{"metadata":{"_kg_hide-input":true,"trusted":true,"id":"HX0JsXcDG2sg"},"cell_type":"code","source":"SampleSubmission = \"../input/cassava-leaf-disease-classification/sample_submission.csv\"\nlabels_desease = \"../input/cassava-leaf-disease-classification/label_num_to_disease_map.json\"\ntraintablepath = \"../input/cassava-leaf-disease-merged/merged.csv\"\n\ntrain_tfr_path = \"../input/cassava-leaf-disease-classification/train_tfrecords\"\ntest_tfr_path = \"../input/cassava-leaf-disease-classification/train_tfrecords\"\n\ntrain_img_path = \"../input/cassava-leaf-disease-merged/train\"\ntest_img_path = \"../input/cassava-leaf-disease-classification/test_images\"\n\nmodels_path = \"\"","execution_count":null,"outputs":[]},{"metadata":{"id":"XoijpxUUG2sg"},"cell_type":"markdown","source":"### Open files šŸ“š"},{"metadata":{"trusted":true,"id":"pWIPf2x6G2sg"},"cell_type":"code","source":"train_table = pd.read_csv(traintablepath)\nclasses_label= pd.read_json(labels_desease, orient='index')\nsampSum = pd.read_csv(SampleSubmission)","execution_count":null,"outputs":[]},{"metadata":{"id":"9qhbTM2IG2sh"},"cell_type":"markdown","source":"## Train file overview\n\n1. Sample train table, only five rows.\n1. Because this is a deeplearning problem, our train table doesn't have any issue\n1. We need to analyse how many classes there are in our dataset, and how they are introduce to us, like hot encoder or numerical.\n1. There are only one type of deseasse by image, we have 21397 uniques images and 21397 labeled data points."},{"metadata":{"_kg_hide-input":true,"trusted":true,"id":"qaFkjfmYG2sh","outputId":"f9f58334-c860-493e-eace-056121159f0b","_kg_hide-output":true},"cell_type":"code","source":"display.display(train_table.info())\ndisplay.display(train_table.nunique())\ntrain_table.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"nTsRnyabG2sh"},"cell_type":"markdown","source":"### Desease Numerical class ā˜ ļø"},{"metadata":{"_kg_hide-input":true,"trusted":true,"id":"HtMdMgR2G2sh","outputId":"6c952809-db55-4ea1-825f-227ddb3ef688"},"cell_type":"code","source":"classes_label = classes_label.reset_index()\nclasses_label.columns = [\"label\", \"Desease\"]\nclasses_label[\"short\"] = classes_label[\"Desease\"].apply(lambda x: x.split(\" \")[-1])\ndisplay.display(classes_label)","execution_count":null,"outputs":[]},{"metadata":{"id":"Yq2dfXn0G2si"},"cell_type":"markdown","source":"### Diseases representation šŸ’€"},{"metadata":{"_kg_hide-input":true,"trusted":true,"id":"r0G0LfEQG2si","outputId":"d3c2974d-80d1-465e-e673-bee5e45363c6"},"cell_type":"code","source":"temp = train_table.join(classes_label[[\"label\", \"short\"]].set_index('label'), on= \"label\")[[\"label\", \"short\"]]\ntemp[\"count\"] = 1\ntemp = temp.groupby([\"label\", \"short\"]).sum()\nclass_count = temp.reset_index()\nfig = px.bar(class_count, x=\"short\", y=\"count\", title=\"Count per leaf desease\",text=class_count['count'])\nfig.update_traces(texttemplate='%{text:.2s}',textposition='outside')\nfig.update_layout(template= \"plotly_dark\" , \n xaxis = dict(title = \"Cassava Leaf Desease\"))\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"UevqLQftG2si"},"cell_type":"markdown","source":"### Weighting classes šŸƒ šŸƒ šŸƒ šŸƒ \n\nBecause this dataset is imbalanced, I will need to consider create the inverse weight to get better performance on my model."},{"metadata":{"_kg_hide-input":true,"trusted":true,"id":"cF8tjL79G2si","outputId":"c888bac7-b30f-44f3-befd-9693a7e2c5fe","_kg_hide-output":true},"cell_type":"code","source":"from sklearn.utils import class_weight\nclass_weights = class_weight.compute_class_weight('balanced',\n np.unique(train_table[\"label\"]),\n train_table[\"label\"])\nclass_weights = dict(enumerate(class_weights)) \ndisplay.display(class_weights)","execution_count":null,"outputs":[]},{"metadata":{"id":"2cyFB8iwG2si"},"cell_type":"markdown","source":"\n# Chapter 4 - Exploring Image Augmentation"},{"metadata":{"id":"45Frr1V6G2sj"},"cell_type":"markdown","source":"# Exploring Images"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"id":"hf1jVxGiG2sj"},"cell_type":"code","source":"def print_leaf(path_folder, table, augg_funct=None, augg = False,prefix=\"Orig\" ):\n temp = table.iloc[[rd.randint(0, len(train_table)) for _ in range(6)]]\n \n plt.figure(figsize=(15,9))\n for i, (name, label )in enumerate(zip(temp[\"image_id\"], temp[\"label\"])):\n plt.subplot(2,3, i%6 +1)\n plt.axis('off')\n if not augg:\n img=Image.open(join(path_folder,name))\n else:\n image = tf.expand_dims(np.array(Image.open(join(path_folder,name))), 0)\n img=augg_funct(image)[0]\n plt.imshow(img)\n img_size = np.asarray(img).shape\n plt.title(\"{} {} - size {} \".format(prefix,classes_label[\"short\"][int(label)], img_size) )\n\ndef print_leaf_mix_cutup(x,y,prefix ):\n plt.figure(figsize=(15,9))\n for i, (image, label )in enumerate(zip(x,y)):\n if i > 5:\n continue\n plt.subplot(2,3, i%6 +1)\n plt.axis('off')\n img=image.numpy()\n plt.imshow(img)\n img_size = np.asarray(img).shape\n plt.title(\"{} {} - size {} \".format(prefix,classes_label[\"short\"][np.where(label>0)[0].tolist()].tolist(), img_size) )","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"id":"mrE-tpDwG2sj","outputId":"097ffca3-1210-4ed2-eca0-c06ec87e6086"},"cell_type":"code","source":"print_leaf(train_img_path, train_table)","execution_count":null,"outputs":[]},{"metadata":{"id":"U0HuhfrdG2sk"},"cell_type":"markdown","source":"## Exploring Image Aumentation"},{"metadata":{"id":"E1LXPHMPG2sk"},"cell_type":"markdown","source":"Tensorflow keras, provide us a set of tools and techniques that we can use direct form the box like\nany other layer. On this proyect y going to use 6 types of preprocessing techniques.\n1. [RandomCrop](https://www.tensorflow.org/api_docs/python/tf/keras/layers/experimental/preprocessing/RandomCrop)\n Randomly crop the images to target height and width.\n1. [RandomTranslation](https://www.tensorflow.org/api_docs/python/tf/keras/layers/experimental/preprocessing/RandomTranslation)\n Randomly translate each image during training.\n1. [RandomZoom](https://www.tensorflow.org/api_docs/python/tf/keras/layers/experimental/preprocessing/RandomZoom)\n Randomly zoom each image during training.\n1. [RandomFlip](https://www.tensorflow.org/api_docs/python/tf/keras/layers/experimental/preprocessing/RandomFlip)\n Randomly flip each image horizontally and vertically.\n1. [RandomRotation](https://www.tensorflow.org/api_docs/python/tf/keras/layers/experimental/preprocessing/RandomRotation)\n Randomly rotate each image\n1. [RandomContrast](https://www.tensorflow.org/api_docs/python/tf/keras/layers/experimental/preprocessing/RandomContrast)\n Adjust the contrast of an image or images by a random factor."},{"metadata":{"_kg_hide-input":true,"trusted":true,"id":"SO0259RdG2sk"},"cell_type":"code","source":"img_augmentation = tf.keras.Sequential(\n [\n #tf.keras.layers.experimental.preprocessing.RandomCrop(image_size, image_size), #during inference will work like a resize layer\n #tf.keras.layers.experimental.preprocessing.RandomTranslation((-.1,.1), (-.1,-.1)),\n tf.keras.layers.experimental.preprocessing.RandomZoom((-.1,-.1)),\n tf.keras.layers.experimental.preprocessing.RandomFlip(\"horizontal_and_vertical\"),\n tf.keras.layers.experimental.preprocessing.RandomRotation(0.2),\n tf.keras.layers.experimental.preprocessing.RandomContrast(0.1),\n ])","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"id":"P7Dc4LKoG2sk","outputId":"5708eb6d-ea94-4c08-dda2-3017f69ea551"},"cell_type":"code","source":"print_leaf(train_img_path, train_table,img_augmentation,True, prefix= \"Augm-\" )","execution_count":null,"outputs":[]},{"metadata":{"id":"q6SIrLLfN1CV"},"cell_type":"markdown","source":"\n# Chapter 4.1 - CutMix and MixUp\n\nAcknowledge to @cdotte Chris Deotte for his amazing [notebook](https://www.kaggle.com/cdeotte/cutmix-and-mixup-on-gpu-tpu)\n\n### CutMix Augmentation\nYou can see more about cutmix [here](https://arxiv.org/abs/1905.04899)\n\nCut a portion form an images and paste randomly into another image.\nThe same with the labels. \n\n### MixUp Augmentation\nYou can see more about mixup [here](https://arxiv.org/abs/1710.09412)\n\n______________________________\n#### Both methods use the same logic, lambda determines how much to mix\n```\nX_i1 = tensor_images\ny_i1 = label\n\nX_i2 = tensor_images\ny_i2 = label\n\nlambda = beta_distribution\n\nx_cutmix = x_i1*lambda + x_i2*(1-lambda)\ny_cutmix = y_i1*lambda + y_i2*(1-lambda)\n```"},{"metadata":{"id":"i-Qp3xJrN3vQ","trusted":true},"cell_type":"code","source":"class ImageCustomGenerator(tf.keras.utils.Sequence):\n \"\"\"\n df_table, dataframe\n col_name, column name for file images\n col_label, column label\n indexs, index to extract from dataframe\n path_files, folder path where images are stored\n batch_size,\n target_size, output images\n \"\"\"\n def __init__(self, \n df_table,\n col_name,\n col_label,\n path_files,\n batch_size,\n target_size,\n augmented_seq=None,\n indexs=None,\n shuffle = False,\n TEST = False\n ):\n self.df_table = df_table\n self.augmented_seq = augmented_seq\n self.indexs = indexs if not TEST else df_table.index.tolist()\n self.col_name = col_name\n self.col_label = col_label\n self.path_files = path_files\n self.batch_size = batch_size\n self.target_size = target_size\n self.shuffle = shuffle\n self.TEST = TEST\n self.n_class = df_table[col_label].nunique() if not TEST else 1\n self.on_epoch_end()\n self.__info()\n \n def __info(self):\n print(\"Custom Image generator {:4d} images\".format(len(self.indexs)))\n\n def __len__(self):\n #batches per epochs\n return len(self.indexs) // self.batch_size + 1 if len(self.indexs)%self.batch_size != 0 else len(self.indexs) // self.batch_size\n\n def __getitem__(self, index):\n indexs_step = self.index_gen[index * self.batch_size:(index + 1) * self.batch_size]\n batch = [self.indexs[k] for k in indexs_step]\n X, y = self.__get_data(batch)\n return X, y\n\n def on_epoch_end(self):\n self.index_gen = np.arange(len(self.indexs))\n if self.shuffle == True and self.TEST == False:\n np.random.shuffle(self.index_gen)\n \n def __get_data(self, batch):\n X = []\n if not self.TEST:\n y = self.df_table.iloc[batch,:][self.col_label].to_numpy().astype(int)\n file_list = self.df_table.iloc[batch,:][self.col_name]\n file_list = file_list.map(lambda x: os.path.join(self.path_files,x))\n\n for name in file_list:\n img_ = tf.image.resize(np.array(np.array(Image.open(name))), self.target_size, method = \"bilinear\")\n X.append(img_)\n return self.transform(tf.stack(X), tf.stack(y))\n \n else:#################### TEST BEHAVIOR ################\n file_list = self.df_table.iloc[batch,:][self.col_name]\n file_list = file_list.map(lambda x: os.path.join(self.path_files,x))\n for name in file_list:\n img_ = tf.image.resize(np.array(np.array(Image.open(name))),[600,800], method = \"bilinear\")\n img_ = tf.image.random_crop(img_, [*self.target_size,3])\n X.append(img_)\n return self.augmented_seq(tf.stack(X))/255, tf.zeros([len(batch)], tf.float32)\n \n \n \n def cutmix(self, image, label, PROBABILITY = 1.0):\n # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n # output - a batch of images with cutmix applied\n IMAGE_SIZE = self.target_size\n CLASSES = self.n_class\n AUG_BATCH = self.batch_size\n DIM = IMAGE_SIZE[0]\n \n imgs = []; labs = []\n for j in range(AUG_BATCH):\n # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n # CHOOSE RANDOM LOCATION\n x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n ya = tf.math.maximum(0,y-WIDTH//2)\n yb = tf.math.minimum(DIM,y+WIDTH//2)\n xa = tf.math.maximum(0,x-WIDTH//2)\n xb = tf.math.minimum(DIM,x+WIDTH//2)\n # MAKE CUTMIX IMAGE\n one = image[j,ya:yb,0:xa,:]\n two = image[k,ya:yb,xa:xb,:]\n three = image[j,ya:yb,xb:DIM,:]\n middle = tf.concat([one,two,three],axis=1)\n img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n imgs.append(img)\n # MAKE CUTMIX LABEL\n a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n if len(label.shape)==1:\n lab1 = tf.one_hot(label[j],CLASSES)\n lab2 = tf.one_hot(label[k],CLASSES)\n else:\n lab1 = label[j,]\n lab2 = label[k,]\n labs.append((1-a)*lab1 + a*lab2)\n\n # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n return image2,label2\n \n\n def mixup(self,image, label, PROBABILITY = 1.0):\n # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n # output - a batch of images with mixup applied\n IMAGE_SIZE = self.target_size\n CLASSES = self.n_class\n AUG_BATCH = self.batch_size\n \n DIM = IMAGE_SIZE[0]\n\n imgs = []; labs = []\n for j in range(AUG_BATCH):\n # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n # CHOOSE RANDOM\n k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n # MAKE MIXUP IMAGE\n img1 = image[j,]\n img2 = image[k,]\n imgs.append((1-a)*img1 + a*img2)\n # MAKE CUTMIX LABEL\n if len(label.shape)==1:\n lab1 = tf.one_hot(label[j],CLASSES)\n lab2 = tf.one_hot(label[k],CLASSES)\n else:\n lab1 = label[j,]\n lab2 = label[k,]\n labs.append((1-a)*lab1 + a*lab2)\n\n # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n return image2,label2\n\n def transform(self,image,label):\n # THIS FUNCTION APPLIES BOTH CUTMIX AND MIXUP\n DIM = self.target_size[0]\n CLASSES = self.n_class\n AUG_BATCH = self.batch_size\n \n SWITCH = 0.5\n CUTMIX_PROB = 0.666\n MIXUP_PROB = 0.666\n # FOR SWITCH PERCENT OF TIME WE DO CUTMIX AND (1-SWITCH) WE DO MIXUP\n image2, label2 = self.cutmix(image, label, CUTMIX_PROB)\n image3, label3 = self.mixup(image, label, MIXUP_PROB)\n imgs = []; labs = []\n for j in range(AUG_BATCH):\n P = tf.cast( tf.random.uniform([],0,1)<=SWITCH, tf.float32)\n imgs.append(P*image2[j,]+(1-P)*image3[j,])\n labs.append(P*label2[j,]+(1-P)*label3[j,])\n # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n return image4,label4","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"migen_test = ImageCustomGenerator(\n df_table = train_table,\n col_name = \"image_id\",\n col_label = \"label\",\n augmented_seq = img_augmentation,\n path_files=train_img_path ,\n batch_size = 6,\n target_size = (450, 450),\n TEST = True,\n \n)\nx, y = migen_test.__getitem__(2)\nprint_leaf_mix_cutup(x,y,\"Test Format\")","execution_count":null,"outputs":[]},{"metadata":{"id":"ED-NpBE7G2sk"},"cell_type":"markdown","source":"\n# Chapter 5 - Data Generator"},{"metadata":{"id":"epauKIMfG2sl"},"cell_type":"markdown","source":"OOF require that we create a new data generator for every fold. So, this image data generator take that idea.\n"},{"metadata":{"_kg_hide-input":true,"trusted":true,"id":"wY9m5AObG2sl"},"cell_type":"code","source":"def img_data_generator(df,\n col_name, \n col_label, \n train_index, \n val_index,\n path_files,\n df_test = None,\n path_test_files= None,\n augmented_seq = None,\n target_size=(300,300),\n batch_size=32, \n seed=10):\n data_gen_args = dict(\n rotation_range=20,\n fill_mode = \"reflect\",\n horizontal_flip = True,\n vertical_flip = True,\n width_shift_range=0.05,\n height_shift_range=0.05,\n shear_range = .1,\n zoom_range=0.1)\n \n train_datagen = ImageCustomGenerator(\n df_table = df,\n col_name = col_name,\n col_label = col_label,\n indexs = train_index,\n path_files=path_files ,\n batch_size = batch_size,\n target_size = target_size,)\n \n # train_datagen = ImageDataGenerator().flow_from_dataframe(\n # dataframe = df.iloc[train_index],\n # directory= path_files,\n # x_col=col_name,\n # y_col=col_label,\n # color_mode=\"rgb\",\n # target_size=target_size,\n # batch_size=batch_size,\n # shuffle=True,\n # class_mode='categorical',\n # validate_filenames=True,\n # seed=seed)\n\n val_datagen = ImageDataGenerator().flow_from_dataframe(\n dataframe = df.iloc[val_index],\n directory = path_files,\n x_col=col_name,\n y_col=col_label,\n color_mode=\"rgb\",\n target_size=target_size,\n batch_size=batch_size,\n shuffle=False,\n class_mode='categorical',\n validate_filenames=True,\n seed=seed)\n if df_test is not None :\n test_datagen = ImageCustomGenerator(\n df_table = df_test,\n col_name = col_name,\n col_label = col_label,\n augmented_seq = augmented_seq,\n path_files=path_test_files ,\n batch_size = batch_size,\n target_size = target_size,\n TEST = True,\n )\n else:\n test_datagen = None\n return train_datagen, val_datagen, test_datagen","execution_count":null,"outputs":[]},{"metadata":{"id":"fIQetnxWG2sl"},"cell_type":"markdown","source":"\n# Chapter 6 - Model Creation\n\nI will create a class for my model, this class have all the good stuff for a deeplearing model, transfern learning model.\n\nThe class create de model, manage training and change from static pretraining models to whole trainable model.\n\nI've considered three principals call backs,\n1. **ModelCheckpoint** : save my progress\n1. **EarlyStopping** : Save time\n1. **ReduceLROnPlateau** : Avoid early stoping and keep improving the model"},{"metadata":{"id":"q7SI4HZsgc4n"},"cell_type":"markdown","source":"### Learing Rate Scheduler\n\nI've saw this lr scheduler [here](https://colab.research.google.com/notebooks/tpu.ipynb#scrollTo=iXJXG-Ufdbnu) and in so many notebooks here on kaggle. Using for pretrained models. \n"},{"metadata":{"id":"FiTpfnopgcPS","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def lrfn(epoch):\n \"\"\"\n Learing rate schedule for pretrained models\n \"\"\"\n start_lr = 0.00001\n min_lr = 0.00001\n max_lr = 0.00005 * 3\n rampup_epochs = 2\n sustain_epochs = 0\n exp_decay = .35\n if epoch < rampup_epochs:\n return (max_lr - start_lr)/rampup_epochs * epoch + start_lr\n elif epoch < rampup_epochs + sustain_epochs:\n return max_lr\n else:\n return (max_lr - min_lr) * exp_decay**(epoch-rampup_epochs-sustain_epochs) + min_lr\n","execution_count":null,"outputs":[]},{"metadata":{"id":"D3qkSOisONr8","outputId":"2f643566-54f3-4010-cd91-fb0c2f825f79","trusted":true},"cell_type":"code","source":"n_epochs = 6\nrang = np.arange(n_epochs)\ny = [lrfn(x) for x in rang]\nplt.plot(rang, y,linewidth=2, color = \"blue\")\nplt.xlabel(\"NĀ° epoch\")\nplt.ylabel(\"LR\")\nplt.grid(True)\n_ = plt.title('Learning rate per epoch:')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"dUTWw-2wG2sl","_kg_hide-output":true},"cell_type":"code","source":"class cassava_model(object):\n def __init__(self, \n input_shape,\n n_classes, \n augment_layer, \n lr=1e-4, \n saved_file = None,\n model_name= \"cassavaleaf\",\n model_suff = 0,\n tb_file = \"log1\",\n metric_name = False,\n ):\n self.input_shape = input_shape\n self.n_classes = n_classes\n self.name = model_name\n self.tb_file = tb_file\n self.name_suff = model_suff\n self.augm_layer = augment_layer # layer for image augmentation\n self.saved_file = saved_file\n self.raw_name = model_name + \"_\" + str(model_suff) + '.hdf5'\n self.metric_name = metric_name\n self.n_epochs = 0\n self.lr = lr\n self.create_model()\n self.load_weights() if saved_file is not None else None\n \n def create_model(self,):\n input1 = Input(self.input_shape)\n aug_input = self.augm_layer(input1)\n x = Lambda(lambda x: x/255)(aug_input)\n \n x = efn.EfficientNetB5(weights = \"noisy-student\", include_top = False )(x)\n x = GlobalAveragePooling2D()(x)\n x = Dropout(0.2)(x)\n\n z = Dense(5, activation=\"softmax\", dtype='float32')(x)\n self.model = Model(inputs=input1, outputs=z)\n self.compile_model()\n \n def compile_model(self,lr=None):\n \"\"\"\n MODEL compile, UTIL AFTER ANY CHANGE IN THE MODEL\n \"\"\"\n if lr is not None:\n self.lr = lr\n self.model.summary()\n self.model.compile(loss='categorical_crossentropy',\n optimizer= optimizers.Adam(learning_rate=self.lr),\n metrics=[tf.keras.metrics.CategoricalAccuracy(),\n tf.keras.metrics.PrecisionAtRecall(recall=0.8),\n tf.keras.metrics.AUC(name='auc', multi_label= True),\n tf.keras.metrics.Recall(name='recall')])\n\n def load_weights(self,model_path_file=None):\n \"\"\"\n LOAD THE MODEL BY PATH\n \"\"\"\n if (self.saved_file) or model_path_file:\n try:\n #model.load_model(saved_file)\n self.model.load_weights(self.saved_file if self.saved_file else model_path_file )\n print(\"Success : Loaded model weights\")\n except:\n print(\"Fail: Weights not loaded\")\n \n def fit_model(self,**kwarg):\n \"\"\"\n TRAIN THE MODEL\n logdir: log dir for tensorboard files\n epoch_add: number of epochs to train\n train_data: datagen for the trainig\n val_data: datagen for validation\n \n \"\"\"\n logdir= kwarg[\"logdir\" ]\n epoch_add = kwarg[\"epoch_add\" ]\n file2save = str(self.name) + \"_\" + str(self.name_suff) \n if self.metric_name :\n file2save = file2save + \"_\" +'{val_loss:.4f}acc_val{val_categorical_accuracy:.4f}acc_train{categorical_accuracy:.4f}.hdf5'\n else: \n file2save = file2save + '.hdf5'\n\n callbacks_ = []\n callbacks_.append( TensorBoard(log_dir=self.tb_file))\n #csv_logger = CSVLogger('training.log')\n #callbacks.append( reduce_LR = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=2,))\n callbacks_.append( ModelCheckpoint( file2save , monitor='val_loss',verbose=1, save_best_only=True) )\n callbacks_.append( EarlyStopping(monitor='val_loss',patience=2,min_delta=0))\n callbacks_.append( tf.keras.callbacks.LearningRateScheduler(lambda epoch: self.lrfn(epoch), verbose=True)) if type(self.lr) == float else None\n \n self.model.fit(kwarg[\"train_data\" ],\n #steps_per_epoch = 8, #\n #batch_size=6, # \n epochs=self.n_epochs+kwarg[\"epoch_add\" ],\n initial_epoch = self.n_epochs,\n callbacks=callbacks_, \n validation_data = kwarg[\"val_data\" ],\n class_weight = kwarg[\"class_weight\" ],\n workers=-1)\n self.n_epochs=self.n_epochs+epoch_add\n @staticmethod \n def lrfn(epoch):\n \"\"\"\n Learning rate schedule applied to pretrained models\n Design for 6 epochs, fresh train\n \"\"\"\n start_lr = 0.00001\n min_lr = 0.00001\n max_lr = 0.00005 * 3\n rampup_epochs = 2\n sustain_epochs = 0\n exp_decay = .35\n\n if epoch < rampup_epochs:\n return (max_lr - start_lr)/rampup_epochs * epoch + start_lr\n elif epoch < rampup_epochs + sustain_epochs:\n return max_lr\n else:\n return (max_lr - min_lr) * exp_decay**(epoch-rampup_epochs-sustain_epochs) + min_lr\n","execution_count":null,"outputs":[]},{"metadata":{"id":"Y61o-ZyfG2sl"},"cell_type":"markdown","source":"\n# Chapter 7. Training the Base - Model\n\nThis will be the main training, for stablish the first training weights. Thes best model posible."},{"metadata":{"trusted":true,"id":"pDz4YFZKG2sm"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_table[\"label\"] = train_table[\"label\"].astype(str)\nsampSum[\"label\"] = sampSum[\"label\"].astype(str)\nindex_train, index_val, _, _ = train_test_split(list(train_table.index),\n train_table[\"label\"], \n test_size=0.20,\n random_state =50,\n stratify= train_table[\"label\"])\n\ntarget_size = (456, 456) # recomended size for EffientnetB5, please refer to the links in the beggining.","execution_count":null,"outputs":[]},{"metadata":{"id":"JvkSRfMBG2sm"},"cell_type":"markdown","source":"### Training Parameters"},{"metadata":{"trusted":true,"_kg_hide-input":true,"id":"MjCl6TMJG2sm","outputId":"f4b89ea1-599d-418a-824b-8926d903dd9e","_kg_hide-output":true},"cell_type":"code","source":"\"\"\"\ntrain_param_data = {\n \"df\": train_table,\n \"col_name\": \"image_id\",\n \"col_label\": \"label\",\n \"train_index\": index_train, ### change for every kfold\n \"val_index\": index_val, ### change for every kfold\n \"path_files\":train_img_path,\n \"batch_size\":6,\n \"target_size\": target_size,\n \"seed\": 16,\n \"df_test\": sampSum,\n \"path_test_files\": test_img_path, \n}\ntrain, val, test = img_data_generator(**train_param_data)\nfit_params = {\n \"logdir\": \"log2\",\n \"epoch_add\": 3,\n \"train_data\": train,\n \"val_data\": val,\n \"class_weight\": class_weights,\n}\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"Iw1H_aKbG2sm","outputId":"d2bbb40f-acac-4a71-ca01-268b0c74aac5","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"\"\"\"\ntf.keras.backend.clear_session()\nmodel = cassava_model(input_shape = (*target_size,3),n_classes= 5, augment_layer=img_augmentation,saved_file = \"k_model_0.hdf5\")\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"id":"PDHt8aIGG2sn"},"cell_type":"markdown","source":"### Training Time"},{"metadata":{"trusted":true,"id":"3gUTtqLTG2sn","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"#model.fit_model(**fit_params)","execution_count":null,"outputs":[]},{"metadata":{"id":"oYannwGIG2sn"},"cell_type":"markdown","source":"### Evaluation Metrics"},{"metadata":{"trusted":true,"id":"YoTENCImG2sn","_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"\"\"\"\nval_predict = model.model.predict(val, verbose=1) \ntest_labels = pd.get_dummies(train_table.iloc[index_val][\"label\"]).to_numpy()\nplot_confusion_matrix(test_labels, val_predict,classes_label[\"short\"],normalize=False, onehot= True)\nprint(classification_report([y_i.argmax() for y_i in test_labels], \n [y_ip.argmax() for y_ip in val_predict],\n digits=4 ) )\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"id":"SqP6ro2uT32B"},"cell_type":"markdown","source":"# Chapter 8 - Training Kfold Model"},{"metadata":{"id":"oxpkW0ICE0lQ"},"cell_type":"markdown","source":"### General RUN"},{"metadata":{"trusted":true,"id":"vghdZBeEG2so","outputId":"54bd628c-9b63-4a56-adad-fd872e4515a4","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"\"\"\"\ntf.keras.backend.clear_session() \nsampSum[\"label\"] = sampSum[\"label\"].astype(str)\nn_splits = 5\nskf = StratifiedKFold(n_splits=n_splits)\nk_ready = [0,1]\nfor ind_k, (index_train, index_val) in enumerate(skf.split(train_table[\"image_id\"], train_table[\"label\"])):\n if ind_k in k_ready:\n continue\n # Creating generators\n train_param_data[\"train_index\"]= index_train\n train_param_data[\"val_index\"]= index_val\n train_param_data[\"batch_size\"]= 6\n train, val, _ = img_data_generator(**train_param_data)\n\n # Creating the k model\n model_params = {\n \"input_shape\": (*target_size,3),\n \"n_classes\" : 4,\n \"augment_layer\": img_augmentation,\n \"model_name\": \"k_model\",\n \"model_suff\": ind_k,\n \"tb_file\": \"logk_\" + str(ind_k),\n \"saved_file\": \"k_model\" + \"_\" + str(ind_k) + '.hdf5',\n }\n model = cassava_model(**model_params)\n\n # Creating Training parameters\n fit_params = {\n \"logdir\": \"log_\" + str(ind_k),\n \"epoch_add\": 4,\n \"train_data\": train,\n \"val_data\": val,\n \"class_weight\": class_weights,\n }\n ###### Automatic best model and tensorboard saved #########\n # Training the k model - first round , lr linear incremental then exp decay\n model.fit_model(**fit_params) \n\n # Training the k model - second round, lr cosine decay\n fit_params[\"epoch_add\"] = 4\n lrcos = tf.keras.experimental.CosineDecay(initial_learning_rate=5e-5, \n decay_steps=int(len(train))*fit_params[\"epoch_add\"] )\n model.compile_model(lr=lrcos)\n model.fit_model(**fit_params) \n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"id":"aCfYuqyXcNIb"},"cell_type":"markdown","source":"# Chapter 9 - Submission - Test"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"def create_model(model_id,CLASSES=5,IMAGE_SIZE=(450,450)):\n if model_id==0:\n pretrained_model = tf.keras.applications.Xception(weights = None, input_shape=[*IMAGE_SIZE, 3], include_top=False)\n elif model_id==1:\n pretrained_model = efn.EfficientNetB5(weights = None, include_top = False,input_shape=[*IMAGE_SIZE, 3] )\n elif model_id ==2:\n pretrained_model = efn.EfficientNetB0(weights = None, include_top = False,input_shape=[*IMAGE_SIZE, 3] )\n \n pretrained_model.trainable = True\n model = tf.keras.Sequential([\n pretrained_model,\n GlobalAveragePooling2D(),\n Dropout(0.3),\n Dense(CLASSES, activation='softmax')\n ])\n model.compile(\n optimizer='adam',\n loss = 'categorical_crossentropy',\n metrics=[tf.keras.metrics.CategoricalAccuracy(),\n tf.keras.metrics.PrecisionAtRecall(recall=0.8),\n tf.keras.metrics.AUC(name='auc', multi_label= True),\n tf.keras.metrics.Recall(name='recall')])\n return model","execution_count":null,"outputs":[]},{"metadata":{"id":"Wn3eRSxNY3VS","trusted":true},"cell_type":"code","source":"%time\nimport glob\npath_model_efnB5 = glob.glob(\"../input/cassava-efficientnetb5-450px/*.hdf5\")\npath_model_xcept = glob.glob(\"../input/cassava-xception-450px/*.hdf5\")\npath_model_efnB0 = glob.glob(\"../input/cassava-efficientnetb0-450/*.hdf5\")\n\nTTA_n = 4\n\n#table_test_dummy = sampSum.copy()\ntable_test_dummy = pd.DataFrame()\ntable_test_dummy['image_id'] = list(os.listdir(test_img_path))\n\nn_splits = 5\n#### Justo for testing purpose\ntest_gen = ImageCustomGenerator(\n df_table = table_test_dummy, # sampSum\n col_name = \"image_id\",\n col_label = \"label\",\n augmented_seq = img_augmentation, \n path_files= test_img_path , # test_img_path\n batch_size = 6,\n target_size = (450, 450),\n TEST = True,\n )\ny_predict = np.zeros((len(table_test_dummy),5)) #np.zeros((len(sampSum),5))\nmodel_name = [\"Xception\", \"EfficientNet B5\", \"EfficientNet B0\"]\nfor id_model, list_kfold_models in enumerate([path_model_xcept, path_model_efnB5,path_model_efnB0 ]):\n y_predict_model = np.zeros((len(table_test_dummy),5))\n print(\"Inference by model - {}\".format(model_name[id_model]))\n for ki, model_kfold_path in enumerate(list_kfold_models):\n print(\"{} - {} Kfold pre trained\".format(model_name[id_model], ki))\n tf.keras.backend.clear_session() \n model = create_model(id_model)\n model.load_weights(model_kfold_path)\n y_predict_tta = np.zeros((len(table_test_dummy),5))\n for t in range(TTA_n):\n print(\"NĀ°{} TTA\".format(t))\n y_predict_tta += model.predict(test_gen, verbose=1)\n \n y_predict_model += y_predict_tta/TTA_n\n y_predict += y_predict_model/5*.333\n\n \ny_predict = np.argmax(y_predict, axis = 1)\ntable_test_dummy[\"label\"] = y_predict\ntable_test_dummy.to_csv(\"submission.csv\", index=False)\ntable_test_dummy.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}