{"cells":[{"metadata":{},"cell_type":"markdown","source":"# [Leaf pathology detection 2021](https://www.kaggle.com/c/plant-pathology-2021-fgvc8/data)\n## This notebook compares state of the art DL models as feature extractors for the plant pathology detection\n## During the training, _on the fly_ data augmentation was used with tf ImageDataGenerator\n\n\nRelated notebooks (thanks for sharing) : \n* Used the output of [PP2021 - Duplicates Revealing](https://www.kaggle.com/nickuzmenkov/pp2021-duplicates-revealing) for duplicates in the dataset\n\nFirst, lets import the relevant libraries and put the few things we want to configure"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport os\nimport glob\nimport pandas as pd\n\nimport tensorflow as tf\n\nimport tensorflow_addons as tfa\nprint(\"Using tensorflow \", tf.__version__)\n\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    classes = [\n        'complex', \n        'frog_eye_leaf_spot', \n        'powdery_mildew', \n        'rust', \n        'scab',\n        'healthy']\n    batch_size = 16\n    test_size = 0.25\n    img_size = 512 # image size\n    seed = 42 # random seed \n    retrain = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check if files exists and if labeled files in train_csv are the same files than in train_images\n\nIf you don't have a png dir, consider adding one with : _mkdir /kaggle/pngs_"},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir /kaggle/working/pngs/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir = '../input/plant-pathology-2021-fgvc8/train_images/'\ntrain_csv = '../input/plant-pathology-2021-fgvc8/train.csv'\ntest_dir = '../input/plant-pathology-2021-fgvc8/test_images/'\nduplicates = '../input/duplicatescsv/duplicates.csv'\npng_dir = \"/kaggle/working/pngs/\" \nmodel_dir = \"../input/models/\"\n\nprint(os.path.exists(train_dir))\nprint(os.path.exists(train_csv))\nprint(os.path.exists(test_dir))\nprint(os.path.exists(duplicates))\nprint(os.path.exists(png_dir))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inspecting data :\n* Problem statement\n* Size and number of files of train and test datasets\n* Number of true duplicates and mistakenly classed same images"},{"metadata":{"trusted":true},"cell_type":"code","source":"imgs = os.listdir(train_dir)\ndf_train = pd.read_csv(train_csv)\nfor ind in range(df_train.shape[0]):\n    if df_train['image'][ind] not in imgs:\n        print(\"{} not in train_images\".foramt(df_train['image'][id]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_duplicates = pd.read_csv(duplicates,header=None)\ndf_duplicates.columns = ['img1','img2']\n\n\nprint(\"The train dataset is composed of {} labeled images\".format(df_train.shape[0]))\n\nprint(\"The test dataset is composed of {} unlabeled images\".format(len(os.listdir(test_dir))))\nprint(df_train.head())\n\nprint(\"\\nThere are {} duplicated images.\\n\".format(df_duplicates.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"true_duplicates = []\nfalse_duplicates = []\nfor ind in range(df_duplicates.shape[0]):\n    # First, check if all images are in the train dataset\n    if not df_duplicates['img1'][ind] in list(df_train[\"image\"])  : print(\"{} not in training dataset\".format(df_duplicates['img1'][ind]))\n    elif not df_duplicates['img2'][ind] in list(df_train[\"image\"])  : print(\"{} not in training dataset\".format(df_duplicates['img2'][ind]))\n    else : \n        # Check wether it is a True duplicate -> same labels\n        # Or not and then plot those\n        if np.all(df_train[df_train[\"image\"]==df_duplicates['img2'][ind]].reset_index()[\"labels\"] == df_train[df_train[\"image\"]==df_duplicates['img1'][ind]].reset_index()[\"labels\"]):\n            true_duplicates.append(df_duplicates['img1'][ind])\n        else :\n            false_duplicates.append((df_duplicates['img1'][ind],df_duplicates['img2'][ind]))\n\nprint('There are {} true duplicates and {} false ones.'.format(len(true_duplicates),len(false_duplicates)))\nprint('Lets display the false duplicates')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count = 0\nfor img1, img2 in false_duplicates[:5]:\n    fig, axs = plt.subplots(1,2)\n    axs[0].imshow(plt.imread(train_dir+img1))\n    axs[0].set_title(df_train[df_train[\"image\"]==img1].reset_index()[\"labels\"][0])\n    axs[0].axis('off')\n    axs[1].imshow(plt.imread(train_dir+img2))\n    axs[1].set_title(df_train[df_train[\"image\"]==img2].reset_index()[\"labels\"][0])\n    axs[1].axis('off')\n    plt.savefig(png_dir+\"compare_false_dup\"+str(count)+\".png\")\n    count += 1\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inspection results : \n* The problem is a multi label problem -> Using F1-score micro as metric (see [here](https://towardsdatascience.com/multi-class-metrics-made-simple-part-ii-the-f1-score-ebe8b2c2ca1) why) and  modifying dataframe\n* There are some duplicated images with same labels => may harm training => deleting one of them \n* There are some duplicated images with different labels => produce noise in data => deleting the two\n\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Just check that every possible label is in CFG.classes : \nlabels = [x.split(' ') for x in df_train['labels']]\nlabels = [l for label in labels for l in label ]\n\nuniques = np.unique(labels)\nassert len(uniques)==len(CFG.classes), 'ERROR : labels and CFG.classes mismatch'\nfor unique in uniques : \n    assert unique in CFG.classes , 'ERROR : labels and CFG.classes mismatch'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['labels'] = [x.split(' ') for x in df_train['labels']]\nlabels = MultiLabelBinarizer(classes=CFG.classes).fit_transform(df_train['labels'].values)\nlabels = pd.DataFrame(columns=CFG.classes, data=labels, index=df_train.index)\ndf_train.drop('labels', axis = 1, inplace = True)\nfor col in labels.columns:\n    df_train[col] = labels[col]\nprint(df_train.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"init = df_train.shape[0]\n\nfor img1, img2 in false_duplicates:\n    df_train = df_train[df_train[\"image\"]!=img1]\n    df_train = df_train[df_train[\"image\"]!=img2]\nfor img in true_duplicates:\n    df_train = df_train[df_train[\"image\"]!=img]\n\nend = df_train.shape[0]\ndf_train.reset_index(drop = True, inplace = True)\nprint(\"Deleted {} files\".format(init-end))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lets now take a look at the proportion of labels in the train data"},{"metadata":{"trusted":true},"cell_type":"code","source":"value_counts = lambda x: pd.Series.value_counts(x, normalize=True)\ndf_occurence = pd.DataFrame({\n    'origin': df_train[CFG.classes].apply(value_counts).loc[1]})\n\nbar = df_occurence.plot.barh(figsize=[15, 5], colormap='plasma')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## It is important to keep the proportion of labels in both train and validation datasets \nLets use [StratifiedShuffleSplit](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.StratifiedShuffleSplit.html) and compare with the usual train_test_split"},{"metadata":{"trusted":true},"cell_type":"code","source":"sss = StratifiedShuffleSplit(n_splits=1, test_size= CFG.test_size, random_state=CFG.seed)\nX = df_train['image']\ny = df_train[CFG.classes]\nfor train_index, test_index in sss.split(X, y):\n    X_train, X_test = df_train.loc[train_index], df_train.loc[test_index]\n\n## Just for comparison : \ncompare_X_train, compare_X_test = train_test_split( df_train,  test_size= CFG.test_size, random_state= CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_occurence = pd.DataFrame({\n    'origin': df_train[CFG.classes].apply(value_counts).loc[1],\n    'stratified_train': X_train[CFG.classes].apply(value_counts).loc[1],\n    'stratified_test': X_test[CFG.classes].apply(value_counts).loc[1],\n    'compare_train': compare_X_train[CFG.classes].apply(value_counts).loc[1],\n    'compare_test': compare_X_test[CFG.classes].apply(value_counts).loc[1]\n})\n\nbar = df_occurence.plot.barh(figsize=[15, 5], colormap='plasma')\nplt.savefig(png_dir+'comparison_stratified.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Now files are selected for train and test, lets create a tf.data.dataset \nIndeed, we cant load every image in memory and this will help during training.\n\nNote that parse_image does : \n* Load the image\n* Decode it \n* Convert into floats in 0 and 1 \n* Resize the image \n\nDuring training, this will be done implicitely.\nDoing so would imply recoding the logic in a server if we wanted to export the model.\n## We also use data augmentation for the training dataset :\nThe following steps are ramdomely used only on the training dataset : \n* Vertical/Horizontal flip \n* Rotate \n* Change contrast \n* Vertical/horizontal shift (translation)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def pred2labels(pred, thresh = 0.5, labels = CFG.classes):\n\n    assert len(pred)==len(labels), 'Predictions must have shape : ({},)'.format(len(labels))\n    pred = [labels[i] for i in range(len(labels)) if pred[i]>thresh]\n    pred = np.array(pred)\n    res = ''\n    for p in pred :\n        if res == '':\n            res += p\n        else:\n            res += ' '+p\n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\ndata_augmentation = tf.keras.Sequential([tf.keras.layers.experimental.preprocessing.RandomFlip(\"horizontal_and_vertical\", seed = CFG.seed),\n                                         tf.keras.layers.experimental.preprocessing.RandomRotation(0.2),\n                                         tf.keras.layers.experimental.preprocessing.RandomContrast([0,0.3], seed= CFG.seed ),\n                                         tf.keras.layers.experimental.preprocessing.RandomTranslation(height_factor=0.2, width_factor=0.2)\n                                        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def parse_image(file_path):\n    # load the raw data from the file as a string\n    img = tf.io.read_file(train_dir + file_path)\n    # convert the compressed string to a 3D uint8 tensor\n    img = tf.image.decode_jpeg(img, channels=3)\n    # Use `convert_image_dtype` to convert to floats in the [0,1] range\n    img = tf.image.convert_image_dtype(img, tf.float32)\n    # resize the image to the desired size.\n    img = tf.image.resize(img, [CFG.img_size,CFG.img_size])\n    return img\n\ndef prepare_dataset(X, augmentation = False):\n    dataset = tf.data.Dataset.from_tensor_slices((X['image'].values, X[CFG.classes].values ))\n    dataset = dataset.map(lambda x ,y : (parse_image(x),y) )\n    dataset = dataset.batch(CFG.batch_size)\n    \n    if augmentation :\n        dataset = dataset.map(lambda x, y: (data_augmentation(x, training=True), y), \n                                            num_parallel_calls=AUTOTUNE)\n    dataset = dataset.repeat().prefetch(buffer_size=AUTOTUNE)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds_train = prepare_dataset(X_train, augmentation = True)\nds_test = prepare_dataset(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for inputs, outputs in ds_train.as_numpy_iterator():\n    # Verify the shapes are still as we expect\n    print(\"Input shape is:\", inputs.shape, \"output shape is:\", outputs.shape)\n\n    # Print the first element and the label\n    plt.imshow(inputs[0])\n    plt.show()\n    print('label of this input is', outputs[0], 'corresponding to', pred2labels(outputs[0]))\n\n    # Break now. We only want to visualise the first example\n    break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Everything ready, lets create a function to create the model and easily tweak it if needed \nThe model_transfert should be a pretrained model without the output layer so we can use it as feature extractor"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_cnn(input_shape, output_length,\n               nb_cnn=3, nb_filters = 64, activation_cnn = 'relu', \n               model_transfert = None, fine_tune = False, \n               nb_FC_layer = 3, nb_FC_neurons = 512, reducing = False, activation_FC = 'relu',\n               dropout = 0.0,\n               activation_output = 'sigmoid',\n               name = 'my_cnn_model'\n               ):\n    '''Create a CNN based model is model_transfert is None. Else, the model_transfert is used for feature extraction. \n    If reducing is not False, nb_FC_neurons must be multiple of 2**nb_FC_layer '''\n    \n    assert input_shape[-1] == 3, 'For the moment only models with rgb input is dealt'\n    #for shape in input_shape[:-1] : assert shape % 2**nb_cnn ==  0 , 'Each dimension of input must be a multiple of 2**nb_cnn'\n    if reducing : assert nb_FC_neurons % 2**nb_FC_layer == 0 , 'If reducing, nb_FC_neurons must be multiple of 2**nb_FC_layer '\n        \n    model = tf.keras.models.Sequential(name=name)\n    model.add(tf.keras.layers.InputLayer(input_shape=input_shape, name = 'Input_layer'))\n    \n    if model_transfert == None: \n        for cnn in range(nb_cnn):\n            model.add(tf.keras.layers.Conv2D( filters = nb_filters, kernel_size = (3,3), padding='same', activation = activation_cnn, name ='Conv2D_'+str(cnn+1) ))\n            model.add(tf.keras.layers.MaxPooling2D( pool_size=(2, 2), name ='MaxPool_'+str(cnn+1)))\n    else : \n        if not fine_tune : model_transfert.trainable = False\n            \n        model.add(model_transfert)\n        model.add(tf.keras.layers.MaxPooling2D( pool_size=(2, 2), name ='MaxPool_transfer'))\n        \n    model.add(tf.keras.layers.Flatten())\n    \n    if reducing : \n        for FC in range(nb_FC_layer):\n            model.add(tf.keras.layers.Dense(nb_FC_neurons/2**FC, activation= activation_FC, name='FC_layer_'+str(FC+1)))\n            \n            if dropout != 0.0: \n                model.add(tf.keras.layers.Dropout(dropout, name = 'Dropout_'+str(FC+1)))\n    else:\n        for FC in range(nb_FC_layer):\n            model.add(tf.keras.layers.Dense(nb_FC_neurons, activation= activation_FC, name='FC_layer_'+str(FC+1)))\n        if dropout != 0.0:  \n            model.add(tf.keras.layers.Dropout(dropout, name = 'Dropout_'+str(FC+1)))\n\n    model.add(tf.keras.layers.Dense(output_length, activation = activation_output ,name='Output_layer'))\n\n    return model\n\n\ndef get_callbacks(monitor='val_loss',save_name=None,patience=8):\n    '''Returns the wanted callbacks to save models and avoid overfitting.\n    monitor (str, optional): the monitor to check for the early stopping. Default is 'val_loss'\n    save_name (str, optional): if not None, uses modelcheckpoint and saves checkpoints at the save_name. Default is None.\n    patience (int, optional): number of epoch to wait for improvment of monitor. Default is 8.'''\n    if save_name :\n        return [tf.keras.callbacks.ModelCheckpoint(filepath=save_name,\n                                                   monitor=monitor, \n                                                   save_best_only=True,\n                                                   verbose=0),\n                tf.keras.callbacks.EarlyStopping(monitor=monitor, \n                                                 patience=patience,\n                                                 restore_best_weights=True)\n                ]\n    else:\n        return [tf.keras.callbacks.EarlyStopping(monitor=monitor, \n                                                 patience=patience,\n                                                 restore_best_weights=True)\n                ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try : \n    base = tf.keras.applications.Xception( include_top=False, weights='imagenet', input_shape=(CFG.img_size, CFG.img_size, 3), classes=len(CFG.classes) )\n\n    model = create_cnn(input_shape=(CFG.img_size, CFG.img_size, 3), output_length=len(CFG.classes),\n                   model_transfert = base, fine_tune = True, \n                   nb_FC_layer = 2, nb_FC_neurons = 512, reducing = True, activation_FC = 'relu',\n                   dropout = 0,\n                   activation_output = 'sigmoid',\n                   name='my_model'\n                   )\n    optimizer = tf.keras.optimizers.Adam(learning_rate=3.5e-5)\n\n    model.compile(optimizer=optimizer,\n                      loss=tf.keras.losses.BinaryCrossentropy(),\n                      metrics=[\n                        tf.keras.metrics.BinaryAccuracy(name='acc'), \n                        tfa.metrics.F1Score(\n                            num_classes=len(CFG.classes), \n                            average='micro', name = 'micro-F1'),\n                        tfa.metrics.F1Score(\n                            num_classes=len(CFG.classes), \n                            average='macro', name = 'macro-F1'),\n                        tfa.metrics.F1Score(\n                            num_classes=len(CFG.classes), \n                            average='weighted', name = 'weighted-F1')])\n\n    model.summary()\nexcept : \n    print('Internet not available')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if os.path.exists(model_dir + 'model.h5') and not CFG.retrain : \n    print('Loading model from file')\n    model = tf.keras.models.load_model(model_dir+'model.h5')\n    history = None\nelse :\n    history = model.fit(ds_train,\n                          validation_data=ds_test,\n                          steps_per_epoch=(X_train.shape[0]*0.8)//CFG.batch_size, \n                          validation_steps= (X_test.shape[0]*0.2)//CFG.batch_size,\n                          callbacks = get_callbacks(monitor = 'val_micro-F1', save_name = '/kaggle/working/model.h5', patience = 4),\n                        epochs = 10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lets plot the training curves"},{"metadata":{"trusted":true},"cell_type":"code","source":"if history : \n    fig, axes = plt.subplots(1, 3, figsize=(30, 5))\n\n    axes[0].plot(history.history['loss'], label = 'Train loss')\n    axes[0].plot(history.history['val_loss'], label = 'Validation loss')\n    axes[0].set_title('Loss')\n    axes[0].legend()\n\n    axes[1].plot(history.history['acc'], label = 'Train accuracy')\n    axes[1].plot(history.history['val_acc'], label = 'Validation accuracy')\n    axes[1].set_title('Accuracy')\n    axes[1].legend()\n\n    axes[2].plot(history.history['micro-F1'], label = 'Train micro-F1',color='lightblue')\n    axes[2].plot(history.history['val_micro-F1'], label = 'Validation micro-F1',color='darkblue')\n    \n    axes[2].plot(history.history['macro-F1'], label = 'Train macro-F1',color='red')\n    axes[2].plot(history.history['val_macro-F1'], label = 'Validation macro-F1',color='darkred')\n    \n    axes[2].plot(history.history['weighted-F1'], label = 'Train weighted-F1',color='lightgreen')\n    axes[2].plot(history.history['val_weighted-F1'], label = 'Validation weighted-F1',color='darkgreen')\n    axes[2].set_title('F1-score')\n    axes[2].legend()\n    \n    plt.savefig(png_dir+'history_xception.png')\n\n    plt.show()\nelse : \n    print('There is no history')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Now we are able to create the submission file"},{"metadata":{"trusted":true},"cell_type":"code","source":"def parse_test_image(file_path):\n    # load the raw data from the file as a string\n    img = tf.io.read_file(file_path)\n    # convert the compressed string to a 3D uint8 tensor\n    img = tf.image.decode_jpeg(img, channels=3)\n    # Use `convert_image_dtype` to convert to floats in the [0,1] range\n    img = tf.image.convert_image_dtype(img, tf.float32)\n    # resize the image to the desired size.\n    img = tf.image.resize(img, [CFG.img_size,CFG.img_size])\n    return img\n\n\n\ndef predict_new(path, model):\n    img = parse_test_image(path)\n    img = tf.expand_dims(img,axis = 0)\n    pred = model.predict(img)\n    return pred2labels(pred[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub = pd.DataFrame(columns=['image','labels'])\nfor path in os.listdir(test_dir):\n    pred = predict_new(test_dir+path, model)\n    \n    df_sub = df_sub.append( {'image': path, 'labels': pred}, ignore_index = True )\n    \nprint(df_sub.head())\ndf_sub.to_csv('submission.csv', index=False)\nprint('Submission completed')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}