{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### Necessary Imports ","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n\nimport os\nimport matplotlib.pyplot as plt\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nimport tensorflow.keras.backend as K\nfrom skimage import io\n\n\nfrom sklearn.preprocessing import MultiLabelBinarizer ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-26T16:04:04.312175Z","iopub.execute_input":"2021-05-26T16:04:04.312511Z","iopub.status.idle":"2021-05-26T16:04:04.317977Z","shell.execute_reply.started":"2021-05-26T16:04:04.312476Z","shell.execute_reply":"2021-05-26T16:04:04.317047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Load Training Data","metadata":{}},{"cell_type":"code","source":"train_read = pd.read_csv(\"../input/plant-pathology-2021-fgvc8/train.csv\", sep=',')\nprint ('dataframe shape: ', train_read.shape)\ntrain_read.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:04:12.935059Z","iopub.execute_input":"2021-05-26T16:04:12.935417Z","iopub.status.idle":"2021-05-26T16:04:13.052715Z","shell.execute_reply.started":"2021-05-26T16:04:12.935383Z","shell.execute_reply":"2021-05-26T16:04:13.040983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Visualize Images with Different Labels \n\n1. Healthy","metadata":{}},{"cell_type":"code","source":"train_im_path = \"../input/plant-pathology-2021-fgvc8/train_images\"\n\nfig = plt.figure(figsize=(15, 10))\nnpics= 6\n\ncount = 1\nimage_list = train_read[train_read['labels'] == 'healthy']['image'].sample(frac=1)[:npics].to_list()  \nfor i, img in enumerate(image_list):\n    \n    sample = os.path.join(train_im_path, img) \n    sample_img = io.imread(sample)   \n    ax = fig.add_subplot(npics/2 , 3, count, xticks=[],yticks=[])   \n    plt.imshow(sample_img)\n    count +=1\nfig.suptitle('All Healthy')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:04:33.80499Z","iopub.execute_input":"2021-05-26T16:04:33.805307Z","iopub.status.idle":"2021-05-26T16:04:40.96048Z","shell.execute_reply.started":"2021-05-26T16:04:33.805277Z","shell.execute_reply":"2021-05-26T16:04:40.959579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2. Scab","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(15, 10))\nnpics= 6\n\ncount = 1\nimage_list = train_read[train_read['labels'] == 'scab']['image'].sample(frac=1)[:npics].to_list()  \nfor i, img in enumerate(image_list):\n    \n    sample = os.path.join(train_im_path, img) \n    sample_img = io.imread(sample)   \n    ax = fig.add_subplot(npics/2 , 3, count, xticks=[],yticks=[])   \n    plt.imshow(sample_img)\n    count +=1\nfig.suptitle('All Scabs')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3. Rust","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(15, 10))\nnpics= 6\n\ncount = 1\nimage_list = train_read[train_read['labels'] == 'rust']['image'].sample(frac=1)[:npics].to_list()  \nfor i, img in enumerate(image_list):\n    \n    sample = os.path.join(train_im_path, img) \n    sample_img = io.imread(sample)   \n    ax = fig.add_subplot(npics/2 , 3, count, xticks=[],yticks=[])   \n    plt.imshow(sample_img)\n    count +=1\nfig.suptitle('All Rusty')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4. Complex","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(15, 10))\nnpics= 6\n\ncount = 1\nimage_list = train_read[train_read['labels'] == 'complex']['image'].sample(frac=1)[:npics].to_list()  \nfor i, img in enumerate(image_list):\n    \n    sample = os.path.join(train_im_path, img) \n    sample_img = io.imread(sample)   \n    ax = fig.add_subplot(npics/2 , 3, count, xticks=[],yticks=[])   \n    plt.imshow(sample_img)\n    count +=1\nfig.suptitle('All Complex')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5. Frog Eye Leaf Spot","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(15, 10))\nnpics= 6\n\ncount = 1\nimage_list = train_read[train_read['labels'] == 'frog_eye_leaf_spot']['image'].sample(frac=1)[:npics].to_list()  \nfor i, img in enumerate(image_list):\n    \n    sample = os.path.join(train_im_path, img) \n    sample_img = io.imread(sample)   \n    ax = fig.add_subplot(npics/2 , 3, count, xticks=[],yticks=[])   \n    plt.imshow(sample_img)\n    count +=1\nfig.suptitle('All Frog Eye Leaf Spot')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"6. Powdery Mildew","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(15, 10))\nnpics= 6\n\ncount = 1\nimage_list = train_read[train_read['labels'] == 'powdery_mildew']['image'].sample(frac=1)[:npics].to_list()  \nfor i, img in enumerate(image_list):\n    \n    sample = os.path.join(train_im_path, img) \n    sample_img = io.imread(sample)   \n    ax = fig.add_subplot(npics/2 , 3, count, xticks=[],yticks=[])   \n    plt.imshow(sample_img)\n    count +=1\nfig.suptitle('All Powdery Mildew')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Clean the Labels \n\nSplit the Original Labels based on space (' ') to properly take into account of available multi-labels for a sinlge image.   ","metadata":{}},{"cell_type":"code","source":"train_read['labels'] = train_read['labels'].apply(lambda s: s.split(' '))\ntrain_read.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:05:02.37287Z","iopub.execute_input":"2021-05-26T16:05:02.373201Z","iopub.status.idle":"2021-05-26T16:05:02.392558Z","shell.execute_reply.started":"2021-05-26T16:05:02.37317Z","shell.execute_reply":"2021-05-26T16:05:02.391414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### How many training images that have multi-labels ? ","metadata":{}},{"cell_type":"code","source":"### check how many images are actually multi-labelled \nmulti_label_count = 0\nfor i in range(train_read.shape[0]):\n    if len(train_read['labels'][i])>1:\n        multi_label_count +=1\nprint ('number of images having multi-labels: ', multi_label_count)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:05:05.631017Z","iopub.execute_input":"2021-05-26T16:05:05.631346Z","iopub.status.idle":"2021-05-26T16:05:05.770842Z","shell.execute_reply.started":"2021-05-26T16:05:05.631314Z","shell.execute_reply":"2021-05-26T16:05:05.769785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Multi-Label Classification \n\nWe have seen that 18632 images only 1355 images have more than a label. This marks the question whether it is indeed necessary to treat this problem as multi-label problem; To quote specific objectives from this year competition, we refer to the detailed objectives given in last year's competition.   \n\n>Specific Objectives: \nObjectives of ‘Plant Pathology Challenge’ are to train a model using images of training dataset to 1) Accurately classify a given image from testing dataset into different diseased category or a healthy leaf; 2) Accurately distinguish between many diseases, sometimes more than one on a single leaf; 3) Deal with rare classes and novel symptoms; 4) Address depth perception—angle, light, shade, physiological age of the leaf; and 5) Incorporate expert knowledge in identification, annotation, quantification, and guiding computer vision to search for relevant features during learning. \n\nThe first 2 objectives clearly state that the problem at hand should be considered as a multi-label problem. \n>Distinguish many diseases, **sometimes more than one on a single leaf**.\n\nFrom here onwards, we decide to go via multi-label classification route. ","metadata":{}},{"cell_type":"code","source":"dfcheck = pd.DataFrame({'a':[y for x in train_read['labels'] for y in x]})\ndfcheck.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:37:47.326824Z","iopub.execute_input":"2021-05-26T14:37:47.327151Z","iopub.status.idle":"2021-05-26T14:37:47.34426Z","shell.execute_reply.started":"2021-05-26T14:37:47.32712Z","shell.execute_reply":"2021-05-26T14:37:47.343307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = plt.figure(figsize=(12, 8))\n\nsns.set(font_scale=1.1)\nyear_count = sns.countplot(x='a', data=dfcheck, order = dfcheck['a'].value_counts().index)\nyear_count.set_xticklabels(year_count.get_xticklabels(), rotation=80)\nplt.ylabel('Count', fontsize=12)\nplt.xlabel('Labels', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:37:50.302565Z","iopub.execute_input":"2021-05-26T14:37:50.3029Z","iopub.status.idle":"2021-05-26T14:37:50.476985Z","shell.execute_reply.started":"2021-05-26T14:37:50.302868Z","shell.execute_reply":"2021-05-26T14:37:50.475988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 2 Main Characteristics of Multi-Label Classification \n\nMulti-Label : Labls are mutually exclusive. \n\n1. Final Activation Function irrespective of neural-net structure should be `softmax`.  \n2. Loss Function needs to be Binary Cross Entropy, and not categorical cross entropy.  It is independent for each vector component (class/label), meaning that the loss computed for every CNN output vector component is not affected by other component values. ","metadata":{}},{"cell_type":"markdown","source":"#### Binarize the Labels \n\n1. We will use Scikit-Learn [`MultiLabelBinarizer`](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MultiLabelBinarizer.html) to create the tabulated matrix \n2. Turn this matrix into a dataframe. \n3. Concatenate with the original dataframe, so our final training data is of the form images col and binarized labels cols","metadata":{}},{"cell_type":"code","source":"### it is important now to binarize the labels \n\nall_labels = list(train_read['labels'])\nmlb = MultiLabelBinarizer()\n\ntrain_labels_binary = pd.DataFrame(mlb.fit_transform(all_labels), columns=mlb.classes_, index=train_read.index)\ntrain_labels_binary.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:05:17.65832Z","iopub.execute_input":"2021-05-26T16:05:17.658701Z","iopub.status.idle":"2021-05-26T16:05:17.694687Z","shell.execute_reply.started":"2021-05-26T16:05:17.658669Z","shell.execute_reply":"2021-05-26T16:05:17.69394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_labels = list(mlb.classes_)\nprint (class_labels)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:05:21.514797Z","iopub.execute_input":"2021-05-26T16:05:21.51515Z","iopub.status.idle":"2021-05-26T16:05:21.519593Z","shell.execute_reply.started":"2021-05-26T16:05:21.515119Z","shell.execute_reply":"2021-05-26T16:05:21.518491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_train_read = pd.concat([train_read['image'], train_labels_binary], axis=1)\nfinal_train_read.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:05:24.482726Z","iopub.execute_input":"2021-05-26T16:05:24.483059Z","iopub.status.idle":"2021-05-26T16:05:24.500179Z","shell.execute_reply.started":"2021-05-26T16:05:24.483028Z","shell.execute_reply":"2021-05-26T16:05:24.499378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # calculate class weight \n\n# positive_weights = {}\n# negative_weights = {}\n# for c in mlb.classes_:\n#     positive_weights[c] = final_train_read.shape[0]/(2*np.count_nonzero(final_train_read[c]==1))\n#     negative_weights[c] = final_train_read.shape[0]/(2*np.count_nonzero(final_train_read[c]==0))\n\n# print(positive_weights)\n# print ('\\n')\n# print(negative_weights)    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_im_path = \"../input/plant-pathology-2021-fgvc8/train_images\" \n\n# # let's load some images \n# fig = plt.figure(figsize=(15, 10))\n# npics= 16\n# count = 1\n# for i in range(npics):\n# #   ipic = i # use this to see original and augmented image side by side\n#   ipic = np.random.choice(train_read.shape[0])\n#   sample = os.path.join(train_im_path, train_read['image'][ipic])\n#   sample_img = io.imread(sample)   \n#   ax = fig.add_subplot(npics/4 , 4, count, xticks=[],yticks=[])\n#   title_string = ''\n#   if len(train_read['labels'][ipic]) > 1:  \n#       for x in train_read['labels'][ipic]:            \n#         title_string = title_string + x +',' +f\"{sample_img.shape}\"\n#   else: \n#     title_string += train_read['labels'][ipic][0] + f\"{sample_img.shape}\"\n#   ax.set_title(title_string, fontsize=10)  \n#   plt.imshow(sample_img)\n#   count = count + 1  \n\n# plt.tight_layout()\n# plt.show()   ","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:05:30.627656Z","iopub.execute_input":"2021-05-26T16:05:30.627978Z","iopub.status.idle":"2021-05-26T16:05:30.631648Z","shell.execute_reply.started":"2021-05-26T16:05:30.627949Z","shell.execute_reply":"2021-05-26T16:05:30.630821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Problem with Original Image Size: Adding New Data-Set\n\nOriginal images are huge, wouldn't it be great to have infinite time and resource to directly feed these high resolution images to our network? let's discuss some other time, but for now we need to now deal with smaller versions of them. \n\nI found resized imaged for this dataset [resized-plant2021](https://www.kaggle.com/ankursingh12/resized-plant2021) by Ankur Singh. He has already downsampled the images into size of 256, 384, 512 & 640px. Let's get started with the 384 ones.  ","metadata":{}},{"cell_type":"code","source":"resized_train_im_path = \"../input/resized-plant2021/img_sz_384\"\n\nfig = plt.figure(figsize=(15, 10))\nnpics= 16\ncount = 1\nfor i in range(npics):\n#   ipic = i # use this to see original and augmented image side by side\n  ipic = np.random.choice(train_read.shape[0])\n  sample = os.path.join(resized_train_im_path, train_read['image'][ipic])\n  sample_img = io.imread(sample)   \n  ax = fig.add_subplot(npics/4 , 4, count, xticks=[],yticks=[])\n  title_string = ''\n  if len(train_read['labels'][ipic]) > 1:  \n      for x in train_read['labels'][ipic]:            \n        title_string = title_string + x +',' +f\"{sample_img.shape}\"\n  else: \n    title_string += train_read['labels'][ipic][0] + f\"{sample_img.shape}\"\n  ax.set_title(title_string, fontsize=10)  \n  plt.imshow(sample_img)\n  count = count + 1  \n\nplt.tight_layout()\nplt.show()   ","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:05:34.739748Z","iopub.execute_input":"2021-05-26T16:05:34.740062Z","iopub.status.idle":"2021-05-26T16:05:36.141Z","shell.execute_reply.started":"2021-05-26T16:05:34.740032Z","shell.execute_reply":"2021-05-26T16:05:36.14007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nGreat ! so this will be our input image data-set. \n\n\n#### Augmentation and `ImageDataGenerator`\nNext step is to include augmentation, for augmentation we can directly use [ImageDataGenerator](https://keras.io/api/preprocessing/image/) as this is not so complicated like facial keypoints data. I have also added blurring of images as a preprocessing function , because for one of the particular labels 'powdery-mildew' looked a bit blurry. ","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nimport cv2\ndef blur(img):\n    return (cv2.blur(img,(5,5)))\n\ntrain_DataGen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1/255.0, \n                                                                zoom_range=0.2, \n                                                                width_shift_range=0.1, \n                                                                height_shift_range = 0.1, \n                                                                rotation_range=15,\n                                                                fill_mode=\"nearest\",\n                                                                preprocessing_function=blur,\n                                                                validation_split=0.20)\n \n\n\n    \n    \n    \ntrain_data_flow = train_DataGen.flow_from_dataframe(final_train_read,\n    directory = resized_train_im_path,\n    x_col = 'image',\n    y_col = class_labels,\n    subset=\"training\",\n    color_mode=\"rgb\",\n    batch_size=32,\n    class_mode=\"raw\",                                                \n    shuffle=True,\n    seed=40)\n\nvalid_data_flow = train_DataGen.flow_from_dataframe(final_train_read,\n    directory = resized_train_im_path,\n    x_col = 'image',\n    y_col = class_labels,\n    subset=\"validation\",\n    color_mode=\"rgb\",\n    batch_size=32,\n    class_mode=\"raw\",                                                \n    shuffle=True,\n    seed=40)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:05:40.818651Z","iopub.execute_input":"2021-05-26T16:05:40.818975Z","iopub.status.idle":"2021-05-26T16:06:17.444972Z","shell.execute_reply.started":"2021-05-26T16:05:40.818945Z","shell.execute_reply":"2021-05-26T16:06:17.443469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check Some Augmented Images","metadata":{}},{"cell_type":"code","source":"# Visualize Augmented Images \n# x,y = train_data_flow.next()\n# for i in range(0,1):\n#     image = x[i]\n#     plt.imshow(image)\n#     plt.show()\nfig = plt.figure(figsize=(15, 10))\nnpics= 16\ncount = 1\nfor i in range(npics):\n  x,y = train_data_flow.next()\n  \n  image = x[0]\n#   print (image.shape)\n  label = y[0]  \n  ax = fig.add_subplot(npics/4 , 4, count, xticks=[],yticks=[])\n  ax.set_title(label, fontsize=10)  \n  plt.imshow(image)\n  count = count + 1  \n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:06:17.446511Z","iopub.execute_input":"2021-05-26T16:06:17.446889Z","iopub.status.idle":"2021-05-26T16:06:28.104879Z","shell.execute_reply.started":"2021-05-26T16:06:17.446852Z","shell.execute_reply":"2021-05-26T16:06:28.104055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Deep Neural Net: \n\nFirst I started by building ['Inception Like'](https://www.kaggle.com/suvoooo/facial-key-points-runthrough) from scratch and training it. Even with 50 epochs (due to large training data this takes a lot of time) it wasn't possible to reach F1 score of 77%. \n\nI checked some other notebooks and realized almost every one of them used pre-trained networks and corresponding weights and biases. Based on these, I finally concluded on InceptionV3 network pre-trained with Imagenet. Added some dense layers at the bottom of the original structure and as discussed before I've used sigmoid activation in the final layer.   \n\nTotal params: 21,935,062\n\nTrainable params: 21,900,630\n\nNon-trainable params: 34,432","metadata":{}},{"cell_type":"code","source":" \nfrom tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, Dropout, \\\n     Flatten, BatchNormalization, Dense, Concatenate, GlobalAveragePooling2D\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.activations import elu, relu\nfrom tensorflow.keras.optimizers import Adam, RMSprop, SGD\n# from tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.regularizers import l2\n\nfrom tensorflow.keras.applications import InceptionV3","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:06:28.106338Z","iopub.execute_input":"2021-05-26T16:06:28.106774Z","iopub.status.idle":"2021-05-26T16:06:28.112164Z","shell.execute_reply.started":"2021-05-26T16:06:28.106738Z","shell.execute_reply":"2021-05-26T16:06:28.111341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#### build an inception like model\n\n# def inception_like(input_layer, filter1, filter2, filter3):\n#   # 1x1 conv\n#   conv1 = Conv2D(filter1, (1,1), padding='same', activation='relu')(input_layer)\n#   bn1 = BatchNormalization()(conv1)\n#   # 3x3 conv\n#   conv3 = Conv2D(filter2, (3,3), padding='same', activation='relu')(input_layer)\n#   bn3 = BatchNormalization()(conv3)\n#   # 5x5 conv\n#   conv5 = Conv2D(filter3, (5,5), padding='same', activation='relu')(input_layer)\n#   bn5 = BatchNormalization()(conv5)\n#   # 3x3 max pooling\n# #   pool = MaxPooling2D((3,3), strides=(1,1), padding='same')(input_layer)\n#   pool = MaxPooling2D((2,2), strides=(1,1), padding='same')(input_layer)\n#   # concatenate filters, assumes filters/channels last\n#   layer_out = Concatenate(axis=-1)([ bn3, bn5, pool])\n#   return layer_out","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# input_im = Input(shape=(256, 384, 3))\n# def model2():\n#   x = Conv2D(64, (3, 3), padding='same', strides=(2, 2), activation='relu', )(input_im)\n# #   x = MaxPooling2D((2, 2))(x)\n# #   x = Conv2D(64, (1, 1), padding='same', strides=(1, 1), activation='relu', )(x)\n# # #   x =  Conv2D(64, (3, 3), padding='same', strides=(1, 1), activation='relu', )(x) \n#   x = Conv2D(96, (3, 3), padding='same', strides=(1, 1), activation='relu', )(x)\n#   x = MaxPooling2D((2, 2))(x)\n# #   x = Conv2D(16, (3, 3), padding='same', activation='relu', )(input_im)\n# #   x = Conv2D(32, (3, 3), padding='same', activation='relu', )(input_im)  \n# #   x = Conv2D(64, (3, 3), padding='same', activation='relu', )(x)  \n#   x1 = inception_like(x, 64, 64, 32)\n#   x1 = MaxPooling2D((3, 3), padding='same', strides=(2, 2) )(x1)\n\n#   x2 = inception_like(x1, 64, 64, 32)\n#   x2 = MaxPooling2D((3, 3), padding='same', strides=(2, 2) )(x2)\n  \n#   x2_1 = inception_like(x2, 96, 96, 64)\n#   x2_1 = MaxPooling2D((3, 3), padding='same', strides=(2, 2) )(x2_1)  \n\n#   x3 = inception_like(x2_1, 96, 128, 64)\n#   #x3 = MaxPooling2D((3, 3), padding='same', strides=(2, 2) )(x3)\n#   x3 = MaxPooling2D()(x3)\n\n#   x3_1 = inception_like(x3, 128, 192, 128)\n#   x3_1 = MaxPooling2D((3, 3), padding='same', strides=(2, 2) )(x3_1) \n#   x4   = inception_like(x3_1, 128, 256, 128)  \n#   x4 = GlobalAveragePooling2D()(x4)\n\n#   x4 = Flatten()(x4)\n#   x4 = Dense(256, kernel_regularizer=l2(l2=0.03))(x4)\n#   x4 = Dropout(0.2)(x4)\n\n#   #x5 = Dense(128, kernel_regularizer=l2(l2=0.02))(x4)\n#   #x5 = Dropout(0.1)(x5)\n\n#   pred = Dense(len(mlb.classes_), activation='sigmoid')(x4)\n#   model = Model(inputs=input_im, outputs=pred, name='Inception_Like')\n\n#   return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pretrain_model = InceptionV3(input_shape=(256, 256, 3), include_top=False, \n                             weights=\"imagenet\")\n\nx = pretrain_model.output\nx = GlobalAveragePooling2D()(x)\n#fully connected layer\nx = Dense(64, activation='relu')(x)\nx = Dropout(0.5)(x)\nx = Dense(16, activation='relu')(x)\npreds = Dense(len(mlb.classes_), activation='sigmoid')(x)\n\nfinal_Inception_model = Model(inputs=pretrain_model.input, \n                              outputs=preds)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:06:28.113556Z","iopub.execute_input":"2021-05-26T16:06:28.114031Z","iopub.status.idle":"2021-05-26T16:06:34.803798Z","shell.execute_reply.started":"2021-05-26T16:06:28.113989Z","shell.execute_reply":"2021-05-26T16:06:34.802988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plant_path_model2 = model2()\n# plant_path_model2.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_Inception_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-05-26T14:39:06.633491Z","iopub.execute_input":"2021-05-26T14:39:06.633823Z","iopub.status.idle":"2021-05-26T14:39:06.771751Z","shell.execute_reply.started":"2021-05-26T14:39:06.633791Z","shell.execute_reply":"2021-05-26T14:39:06.770915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tf.keras.utils.plot_model(final_Inception_model, show_shapes=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class customCallbacks(tf.keras.callbacks.Callback):\n  def on_epoch_end(self, epoch, logs=None):\n    self.epoch = epoch + 1\n    if self.epoch % 2 == 0:\n      print (\n          'epoch num {}, train loss: {}, validation loss: {}'.format(epoch, logs['loss'], logs['val_loss']))\n\nreduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_f1_score', factor=0.8,\n                              patience=5, min_lr=1e-5, verbose=1)\n\nf1 = tfa.metrics.F1Score(num_classes=len(class_labels), average='macro')\nadam = Adam(learning_rate=4e-4)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:06:38.861103Z","iopub.execute_input":"2021-05-26T16:06:38.861446Z","iopub.status.idle":"2021-05-26T16:06:38.875292Z","shell.execute_reply.started":"2021-05-26T16:06:38.861411Z","shell.execute_reply":"2021-05-26T16:06:38.874173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# positive_weights = {}\n# negative_weights = {}\n# for c in class_labels:\n#     positive_weights[c] = final_train_read.shape[0]/(2*np.count_nonzero(final_train_read[c]==1))\n#     negative_weights[c] = final_train_read.shape[0]/(2*np.count_nonzero(final_train_read[c]==0))\n# print(positive_weights)\n# print(negative_weights)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def loss_fn(y_true,y_pred):\n#     loss = 0\n#     loss -= (positive_weights['complex'])*y_true[0]*K.log(y_pred[0]) + negative_weights['complex']*(1-y_true[0])*K.log(1-y_pred[0]))\n#     loss -= (positive_weights['frog_eye_leaf_spot']*y_true[1]*K.log(y_pred[1]) + negative_weights['frog_eye_leaf_spot']*(1-y_true[1])*K.log(1-y_pred[1]))\n#     loss -= (positive_weights['healthy']*y_true[2]*K.log(y_pred[2]) + negative_weights['healthy']*(1-y_true[2])*K.log(1-y_pred[2]))\n#     loss -= (positive_weights['powdery_mildew']*y_true[3]*K.log(y_pred[3]) + negative_weights['powdery_mildew']*(1-y_true[3])*K.log(1-y_pred[3]))\n#     loss -= (positive_weights['rust']*y_true[4]*K.log(y_pred[4]) + negative_weights['rust']*(1-y_true[4])*K.log(1-y_pred[4]))\n#     loss -= (positive_weights['scab']*y_true[5]*K.log(y_pred[5]) + negative_weights['scab']*(1-y_true[5])*K.log(1-y_pred[5]))    \n\n#     return loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def multi_category_focal_loss2(gamma=2., alpha=.25):\n    \"\"\"\n    focal loss for multi category of multi label problem\n    Focal loss for multi-class or multi-label problems\n         Alpha controls the weight when the true value y_true is 1/0\n                 The weight of 1 is alpha, and the weight of 0 is 1-alpha.\n         When your model is under-fitting and you have difficulty learning, you can try to apply this function as a loss.\n         When the model is too aggressive (whenever it tends to predict 1), try to reduce the alpha\n         When the model is too inert (whenever it always tends to predict 0, or a fixed constant, it means that no valid features are learned)\n                 Try to increase the alpha and encourage the model to predict 1.\n    Usage:\n     model.compile(loss=[multi_category_focal_loss2(alpha=0.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    epsilon = 1.e-7\n    gamma = float(gamma)\n    alpha = tf.constant(alpha, dtype=tf.float32)\n\n    def multi_category_focal_loss2_fixed(y_true, y_pred):\n        y_true = tf.cast(y_true, tf.float32)\n        y_pred = tf.clip_by_value(y_pred, epsilon, 1. - epsilon)\n    \n        alpha_t = y_true*alpha + (tf.ones_like(y_true)-y_true)*(1-alpha)\n        y_t = tf.multiply(y_true, y_pred) + tf.multiply(1-y_true, 1-y_pred)\n        ce = -tf.math.log(y_t)\n        weight = tf.pow(tf.subtract(1., y_t), gamma)\n        fl = tf.multiply(tf.multiply(weight, ce), alpha_t)\n        loss = tf.reduce_mean(fl)\n        return loss\n    return multi_category_focal_loss2_fixed\n# plant_path_model2.compile(optimizer = adam, \n#               loss = \"binary_crossentropy\",\n#               metrics = [tf.keras.metrics.BinaryAccuracy(), tf.keras.metrics.AUC()])\n# plant_path_model2.compile(optimizer = adam, \n#               loss=[multi_category_focal_loss2(alpha=0.25, gamma=2)],\n#               metrics = [tf.keras.metrics.AUC(), f1])\n\n# plant_path_model2.compile(optimizer = adam, \n#               loss= \"binary_crossentropy\",\n#               metrics = [tf.keras.metrics.AUC(), f1])\n\nfinal_Inception_model.compile(optimizer = adam, \n              loss= \"binary_crossentropy\",\n              metrics = [tf.keras.metrics.AUC(), f1])","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:06:44.653859Z","iopub.execute_input":"2021-05-26T16:06:44.65421Z","iopub.status.idle":"2021-05-26T16:06:44.687708Z","shell.execute_reply.started":"2021-05-26T16:06:44.654177Z","shell.execute_reply":"2021-05-26T16:06:44.686925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history = plant_path_model2.fit(train_data_flow, \n#                     epochs = 5, \n#                     verbose = 1, \n#                     validation_data = valid_data_flow,            \n#                     shuffle = True, callbacks=[reduce_lr])\n\nhistory = final_Inception_model.fit(train_data_flow, \n                    epochs = 30, \n                    verbose = 1, \n                    validation_data = valid_data_flow,            \n                    shuffle = True, callbacks=[reduce_lr]) # can also add the customcallback ","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:06:51.004017Z","iopub.execute_input":"2021-05-26T16:06:51.004377Z","iopub.status.idle":"2021-05-26T18:30:27.583112Z","shell.execute_reply.started":"2021-05-26T16:06:51.004325Z","shell.execute_reply":"2021-05-26T18:30:27.582232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Plot Training Curves \n\nPlot all the metrics vs epochs for training and validation. ","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(12, 5))\nfig.add_subplot(131)\nplt.plot(history.history['f1_score'], label='Train-f1-Score')\nplt.plot(history.history['val_f1_score'], label='Valid-f1-Score')\n# plt.title('Model Accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(fontsize=12)\n# \"Loss\"\nfig.add_subplot(132)\nplt.plot(history.history['loss'], label='Train-Loss')\nplt.plot(history.history['val_loss'], label='Valid-Loss')\nplt.yscale('log')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(fontsize=12)\n# \"AUC\"\nfig.add_subplot(133)\nplt.plot(history.history['auc'], label='Train-AUC')\nplt.plot(history.history['val_auc'], label='Valid-AUC')\nplt.ylabel('AUC')\nplt.xlabel('Epoch')\nplt.legend(fontsize=12)\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:35:46.230084Z","iopub.execute_input":"2021-05-26T18:35:46.230706Z","iopub.status.idle":"2021-05-26T18:35:47.047024Z","shell.execute_reply.started":"2021-05-26T18:35:46.230652Z","shell.execute_reply":"2021-05-26T18:35:47.046257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the final scores on the validation data\n\n# loss, auc, f1 = plant_path_model2.evaluate(valid_data_flow,verbose=1)\n\nloss, auc, f1 = final_Inception_model.evaluate(valid_data_flow,verbose=1)\nprint ('check final loss, auc and f1 score: ', loss, auc, f1)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:36:01.407123Z","iopub.execute_input":"2021-05-26T18:36:01.407477Z","iopub.status.idle":"2021-05-26T18:36:53.745056Z","shell.execute_reply.started":"2021-05-26T18:36:01.407444Z","shell.execute_reply":"2021-05-26T18:36:53.744228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prepare for Submission  ","metadata":{}},{"cell_type":"code","source":"test_csv =\"../input/plant-pathology-2021-fgvc8/sample_submission.csv\"\ntest_df = pd.read_csv(test_csv)\ntest_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:37:15.645925Z","iopub.execute_input":"2021-05-26T18:37:15.64625Z","iopub.status.idle":"2021-05-26T18:37:15.666693Z","shell.execute_reply.started":"2021-05-26T18:37:15.64622Z","shell.execute_reply":"2021-05-26T18:37:15.665715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For Test Data we only do scaling and no other augmentation (of course!) ","metadata":{}},{"cell_type":"code","source":"test_datagen = tf.keras.preprocessing.image.ImageDataGenerator(rescale = 1./255)\ntest_im_path = '../input/plant-pathology-2021-fgvc8/test_images/'\n\ntest_data_flow = test_datagen.flow_from_dataframe(test_df,\n    directory = test_im_path,\n    x_col = 'image',\n    y_col = None,\n    subset=\"training\",\n    color_mode=\"rgb\",\n    batch_size=32,\n    target_size = (256,256), \n    class_mode=None,                                                \n    shuffle=True,\n    seed=40)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:37:23.86258Z","iopub.execute_input":"2021-05-26T18:37:23.862896Z","iopub.status.idle":"2021-05-26T18:37:23.881014Z","shell.execute_reply.started":"2021-05-26T18:37:23.862864Z","shell.execute_reply":"2021-05-26T18:37:23.880218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Get the Predicitions on the Test Data ","metadata":{}},{"cell_type":"code","source":"# predictions = plant_path_model2.predict(test_data_flow)\npredictions = final_Inception_model.predict(test_data_flow)\n\nprint ('num predictions: ', predictions.shape)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:37:28.929168Z","iopub.execute_input":"2021-05-26T18:37:28.929514Z","iopub.status.idle":"2021-05-26T18:37:31.247294Z","shell.execute_reply.started":"2021-05-26T18:37:28.929482Z","shell.execute_reply":"2021-05-26T18:37:31.24645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print ('check predictions: ', predictions)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:37:34.205155Z","iopub.execute_input":"2021-05-26T18:37:34.205501Z","iopub.status.idle":"2021-05-26T18:37:34.210709Z","shell.execute_reply.started":"2021-05-26T18:37:34.205468Z","shell.execute_reply":"2021-05-26T18:37:34.20966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Predictions to Labels \n\n1. Set the threshold (like a hyperparameter) and get back indices for which the predictions are over this threshold for an image. \n2. From the indices get back the original labels. \n3. If more than 2 labels--- join them as a string with a space in between ","metadata":{}},{"cell_type":"code","source":"set_threshold = 0.30\npredictions_list = predictions.tolist()\n\n#['complex', 'frog_eye_leaf_spot', 'healthy', 'powdery_mildew', 'rust', 'scab']\n\n\nindices = []\nfor pred in predictions_list:\n    temp = []\n    for category in pred:\n        if category>=set_threshold:\n            temp.append(pred.index(category))\n    if temp!=[]:\n        indices.append(temp)\n    else:\n        temp.append(np.argmax(pred))\n        indices.append(temp)\n    \nprint(indices)\n\n\nclass_labels_dict = dict(list(enumerate(class_labels)))\nprint (class_labels_dict)\n\ntestlabels = []\n\n\nfor im in indices:\n    temp1 = []\n    for i in im:\n        temp1.append(str(class_labels_dict[i]))\n    testlabels.append(' '.join(temp1))\n\nprint(testlabels)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:38:06.478308Z","iopub.execute_input":"2021-05-26T18:38:06.478667Z","iopub.status.idle":"2021-05-26T18:38:06.490183Z","shell.execute_reply.started":"2021-05-26T18:38:06.478637Z","shell.execute_reply":"2021-05-26T18:38:06.489394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### View the Test Images and Corresponding Predictions","metadata":{}},{"cell_type":"code","source":"\nfig = plt.figure(figsize=(9, 6))\nnpics= 3\ncount = 1\nfor i in range(npics):\n  ipic = i\n#   ipic = np.random.choice(test_df.shape[0])\n  sample = os.path.join(test_im_path, test_df['image'][ipic])\n  sample_img = io.imread(sample)   \n  ax = fig.add_subplot(npics/1 , 3, count, xticks=[],yticks=[])\n#   title_string = ''\n#   if len(train_read['labels'][ipic]) > 1:  \n#       for x in train_read['labels'][ipic]:            \n#         title_string = title_string + x +',' +f\"{sample_img.shape}\"\n#   else: \n#     title_string += train_read['labels'][ipic][0] + f\"{sample_img.shape}\"\n  ax.set_title(testlabels[ipic], fontsize=10)  \n  plt.imshow(sample_img)\n  count = count + 1  \n\nplt.tight_layout()\nplt.show()   ","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:38:12.979209Z","iopub.execute_input":"2021-05-26T18:38:12.979558Z","iopub.status.idle":"2021-05-26T18:38:15.793042Z","shell.execute_reply.started":"2021-05-26T18:38:12.979527Z","shell.execute_reply":"2021-05-26T18:38:15.79227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### submission\n\ntest_df['labels'] = testlabels\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:38:21.288548Z","iopub.execute_input":"2021-05-26T18:38:21.288897Z","iopub.status.idle":"2021-05-26T18:38:21.300385Z","shell.execute_reply.started":"2021-05-26T18:38:21.288864Z","shell.execute_reply":"2021-05-26T18:38:21.299307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_csv('submission.csv', index=False)\ntest_df","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:38:25.15411Z","iopub.execute_input":"2021-05-26T18:38:25.154443Z","iopub.status.idle":"2021-05-26T18:38:25.38608Z","shell.execute_reply.started":"2021-05-26T18:38:25.154411Z","shell.execute_reply":"2021-05-26T18:38:25.385123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}