{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import the necessary libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport tensorflow.keras as keras\nimport PIL\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nimport random\nfrom tqdm import tqdm\nimport tensorflow_addons as tfa\nfrom sklearn.preprocessing import MultiLabelBinarizer\nimport matplotlib.pyplot as plt \n\npd.set_option(\"display.max_columns\", None)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-06T13:45:40.166654Z","iopub.execute_input":"2022-04-06T13:45:40.167025Z","iopub.status.idle":"2022-04-06T13:45:45.685969Z","shell.execute_reply.started":"2022-04-06T13:45:40.166930Z","shell.execute_reply":"2022-04-06T13:45:45.685065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's explore the data.\nHow many images are in the datset, the labels and their frequencies.","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/plant-pathology-2021-fgvc8/train.csv')\n#print(len(train))\n#print(train.columns)\n#print(train['labels'].value_counts())\n#print(train['labels'].value_counts().plot.bar())","metadata":{"execution":{"iopub.status.busy":"2022-04-06T13:45:49.337247Z","iopub.execute_input":"2022-04-06T13:45:49.337577Z","iopub.status.idle":"2022-04-06T13:45:49.370324Z","shell.execute_reply.started":"2022-04-06T13:45:49.337546Z","shell.execute_reply":"2022-04-06T13:45:49.369494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['labels'] = train['labels'].apply(lambda string: string.split(' '))\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-04-06T13:45:51.433175Z","iopub.execute_input":"2022-04-06T13:45:51.433540Z","iopub.status.idle":"2022-04-06T13:45:51.472291Z","shell.execute_reply.started":"2022-04-06T13:45:51.433509Z","shell.execute_reply":"2022-04-06T13:45:51.471530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First I convert the labels representation into **one hot encoded format** using MultilabelBinarizer from sklearn. Now we can see and plot the frequencies of each label. ","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt \n\nplt.rc('font', size=20)        # 기본 폰트 크기\n\ns = list(train['labels'])\nmlb = MultiLabelBinarizer()\ntrainx = pd.DataFrame(mlb.fit_transform(s), columns=mlb.classes_, index=train.index)\nprint(trainx.columns)\nprint(trainx.sum())\n\nlabels = list(trainx.sum().keys())\nprint(labels)\nlabel_counts = trainx.sum().values.tolist()\n\nfig, ax = plt.subplots(1,1, figsize=(20,6))\n\nsns.barplot(x= labels, y= label_counts, ax=ax)","metadata":{"execution":{"iopub.status.busy":"2022-04-06T13:46:08.079254Z","iopub.execute_input":"2022-04-06T13:46:08.079581Z","iopub.status.idle":"2022-04-06T13:46:08.279163Z","shell.execute_reply.started":"2022-04-06T13:46:08.079549Z","shell.execute_reply":"2022-04-06T13:46:08.278270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's view some of the images","metadata":{}},{"cell_type":"code","source":"fig1 = plt.figure(figsize=(26,10))\n\nfor i in range(1, 13):\n    \n    rand =  random.randrange(1, 18000)\n    sample = os.path.join('../input/plant-pathology-2021-fgvc8/train_images/', train['image'][rand])\n    \n    img = PIL.Image.open(sample)\n    \n    ax = fig1.add_subplot(4,3,i)\n    ax.imshow(img)\n    \n    title = f\"{train['labels'][rand]}{img.size}\"\n    plt.title(title)\n    \n    fig1.tight_layout()\n","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:06:10.977739Z","iopub.execute_input":"2022-04-02T16:06:10.978007Z","iopub.status.idle":"2022-04-02T16:06:24.476811Z","shell.execute_reply.started":"2022-04-02T16:06:10.97798Z","shell.execute_reply":"2022-04-02T16:06:24.474697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imaze Size & Processing\nfrom the titles we can see some random image sizes - (4000, 2672). Larger images are harder to process hence takes much longer to train the CNN. Downsampling all these 18632 images is also a time consuming task. This is I am going to use the resized imaged for this dataset [resized-plant2021](https://www.kaggle.com/ankursingh12/resized-plant2021) by Ankur Singh. He has already downsampled the images into size of 256, 384, 512 & 640px.\n\nThere are 18632 images in the training set. Even after using the downsampled images we cant fit all of the images into memory at once. So I have used the flow_from_dataframe method from keras. This method reads images in batch size from the storage without loading all the images at once and saving us from **GPU Out of Memory (OOM)** issue. ","metadata":{}},{"cell_type":"code","source":"df=pd.read_csv(\"../input/plant-pathology-2021-fgvc8/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:06:24.478858Z","iopub.execute_input":"2022-04-02T16:06:24.479165Z","iopub.status.idle":"2022-04-02T16:06:24.50323Z","shell.execute_reply.started":"2022-04-02T16:06:24.479134Z","shell.execute_reply":"2022-04-02T16:06:24.502281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\ndatagen = ImageDataGenerator(rescale=1/255.0,\n                            rotation_range=5,\n                            zoom_range=0.1,\n                            shear_range=0.05,\n                            horizontal_flip=True,\n                            validation_split=0.2)\n\ntrain_generator = datagen.flow_from_dataframe(\n    train,\n    directory='../input/resized-plant2021/img_sz_256',\n    subset='training',\n    x_col='image',\n    y_col='labels',\n    target_size=(224,224),\n    color_mode='rgb',\n    class_mode='categorical',\n    batch_size=32,\n    shuffle=True,\n    seed=444\n    )\n\n#'../input/plant-pathology-2021-fgvc8/train_images'\nvalid_generator = datagen.flow_from_dataframe(\n    train,\n    directory='../input/resized-plant2021/img_sz_256',\n    subset='validation',\n    x_col='image',\n    y_col='labels',\n    target_size=(224,224),\n    color_mode='rgb',\n    class_mode='categorical',\n    batch_size=32,\n    shuffle=True,\n    seed=444\n    )","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:06:24.504693Z","iopub.execute_input":"2022-04-02T16:06:24.505258Z","iopub.status.idle":"2022-04-02T16:06:49.08806Z","shell.execute_reply.started":"2022-04-02T16:06:24.50522Z","shell.execute_reply":"2022-04-02T16:06:49.078474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transfer Learning\nTransfer learning is the process of using frozen weights from a large pre-trained model for a downstream task which is in our case classifying leaf diseases. As we can't use internet in this notebook, I will use the dataset of keras's pretrained models containing the weights of 'imagenet'. The output/top layer of a pretrained layer is a dense layer containing number of nodes = number of output classes. All the models here are pre-trained on 'imagenet' hence they have a output/top layer of 1000 nodes. We will have to replace the output/top layer with our own dense layer with 6 nodes (for 6 classes). \n\nI am going to be using **Inception ResNet v2**.\n","metadata":{}},{"cell_type":"code","source":"seed = 1200\ntf.random.set_seed(seed)\n\nweights_path = '../input/keras-pretrained-models/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5'\nmodel = keras.applications.InceptionResNetV2(weights=weights_path, include_top=False, input_shape=(224, 224, 3))\n\nprint(model.input)\nprint(model.output)","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:06:49.092427Z","iopub.execute_input":"2022-04-02T16:06:49.092917Z","iopub.status.idle":"2022-04-02T16:06:59.151487Z","shell.execute_reply.started":"2022-04-02T16:06:49.092851Z","shell.execute_reply":"2022-04-02T16:06:59.150544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Activation, Losses & Metrices\n\nAs this is a multilabel classification problem, we can't use softmax here, hence the sigmoid activation.\n\nBinary crossentropy is used instead of categorical crossentropy. We use categorical cross-entropy in multi-class problems, but for multi-label problems, we use binary cross-entropy. Think of it this way, an image may have multiple labels, and we need the probabilities that each of these labels corresponds to the given image - this can be considered as n independent binary classifiers for the n labels.\n","metadata":{}},{"cell_type":"code","source":"new_model = tf.keras.Sequential([\n    model,\n    keras.layers.GlobalAveragePooling2D(),\n    keras.layers.Dense(6, \n        kernel_initializer=keras.initializers.RandomUniform(seed=seed),\n        bias_initializer=keras.initializers.Zeros(), name='dense_top', activation='sigmoid')\n])\n\n# Freezing the weights\nfor layer in new_model.layers[:-1]:\n    layer.trainable=False\n    \nnew_model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:06:59.152735Z","iopub.execute_input":"2022-04-02T16:06:59.153241Z","iopub.status.idle":"2022-04-02T16:07:00.534387Z","shell.execute_reply.started":"2022-04-02T16:06:59.153203Z","shell.execute_reply":"2022-04-02T16:07:00.533482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metrics = [  \n        keras.metrics.CategoricalAccuracy(name='accuracy'),\n        keras.metrics.Precision(name='precision'),\n        keras.metrics.Recall(name='recall')\n    ]","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:07:00.535637Z","iopub.execute_input":"2022-04-02T16:07:00.535972Z","iopub.status.idle":"2022-04-02T16:07:00.558475Z","shell.execute_reply.started":"2022-04-02T16:07:00.535944Z","shell.execute_reply":"2022-04-02T16:07:00.557742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping\n\nf1 = tfa.metrics.F1Score(num_classes=6,average='macro')\n\nnew_model.compile(optimizer='adam', loss='binary_crossentropy',metrics=[metrics,f1])\ncallbacks = keras.callbacks.EarlyStopping(monitor=f1, patience=4, mode='max', restore_best_weights=True)\nhistory = new_model.fit_generator(generator=train_generator,\n                    validation_data=valid_generator,\n                    epochs=200,\n                    steps_per_epoch=train_generator.samples//128, \n                    validation_steps=valid_generator.samples//128,\n                    callbacks=callbacks)","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:07:00.559853Z","iopub.execute_input":"2022-04-02T16:07:00.560215Z","iopub.status.idle":"2022-04-02T16:08:52.222226Z","shell.execute_reply.started":"2022-04-02T16:07:00.560178Z","shell.execute_reply":"2022-04-02T16:08:52.218982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\nFor submission I will resize the test images and then predict the labels for them.","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('../input/plant-pathology-2021-fgvc8/sample_submission.csv')\n\nfor img_name in tqdm(test['image']):\n    path = '../input/plant-pathology-2021-fgvc8/test_images/'+str(img_name)\n    with PIL.Image.open(path) as img:\n        img = img.resize((256,256))\n        img.save(f'./{img_name}')","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:08:52.224264Z","iopub.status.idle":"2022-04-02T16:08:52.224759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = datagen.flow_from_dataframe(\n    test,\n    directory = './',\n    x_col=\"image\",\n    y_col= None,\n    color_mode=\"rgb\",\n    target_size = (256,256),\n    classes=None,\n    class_mode=None,\n    batch_size=32,\n    shuffle=False,\n    seed=40,\n)\n\npreds = new_model.predict(test_data)\nprint(preds)\npreds = preds.tolist()\n\nindices = []\nfor pred in preds:\n    temp = []\n    for category in pred:\n        if category>=0.3:\n            temp.append(pred.index(category))\n    if temp!=[]:\n        indices.append(temp)\n    else:\n        temp.append(np.argmax(pred))\n        indices.append(temp)\n    \nprint(indices)","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:08:52.225963Z","iopub.status.idle":"2022-04-02T16:08:52.230732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = (train_generator.class_indices)\nlabels = dict((v,k) for k,v in labels.items())\nprint(labels)\n\ntestlabels = []\n\n\nfor image in indices:\n    temp = []\n    for i in image:\n        temp.append(str(labels[i]))\n    testlabels.append(' '.join(temp))\n\nprint(testlabels)\n","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:08:52.231892Z","iopub.status.idle":"2022-04-02T16:08:52.23267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Remove the resized images from output before submission. if there are any other files present except 'submission.csv' it will throw an error when submitting.","metadata":{}},{"cell_type":"code","source":"delfiles = tf.io.gfile.glob('./*.jpg')\n\nfor file in delfiles:\n    os.remove(file)","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:08:52.233877Z","iopub.status.idle":"2022-04-02T16:08:52.240779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# accuracy\nplt.rc('font', size=20)\nplt.figure(figsize=(15,12))\nepoch_list = list(range(1, len(history.history['accuracy']) + 1))\nplt.plot(epoch_list, history.history['accuracy'],label='accuracy')\n#plt.plot(epoch_list, history.history['val_accuracy'],label='val_accuracy')\nplt.xlabel('epoches')\nplt.ylabel('accuracy')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:08:52.241965Z","iopub.status.idle":"2022-04-02T16:08:52.242756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loss\nplt.rc('font', size=20)\nplt.figure(figsize=(15,12))\nepoch__list = list(range(1,len(history.history['loss'])+1))\nplt.plot(epoch__list, history.history['loss'],label='loss')\n#plt.plot(epoch__list, history.history['val_loss'],label='val_loss')\nplt.xlabel('epoches')\nplt.ylabel('loss')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:08:52.243966Z","iopub.status.idle":"2022-04-02T16:08:52.244809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n# f1  score\nplt.rc('font', size=20)\nplt.figure(figsize=(15,12))\nepoch__list = list(range(1,len(history.history['f1_score'])+1))\nplt.plot(epoch__list, history.history['f1_score'],label='f1_score')\n#plt.plot(epoch__list, history.history['val_f1_score'],label='val_f1_score')\nplt.xlabel('epoches')\nplt.ylabel('f1')\nplt.legend()\nplt.show()\n'''","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:08:52.247111Z","iopub.status.idle":"2022-04-02T16:08:52.247909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# precision\nplt.rc('font', size=20)\nplt.figure(figsize=(15,12))\nepoch_list = list(range(1, len(history.history['precision']) + 1))\nplt.plot(epoch_list, history.history['precision'],label='precision')\n#plt.plot(epoch_list, history.history['val_precision'],label='val_precision')\nplt.xlabel('epoches')\nplt.ylabel('precision')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:08:52.249295Z","iopub.status.idle":"2022-04-02T16:08:52.250074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# recall\nplt.rc('font', size=20)\nplt.figure(figsize=(15,12))\nepoch_list = list(range(1, len(history.history['recall']) + 1))\nplt.plot(epoch_list, history.history['recall'],label='recall')\n#plt.plot(epoch_list, history.history['val_recall'],label='val_recall')\nplt.xlabel('epoches')\nplt.ylabel('recall')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:08:52.251681Z","iopub.status.idle":"2022-04-02T16:08:52.256424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\narr1 = history.history['loss']\nresult1 = sum(arr1)\nprint(f\"loss_av : {result1 / len(arr1)}\")\n\narr2 = history.history['accuracy']\nresult2 = sum(arr2)\nprint(f\"accuracy_av : {result2 / len(arr2)}\")\n\narr3 = history.history['val_loss']\nresult3 = sum(arr3)\nprint(f\"val_loss_av : {result3 / len(arr3)}\")\n\narr4 = history.history['val_accuracy']\nresult4 = sum(arr4)\nprint(f\"val_accuracy_av : {result4 / len(arr4)}\")\n'''","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:08:52.257851Z","iopub.status.idle":"2022-04-02T16:08:52.258538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/plant-pathology-2021-fgvc8/sample_submission.csv')\nsub['labels'] = testlabels\nsub.to_csv('submission.csv', index=False)\nsub","metadata":{"execution":{"iopub.status.busy":"2022-04-02T16:08:52.264029Z","iopub.status.idle":"2022-04-02T16:08:52.264702Z"},"trusted":true},"execution_count":null,"outputs":[]}]}