{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Plant Pathology 2021 - FGVC8","metadata":{}},{"cell_type":"markdown","source":"A competição teve por objetivo encontrar doenças que acometem plantas, mais especificamente folhas de macieiras, através de imagens RGB de alta qualidade.\n","metadata":{}},{"cell_type":"markdown","source":"A métrica de avaliação utilizada na competição é a F1 Score, definida como a média harmônica entre a precisão e a sensibilidade.","metadata":{}},{"cell_type":"code","source":"import keras\nimport tensorflow\nimport numpy as np\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPool2D, Flatten, Dense, Dropout, BatchNormalization\nimport pandas as pd\npd.set_option('display.max_columns', None)\nimport os\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nsns.set(style=\"whitegrid\")\nfrom PIL import Image\nimport warnings\nwarnings.filterwarnings('ignore')\n","metadata":{"execution":{"iopub.status.busy":"2022-10-06T21:38:35.143326Z","iopub.execute_input":"2022-10-06T21:38:35.143991Z","iopub.status.idle":"2022-10-06T21:38:35.164523Z","shell.execute_reply.started":"2022-10-06T21:38:35.143941Z","shell.execute_reply":"2022-10-06T21:38:35.162562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Análise Exploratória","metadata":{}},{"cell_type":"markdown","source":"Os dados são compostos por uma pasta com as imagens junto com um arquivo com a classificação da imagem e seu título.","metadata":{}},{"cell_type":"code","source":"data_path = '../input/plant-pathology-2021-fgvc8'\nlabels_file_path = os.path.join(data_path, 'train.csv')\ntrain_images_path = os.path.join(data_path, 'train_images')\ntest_images_path = os.path.join(data_path, 'test_images')\n","metadata":{"execution":{"iopub.status.busy":"2022-10-06T21:38:35.168571Z","iopub.execute_input":"2022-10-06T21:38:35.169742Z","iopub.status.idle":"2022-10-06T21:38:35.180657Z","shell.execute_reply.started":"2022-10-06T21:38:35.169676Z","shell.execute_reply":"2022-10-06T21:38:35.179252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/plant-pathology-2021-fgvc8/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-10-06T21:38:35.183434Z","iopub.execute_input":"2022-10-06T21:38:35.185008Z","iopub.status.idle":"2022-10-06T21:38:35.197917Z","shell.execute_reply.started":"2022-10-06T21:38:35.184963Z","shell.execute_reply":"2022-10-06T21:38:35.196382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(labels_file_path)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T21:38:35.202228Z","iopub.execute_input":"2022-10-06T21:38:35.202540Z","iopub.status.idle":"2022-10-06T21:38:35.230289Z","shell.execute_reply.started":"2022-10-06T21:38:35.202512Z","shell.execute_reply":"2022-10-06T21:38:35.228828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-06T21:38:35.234010Z","iopub.execute_input":"2022-10-06T21:38:35.234493Z","iopub.status.idle":"2022-10-06T21:38:35.244299Z","shell.execute_reply.started":"2022-10-06T21:38:35.234450Z","shell.execute_reply":"2022-10-06T21:38:35.242876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Possuímos então 18632 imagens no total","metadata":{}},{"cell_type":"code","source":"train_df['labels'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T21:38:35.246315Z","iopub.execute_input":"2022-10-06T21:38:35.247922Z","iopub.status.idle":"2022-10-06T21:38:35.264170Z","shell.execute_reply.started":"2022-10-06T21:38:35.247878Z","shell.execute_reply":"2022-10-06T21:38:35.262419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Possuímos então 5 diferentes tipos de doença:\n\n1. Scab\n2. Frogeye leaf spot\n3. Rust\n4. Complex\n5. Powdery mildew\n\nAlgumas combinações entre elas existem, como **Scab** com **Frogeye Leaf Spot**\n","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,12))\nlabels = sns.barplot(train_df.labels.value_counts().index,train_df.labels.value_counts(),palette=\"turbo\")\nfor item in labels.get_xticklabels():\n    item.set_rotation(90)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T21:38:35.266527Z","iopub.execute_input":"2022-10-06T21:38:35.267755Z","iopub.status.idle":"2022-10-06T21:38:35.681656Z","shell.execute_reply.started":"2022-10-06T21:38:35.267692Z","shell.execute_reply":"2022-10-06T21:38:35.680379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizando as imagens","metadata":{}},{"cell_type":"code","source":"def show_image(class_name = 'healthy', examples=2):\n    image_list = train_df[train_df['labels'] == class_name]['image'].sample(frac=1)[:examples].to_list()\n    plt.figure(figsize=(20,10))\n    for i, img in enumerate(image_list):\n        full_path = os.path.join(train_images_path, img)\n        img = Image.open(full_path)\n        plt.subplot(1 ,examples, i%examples +1)\n        plt.axis('off')\n        plt.imshow(img)\n        plt.title(class_name)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T21:38:35.683757Z","iopub.execute_input":"2022-10-06T21:38:35.684259Z","iopub.status.idle":"2022-10-06T21:38:35.693657Z","shell.execute_reply.started":"2022-10-06T21:38:35.684213Z","shell.execute_reply":"2022-10-06T21:38:35.691811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Healthy ","metadata":{}},{"cell_type":"code","source":"show_image('healthy', examples = 3)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T21:38:35.695198Z","iopub.execute_input":"2022-10-06T21:38:35.696098Z","iopub.status.idle":"2022-10-06T21:38:40.758760Z","shell.execute_reply.started":"2022-10-06T21:38:35.696053Z","shell.execute_reply":"2022-10-06T21:38:40.757602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Bem fácil de distinguir das demais, uma folha considerada saudável não apresenta nenhuma irregularidade na superfície.","metadata":{}},{"cell_type":"markdown","source":"## Scab","metadata":{}},{"cell_type":"markdown","source":"Scab é uma doença causada por fungos e deixa as folhas da planta afetada com \"crostas\" na superfície.","metadata":{}},{"cell_type":"code","source":"show_image('scab', examples = 3)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T21:38:40.765373Z","iopub.execute_input":"2022-10-06T21:38:40.766038Z","iopub.status.idle":"2022-10-06T21:38:43.908985Z","shell.execute_reply.started":"2022-10-06T21:38:40.765997Z","shell.execute_reply":"2022-10-06T21:38:43.907537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"É possível observar alguns pontos mais claros nas folhas, relativamente amarelados no centro.","metadata":{}},{"cell_type":"markdown","source":"## Rust","metadata":{}},{"cell_type":"markdown","source":"Outra doença causada por fungos, afeta principalmente as folhas da planta, deixando pontos amarelados na planta, evoluindo para grandes manchas alaranjadas e pretas.","metadata":{}},{"cell_type":"code","source":"show_image('rust', examples = 3)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T21:38:43.911034Z","iopub.execute_input":"2022-10-06T21:38:43.912328Z","iopub.status.idle":"2022-10-06T21:38:49.460942Z","shell.execute_reply.started":"2022-10-06T21:38:43.912281Z","shell.execute_reply":"2022-10-06T21:38:49.459586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Complex","metadata":{}},{"cell_type":"markdown","source":"As plantas classificadas como complexas são plantas com doenças difíceis de se classificar visualmente, muitas vezes possuindo inúmeras doenças diferentes.","metadata":{}},{"cell_type":"code","source":"show_image('complex', examples = 3)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T21:38:49.462863Z","iopub.execute_input":"2022-10-06T21:38:49.463865Z","iopub.status.idle":"2022-10-06T21:38:54.557290Z","shell.execute_reply.started":"2022-10-06T21:38:49.463822Z","shell.execute_reply":"2022-10-06T21:38:54.556177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Frogeye Leaf Spot","metadata":{}},{"cell_type":"markdown","source":"Outra doença causada por fungos, começa com uma leve coloração roxa, ficando amareladas na medida que crescem com um ponto branco no centro, podendo levar a queda das folhas se não tratadas.","metadata":{}},{"cell_type":"code","source":"show_image('frog_eye_leaf_spot', examples = 3)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T21:38:54.559210Z","iopub.execute_input":"2022-10-06T21:38:54.560007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Powdery Mildew","metadata":{}},{"cell_type":"markdown","source":"Este tipo de mofo causa um aspecto de pó branco nas folhas, é o tipo de doença mais fácil de identificar devido sua característica aparência.","metadata":{}},{"cell_type":"code","source":"show_image('powdery_mildew', examples = 3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pré-Processamento","metadata":{}},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels=np.zeros((18632,6))\nlabels=pd.DataFrame(columns=[\"healthy\",\"scab\",\"frog_eye_leaf_spot\",\n                             \"complex\",\"rust\",\"powdery_mildew\"],data=labels)\nfor i in range(train_df.shape[0]):\n    full_lab=train_df.loc[i,\"labels\"]\n    for j in range(6):\n        lab=labels.columns[j]\n        if lab in full_lab:\n            labels.loc[i,lab]=1\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.index = train_df.index\ntrain_df.drop(\"labels\",axis=1,inplace=True)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=pd.concat([train_df,labels],axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Augmentation ","metadata":{}},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale = 1./255.,\n                                   samplewise_center=True, \n                                   samplewise_std_normalization=True,\n                                   validation_split = 0.2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = train_datagen.flow_from_dataframe(dataframe = data,\n                                                   directory = '../input/resized-plant2021/img_sz_256',\n                                                   target_size = (128,128),\n                                                   x_col = 'image',\n                                                   y_col = list(labels.columns),\n                                                   batch_size = 32,\n                                                   color_mode = 'rgb',\n                                                   class_mode = 'raw',\n                                                   subset = 'training')\n\ntest_generator = train_datagen.flow_from_dataframe(dataframe = data,\n                                                 directory = '../input/resized-plant2021/img_sz_256',\n                                                 target_size = (128,128),\n                                                 x_col = 'image',\n                                                 y_col = list(labels.columns),\n                                                 batch_size = 32,\n                                                 color_mode = 'rgb',\n                                                 class_mode = 'raw',\n                                                 subset = 'validation')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"convnet=Sequential([\n\n    Conv2D(filters=16,kernel_size=5,strides=3,padding=\"same\",activation=\"relu\",name=\"conv1\",input_shape=(128,128,3)),\n    BatchNormalization(name=\"BN1\"),\n    MaxPool2D(pool_size=(2,2),name=\"Pool1\"),\n\n    Conv2D(filters=32,kernel_size=4,strides=2,padding=\"same\",name=\"conv2\",activation=\"relu\"),\n    BatchNormalization(name=\"BN2\"),\n    MaxPool2D(pool_size=(2,2),name=\"Pool2\"),\n\n    Conv2D(filters=64,kernel_size=3,strides=1,padding=\"same\",name=\"conv3\",activation=\"relu\"),\n    BatchNormalization(name=\"BN3\"),\n    MaxPool2D(pool_size=(2,2),name=\"Pool3\"),\n    \n    Conv2D(filters=32,kernel_size=1,strides=1,padding=\"valid\",name=\"conv4\",activation=\"relu\"),\n    BatchNormalization(name=\"BN4\"),\n\n    Flatten(name=\"Flatten\"),\n    \n    Dense(64,activation=\"relu\",name=\"FullyConnected1\"),\n    Dropout(0.3,name=\"DropOut1\"),\n    BatchNormalization(name=\"BN5\"),\n    \n    Dense(32,activation=\"relu\",name=\"FullyConnected2\"),\n    Dropout(0.3,name=\"DropOut2\"),\n    BatchNormalization(name=\"BN6\"),\n    \n    Dense(6,activation=\"softmax\",name=\"OutputDense\")\n])\n\nconvnet.compile(optimizer=\"adam\",loss=\"binary_crossentropy\",metrics=['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"convnet.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = convnet.fit(train_generator, validation_data=test_generator, epochs=10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_datagen = ImageDataGenerator(\n    rescale = 1./255\n)\nINPUT_SIZE = (128,128,3)\ntest_generator =  test_datagen.flow_from_dataframe(\n    submission,\n    directory=\"../input/plant-pathology-2021-fgvc8/test_images\",\n    x_col='image',\n    y_col=None,\n    class_mode=None,\n    target_size=INPUT_SIZE[:2]\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = convnet.predict(test_generator) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predtest = []\nlabeltest = []\npossible = [\"healthy\",\"scab\",\"frog_eye_leaf_spot\",\n            \"complex\",\"rust\",\"powdery_mildew\"]\n\n\nfor row in preds:\n    placeholder = 0\n    for cell in row:\n        if cell >= placeholder:\n            placeholder = cell\n    predtest.append(np.where(preds == placeholder)[1][0].astype(int))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index in predtest:\n    labeltest.append(possible[index])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['labels'] = labeltest\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}