{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-08T19:42:21.263802Z","iopub.execute_input":"2022-08-08T19:42:21.264617Z","iopub.status.idle":"2022-08-08T19:42:21.274482Z","shell.execute_reply.started":"2022-08-08T19:42:21.264485Z","shell.execute_reply":"2022-08-08T19:42:21.273553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import basic libraries from tensorflow\nimport tensorflow as tf\nfrom tensorflow import keras","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:21.298893Z","iopub.execute_input":"2022-08-08T19:42:21.299169Z","iopub.status.idle":"2022-08-08T19:42:22.778120Z","shell.execute_reply.started":"2022-08-08T19:42:21.299144Z","shell.execute_reply":"2022-08-08T19:42:22.777134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Analysis Stage\n\nSee the dataframe features\n\nThe dataframe has in the same row all the possible diseases, we are going to split the labels","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/plant-pathology-2021-fgvc8/train.csv')\ntrain.tail()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-08T19:42:22.780348Z","iopub.execute_input":"2022-08-08T19:42:22.780966Z","iopub.status.idle":"2022-08-08T19:42:22.812014Z","shell.execute_reply.started":"2022-08-08T19:42:22.780927Z","shell.execute_reply":"2022-08-08T19:42:22.810958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Extract all the labels available in the dataframe","metadata":{}},{"cell_type":"code","source":"a = train['labels'].unique()\nfor i in range(len(a)):\n    a[i] = a[i].split()\n    ","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:22.813744Z","iopub.execute_input":"2022-08-08T19:42:22.814122Z","iopub.status.idle":"2022-08-08T19:42:22.822138Z","shell.execute_reply.started":"2022-08-08T19:42:22.814084Z","shell.execute_reply":"2022-08-08T19:42:22.820994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_unique = []\nfor i in range(len(a)):\n    for j in range(len(a[i])):\n        if a[i][j] not in label_unique:\n            label_unique.append(a[i][j])\n        else: \n            pass\n        \n            ","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:22.825141Z","iopub.execute_input":"2022-08-08T19:42:22.825617Z","iopub.status.idle":"2022-08-08T19:42:22.832220Z","shell.execute_reply.started":"2022-08-08T19:42:22.825577Z","shell.execute_reply":"2022-08-08T19:42:22.831106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_unique\ntrain[label_unique] = 0","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:22.833773Z","iopub.execute_input":"2022-08-08T19:42:22.834687Z","iopub.status.idle":"2022-08-08T19:42:22.849965Z","shell.execute_reply.started":"2022-08-08T19:42:22.834650Z","shell.execute_reply":"2022-08-08T19:42:22.846967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"labels\"] = train[\"labels\"].apply(lambda x:x.split(\" \")) ","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:22.852633Z","iopub.execute_input":"2022-08-08T19:42:22.853116Z","iopub.status.idle":"2022-08-08T19:42:22.882137Z","shell.execute_reply.started":"2022-08-08T19:42:22.853076Z","shell.execute_reply":"2022-08-08T19:42:22.881230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"See the results of the transformation","metadata":{}},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:22.883644Z","iopub.execute_input":"2022-08-08T19:42:22.884301Z","iopub.status.idle":"2022-08-08T19:42:22.916917Z","shell.execute_reply.started":"2022-08-08T19:42:22.884263Z","shell.execute_reply":"2022-08-08T19:42:22.915749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(train)):\n    for item in label_unique:\n        if item in train.iloc[i,1]:\n            train.loc[i,item] = 1\n        else: \n            pass","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:22.918484Z","iopub.execute_input":"2022-08-08T19:42:22.919140Z","iopub.status.idle":"2022-08-08T19:42:31.157481Z","shell.execute_reply.started":"2022-08-08T19:42:22.919104Z","shell.execute_reply":"2022-08-08T19:42:31.156466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:31.159075Z","iopub.execute_input":"2022-08-08T19:42:31.159450Z","iopub.status.idle":"2022-08-08T19:42:31.180550Z","shell.execute_reply.started":"2022-08-08T19:42:31.159410Z","shell.execute_reply":"2022-08-08T19:42:31.179510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Statistics\nSee the features of the dataframe","metadata":{}},{"cell_type":"markdown","source":"See the general information of the dataframe","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:31.184268Z","iopub.execute_input":"2022-08-08T19:42:31.184806Z","iopub.status.idle":"2022-08-08T19:42:31.202576Z","shell.execute_reply.started":"2022-08-08T19:42:31.184760Z","shell.execute_reply":"2022-08-08T19:42:31.201680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Calculate the mean, std, max and distributions","metadata":{}},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:31.203866Z","iopub.execute_input":"2022-08-08T19:42:31.204741Z","iopub.status.idle":"2022-08-08T19:42:31.234887Z","shell.execute_reply.started":"2022-08-08T19:42:31.204706Z","shell.execute_reply":"2022-08-08T19:42:31.233889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" See the distribution of the variables","metadata":{}},{"cell_type":"code","source":"for item in label_unique:\n    print (train[item].value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:31.236296Z","iopub.execute_input":"2022-08-08T19:42:31.237009Z","iopub.status.idle":"2022-08-08T19:42:31.248483Z","shell.execute_reply.started":"2022-08-08T19:42:31.236974Z","shell.execute_reply":"2022-08-08T19:42:31.247224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The categorical variable rust than the others","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\n#  Categorical Data\na = 2  # number of rows\nb = 3  # number of columns\nc = 1  # initialize plot counter\n\nfig = plt.figure(figsize=(10,8))\n\nfor i in label_unique:\n    plt.subplot(a, b, c)\n    plt.xlabel(i)\n    sns.countplot(train[i])\n    plt.title(i)\n    c = c + 1\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:31.250176Z","iopub.execute_input":"2022-08-08T19:42:31.250549Z","iopub.status.idle":"2022-08-08T19:42:32.246332Z","shell.execute_reply.started":"2022-08-08T19:42:31.250494Z","shell.execute_reply":"2022-08-08T19:42:32.244110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Engineering - Part 2\n","metadata":{}},{"cell_type":"markdown","source":"### Split data into test and validation\n","metadata":{}},{"cell_type":"code","source":"val = train[12000:]\ntrain = train [:12000]","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:32.248326Z","iopub.execute_input":"2022-08-08T19:42:32.249306Z","iopub.status.idle":"2022-08-08T19:42:32.254368Z","shell.execute_reply.started":"2022-08-08T19:42:32.249264Z","shell.execute_reply":"2022-08-08T19:42:32.253371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:32.255646Z","iopub.execute_input":"2022-08-08T19:42:32.256383Z","iopub.status.idle":"2022-08-08T19:42:32.276228Z","shell.execute_reply.started":"2022-08-08T19:42:32.256346Z","shell.execute_reply":"2022-08-08T19:42:32.275153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create an ImageDataGenerator\n\nThis is made because the dataset does not have enough data in some targets\n\nAlso, this generator helps to load the files","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\ntrain_datagen = ImageDataGenerator(rescale = 1/255,\n                                rotation_range=20,\n                                width_shift_range=0.2,\n                                height_shift_range=0.2,\n                                horizontal_flip=True,\n                                vertical_flip = True)\n\nval_datagen = ImageDataGenerator(rescale = 1/255,\n                                 rotation_range=20,\n                                width_shift_range=0.2,\n                                height_shift_range=0.2,\n                                horizontal_flip=True,\n                                vertical_flip = True)\n\ntrain_dataset = train_datagen.flow_from_dataframe(\n    train,\n    directory = '../input/plant-pathology-2021-fgvc8/train_images',\n    x_col = \"image\",\n    y_col = 'labels',\n    target_size = (300,300),\n    class_mode='categorical',\n    batch_size = 64,\n    shuffle = True,\n)\n\nval_dataset = val_datagen.flow_from_dataframe(\n    val,\n    directory = '../input/plant-pathology-2021-fgvc8/train_images',\n    x_col = \"image\",\n    y_col = 'labels',\n    target_size = (300,300),\n    class_mode='categorical',\n    batch_size = 64,\n    shuffle = True,\n)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:32.277612Z","iopub.execute_input":"2022-08-08T19:42:32.278094Z","iopub.status.idle":"2022-08-08T19:42:40.644423Z","shell.execute_reply.started":"2022-08-08T19:42:32.278049Z","shell.execute_reply":"2022-08-08T19:42:40.643432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_dataset.class_indices","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:40.646359Z","iopub.execute_input":"2022-08-08T19:42:40.647801Z","iopub.status.idle":"2022-08-08T19:42:40.654832Z","shell.execute_reply.started":"2022-08-08T19:42:40.647760Z","shell.execute_reply":"2022-08-08T19:42:40.653687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create the CNN architecture","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization, Dense, Flatten, Dropout","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:40.656905Z","iopub.execute_input":"2022-08-08T19:42:40.657828Z","iopub.status.idle":"2022-08-08T19:42:40.665017Z","shell.execute_reply.started":"2022-08-08T19:42:40.657790Z","shell.execute_reply":"2022-08-08T19:42:40.663822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\n# Convolutional layer #1\nmodel.add(Conv2D(filters=32,kernel_size=(3,3),padding='same',activation='relu',strides=(1,1),input_shape=(300,300,3)))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.2))\n# Convolutional layer #2\nmodel.add(Conv2D(filters=64,kernel_size=(3,3),padding='same',activation='relu',strides=(1,1)))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.2))\n# Convolutional layer #3\nmodel.add(Conv2D(filters=128,kernel_size=(3,3),padding='same',activation='relu',strides=(1,1)))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.2))\n# Convolutional layer #4\nmodel.add(Conv2D(filters=264,kernel_size=(3,3),padding='same',activation='relu',strides=(1,1)))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.2))\n# Convolutional layer #5\nmodel.add(Conv2D(filters=264,kernel_size=(3,3),padding='same',activation='relu',strides=(1,1)))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.2))\n# add a flatten layers\nmodel.add(Flatten())\n# add dense layer #1\nmodel.add(Dense(units=50, activation='relu'))\nmodel.add(Dropout(0.2))\n# add dense layer #2\nmodel.add(Dense(units=6, activation='softmax'))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:40.666872Z","iopub.execute_input":"2022-08-08T19:42:40.667184Z","iopub.status.idle":"2022-08-08T19:42:41.883051Z","shell.execute_reply.started":"2022-08-08T19:42:40.667158Z","shell.execute_reply":"2022-08-08T19:42:41.882066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ModelCheckpoint\nchekcpoint = ModelCheckpoint('mi_mejor_modelo.hdf5',verbose=1,save_best_only=True, monitor = 'val_accuracy')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:41.884567Z","iopub.execute_input":"2022-08-08T19:42:41.885760Z","iopub.status.idle":"2022-08-08T19:42:41.891555Z","shell.execute_reply.started":"2022-08-08T19:42:41.885718Z","shell.execute_reply":"2022-08-08T19:42:41.890439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam',loss='categorical_crossentropy', metrics='accuracy')\nmodel.fit(train_dataset,epochs=5,verbose=2,validation_data=val_dataset)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:42:41.893446Z","iopub.execute_input":"2022-08-08T19:42:41.894922Z","iopub.status.idle":"2022-08-08T19:46:26.579097Z","shell.execute_reply.started":"2022-08-08T19:42:41.894886Z","shell.execute_reply":"2022-08-08T19:46:26.577424Z"},"trusted":true},"execution_count":null,"outputs":[]}]}