{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport time\nimport json\nimport keras\nimport numpy as np\nimport pandas as pd\nimport seaborn as sn\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom PIL import Image\nfrom keras.utils import plot_model\nfrom keras.optimizers import RMSprop, Adam\nfrom keras.models import Sequential, load_model\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.applications import EfficientNetB3, EfficientNetB7\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom keras.layers import GlobalAveragePooling2D, Flatten, Dense, Dropout, BatchNormalization","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dados\n## Explorando dados"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Importando csv do dataset\ndata = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv', sep = ',')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Lendo json e crinado mapping para para classe\nwith open('../input/cassava-leaf-disease-classification/label_num_to_disease_map.json') as f:\n    mapping = json.loads(f.read())\n    mapping = {int(k): v for k, v in mapping.items()}\n\nmapping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Adicionando label_name ao conjunto de dados, e ondenando colunas\ndata['label_name'] = data['label'].map(mapping)\ndata = data[['image_id', 'label_name', 'label']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Verificando frequencia\ndata.label_name.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Verificando proporção \ndata.label_name.value_counts(normalize = True) * 100","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparando os dados\n## HoldOut\nSeparando dados de treino e teste"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Definindo paths de treino e teste\ntrain_path = '../input/cassava-leaf-disease-classification/train_images/'\ntest_path = '../input/cassava-leaf-disease-classification/test_images/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Criando amostra para treino e teste estrafiticada\ntrain, test = train_test_split(data, test_size = 0.10, shuffle = True, random_state = 0, stratify = data['label_name'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.label_name.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.label_name.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tratamento das imagens\nNossas imagens não estão no formato adequado para usarmos em uma rede neural, usaremos a classe **ImageDataGenerator** com o method **flow_from_dataframe** do keras. Ele irar percorrer nosso cvs e diretório de dados, carregando os dados de imagem e retornar a entrada (matrizes de pixels) e a saída (número inteiro de classe)."},{"metadata":{},"cell_type":"markdown","source":"### Definindo parâmetros para tratamnetos das imagens"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator = ImageDataGenerator(\n    preprocessing_function = tf.keras.applications.efficientnet.preprocess_input,\n    rotation_range = 40,\n    width_shift_range = 0.2,\n    height_shift_range = 0.2,\n    shear_range = 0.2,\n    zoom_range = 0.2,\n    horizontal_flip = True,\n    vertical_flip = True,\n    fill_mode = 'nearest',\n)\n\ntest_generator = ImageDataGenerator(\n    preprocessing_function = tf.keras.applications.efficientnet.preprocess_input\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_set = train_generator.flow_from_dataframe(\n    dataframe = train,\n    directory = train_path,\n    seed = 42,\n    x_col = 'image_id',\n    y_col = 'label_name',\n    target_size = (456, 456),\n    class_mode = 'categorical',\n    interpolation = 'nearest',\n    shuffle = True,\n    batch_size = 15\n)\n\ntest_set = test_generator.flow_from_dataframe(\n    dataframe = test,\n    directory= train_path,\n    seed = 42,\n    x_col = 'image_id',\n    y_col = 'label_name',\n    target_size = (456, 456),\n    class_mode = 'categorical',\n    interpolation = 'nearest',\n    shuffle = True,\n    batch_size = 15\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modelo"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Definindo parâmetros para treino e teste\nSTEP_SIZE_TRAIN = train_set.n // train_set.batch_size\nSTEP_SIZE_TEST = test_set.n // test_set.batch_size\n\n# Definindo loss function\nloss = tf.keras.losses.CategoricalCrossentropy(from_logits = False, label_smoothing = 0.0001, name = 'categorical_crossentropy')\n# Pare de treinar quando o val_loss parar de diminuir por 3 épocas\nes = EarlyStopping(monitor = 'val_loss', patience = 3, verbose = 1, mode = 'min', restore_best_weights = True)\n# Salve o modelo com o mínimo de perda de validação\ncheckpoint = ModelCheckpoint(\"CassavaLeafDisease.h5\", monitor = 'val_loss', save_best_only = True, mode = 'min')\n# Reduza a taxa de aprendizagem quando a aprendizagem estagnar\nreduce_lr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, patience = 2, min_lr = 1e-6, mode = 'min', verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Criando modelo de rede neural\nmodelo = Sequential()\n#modelo.add(EfficientNetB7(input_shape = (456, 456, 3), include_top = False, weights = 'imagenet', drop_connect_rate = 0.6))\nmodelo.add(EfficientNetB3(input_shape = (456, 456, 3), include_top = False, weights = 'imagenet', drop_connect_rate = 0.6))\nmodelo.add(GlobalAveragePooling2D())\nmodelo.add(Flatten())\nmodelo.add(Dense(256, activation = 'relu', bias_regularizer = tf.keras.regularizers.L1L2(l1 = 0.01, l2 = 0.001)))\nmodelo.add(Dropout(0.5))\nmodelo.add(Dense(5, activation = 'softmax'))\n# Compilando rede neural\nmodelo.compile(optimizer = Adam(lr = 0.001, decay = 0.0001, clipvalue = 0.5), loss = loss, metrics = ['categorical_accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Informações de parâmetros do modelo \nmodelo.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inicio = time.time()\n\n# Treinamento e teste do modelo\nresults = modelo.fit(\n        train_set, \n        validation_data = test_set, \n        epochs = 15, \n        batch_size = 15, \n        steps_per_epoch = STEP_SIZE_TRAIN,\n        validation_steps = STEP_SIZE_TEST, \n        callbacks = [es, checkpoint, reduce_lr]\n)\n\nprint()\nfim = time.time()\nprint(fim - inicio)\n\n# Salvando modelo\nmodelo.save('CassavaLeafDisease.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Carregando modelo\nleaf_model = load_model('CassavaLeafDisease.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Realizando predição com imagem teste \nTEST_DIR = '../input/cassava-leaf-disease-classification/test_images/'\ntest_images = os.listdir(TEST_DIR)\npredictions = []\n\nfor image in test_images:\n    img = Image.open(TEST_DIR + image)\n    img = img.resize((456, 456), resample = Image.NEAREST) \n    img = np.expand_dims(img, axis = 0)\n    predictions.extend(leaf_model.predict(img).argmax(axis = 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Gerando arquivo submission.csv\nsubmit = pd.DataFrame({'image_id': test_images, 'label': predictions})\nsubmit.to_csv('submission.csv', index = False)\ndisplay(submit)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}