{"cells":[{"metadata":{},"cell_type":"raw","source":"Norayda N'SIEMO\nMounir AMIA\nAlpha SOW\nIsmaîl MANE\n\nNous disposons d'un ensemble de données de 21 367 images étiquetées, nous allons créer et optimiser un modèle pour avoir le maximum de prediction des maladies qui touchent les feuilles de manioc. "},{"metadata":{},"cell_type":"markdown","source":"Nous avons tenté de transformer les images directement en matrice de valeurs représentant les pixels."},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom PIL import Image\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom functools import partial\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Création listes d'image (X) et de catégorie (Y)\nimages=[]\ncategories=[]\n\n# Chargement des données\ndata = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\n\n(listeIMG,listeCat) = data[\"image_id\"],data[\"label\"]\n\nPATH = '../input/cassava-leaf-disease-classification/train_images'\n\n#Limité à 1300 image parcequ'au-dela nous avions une erreur\nlisteIMG= listeIMG[:1000]\nlisteCat= listeCat[:1000]\n\n\n# Parcoure le DataSet d'images et remplis les listes X et Y\nfor x in listeIMG:\n    img = Image.open(f'{PATH}/{x}').convert(\"L\")\n    images.append(np.array(img))\n\n\n#print(X)\n#print(len(listeIMG))\n\n# Normalisation\nimages = np.array(images, dtype=np.float) / 255.0\n\n# Transforme nos catégories en vecteurs de 0 et 1, l'index du 1 correspondant à la catégorie de notre image\ncategories = keras.utils.to_categorical(listeCat, 5)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Test de récupération et affichage d'image\nimgFILE = '../input/cassava-leaf-disease-classification/train_images/' + listeIMG[0]\nimgSRC = Image.open(imgFILE)\nimgSRC","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Création du modèle \ndef build_and_train_model1(x_train, y_train, x_test, y_test,batch_nb,epok):\n    model = keras.models.Sequential()\n    model.add(keras.layers.Flatten())\n    model.add(keras.layers.Dense(1800, activation=keras.activations.relu))\n    model.add(keras.layers.Dense(1800, activation=keras.activations.relu))\n    model.add(keras.layers.Dense(1400, activation=keras.activations.relu))\n    model.add(keras.layers.Dense(1200, activation=keras.activations.relu))\n    model.add(keras.layers.Dense(5, activation=\"sigmoid\"))\n    #model.add(keras.layers.Dense(5, activation=\"softmax\"))\n\n    # Compilation du modèle\n    model.compile(\n        loss=keras.losses.mse,  # Calcul le loss\n        optimizer=keras.optimizers.SGD(learning_rate),  # Minimise le loss\n        # optimizer=keras.optimizers.Adam()\n        metrics=keras.metrics.categorical_accuracy\n    )\n    \n    # Entraînement du modèle\n    model.fit(x_train, y_train, validation_data=(x_test, y_test), epochs=epok, batch_size = batch_nb)\n    model.summary()\n    \n    score = model.evaluate(x_test, y_test, verbose=0)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epok = 100\nnbBatch=300\nlearning_rate = 0.95\n\n# Split des données d'entrainement et de test\nIMG_train, IMG_test, Categories_train, Categories_test = train_test_split(images, categories, test_size=0.33, random_state=42)\n\n# Lancement du programme de notre modèle\nbuild_and_train_model1( IMG_train , Categories_train,IMG_test,Categories_test, nbBatch,epok)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Nous avons abandonné cette méthodes pour essayer de déchiffrer les TFrecords."},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(\"Device:\", tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint(\"Number of replicas:\", strategy.num_replicas_in_sync)\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nPATH = \"../input/cassava-leaf-disease-classification\"\nBATCH_SIZE = 300\nIMAGE_SIZE = [512, 512,3]\nHEIGHT = 100\nWIDTH = 100\nCHANNELS = 3\nEPOCH = 100\nlearning_rate = 0.001","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FILENAMES = tf.io.gfile.glob(PATH + \"/train_tfrecords/*.tfrec\")\nsplit_ind = int(0.8 * len(FILENAMES))\nTRAINING_FILENAMES, VALID_FILENAMES = FILENAMES[:split_ind], FILENAMES[split_ind:]\n\nTEST_FILENAMES = tf.io.gfile.glob(PATH + \"/test_tfrecords/*.tfrec\")\nprint(\"Train TFRecord Files:\", len(TRAINING_FILENAMES))\nprint(\"Validation TFRecord Files:\", len(VALID_FILENAMES))\nprint(\"Test TFRecord Files:\", len(TEST_FILENAMES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize(x):\n    x = tf.image.resize(x, (HEIGHT,WIDTH))\n    return x\n\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32)\n    return resize(image)#Modifie la taille de l'image en [HEIGHT,WIDTH,3], ce qi nous permettra de travailler avec de plus petites images\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_tfrecord(example, labeled):\n    tfrecord_format = (\n        {\n            \"image\": tf.io.FixedLenFeature([], tf.string),\n            \"target\": tf.io.FixedLenFeature([], tf.int64),\n        }\n        if labeled\n        else {\"image\": tf.io.FixedLenFeature([], tf.string),}\n    )\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example[\"image\"])\n\n    if labeled:\n        label = tf.cast(example[\"target\"], tf.int32)\n        return image, label\n    return image\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False  # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames )  # automatically interleaves reads from multiple files\n    dataset = dataset.with_options( ignore_order)  # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map( partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE )\n    # returns a dataset of (image, label) pairs if labeled=True or just images if labeled=False\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(filenames, labeled=True):\n    dataset = load_dataset(filenames, labeled=labeled)\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.prefetch(buffer_size=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_dataset(TRAINING_FILENAMES)\nvalid_dataset = get_dataset(VALID_FILENAMES)\ntest_dataset = get_dataset(TEST_FILENAMES, labeled=False)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for images in test_dataset.take(1):\n    plt.imshow(images[0]\n               .numpy().astype(\"uint8\"))\n    plt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#AFFICHES des images et leurs catégories\n\ndef show_batch(data):\n    plt.figure(figsize=(10, 10))\n\n    for images, labels in data.take(1):\n        for i in range(25):\n            ax = plt.subplot(5, 5, i + 1)\n            plt.imshow(images.numpy()[i].astype(\"uint8\"))\n            plt.axis(\"off\")\n            \n        \nshow_batch(train_dataset)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_and_train_model(addlayers,train_dataset, valid_dataset):\n    model = keras.models.Sequential()\n    addlayers(model)\n    \n    model.add(keras.layers.Flatten())\n\n    model.add(keras.layers.Dense(5, activation=\"sigmoid\"))\n    model.add(keras.layers.Dense(5, activation=\"softmax\"))\n\n\n    model.compile(\n        loss=keras.losses.binary_crossentropy,  # calcul l'erreur\n        optimizer=keras.optimizers.SGD(learning_rate),  # minimise l'erreur\n        #optimizer=keras.optimizers.Adam(),\n        metrics=['accuracy']\n    )\n    \n    logs = history = model.fit(\n        train_dataset,\n        epochs= EPOCH,\n        batch_size = BATCH_SIZE,\n        validation_data = valid_dataset,\n    )\n    model.summary()\n    \n    score = model.evaluate(train_dataset, verbose=0)\n    model.reset_states()\n\n\n    return logs\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pmc(model):\n    model.add(keras.layers.Flatten())\n    model.add(keras.layers.Dense(800, activation='relu'))\n    model.add(keras.layers.Dense(800, activation='relu'))\n    model.add(keras.layers.Dense(400, activation='relu'))\n    model.add(keras.layers.Dense(200, activation='relu'))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convnet(model):\n    model.add(keras.layers.Conv2D(64, (4, 4), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.MaxPool2D())\n    model.add(keras.layers.Conv2D(32, (4, 4), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.MaxPool2D())\n    model.add(keras.layers.Conv2D(16, (4, 4), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.MaxPool2D())\n    model.add(keras.layers.Dropout(0.5))\n    model.add(keras.layers.Dropout(0.5))\n    \n    model.add(keras.layers.Flatten())\n    \n#le loss et l'accuracy gardent la même valeur à chaque epoch\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resNet(model):\n    model.add(keras.layers.Conv2D(32, (3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.BatchNormalization())\n\n    model.add(keras.layers.Conv2D(64, (3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.BatchNormalization())\n\n\n    model.add(keras.layers.Conv2D(64, (3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.BatchNormalization())\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#import torchvision.models as models\n#vgg16 = models.vgg16(pretrained=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#def vgg16_pytorch(model):\n    #model = vgg16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#On a divisé le nombre initial de couche à cause d'un problème de mémoire\n\ndef vgg16(model):\n    model.add(keras.layers.Conv2D(64,(3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.Conv2D(64,(3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.MaxPool2D())\n    model.add(keras.layers.Dropout(0.5))\n\n    \n    model.add(keras.layers.Conv2D(128,(3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.Conv2D(128,(3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.MaxPool2D())\n    model.add(keras.layers.Dropout(0.5))\n\n    \n    model.add(keras.layers.Conv2D(256,(3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.Conv2D(256,(3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.Conv2D(256,(3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.MaxPool2D())\n    model.add(keras.layers.Dropout(0.5))\n\n    \n    model.add(keras.layers.Conv2D(512,(3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.Conv2D(512,(3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.Conv2D(512,(3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.MaxPool2D())\n    model.add(keras.layers.Dropout(0.5))\n\n    \n    model.add(keras.layers.Conv2D(512,(3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.Conv2D(512,(3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.Conv2D(512,(3, 3), padding=\"same\", activation=keras.activations.relu))\n    model.add(keras.layers.MaxPool2D())\n    model.add(keras.layers.Dropout(0.5))\n\n    \n    model.add(keras.layers.Flatten())\n\n    # Ajout des couches fully-connected, suivie de couche ReLU\n    model.add(keras.layers.Dense(4096, activation='relu'))\n    model.add(keras.layers.Dense(4096, activation='relu'))\n    model.add(keras.layers.Dense(1000, activation='relu'))\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nall_logs = [(build_and_train_model(pmc,train_dataset , valid_dataset),\"modèle pmc\"),\n           (build_and_train_model(convnet,train_dataset , valid_dataset),\"modèle convnet\"),\n            (build_and_train_model(vgg16,train_dataset , valid_dataset),\"modèle vgg-16\")\n\n           ]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Pour l'affichage\ndef plot_all_logs(all_logs):\n    # Loss\n    for logs in all_logs:\n        y_coords = logs[0].history[\"loss\"]\n        x_coords = list(range(len(y_coords)))\n        plt.plot(x_coords, y_coords,label=logs[1])\n        plt.legend()\n        plt.title(\"Loss\")\n\n    plt.show()\n    \n        # accuracy\n    for logs in all_logs:\n        y_coords = logs[0].history[\"categorical_accuracy\"]\n        x_coords = list(range(len(y_coords)))\n        plt.plot(x_coords, y_coords,label=logs[1])\n        plt.legend()\n        plt.title(\"Accuracy\")\n        \n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_all_logs(all_logs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}