{"cells":[{"metadata":{"_uuid":"a99e1f75bc817bf7fbb6f9b9a0bdca6fe81a108d"},"cell_type":"markdown","source":"# Projet commencé le 04/02/2019"},{"metadata":{"_uuid":"f8a17a649d4c4044af9cb18c4b62cf6c54d881ba"},"cell_type":"markdown","source":"## Imports :"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom __future__ import division\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom io import BytesIO\nimport requests\nimport bq_helper\nfrom sklearn.model_selection import train_test_split\nimport keras.backend as K\nimport keras_rcnn as KC\nimport keras\nfrom keras.models import Model, load_model\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Conv2D, Convolution2D, Flatten, MaxPooling2D, Dropout, Activation, Reshape, Input\nfrom keras.utils import to_categorical\nfrom keras.models import load_model\nfrom keras.models import Sequential\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.applications.vgg16 import decode_predictions, VGG16\nimport tensorflow as tf\nimport queue as Q\nimport math\nimport random\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8040e9abddde8aab94858acb127e81d57a53cba8"},"cell_type":"markdown","source":"# DEFINITION DES FONCTIONS"},{"metadata":{"trusted":true,"_uuid":"cab4e20497b91f933f67ee646cc8e2b94ad9d8c7","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"# Return une image depuis son URL\ndef images_from_url(url):\n    try:\n        response = requests.get(url)\n        return Image.open(BytesIO(response.content))\n    except:\n        return False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e28ef46a0db22cd3c70424daa7423aeb8e46566","_kg_hide-input":true},"cell_type":"code","source":"# Permet d'afficher (pour une image, une liste de bboxs et une liste de labels) une image avec les objets labellisés\ndef plot_bbox_label(image, bbox, label):\n    im_dim_y = image.shape[0]\n    im_dim_x = image.shape[1]\n    \n    fig, ax = plt.subplots(1,figsize=(15,20))\n    ax.imshow(image)\n    \n    it = 0\n    for l_bbox in bbox:\n        im_width = l_bbox[2] - l_bbox[0]\n        im_height = l_bbox[3] - l_bbox[1]\n        \n        np.random.seed(seed = int(np.prod(bytearray(label[it], 'utf8'))) %2**32)\n        color = np.random.rand(3,1)\n        color = np.insert(color, 3, 0.7)\n        \n        ax.add_patch(patches.Rectangle((l_bbox[0]*im_dim_x, l_bbox[1]*im_dim_y), im_width*im_dim_x, im_height*im_dim_y, linewidth=8, edgecolor=color, facecolor='none'));\n        text = ax.annotate(label[it], (l_bbox[2]*im_dim_x,l_bbox[1]*im_dim_y), bbox=dict(boxstyle=\"square,pad=0.3\", fc=color, lw=2))\n        text.set_fontsize(18)\n        it = it+1\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calcul l'IoU pour 2 box au format xmin ymin xmax ymax\ndef IoU(bbox1, bbox2):\n    w_intersect = (bbox1[2] - bbox1[0]) + (bbox2[2] - bbox2[0]) - (max(bbox1[2], bbox2[2]) - min(bbox1[0], bbox2[0]))\n    h_intersect = (bbox1[3] - bbox1[1]) + (bbox2[3] - bbox2[1]) - (max(bbox1[3], bbox2[3]) - min(bbox1[1], bbox2[1]))\n    \n    if(w_intersect < 0 or h_intersect < 0):\n        return 0\n    \n    intersect = w_intersect * h_intersect\n\n    union_1 = (bbox1[2]-bbox1[0]) * (bbox1[3]-bbox1[1])\n    union_2 = (bbox2[2]-bbox2[0]) * (bbox2[3]-bbox2[1])\n    \n    union = union_1 + union_2 - intersect\n\n    return intersect/union","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Créer des anchors avec le centre (x,y) et la largeur/hauteur de la convolution (de réduction 16)\ndef generate_anchors(center_x, center_y, conv_w, conv_h):\n    anchor_ratio = [[1, 1], [1, 2], [2, 1]]\n    anchor_coef = [1, 2, 4]\n    anchor_size = 128\n    \n    anchor_list = []\n    \n    for ratio in anchor_ratio:\n        for coef in anchor_coef:\n            anchor_width = (anchor_size*coef*ratio[0]) / (conv_w*16)\n            anchor_height = (anchor_size*coef*ratio[1]) / (conv_h*16)\n            anchor_x = (center_x/conv_w) - (anchor_width/2)\n            anchor_y = (center_y/conv_h) - (anchor_height/2)\n            anchor = [anchor_x, anchor_y, anchor_x+anchor_width, anchor_y+anchor_height]\n            \n            anchor_list.append(anchor)\n    \n    anchor_list = np.array(anchor_list)\n    \n    return anchor_list","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"# Opération de RoI pooling sur un tableau et une taille de shape*shape\n# Il y a 3 lignes similaire en fonction du type d'opération utilisé pour le pooling (apres test la moyenne est mieu que le max)\ndef RoI(array, shape):\n    result = np.zeros((shape, shape, array.shape[2]))\n    for i in range (0, shape):\n        for j in range (0, shape):\n            sub_array = array[int(i*array.shape[0]/shape):int((i+1)*array.shape[0]/shape), int(j*array.shape[1]/shape):int((j+1)*array.shape[1]/shape)]\n            #result[i][j] = np.amax(np.amax(sub_array, axis = 0), axis = 0)\n            result[i][j] = np.mean(np.mean(sub_array, axis = 0), axis = 0)\n            #result[i][j] = np.amin(np.amin(sub_array, axis = 0), axis = 0)\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1ead23e1c91953ac095b3ca7bec92b7c82563d2","_kg_hide-input":false},"cell_type":"code","source":"# Extrait la partie convolution de VGG16\ndef generate_conv():\n    vgg16_net = VGG16(include_top=False, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000)\n    model = Model(input=vgg16_net.layers[0].input, output=vgg16_net.layers[17].output)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Une accuracy custom, elle est légérement capable de dépasser 1 mais sinon avec keras,\n# Pour une loss custom, l'accuracy est bugé (c'est un bug connu)\ndef acc(y_true, y_pred): return K.mean(K.round(y_pred)*y_true + (1.-K.round(y_pred))*(1.-y_true))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"# Loss custom pour le classifier du rpn, on igniore les cas ou la prédiction est (0, 0) et on renforce l'apprentissage lors de la présence d'un objet\ndef custom_loss_rpn_cls(y_true, y_pred):\n    shape = K.shape(y_true)\n    depth = shape[0]\n        \n    # Retire les [0, 0] (c'est à dire entre objet et pas d'objet)\n    new_y_pred = K.zeros((depth, 2))\n    for i in range (0, 9):\n        cond = K.equal(y_true[:, 2*i:2*i+2], [0,0])\n        cond = tf.math.logical_and(cond[:,0], cond[:,1])\n        cond = K.concatenate([cond, cond])\n        cond = K.reshape(cond, (depth,2))\n        \n        temp = K.switch(cond, y_true[:, 2*i:2*i+2], y_pred[:, 2*i:2*i+2])\n        if i == 0:\n            new_y_pred = temp\n        else:\n            new_y_pred = K.concatenate([new_y_pred, temp])\n\n    new_y_pred = K.reshape(new_y_pred, (depth, 18))\n    cls = K.binary_crossentropy(y_true, new_y_pred)\n    \n    # Renforce les [1,0] (c'est à dire la présence d'objet)\n    new_cls = K.zeros((depth, 2))\n    for i in range (0, 9):\n        cond = K.equal(y_true[:, 2*i:2*i+2], [0,1])\n        cond = tf.math.logical_and(cond[:,0], cond[:,1])\n        cond = K.concatenate([cond, cond])\n        cond = K.reshape(cond, (depth,2))\n        \n        temp = K.switch(cond, cls[:, 2*i:2*i+2], cls[:, 2*i:2*i+2]*4.7)\n        if i == 0:\n            new_cls = temp\n        else:\n            new_cls = K.concatenate([new_cls, temp])\n    new_cls = K.reshape(new_cls, (depth, 18))\n    \n    # On re multipli pour compenser les [0,0]\n    # Les coeficients ont était trouvé expérimentalement\n    return K.mean(new_cls*2.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"# Fonction smoothL1 (fonction connue)\ndef smoothL1(y_true, y_pred):\n    x   = K.abs(y_true - y_pred)\n    x   = K.switch(x < 1, x*x, x)\n    return  x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"# Loss custom pour le regresseur du rpn, on igniore les cas ou la prédiction est (0, 0)\ndef custom_loss_rpn_reg(y_true, y_pred):\n    shape = K.shape(y_true)\n    depth = shape[0]\n        \n    # Retire les [0, 0, 0, 0] c'est à dire qu'il n'y a pas de présence d'objet\n    new_y_pred = K.zeros((depth, 4))\n    for i in range (0, 9):\n        cond = K.equal(y_true[:, 2*i:2*i+4], [0,0,0,0])\n        cond = tf.math.logical_and(tf.math.logical_and(cond[:,0], cond[:,1]), tf.math.logical_and(cond[:,2], cond[:,3]))\n        cond = K.concatenate([cond, cond, cond, cond])\n        cond = K.reshape(cond, (depth,4))\n        \n        temp = K.switch(cond, y_true[:, 2*i:2*i+4], y_pred[:, 2*i:2*i+4])\n        if i == 0:\n            new_y_pred = temp\n        else:\n            new_y_pred = K.concatenate([new_y_pred, temp])\n\n    new_y_pred = K.reshape(new_y_pred, (depth, 36))\n    reg = smoothL1(y_true, new_y_pred)\n    \n    # On re multipli pour compenser les [0,0,0,0]\n    # Les coeficients ont était trouvé expérimentalement\n    return K.mean(reg)*9","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1a06de37402235bffcc2c1085b7ccaecaf05749","_kg_hide-input":false},"cell_type":"code","source":"# Return le model utilisé pour le classifier\ndef generate_cls_nn():\n    vgg16_net = VGG16(include_top=True, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000)\n    \n    l_input = Input(shape=(7, 7, 512))\n    l_flatten = vgg16_net.get_layer(\"flatten\")\n    l_fc1 = vgg16_net.get_layer(\"fc1\")\n    l_fc2 = vgg16_net.get_layer(\"fc2\")\n    l_output = vgg16_net.get_layer(\"predictions\")\n    model = Model(input=l_input, output=l_output(l_fc2(l_fc1(l_flatten(l_input)))))\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Return la convolution d'une image\ndef pred_conv(image):\n    return conv_net.predict(np.array([image]))[0]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7d4f49478a654c12b995980decab564d57f69f25"},"cell_type":"markdown","source":"# DEFINITION DES GLOBALS"},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"conv_net = generate_conv()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d3de51127246e8b62b5856470c06cdf853cfe97","_kg_hide-input":false},"cell_type":"code","source":"rpn_cls_net = load_model('../input/cls.h5', custom_objects={'custom_loss_rpn_cls': custom_loss_rpn_cls})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rpn_reg_net = load_model('../input/reg.h5', custom_objects={'custom_loss_rpn_reg': custom_loss_rpn_reg})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cls_net = generate_cls_nn()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PARTIE APPLICATION"},{"metadata":{"trusted":true,"_uuid":"ad87f23f55e840d3b8cfba1cca9b7132c60fba40","_kg_hide-input":false},"cell_type":"code","source":"def predict(URL, threshold_cls_rpn = 0.7, threshold_cls_vgg = 0.5, threshold_iou = 0.8):\n    # L'image est téléchargé et mise au bon format (le format vgg est non normalisé, celui de notre rpn l'est car nous donne de meilleurs résultats)\n    image_test = images_from_url(URL)\n    if(image_test != False):\n        image_test_w, image_test_h = image_test.size\n        taille_max = max(image_test_w, image_test_h)\n        coef = 800/taille_max\n        image_test = image_test.resize((int(coef*image_test_w), int(coef*image_test_h)))\n        image_test_vgg = np.array(image_test)\n        image_test = np.array(image_test)/255\n        if(len(image_test.shape) == 3):\n            anchor_test_valid = []\n\n            image_test_conv_vgg = pred_conv(image_test_vgg)\n            image_test_conv = pred_conv(image_test)\n\n            # On passe sur chaque pixel de la convolution\n            for x in range (1, image_test_conv.shape[1] - 1):\n                for y in range (1, image_test_conv.shape[0] - 1):\n                    anchor_valid = False\n                    anchor_empty = True\n\n                    # On effectue une prédiction sur la fenetre glissant centré sur le pixel actuel\n                    pred_cls = rpn_cls_net.predict(np.array([image_test_conv[y-1:y+2, x-1:x+2]]))[0]\n                    pred_reg = rpn_reg_net.predict(np.array([image_test_conv[y-1:y+2, x-1:x+2]]))[0]\n\n                    # On test pour toutes les anchors si le rpn à detecté un objet\n                    list_anchors = generate_anchors(x, y, image_test_conv.shape[1], image_test_conv.shape[0])\n                    for k in range(0, 9):\n                        # Si on trouve un objet à plus de 70% de sureté\n                        if(pred_cls[k*2] >= threshold_cls_rpn):\n                            # On recupère les infos de l'anchor\n                            anchor = list_anchors[k]\n                            anchor_x, anchor_y, anchor_xm, anchor_ym = anchor\n                            anchor_width = anchor_xm - anchor_x\n                            anchor_height = anchor_ym - anchor_y\n\n                            # On recupère les infos de la prédiction\n                            pred_reg_x, pred_reg_y, pred_reg_w, pred_reg_h = pred_reg[k*4:k*4+4]\n\n                            # On test si l'anchor ne sort pas de l'écrant\n                            cond1 = anchor_x+(pred_reg_x*anchor_width) >= 0\n                            cond2 = anchor_y+(pred_reg_y*anchor_height) >= 0\n                            cond3 = anchor_x+(pred_reg_x*anchor_width) + (10**pred_reg_w)*anchor_width < 1\n                            cond4 = anchor_y+(pred_reg_y*anchor_height) + (10**pred_reg_h)*anchor_height < 1\n                            if(cond1 and cond2 and cond3 and cond4):\n                                # On calcul le xmin/max ymin/max de la prediction relativement à l'image\n                                it_min_x = int((anchor_x+(pred_reg_x*anchor_width)) * image_test_conv.shape[1])\n                                it_max_x = int((anchor_x+(pred_reg_x*anchor_width) + (10**pred_reg_w)*anchor_width) * image_test_conv.shape[1])\n                                it_min_y = int((anchor_y+(pred_reg_y*anchor_height)) * image_test_conv.shape[0])\n                                it_max_y = int((anchor_y+(pred_reg_y*anchor_height) + (10**pred_reg_h)*anchor_height) * image_test_conv.shape[0])\n\n                                # Si la prédiction est plus large que du 7*7 (minimum du classifier)\n                                if(it_max_y-it_min_y >= 7 and it_max_x-it_min_x >= 7):\n                                    # Prédiction des 5 premières classes que vgg trouve\n                                    label = decode_predictions(cls_net.predict(np.array([RoI(image_test_conv_vgg[it_min_y:it_max_y, it_min_x:it_max_x], 7)])), top=5)[0]\n                                    # Si la confiance accordé à la top classe de vgg est de plus de 50%\n                                    if(label[0][2] >= threshold_cls_vgg):\n                                        # On stock les données au plus simple pour les traiter avec le nonmax\n                                        anchor_test_valid.append([label[0][2], [[label[0][1], label[1][1], label[2][1], label[3][1], label[4][1]], anchor_x+(pred_reg_x*anchor_width), anchor_y+(pred_reg_y*anchor_height), anchor_x+(pred_reg_x*anchor_width) + (10**pred_reg_w)*anchor_width, anchor_y+(pred_reg_y*anchor_height) + (10**pred_reg_h)*anchor_height]])  \n\n            # SECTION NONMAX\n            # Le but est de supprimer les overlaps au dessus d'un seuil pour les mêmes classes (il suffit d'une coresspondance dans les 5 premières classes)\n            anchor_test_valid = np.array(anchor_test_valid)      \n            q = Q.PriorityQueue()\n            for a in anchor_test_valid:\n                q.put((1-a[0] + random.random()/100000,a[1]))\n            anchor_test_valid = []\n            size = q.qsize()\n            for i in range (0, size):\n                var_i = q.get()\n                found_one = False\n                for a in anchor_test_valid:\n                    for labelnb in range(0, 5):\n                        if(IoU(a[1], var_i[1][1:5]) >= 1 - threshold_iou and a[0] == var_i[1][0][labelnb]):\n                            found_one = True\n                if(not found_one):\n                    anchor_test_valid.append([var_i[1][0][0], var_i[1][1:5]])\n            anchor_test_valid = np.array(anchor_test_valid)\n            if(anchor_test_valid.shape[0] != 0):\n                plot_bbox_label(image_test, anchor_test_valid[:, 1], anchor_test_valid[:, 0])\n            else:\n                plt.figure(figsize=(15,20))\n                plt.imshow(image_test_vgg)\n                plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_multiple(list_URL):\n    for URL in list_URL:\n        url_pred = predict(URL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# UNE VOITURE :   https://www.usinenouvelle.com/mediatheque/4/5/4/000626454_image_896x598/dacia-sandero.jpg\n# 2 VOITURES :   https://hips.hearstapps.com/hmg-prod.s3.amazonaws.com/images/s17-2051-fine-1553003760.jpg\n# N VOITURES :   https://cdn-images-1.medium.com/max/1600/1*ICvAO8mPCA_sXOzW9zeM7g.jpeg\n# 2 VOITURES :   https://ischool.syr.edu/infospace/wp-content/files/2015/10/toyota-and-lexus-car-on-road--e1444655872784.jpg\n# ZOO : http://www.mdjunited.com/medias/images/zoo.jpg\n\nurl_images_test = ['https://www.usinenouvelle.com/mediatheque/4/5/4/000626454_image_896x598/dacia-sandero.jpg',\n                   'https://images5.alphacoders.com/393/393962.jpg',\n                   'https://images.unsplash.com/photo-1544776527-68e63addedf7?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&w=1000&q=80',\n                   'https://www.autocar.co.uk/sites/autocar.co.uk/files/styles/gallery_slide/public/images/car-reviews/first-drives/legacy/gallardo-0638.jpg?itok=-So1NoXA', \n                   'http://www.mdjunited.com/medias/images/zoo.jpg']\n\npredict_multiple(url_images_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict('https://cdn.pixabay.com/photo/2017/05/09/21/49/gecko-2299365_960_720.jpg', 0.7, 0.5, 0.8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict('https://cdn.pixabay.com/photo/2017/05/09/21/49/gecko-2299365_960_720.jpg', 0.7, 0.3, 0.8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}