{"cells":[{"metadata":{"_uuid":"a99e1f75bc817bf7fbb6f9b9a0bdca6fe81a108d"},"cell_type":"markdown","source":"> # Projet commencé le 04/02/2019"},{"metadata":{"_uuid":"f8a17a649d4c4044af9cb18c4b62cf6c54d881ba"},"cell_type":"markdown","source":"## Imports :"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom __future__ import division\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom io import BytesIO\nimport requests\nimport bq_helper\nfrom sklearn.model_selection import train_test_split\nimport keras.backend as K\nimport keras_rcnn as KC\nimport keras\nfrom keras.models import Model\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Conv2D, Convolution2D, Flatten, MaxPooling2D, Dropout, Activation, Reshape, Input\nfrom keras.utils import to_categorical\nfrom keras.models import load_model\nfrom keras.models import Sequential\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.applications.vgg16 import decode_predictions, VGG16\nimport tensorflow as tf\nimport queue as Q\nimport math\nimport random\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8040e9abddde8aab94858acb127e81d57a53cba8"},"cell_type":"markdown","source":"# DEFINITION DES FONCTIONS"},{"metadata":{"trusted":true,"_uuid":"cab4e20497b91f933f67ee646cc8e2b94ad9d8c7","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"# Return an image from its URL\ndef images_from_url(url):\n    try:\n        response = requests.get(url)\n        return Image.open(BytesIO(response.content))\n    except:\n        return False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e28ef46a0db22cd3c70424daa7423aeb8e46566","_kg_hide-input":true},"cell_type":"code","source":"# Allows to display (for an image, a list of bboxes and a list of labels) an image with the labeled objects\ndef plot_bbox_label(image, bbox, label):\n    im_dim_y = image.shape[0]\n    im_dim_x = image.shape[1]\n    \n    plt.figure(figsize=(15,20))\n    fig, ax = plt.subplots(1,figsize=(15,20))\n    ax.imshow(image)\n    \n    it = 0\n    for l_bbox in bbox:\n        im_width = l_bbox[2] - l_bbox[0]\n        im_height = l_bbox[3] - l_bbox[1]\n        \n        np.random.seed(seed = int(np.prod(bytearray(label[it], 'utf8'))) %2**32)\n        color = np.random.rand(3,1)\n        color = np.insert(color, 3, 0.7)\n        \n        ax.add_patch(patches.Rectangle((l_bbox[0]*im_dim_x, l_bbox[1]*im_dim_y), im_width*im_dim_x, im_height*im_dim_y, linewidth=8, edgecolor=color, facecolor='none'));\n        text = ax.annotate(label[it], (l_bbox[2]*im_dim_x,l_bbox[1]*im_dim_y), bbox=dict(boxstyle=\"square,pad=0.3\", fc=color, lw=2))\n        text.set_fontsize(18)\n        it = it+1\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9878f1abe10a125c2a251f26ea2ad96a7edbccc","_kg_hide-input":true},"cell_type":"code","source":"# Return the result of the querry having all the information we need\n# [URL, number of objects on the image, label of the object, English label, digital label, x min, x max, y min, y max]\ndef query_dataset(size):\n    print(\"Loading bbox dataset...\")\n    sub_query_images = \"\"\"\n    (SELECT image_id, thumbnail_300k_url\n    FROM `bigquery-public-data.open_images.images`\n    WHERE thumbnail_300k_url IS NOT NULL)\"\"\"\n    \n    sub_query_box = \"\"\"\n    (SELECT image_id, label_name, x_min, x_max, y_min, y_max\n    FROM `bigquery-public-data.open_images.annotations_bbox`\n    ORDER BY image_id\n    LIMIT \"\"\" + str(size) + \"\"\")\"\"\"\n\n    sub_query_occ_img = \"\"\"\n    (SELECT image_id, COUNT(*) AS nb\n    FROM `bigquery-public-data.open_images.annotations_bbox`\n    GROUP BY image_id\n    ORDER BY image_id\n    LIMIT \"\"\" + str(size) + \"\"\")\"\"\"\n    \n    sub_query_word = \"\"\"\n    (SELECT label_name, label_display_name\n    FROM `bigquery-public-data.open_images.dict`)\n    \"\"\"\n\n    sub_query_id_word = \"\"\"\n    (SELECT lab.label_name, ROW_NUMBER() OVER (ORDER BY lab.label_name) - 1 AS id\n    FROM (SELECT DISTINCT(label_name) FROM \"\"\" + sub_query_box + \"\"\") lab)\"\"\"\n\n    main_query = \"\"\"\n    SELECT img.thumbnail_300k_url, occ.nb, box.label_name, wrd.label_display_name, idw.id, box.x_min, box.x_max, box.y_min, box.y_max\n    FROM \"\"\" + sub_query_box + \"\"\" box\n    INNER JOIN \"\"\" + sub_query_occ_img + \"\"\" occ ON occ.image_id = box.image_id\n    INNER JOIN \"\"\" + sub_query_images + \"\"\" img ON occ.image_id = img.image_id\n    INNER JOIN \"\"\" + sub_query_word + \"\"\" wrd ON wrd.label_name = box.label_name\n    INNER JOIN \"\"\" + sub_query_id_word + \"\"\" idw ON idw.label_name = box.label_name\n    ORDER BY thumbnail_300k_url\"\"\"\n    \n    print(\"Dataset loaded\")\n    return open_images.query_to_pandas_safe(main_query)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71dbfb82bc9f6a2cb54892684c762d6283b05be7","_kg_hide-input":false},"cell_type":"code","source":"# Load images and bboxes / labels in the correct format into the ram\n# This part also contains \"vestiges\" to realize the train of a classifier, but we use that of VGG16 which is more powerful\ndef load_data(data_start, data_length):\n    print(\"Loading images from URL... (From \" + str(data_start) + \" to \" + str(data_start + data_length) + \")\")\n    \n    tab_image = []\n    tab_list_bbox = []\n    tab_list_word = []\n    tab_list_labl = []\n    \n    # As long as there are still images in the sub-part of our dataset\n    it_tuple_image = data_start\n    while it_tuple_image < (data_start + data_length):\n        list_bbox = []\n        list_word = []\n        list_labl = []\n        \n        # We get the image from its link\n        ulr_image = dataset.thumbnail_300k_url.loc[[it_tuple_image]].iloc[0]\n        image = images_from_url(ulr_image)\n        print(image.size)\n        nb_bbox = dataset.nb.loc[[it_tuple_image]].iloc[0] # The number of bbox = the number of objects\n        \n        if(image != False):\n            # We resize and normalize the image\n            image_w, image_h = image.size\n            taille_max = max(image_w, image_h)\n            coef = 800/taille_max\n            image = image.resize((int(coef*image_w), int(coef*image_h)))\n            image = np.array(image)/255\n\n            # We only process images that are in RGB (This also removes the more available images)\n            if(len(image.shape) == 3):\n                # We insert the image in tab_image with the normalized pixel value\n                tab_image.append(image)\n\n                # For each bbox of the image, we store it in a list that we store in tab_list_bbox\n                for it_bbox in range (0, nb_bbox):\n                    it_tuple_bbox = it_tuple_image + it_bbox\n                    if(it_tuple_bbox < data_start + data_length):\n                        list_bbox.append([dataset.x_min.loc[[it_tuple_bbox]].iloc[0], dataset.y_min.loc[[it_tuple_bbox]].iloc[0], dataset.x_max.loc[[it_tuple_bbox]].iloc[0], dataset.y_max.loc[[it_tuple_bbox]].iloc[0]])\n\n                        one_hot = np.zeros(600)\n                        one_hot[dataset.id.loc[[it_tuple_bbox]].iloc[0]] = 1\n                        list_word.append(one_hot)\n                        \n                        label = dataset.label_display_name.loc[[it_tuple_bbox]].iloc[0]\n                        list_labl.append(label)\n\n                tab_list_bbox.append(list_bbox)\n                tab_list_word.append(list_word)\n                tab_list_labl.append(list_labl)\n            # To understand this jump you have to understand the structure of dataset_bbox\n        it_tuple_image = it_tuple_image + nb_bbox\n                                                               \n    tab_image = np.array(tab_image)\n    tab_list_bbox = np.array(tab_list_bbox)\n    tab_list_word = np.array(tab_list_word)\n    tab_list_labl = np.array(tab_list_labl)\n    \n    print(\"Image loaded\")\n    \n    return [tab_image, tab_list_bbox, tab_list_word, tab_list_labl]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8df401b2f6f8b6bf3e9da2c5522479be0261742","_kg_hide-input":true},"cell_type":"code","source":"# Calculate the IoU per 2 box in format xmin ymin xmax ymax\ndef IoU(bbox1, bbox2):\n    w_intersect = (bbox1[2] - bbox1[0]) + (bbox2[2] - bbox2[0]) - (max(bbox1[2], bbox2[2]) - min(bbox1[0], bbox2[0]))\n    h_intersect = (bbox1[3] - bbox1[1]) + (bbox2[3] - bbox2[1]) - (max(bbox1[3], bbox2[3]) - min(bbox1[1], bbox2[1]))\n    \n    if(w_intersect < 0 or h_intersect < 0):\n        return 0\n    \n    intersect = w_intersect * h_intersect\n\n    union_1 = (bbox1[2]-bbox1[0]) * (bbox1[3]-bbox1[1])\n    union_2 = (bbox2[2]-bbox2[0]) * (bbox2[3]-bbox2[1])\n    \n    union = union_1 + union_2 - intersect\n\n    return intersect/union","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Create anchors with the center (x, y) and the width / height of the convolution (reduction 16)\ndef generate_anchors(center_x, center_y, conv_w, conv_h):\n    anchor_ratio = [[1, 1], [1, 2], [2, 1]]\n    anchor_coef = [1, 2, 4]\n    anchor_size = 128\n    \n    anchor_list = []\n    \n    for ratio in anchor_ratio:\n        for coef in anchor_coef:\n            anchor_width = (anchor_size*coef*ratio[0]) / (conv_w*16)\n            anchor_height = (anchor_size*coef*ratio[1]) / (conv_h*16)\n            anchor_x = (center_x/conv_w) - (anchor_width/2)\n            anchor_y = (center_y/conv_h) - (anchor_height/2)\n            anchor = [anchor_x, anchor_y, anchor_x+anchor_width, anchor_y+anchor_height]\n            \n            anchor_list.append(anchor)\n    \n    anchor_list = np.array(anchor_list)\n    \n    return anchor_list","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"# RoI pooling operation on an array and a shape size * shape\n# There are 3 similar lines depending on the type of operation used for pooling (after test the average is better than the max)\ndef RoI(array, shape):\n    result = np.zeros((shape, shape, array.shape[2]))\n    for i in range (0, shape):\n        for j in range (0, shape):\n            sub_array = array[int(i*array.shape[0]/shape):int((i+1)*array.shape[0]/shape), int(j*array.shape[1]/shape):int((j+1)*array.shape[1]/shape)]\n            #result[i][j] = np.amax(np.amax(sub_array, axis = 0), axis = 0)\n            result[i][j] = np.mean(np.mean(sub_array, axis = 0), axis = 0)\n            #result[i][j] = np.amin(np.amin(sub_array, axis = 0), axis = 0)\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1ead23e1c91953ac095b3ca7bec92b7c82563d2","_kg_hide-input":false},"cell_type":"code","source":"# Extract the convolution part of VGG16\ndef generate_conv():\n    vgg16_net = VGG16(include_top=False, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000)\n    model = Model(input=vgg16_net.layers[0].input, output=vgg16_net.layers[17].output)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# A custom accuracy, it is slightly able to exceed 1 but otherwise with keras,\n# For a custom loss, the accuracy is bugged (it's a known bug)\ndef acc(y_true, y_pred): return K.mean(K.round(y_pred)*y_true + (1.-K.round(y_pred))*(1.-y_true))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"# Loss custom to classify it from the rpn, we ignore the cases where the prediction is (0, 0) and we reinforce the learning when an object is present\ndef custom_loss_rpn_cls(y_true, y_pred):\n    shape = K.shape(y_true)\n    depth = shape[0]\n        \n    # Remove the [0, 0] (i.e. between object and no object)\n    new_y_pred = K.zeros((depth, 2))\n    for i in range (0, 9):\n        cond = K.equal(y_true[:, 2*i:2*i+2], [0,0])\n        cond = tf.math.logical_and(cond[:,0], cond[:,1])\n        cond = K.concatenate([cond, cond])\n        cond = K.reshape(cond, (depth,2))\n        \n        temp = K.switch(cond, y_true[:, 2*i:2*i+2], y_pred[:, 2*i:2*i+2])\n        if i == 0:\n            new_y_pred = temp\n        else:\n            new_y_pred = K.concatenate([new_y_pred, temp])\n\n    new_y_pred = K.reshape(new_y_pred, (depth, 18))\n    cls = K.binary_crossentropy(y_true, new_y_pred)\n    \n    # Reinforce the [1,0] (ie the presence of object)\n    new_cls = K.zeros((depth, 2))\n    for i in range (0, 9):\n        cond = K.equal(y_true[:, 2*i:2*i+2], [0,1])\n        cond = tf.math.logical_and(cond[:,0], cond[:,1])\n        cond = K.concatenate([cond, cond])\n        cond = K.reshape(cond, (depth,2))\n        \n        temp = K.switch(cond, cls[:, 2*i:2*i+2], cls[:, 2*i:2*i+2]*4.7)\n        if i == 0:\n            new_cls = temp\n        else:\n            new_cls = K.concatenate([new_cls, temp])\n    new_cls = K.reshape(new_cls, (depth, 18))\n    \n    # We multiply again to compensate for [0,0]\n    # The coefficients were found experimentally\n    return K.mean(new_cls*2.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78aafc3f88500a61d0b47ec717934e35c9ebff68","_kg_hide-input":false},"cell_type":"code","source":"# Return the model used to classify it from the rpn\ndef generate_rpn_cls():\n    model = Sequential()\n    \n    model.add(Flatten(input_shape=(3, 3, 512)))\n    \n    model.add(Dense(512))\n    model.add(Activation('relu'))\n    model.add(Dropout(0.4))\n\n    model.add(Dense(18))\n    model.add(Activation('sigmoid'))\n    model.add(Dropout(0.4))\n\n    model.compile(loss=custom_loss_rpn_cls, optimizer='adam', metrics=[acc])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"# Fonction smoothL1 (known function)\ndef smoothL1(y_true, y_pred):\n    x   = K.abs(y_true - y_pred)\n    x   = K.switch(x < 1, x*x, x)\n    return  x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"# Custom loss for the rpn regressor, we ignore the cases where the prediction is (0, 0)\ndef custom_loss_rpn_reg(y_true, y_pred):\n    shape = K.shape(y_true)\n    depth = shape[0]\n        \n    # Remove the [0, 0, 0, 0] that is to say that there is no object presence\n    new_y_pred = K.zeros((depth, 4))\n    for i in range (0, 9):\n        cond = K.equal(y_true[:, 2*i:2*i+4], [0,0,0,0])\n        cond = tf.math.logical_and(tf.math.logical_and(cond[:,0], cond[:,1]), tf.math.logical_and(cond[:,2], cond[:,3]))\n        cond = K.concatenate([cond, cond, cond, cond])\n        cond = K.reshape(cond, (depth,4))\n        \n        temp = K.switch(cond, y_true[:, 2*i:2*i+4], y_pred[:, 2*i:2*i+4])\n        if i == 0:\n            new_y_pred = temp\n        else:\n            new_y_pred = K.concatenate([new_y_pred, temp])\n\n    new_y_pred = K.reshape(new_y_pred, (depth, 36))\n    reg = smoothL1(y_true, new_y_pred)\n    \n    # We multiply to compensate for the [0,0,0,0]\n    # The coefficients were found experimentally\n    return K.mean(reg)*9","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"# Return the model used for the rpn regressor\ndef generate_rpn_reg():\n    model = Sequential()\n    \n    model.add(Flatten(input_shape=(3, 3, 512)))\n    \n    model.add(Dense(512))\n    model.add(Activation('relu'))\n    model.add(Dropout(0.4))\n    \n    model.add(Dense(512))\n    model.add(Activation('relu'))\n    model.add(Dropout(0.4))\n\n    model.add(Dense(36))\n    model.add(Activation('linear'))\n    model.add(Dropout(0.4))\n\n    model.compile(loss=custom_loss_rpn_reg, optimizer='adam', metrics=[acc])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1a06de37402235bffcc2c1085b7ccaecaf05749","_kg_hide-input":false},"cell_type":"code","source":"# Return the model used to classify it\ndef generate_cls_nn():\n    vgg16_net = VGG16(include_top=True, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000)\n    \n    l_input = Input(shape=(7, 7, 512))\n    l_flatten = vgg16_net.get_layer(\"flatten\")\n    l_fc1 = vgg16_net.get_layer(\"fc1\")\n    l_fc2 = vgg16_net.get_layer(\"fc2\")\n    l_output = vgg16_net.get_layer(\"predictions\")\n    model = Model(input=l_input, output=l_output(l_fc2(l_fc1(l_flatten(l_input)))))\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b0c28fc8dd00ca26babf8597d52bbbbac1b1259","_kg_hide-input":false},"cell_type":"code","source":"# Generate data to make them compatible with rpn networks\ndef generate_feature_label_rpn():\n    features = []\n    labels = []\n    \n    nb_feature_map = list_feature_map.shape[0]\n    \n    nb_max = []\n    for i in range(0,9):\n        nb_max.append(0)\n    \n    # For each\n    for it_feature_map in range (0, nb_feature_map):\n        feature_map = list_feature_map[it_feature_map]\n        \n        feature_map_width = feature_map.shape[1]\n        feature_map_height = feature_map.shape[0]\n        \n        # Each sliding window\n        for x in range (1, feature_map_width - 1):\n            for y in range (1, feature_map_height - 1):\n                \n                sub_labels = []\n                sub_anchor = []\n                \n                window_valid = False\n                \n                list_anchors = generate_anchors(x, y, feature_map_width, feature_map_height)\n                \n                it_anch = -1\n                good_it_anch = it_anch\n                \n                # Each anchors\n                for anchor in list_anchors:\n                    it_anch = it_anch + 1\n                    \n                    anchor_cross = not(anchor[0] >= 0 and anchor[1] >= 0 and anchor[0] + anchor[2] < 1 and anchor[1] + anchor[3] < 1)\n                    anchor_cross = False\n                    anchor_valid = False\n                    anchor_empty = True\n\n                    # If the anchor is> 0.7, or> 0.3 once\n                    for bbox in data_bbox[it_feature_map]:\n                        if(IoU(anchor, bbox) > 0.7):\n                            anchor_valid = True\n                            window_valid = True\n                            break\n                        if(IoU(anchor, bbox) > 0.3):\n                            anchor_empty = False\n                    \n                    # Valid anchor case\n                    if(anchor_valid and not(anchor_cross)):\n                        good_it_anch = it_anch\n                        sub_labels.append(1.)\n                        sub_labels.append(0.)\n                        \n                        # Coordinates part relating to the anchor\n                        bbox_x, bbox_y, bbox_xm, bbox_ym = bbox\n                        bbox_width = bbox_xm - bbox_x\n                        bbox_height = bbox_ym - bbox_y\n                        anchor_x, anchor_y, anchor_xm, anchor_ym = anchor\n                        anchor_width = anchor_xm - anchor_x\n                        anchor_height = anchor_ym - anchor_y\n                        \n                        sub_anchor.append((bbox_x - anchor_x)/anchor_width)\n                        sub_anchor.append((bbox_y - anchor_y)/anchor_height)\n                        sub_anchor.append(math.log(bbox_width/anchor_width))\n                        sub_anchor.append(math.log(bbox_height/anchor_height))\n                    # Empty anchor case\n                    elif(anchor_empty and not(anchor_cross)):\n                        sub_labels.append(0.)\n                        sub_labels.append(1.)\n\n                        sub_anchor.append(0.)\n                        sub_anchor.append(0.)\n                        sub_anchor.append(0.)\n                        sub_anchor.append(0.)\n                    # Case anchor between the 2\n                    else:\n                        sub_labels.append(0.)\n                        sub_labels.append(0.)\n\n                        sub_anchor.append(0.)\n                        sub_anchor.append(0.)\n                        sub_anchor.append(0.)\n                        sub_anchor.append(0.)\n                \n                # Normalization (only windows with one object and no more than 400 object per anchor (400 is a number found experimentally))\n                if(window_valid and nb_max[good_it_anch] < 400):\n                    nb_max[good_it_anch] = nb_max[good_it_anch] + 1\n                    \n                    features.append(feature_map[y-1:y+2, x-1:x+2])\n                    labels.append(np.array(sub_labels + sub_anchor))\n            \n    features = np.array(features)\n    labels = np.array(labels)\n\n    return features, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"930e1f1b0286378a8a881a7bf2f1b79da56c201f","_kg_hide-input":true},"cell_type":"code","source":"# Remnant of our classifer (the one to replace vgg)\ndef generate_feature_label_cls():\n    features = []\n    labels = []\n    \n    counter = 0\n    \n    nb_image_conv = data_conv.shape[0]\n    for it_image_conv in range (0, nb_image_conv):\n        for it_bbox in range (0, len(data_bbox[it_image_conv])):\n            list_bbox = data_bbox[it_image_conv]\n            bbox = list_bbox[it_bbox]\n            features.append(add_black_border(data_conv[it_image_conv][int(bbox[0]*13):int(bbox[2]*13), int(bbox[1]*13):int(bbox[3]*13)]))\n            labels.append(data_word[it_image_conv][it_bbox])\n            \n    features = np.array(features)\n    labels = np.array(labels)\n    \n    return features, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Return the convolution of an image\ndef pred_conv(image):\n    return conv_net.predict(np.array([image]))[0]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7d4f49478a654c12b995980decab564d57f69f25"},"cell_type":"markdown","source":"# DEFINITION DES GLOBALS"},{"metadata":{"trusted":true,"_uuid":"e2d8dd56aef8b2eb42577edf7621ef04e13f9008","_kg_hide-input":false},"cell_type":"code","source":"# Variable dataset\nopen_images = bq_helper.BigQueryHelper(active_project=\"bigquery-public-data\", dataset_name=\"open_images\") \n#print(open_images)\nquery_size = 80000\ndataset_size = 1000\ndataset = query_dataset(query_size) \n#print(dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"conv_net = generate_conv()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d3de51127246e8b62b5856470c06cdf853cfe97","_kg_hide-input":false},"cell_type":"code","source":"rpn_cls_net = generate_rpn_cls()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rpn_reg_net = generate_rpn_reg()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cls_net = generate_cls_nn()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PARTIE TRAIN"},{"metadata":{"trusted":true,"_uuid":"cc9b2076b96ef57ed28a269f844f6e0cec1e0df6"},"cell_type":"code","source":"# Load images from 0 to dataset_size (1000) in the ram, as well as their label\ndata_image, data_bbox, data_word, data_labl = load_data(0, dataset_size)\ndel dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Transform images into convolution\nlist_feature_map = np.array(list(map(pred_conv, data_image)))\ndel data_image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"890508378ba6bf14106718d0ba039d4cee8c9c31","scrolled":true},"cell_type":"code","source":"# Generate data to make them compatible with rpn networks\n# (the data is also mixed and a validation part is extracted)\nfeatures_rpn, labels_rpn = generate_feature_label_rpn()\nX_train_rpn, X_valid_rpn, y_train_rpn, y_valid_rpn = train_test_split(features_rpn, labels_rpn, test_size=0.1, random_state=42)\ndel features_rpn\ndel labels_rpn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Learning of the rpn \"regression\" network\nrpn_reg_net.fit(X_train_rpn, y_train_rpn[:, 18:54], validation_data=(X_valid_rpn, y_valid_rpn[:, 18:54]), epochs=100, batch_size=32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce0d79c29d3eb31161f5139d6721762b8cf08d67"},"cell_type":"code","source":"# Learning the rpn \"classifier\" network\nrpn_cls_net.fit(X_train_rpn, y_train_rpn[:, 0:18], validation_data=(X_valid_rpn, y_valid_rpn[:, 0:18]), epochs=200, batch_size=64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We save the rpn networks to be able to use them without train\nrpn_reg_net.save(\"reg.h5\")\nrpn_cls_net.save(\"cls.h5\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PARTIE APPLICATION"},{"metadata":{"trusted":true,"_uuid":"ad87f23f55e840d3b8cfba1cca9b7132c60fba40","_kg_hide-input":false},"cell_type":"code","source":"def predict(URL, threshold_cls_rpn = 0.7, threshold_cls_vgg = 0.5, threshold_iou = 0.8):\n    # The image is downloaded and put in the right format (the vgg format is not standardized, that of our rpn is because it gives us better results)\n    image_test = images_from_url(URL)\n    if(image_test != False):\n        image_test_w, image_test_h = image_test.size\n        taille_max = max(image_test_w, image_test_h)\n        coef = 800/taille_max\n        image_test = image_test.resize((int(coef*image_test_w), int(coef*image_test_h)))\n        image_test_vgg = np.array(image_test)\n        image_test = np.array(image_test)/255\n        if(len(image_test.shape) == 3):\n            anchor_test_valid = []\n\n            image_test_conv_vgg = pred_conv(image_test_vgg)\n            image_test_conv = pred_conv(image_test)\n\n            # We pass on each pixel of the convolution\n            for x in range (1, image_test_conv.shape[1] - 1):\n                for y in range (1, image_test_conv.shape[0] - 1):\n                    anchor_valid = False\n                    anchor_empty = True\n\n                   # We make a prediction on the sliding window centered on the current pixel\n                    pred_cls = rpn_cls_net.predict(np.array([image_test_conv[y-1:y+2, x-1:x+2]]))[0]\n                    pred_reg = rpn_reg_net.predict(np.array([image_test_conv[y-1:y+2, x-1:x+2]]))[0]\n\n                    # We test for all anchors if the rpn has detected an object\n                    list_anchors = generate_anchors(x, y, image_test_conv.shape[1], image_test_conv.shape[0])\n                    for k in range(0, 9):\n                        # If you find an object with more than 70% safety\n                        if(pred_cls[k*2] >= threshold_cls_rpn):\n                            # We get the anchor info\n                            anchor = list_anchors[k]\n                            anchor_x, anchor_y, anchor_xm, anchor_ym = anchor\n                            anchor_width = anchor_xm - anchor_x\n                            anchor_height = anchor_ym - anchor_y\n\n                            # We get the information from the prediction\n                            pred_reg_x, pred_reg_y, pred_reg_w, pred_reg_h = pred_reg[k*4:k*4+4]\n\n                            # We test if the anchor does not come out of the screen\n                            cond1 = anchor_x+(pred_reg_x*anchor_width) >= 0\n                            cond2 = anchor_y+(pred_reg_y*anchor_height) >= 0\n                            cond3 = anchor_x+(pred_reg_x*anchor_width) + (10**pred_reg_w)*anchor_width < 1\n                            cond4 = anchor_y+(pred_reg_y*anchor_height) + (10**pred_reg_h)*anchor_height < 1\n                            if(cond1 and cond2 and cond3 and cond4):\n                               # We calculate the xmin / max ymin / max of the prediction relative to the image\n                                it_min_x = int((anchor_x+(pred_reg_x*anchor_width)) * image_test_conv.shape[1])\n                                it_max_x = int((anchor_x+(pred_reg_x*anchor_width) + (10**pred_reg_w)*anchor_width) * image_test_conv.shape[1])\n                                it_min_y = int((anchor_y+(pred_reg_y*anchor_height)) * image_test_conv.shape[0])\n                                it_max_y = int((anchor_y+(pred_reg_y*anchor_height) + (10**pred_reg_h)*anchor_height) * image_test_conv.shape[0])\n\n                                # If the prediction is larger than 7 * 7 (minimum of the classifier)\n                                if(it_max_y-it_min_y >= 7 and it_max_x-it_min_x >= 7):\n                                    # Prediction of the first 5 classes that vgg finds\n                                    label = decode_predictions(cls_net.predict(np.array([RoI(image_test_conv_vgg[it_min_y:it_max_y, it_min_x:it_max_x], 7)])), top=5)[0]\n                                    # If the trust in the top class of vgg is more than 50%\n                                    if(label[0][2] >= threshold_cls_vgg):\n                                        # We store the data in the simplest way to treat them with the nonmax\n                                        anchor_test_valid.append([label[0][2], [[label[0][1], label[1][1], label[2][1], label[3][1], label[4][1]], anchor_x+(pred_reg_x*anchor_width), anchor_y+(pred_reg_y*anchor_height), anchor_x+(pred_reg_x*anchor_width) + (10**pred_reg_w)*anchor_width, anchor_y+(pred_reg_y*anchor_height) + (10**pred_reg_h)*anchor_height]])  \n\n            # SECTION NONMAX\n            # Le but est de supprimer les overlaps au dessus d'un seuil pour les mêmes classes (il suffit d'une coresspondance dans les 5 premières classes)\n            anchor_test_valid = np.array(anchor_test_valid)      \n            q = Q.PriorityQueue()\n            for a in anchor_test_valid:\n                q.put((1-a[0] + random.random()/100000,a[1]))\n            anchor_test_valid = []\n            size = q.qsize()\n            for i in range (0, size):\n                var_i = q.get()\n                found_one = False\n                for a in anchor_test_valid:\n                    for labelnb in range(0, 5):\n                        if(IoU(a[1], var_i[1][1:5]) >= 1 - threshold_iou and a[0] == var_i[1][0][labelnb]):\n                            found_one = True\n                if(not found_one):\n                    anchor_test_valid.append([var_i[1][0][0], var_i[1][1:5]])\n            anchor_test_valid = np.array(anchor_test_valid)\n            if(anchor_test_valid.shape[0] != 0):\n                plot_bbox_label(image_test, anchor_test_valid[:, 1], anchor_test_valid[:, 0])\n            else:\n                plt.figure(figsize=(15,20))\n                plt.imshow(image_test_vgg)\n                plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_multiple(list_URL):\n    for URL in list_URL:\n        url_pred = predict(URL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# UNE VOITURE :   https://www.usinenouvelle.com/mediatheque/4/5/4/000626454_image_896x598/dacia-sandero.jpg\n# 2 VOITURES :   https://hips.hearstapps.com/hmg-prod.s3.amazonaws.com/images/s17-2051-fine-1553003760.jpg\n# N VOITURES :   https://cdn-images-1.medium.com/max/1600/1*ICvAO8mPCA_sXOzW9zeM7g.jpeg\n# 2 VOITURES :   https://ischool.syr.edu/infospace/wp-content/files/2015/10/toyota-and-lexus-car-on-road--e1444655872784.jpg\n# ZOO : http://www.mdjunited.com/medias/images/zoo.jpg\n\nurl_images_test = ['https://www.usinenouvelle.com/mediatheque/4/5/4/000626454_image_896x598/dacia-sandero.jpg',\n                   'https://images5.alphacoders.com/393/393962.jpg',\n                   'https://images.unsplash.com/photo-1544776527-68e63addedf7?ixlib=rb-1.2.1&ixid=eyJhcHBfaWQiOjEyMDd9&w=1000&q=80',\n                   'https://www.autocar.co.uk/sites/autocar.co.uk/files/styles/gallery_slide/public/images/car-reviews/first-drives/legacy/gallardo-0638.jpg?itok=-So1NoXA', \n                   'http://www.mdjunited.com/medias/images/zoo.jpg']\n\npredict_multiple(url_images_test)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}