{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"data_path =  '../input/3d-object-detection-for-autonomous-vehicles/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import struct\nimport numpy as np\nfrom keras.layers import Conv2D\nfrom keras.layers import Input\nfrom keras.layers import BatchNormalization\nfrom keras.layers import LeakyReLU\nfrom keras.layers import ZeroPadding2D\nfrom keras.layers import UpSampling2D\nfrom keras.layers.merge import add, concatenate\nfrom keras.models import Model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _conv_block(inp, convs, skip=True):\n    x = inp\n    count = 0\n    for conv in convs:\n        if count == (len(convs) - 2) and skip:\n            skip_connection = x\n        count += 1\n        if conv['stride'] > 1: x = ZeroPadding2D(((1,0),(1,0)))(x) # peculiar padding as darknet prefer left and top\n        x = Conv2D(conv['filter'],\n                   conv['kernel'],\n                   strides=conv['stride'],\n                   padding='valid' if conv['stride'] > 1 else 'same', # peculiar padding as darknet prefer left and top\n                   name='conv_' + str(conv['layer_idx']),\n                   use_bias=False if conv['bnorm'] else True)(x)\n        if conv['bnorm']: x = BatchNormalization(epsilon=0.001, name='bnorm_' + str(conv['layer_idx']))(x)\n        if conv['leaky']: x = LeakyReLU(alpha=0.1, name='leaky_' + str(conv['layer_idx']))(x)\n    return add([skip_connection, x]) if skip else x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_yolov3_model():\n    input_image = Input(shape=(None, None, 3))\n    # Layer  0 => 4\n    x = _conv_block(input_image, [{'filter': 32, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 0},\n                                  {'filter': 64, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 1},\n                                  {'filter': 32, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 2},\n                                  {'filter': 64, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 3}])\n    # Layer  5 => 8\n    x = _conv_block(x, [{'filter': 128, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 5},\n                        {'filter':  64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 6},\n                        {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 7}])\n    # Layer  9 => 11\n    x = _conv_block(x, [{'filter':  64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 9},\n                        {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 10}])\n    # Layer 12 => 15\n    x = _conv_block(x, [{'filter': 256, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 12},\n                        {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 13},\n                        {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 14}])\n    # Layer 16 => 36\n    for i in range(7):\n        x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 16+i*3},\n                            {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 17+i*3}])\n    skip_36 = x\n    # Layer 37 => 40\n    x = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 37},\n                        {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 38},\n                        {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 39}])\n    # Layer 41 => 61\n    for i in range(7):\n        x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 41+i*3},\n                            {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 42+i*3}])\n    skip_61 = x\n    # Layer 62 => 65\n    x = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 62},\n                        {'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 63},\n                        {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 64}])\n    # Layer 66 => 74\n    for i in range(3):\n        x = _conv_block(x, [{'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 66+i*3},\n                            {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 67+i*3}])\n    # Layer 75 => 79\n    x = _conv_block(x, [{'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 75},\n                        {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 76},\n                        {'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 77},\n                        {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 78},\n                        {'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 79}], skip=False)\n    # Layer 80 => 82\n    yolo_82 = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 80},\n                              {'filter':  255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 81}], skip=False)\n    # Layer 83 => 86\n    x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 84}], skip=False)\n    x = UpSampling2D(2)(x)\n    x = concatenate([x, skip_61])\n    # Layer 87 => 91\n    x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 87},\n                        {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 88},\n                        {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 89},\n                        {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 90},\n                        {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 91}], skip=False)\n    # Layer 92 => 94\n    yolo_94 = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 92},\n                              {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 93}], skip=False)\n    # Layer 95 => 98\n    x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True,   'layer_idx': 96}], skip=False)\n    x = UpSampling2D(2)(x)\n    x = concatenate([x, skip_36])\n    # Layer 99 => 106\n    yolo_106 = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 99},\n                               {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 100},\n                               {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 101},\n                               {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 102},\n                               {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 103},\n                               {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 104},\n                               {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 105}], skip=False)\n    model = Model(input_image, [yolo_82, yolo_94, yolo_106])\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class WeightReader:\n    def __init__(self, weight_file):\n        with open(weight_file, 'rb') as w_f:\n            major,\t= struct.unpack('i', w_f.read(4))\n            minor,\t= struct.unpack('i', w_f.read(4))\n            revision, = struct.unpack('i', w_f.read(4))\n            if (major*10 + minor) >= 2 and major < 1000 and minor < 1000:\n                w_f.read(8)\n            else:\n                w_f.read(4)\n            transpose = (major > 1000) or (minor > 1000)\n            binary = w_f.read()\n        self.offset = 0\n        self.all_weights = np.frombuffer(binary, dtype='float32')\n \n    def read_bytes(self, size):\n        self.offset = self.offset + size\n        return self.all_weights[self.offset-size:self.offset]\n\n    def load_weights(self, model):\n        for i in range(106):\n            try:\n                conv_layer = model.get_layer('conv_' + str(i))\n                print(\"loading weights of convolution #\" + str(i))\n                if i not in [81, 93, 105]:\n                    norm_layer = model.get_layer('bnorm_' + str(i))\n                    size = np.prod(norm_layer.get_weights()[0].shape)\n                    beta  = self.read_bytes(size) # bias\n                    gamma = self.read_bytes(size) # scale\n                    mean  = self.read_bytes(size) # mean\n                    var   = self.read_bytes(size) # variance\n                    weights = norm_layer.set_weights([gamma, beta, mean, var])\n                if len(conv_layer.get_weights()) > 1:\n                    bias   = self.read_bytes(np.prod(conv_layer.get_weights()[1].shape))\n                    kernel = self.read_bytes(np.prod(conv_layer.get_weights()[0].shape))\n                    kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape)))\n                    kernel = kernel.transpose([2,3,1,0])\n                    conv_layer.set_weights([kernel, bias])\n                else:\n                    kernel = self.read_bytes(np.prod(conv_layer.get_weights()[0].shape))\n                    kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape)))\n                    kernel = kernel.transpose([2,3,1,0])\n                    conv_layer.set_weights([kernel])\n            except ValueError:\n                print(\"no convolution #\" + str(i))\n\n    def reset(self):\n        self.offset = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib import pyplot\nfrom matplotlib.patches import Rectangle\n \nclass BoundBox:\n    def __init__(self, xmin, ymin, xmax, ymax, objness = None, classes = None):\n        self.xmin = xmin\n        self.ymin = ymin\n        self.xmax = xmax\n        self.ymax = ymax\n        self.objness = objness\n        self.classes = classes\n        self.label = -1\n        self.score = -1\n\n    def get_label(self):\n        if self.label == -1:\n            self.label = np.argmax(self.classes)\n\n        return self.label\n\n    def get_score(self):\n        if self.score == -1:\n            self.score = self.classes[self.get_label()]\n \n        return self.score\n \ndef _sigmoid(x):\n    return 1. / (1. + np.exp(-x))\n \ndef decode_netout(netout, anchors, obj_thresh, net_h, net_w):\n    grid_h, grid_w = netout.shape[:2]\n    nb_box = 3\n    netout = netout.reshape((grid_h, grid_w, nb_box, -1))\n    nb_class = netout.shape[-1] - 5\n    boxes = []\n    netout[..., :2]  = _sigmoid(netout[..., :2])\n    netout[..., 4:]  = _sigmoid(netout[..., 4:])\n    netout[..., 5:]  = netout[..., 4][..., np.newaxis] * netout[..., 5:]\n    netout[..., 5:] *= netout[..., 5:] > obj_thresh\n \n    for i in range(grid_h*grid_w):\n        row = i / grid_w\n        col = i % grid_w\n        for b in range(nb_box):\n            # 4th element is objectness score\n            objectness = netout[int(row)][int(col)][b][4]\n            if(objectness.all() <= obj_thresh): continue\n            # first 4 elements are x, y, w, and h\n            x, y, w, h = netout[int(row)][int(col)][b][:4]\n            x = (col + x) / grid_w # center position, unit: image width\n            y = (row + y) / grid_h # center position, unit: image height\n            w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width\n            h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height\n            # last elements are class probabilities\n            classes = netout[int(row)][col][b][5:]\n            box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, objectness, classes)\n            boxes.append(box)\n    return boxes\n \ndef correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w):\n    new_w, new_h = net_w, net_h\n    for i in range(len(boxes)):\n        x_offset, x_scale = (net_w - new_w)/2./net_w, float(new_w)/net_w\n        y_offset, y_scale = (net_h - new_h)/2./net_h, float(new_h)/net_h\n        boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w)\n        boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w)\n        boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h)\n        boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h)\n\ndef _interval_overlap(interval_a, interval_b):\n    x1, x2 = interval_a\n    x3, x4 = interval_b\n    if x3 < x1:\n        if x4 < x1:\n            return 0\n        else:\n            return min(x2,x4) - x1\n    else:\n        if x2 < x3:\n            return 0\n        else:\n            return min(x2,x4) - x3\n\ndef bbox_iou(box1, box2):\n    intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax])\n    intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax])\n    intersect = intersect_w * intersect_h\n    w1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin\n    w2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin\n    union = w1*h1 + w2*h2 - intersect\n    return float(intersect) / union\n \ndef do_nms(boxes, nms_thresh):\n    if len(boxes) > 0:\n        nb_class = len(boxes[0].classes)\n    else:\n        return\n    for c in range(nb_class):\n        sorted_indices = np.argsort([-box.classes[c] for box in boxes])\n        for i in range(len(sorted_indices)):\n            index_i = sorted_indices[i]\n            if boxes[index_i].classes[c] == 0: continue\n            for j in range(i+1, len(sorted_indices)):\n                index_j = sorted_indices[j]\n                if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh:\n                    boxes[index_j].classes[c] = 0\n\n# load and prepare an image\ndef load_image_pixels(filename, shape):\n    # load the image to get its shape\n    image = load_img(filename)\n    width, height = image.size\n    # load the image with the required size\n    image = load_img(filename, target_size=shape)\n    # convert to numpy array\n    image = img_to_array(image)\n    # scale pixel values to [0, 1]\n    image = image.astype('float32')\n    image /= 255.0\n    # add a dimension so that we have one sample\n    image = expand_dims(image, 0)\n    return image, width, height\n \n# get all of the results above a threshold\ndef get_boxes(boxes, labels, thresh):\n    v_boxes, v_labels, v_scores = list(), list(), list()\n    # enumerate all boxes\n    for box in boxes:\n        # enumerate all possible labels\n        for i in range(len(labels)):\n            # check if the threshold for this label is high enough\n            if box.classes[i] > thresh:\n                v_boxes.append(box)\n                v_labels.append(labels[i])\n                v_scores.append(box.classes[i]*100)\n                # don't break, many labels may trigger for one box\n    return v_boxes, v_labels, v_scores\n \n# draw all results\ndef draw_boxes(filename, v_boxes, v_labels, v_scores):\n    # load the image\n    data = pyplot.imread(filename)\n    # plot the image\n    pyplot.imshow(data)\n    # get the context for drawing boxes\n    ax = pyplot.gca()\n    # plot each box\n    for i in range(len(v_boxes)):\n        box = v_boxes[i]\n        # get coordinates\n        y1, x1, y2, x2 = box.ymin, box.xmin, box.ymax, box.xmax\n        # calculate width and height of the box\n        width, height = x2 - x1, y2 - y1\n        # create the shape\n        rect = Rectangle((x1, y1), width, height, fill=False, color='white')\n        # draw the box\n        ax.add_patch(rect)\n        # draw text and score in top left corner\n        label = \"%s (%.3f)\" % (v_labels[i], v_scores[i])\n        pyplot.text(x1, y1, label, color='white')\n    # show the plot\n    pyplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define the model\nmodel = make_yolov3_model()\n\n# load the model weights\n# I have loaded the pretrained weights in a separate dataset\nweight_reader = WeightReader('../input/lyft-3d-recognition/yolov3.weights')\n\n# set the model weights into the model\nweight_reader.load_weights(model)\n\n# save the model to file\nmodel.save('model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import load_model\nmodel = load_model('model.h5')\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Parameters used in the Dataset, on which YOLOv3 was pretrained\nanchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]]\n\n# define the expected input shape for the model\nWIDTH, HEIGHT = 416, 416\n\n# define the probability threshold for detected objects\nclass_threshold = 0.3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfrom matplotlib import pyplot as plt\nimages = os.listdir('../input/3d-object-detection-for-autonomous-vehicles/train_images')[:10]\nfrom numpy import expand_dims\nfrom keras.preprocessing.image import load_img, img_to_array\n\n# load and prepare an image\ndef load_image_pixels(filename, shape):\n\n    # load the image to get its shape\n    image = load_img(filename)\n    width, height = image.size\n    \n    # load the image with the required size\n    image = load_img(filename, target_size=shape)\n    \n    # convert to numpy array\n    image = img_to_array(image)\n    \n    # scale pixel values to [0, 1]\n    image = image.astype('float32')\n    image /= 255.0\n    \n    # add a dimension so that we have one sample\n    image = expand_dims(image, 0)\n    return image, width, height","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for file in images:\n    photo_filename = data_path + 'train_images/' + file\n    \n    # load picture with old dimensions\n    image, image_w, image_h = load_image_pixels(photo_filename, (WIDTH, HEIGHT))\n    \n    # Predict image\n    yhat = model.predict(image)\n    \n    # Create boxes\n    boxes = list()\n    for i in range(len(yhat)):\n        # decode the output of the network\n        boxes += decode_netout(yhat[i][0], anchors[i], class_threshold, HEIGHT, WIDTH)\n    \n    correct_yolo_boxes(boxes, image_h, image_w, HEIGHT, WIDTH)\n\n    # suppress non-maximal boxes\n    do_nms(boxes, 0.5)\n\n    # define the labels (Filtered only the ones relevant for this task, which were used in pretraining the YOLOv3 model)\n    labels = [\"person\", \"bicycle\", \"car\", \"motorbike\", \"aeroplane\", \"bus\", \"train\", \"truck\",\"boat\"]\n\n    # get the details of the detected objects\n    v_boxes, v_labels, v_scores = get_boxes(boxes, labels, class_threshold)\n\n    # summarize what we found\n    for i in range(len(v_boxes)):\n\n        print(v_labels[i], v_scores[i])\n\n    # draw what we found\n    draw_boxes(photo_filename, v_boxes, v_labels, v_scores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}