{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# create a YOLOv3 Keras model and save it to file\n# based on https://github.com/experiencor/keras-yolo3\nimport struct\nimport numpy as np\nfrom keras.layers import Conv2D\nfrom keras.layers import Input\nfrom keras.layers import BatchNormalization\nfrom keras.layers import LeakyReLU\nfrom keras.layers import ZeroPadding2D\nfrom keras.layers import UpSampling2D\nfrom keras.layers.merge import add, concatenate\nfrom keras.models import Model\n \ndef _conv_block(inp, convs, skip=True):\n    x = inp\n    count = 0\n    for conv in convs:\n        if count == (len(convs) - 2) and skip:\n            skip_connection = x\n        count += 1\n        if conv['stride'] > 1: x = ZeroPadding2D(((1,0),(1,0)))(x) # peculiar padding as darknet prefer left and top\n        x = Conv2D(conv['filter'],\n                   conv['kernel'],\n                   strides=conv['stride'],\n                   padding='valid' if conv['stride'] > 1 else 'same', # peculiar padding as darknet prefer left and top\n                   name='conv_' + str(conv['layer_idx']),\n                   use_bias=False if conv['bnorm'] else True)(x)\n        if conv['bnorm']: x = BatchNormalization(epsilon=0.001, name='bnorm_' + str(conv['layer_idx']))(x)\n        if conv['leaky']: x = LeakyReLU(alpha=0.1, name='leaky_' + str(conv['layer_idx']))(x)\n    return add([skip_connection, x]) if skip else x\n\ndef make_yolov3_model():\n    input_image = Input(shape=(None, None, 3))\n    # Layer  0 => 4\n    x = _conv_block(input_image, [{'filter': 32, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 0},\n                                  {'filter': 64, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 1},\n                                  {'filter': 32, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 2},\n                                  {'filter': 64, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 3}])\n    # Layer  5 => 8\n    x = _conv_block(x, [{'filter': 128, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 5},\n                        {'filter':  64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 6},\n                        {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 7}])\n    # Layer  9 => 11\n    x = _conv_block(x, [{'filter':  64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 9},\n                        {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 10}])\n    # Layer 12 => 15\n    x = _conv_block(x, [{'filter': 256, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 12},\n                        {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 13},\n                        {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 14}])\n    # Layer 16 => 36\n    for i in range(7):\n        x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 16+i*3},\n                            {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 17+i*3}])\n    skip_36 = x\n    # Layer 37 => 40\n    x = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 37},\n                        {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 38},\n                        {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 39}])\n    # Layer 41 => 61\n    for i in range(7):\n        x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 41+i*3},\n                            {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 42+i*3}])\n    skip_61 = x\n    # Layer 62 => 65\n    x = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 62},\n                        {'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 63},\n                        {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 64}])\n    # Layer 66 => 74\n    for i in range(3):\n        x = _conv_block(x, [{'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 66+i*3},\n                            {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 67+i*3}])\n    # Layer 75 => 79\n    x = _conv_block(x, [{'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 75},\n                        {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 76},\n                        {'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 77},\n                        {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 78},\n                        {'filter':  512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 79}], skip=False)\n    # Layer 80 => 82\n    yolo_82 = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 80},\n                              {'filter':  255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 81}], skip=False)\n    # Layer 83 => 86\n    x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 84}], skip=False)\n    x = UpSampling2D(2)(x)\n    x = concatenate([x, skip_61])\n    # Layer 87 => 91\n    x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 87},\n                        {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 88},\n                        {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 89},\n                        {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 90},\n                        {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 91}], skip=False)\n    # Layer 92 => 94\n    yolo_94 = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 92},\n                              {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 93}], skip=False)\n    # Layer 95 => 98\n    x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True,   'layer_idx': 96}], skip=False)\n    x = UpSampling2D(2)(x)\n    x = concatenate([x, skip_36])\n    # Layer 99 => 106\n    yolo_106 = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 99},\n                               {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 100},\n                               {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 101},\n                               {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 102},\n                               {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 103},\n                               {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True,  'leaky': True,  'layer_idx': 104},\n                               {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 105}], skip=False)\n    model = Model(input_image, [yolo_82, yolo_94, yolo_106])\n    return model\n\nclass WeightReader:\n    def __init__(self, weight_file):\n        with open(weight_file, 'rb') as w_f:\n            major,\t= struct.unpack('i', w_f.read(4))\n            minor,\t= struct.unpack('i', w_f.read(4))\n            revision, = struct.unpack('i', w_f.read(4))\n            if (major*10 + minor) >= 2 and major < 1000 and minor < 1000:\n                w_f.read(8)\n            else:\n                w_f.read(4)\n            transpose = (major > 1000) or (minor > 1000)\n            binary = w_f.read()\n        self.offset = 0\n        self.all_weights = np.frombuffer(binary, dtype='float32')\n \n    def read_bytes(self, size):\n        self.offset = self.offset + size\n        return self.all_weights[self.offset-size:self.offset]\n\n    def load_weights(self, model):\n        for i in range(106):\n            try:\n                conv_layer = model.get_layer('conv_' + str(i))\n                print(\"loading weights of convolution #\" + str(i))\n                if i not in [81, 93, 105]:\n                    norm_layer = model.get_layer('bnorm_' + str(i))\n                    size = np.prod(norm_layer.get_weights()[0].shape)\n                    beta  = self.read_bytes(size) # bias\n                    gamma = self.read_bytes(size) # scale\n                    mean  = self.read_bytes(size) # mean\n                    var   = self.read_bytes(size) # variance\n                    weights = norm_layer.set_weights([gamma, beta, mean, var])\n                if len(conv_layer.get_weights()) > 1:\n                    bias   = self.read_bytes(np.prod(conv_layer.get_weights()[1].shape))\n                    kernel = self.read_bytes(np.prod(conv_layer.get_weights()[0].shape))\n                    kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape)))\n                    kernel = kernel.transpose([2,3,1,0])\n                    conv_layer.set_weights([kernel, bias])\n                else:\n                    kernel = self.read_bytes(np.prod(conv_layer.get_weights()[0].shape))\n                    kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape)))\n                    kernel = kernel.transpose([2,3,1,0])\n                    conv_layer.set_weights([kernel])\n            except ValueError:\n                print(\"no convolution #\" + str(i))\n\n    def reset(self):\n        self.offset = 0\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot\nfrom matplotlib.patches import Rectangle\n \nclass BoundBox:\n    def __init__(self, xmin, ymin, xmax, ymax, objness = None, classes = None):\n        self.xmin = xmin\n        self.ymin = ymin\n        self.xmax = xmax\n        self.ymax = ymax\n        self.objness = objness\n        self.classes = classes\n        self.label = -1\n        self.score = -1\n\n    def get_label(self):\n        if self.label == -1:\n            self.label = np.argmax(self.classes)\n\n        return self.label\n\n    def get_score(self):\n        if self.score == -1:\n            self.score = self.classes[self.get_label()]\n \n        return self.score\n \ndef _sigmoid(x):\n    return 1. / (1. + np.exp(-x))\n \ndef decode_netout(netout, anchors, obj_thresh, net_h, net_w):\n    grid_h, grid_w = netout.shape[:2]\n    nb_box = 3\n    netout = netout.reshape((grid_h, grid_w, nb_box, -1))\n    nb_class = netout.shape[-1] - 5\n    boxes = []\n    netout[..., :2]  = _sigmoid(netout[..., :2])\n    netout[..., 4:]  = _sigmoid(netout[..., 4:])\n    netout[..., 5:]  = netout[..., 4][..., np.newaxis] * netout[..., 5:]\n    netout[..., 5:] *= netout[..., 5:] > obj_thresh\n \n    for i in range(grid_h*grid_w):\n        row = i / grid_w\n        col = i % grid_w\n        for b in range(nb_box):\n            # 4th element is objectness score\n            objectness = netout[int(row)][int(col)][b][4]\n            if(objectness.all() <= obj_thresh): continue\n            # first 4 elements are x, y, w, and h\n            x, y, w, h = netout[int(row)][int(col)][b][:4]\n            x = (col + x) / grid_w # center position, unit: image width\n            y = (row + y) / grid_h # center position, unit: image height\n            w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width\n            h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height\n            # last elements are class probabilities\n            classes = netout[int(row)][col][b][5:]\n            box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, objectness, classes)\n            boxes.append(box)\n    return boxes\n \ndef correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w):\n    new_w, new_h = net_w, net_h\n    for i in range(len(boxes)):\n        x_offset, x_scale = (net_w - new_w)/2./net_w, float(new_w)/net_w\n        y_offset, y_scale = (net_h - new_h)/2./net_h, float(new_h)/net_h\n        boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w)\n        boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w)\n        boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h)\n        boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h)\n\ndef _interval_overlap(interval_a, interval_b):\n    x1, x2 = interval_a\n    x3, x4 = interval_b\n    if x3 < x1:\n        if x4 < x1:\n            return 0\n        else:\n            return min(x2,x4) - x1\n    else:\n        if x2 < x3:\n            return 0\n        else:\n            return min(x2,x4) - x3\n\ndef bbox_iou(box1, box2):\n    intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax])\n    intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax])\n    intersect = intersect_w * intersect_h\n    w1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin\n    w2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin\n    union = w1*h1 + w2*h2 - intersect\n    return float(intersect) / union\n\ndef do_nms(boxes, nms_thresh):\n    if len(boxes) > 0:\n        nb_class = len(boxes[0].classes)\n    else:\n        return\n    for c in range(nb_class):\n        sorted_indices = np.argsort([-box.classes[c] for box in boxes])\n        for i in range(len(sorted_indices)):\n            index_i = sorted_indices[i]\n            if boxes[index_i].classes[c] == 0: continue\n            for j in range(i+1, len(sorted_indices)):\n                index_j = sorted_indices[j]\n                if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh:\n                    boxes[index_j].classes[c] = 0\n    return boxes\n\n# get all of the results above a threshold\ndef get_boxes(boxes, labels, thresh):\n    v_boxes, v_labels, v_scores = list(), list(), list()\n    # enumerate all boxes\n    for box in boxes:\n        # enumerate all possible labels\n        for i in range(len(labels)):\n            # check if the threshold for this label is high enough\n            if box.classes[i] > thresh:\n                v_boxes.append(box)\n                v_labels.append(labels[i])\n                v_scores.append(box.classes[i]*100)\n                # don't break, many labels may trigger for one box\n    return v_boxes, v_labels, v_scores\n \n# draw all results\ndef draw_boxes(frame, v_boxes, v_labels, v_scores):\n    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n    pyplot.imshow(frame)\n    # get the context for drawing boxes\n    ax = pyplot.gca()\n    # plot each box\n    for i in range(len(v_boxes)):\n        box = v_boxes[i]\n        # get coordinates\n        y1, x1, y2, x2 = box.ymin, box.xmin, box.ymax, box.xmax\n        # calculate width and height of the box\n        width, height = x2 - x1, y2 - y1\n        # create the shape\n        rect = Rectangle((x1, y1), width, height, fill=False, color='white')\n        # draw the box\n        ax.add_patch(rect)\n        # draw text and score in top left corner\n        label = \"%s (%.3f)\" % (v_labels[i], v_scores[i])\n        pyplot.text(x1, y1, label, color='white')\n    # show the plot\n    pyplot.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create YOLOv3","metadata":{}},{"cell_type":"code","source":"# define the model\nmodel = make_yolov3_model()\n\n# load the model weights\n# I have loaded the pretrained weights in a separate dataset\nweight_reader = WeightReader('../input/lyft-3d-recognition/yolov3.weights')\n\n# set the model weights into the model\nweight_reader.load_weights(model)\n\n# save the model to file\nmodel.save('model.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load yolov3 model\nfrom keras.models import load_model\nmodel = load_model('model.h5')\n#model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Parameters used in the Dataset, on which YOLOv3 was pretrained\nanchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]]\n\n# define the expected input shape for the model\nWIDTH, HEIGHT = 416, 416\n\n# define the probability threshold for detected objects\nclass_threshold = 0.3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy import expand_dims\nfrom keras.preprocessing.image import load_img, img_to_array\n\n# load and prepare an image\ndef load_image_pixels(filename, shape):\n    '''\n    Function preprocess the images to 416x416, which is the standard input shape for YOLOv3, \n    and also keeps track of the originl shape, which is later used to draw the boxes.\n    \n    paramters:\n    filename {String}: path to the image\n    shape {tuple}: shape of the input dimensions of the network\n    \n    returns:\n    image {PIL}: image of shape 'shape'\n    width {int}: original width of the picture\n    height {int}: original height of the picture\n    '''\n    # load the image to get its shape\n    image = load_img(filename)\n    width, height = image.size\n    \n    # load the image with the required size\n    image = load_img(filename, target_size=shape)\n    \n    # convert to numpy array\n    image = img_to_array(image)\n    \n    # scale pixel values to [0, 1]\n    image = image.astype('float32')\n    image /= 255.0\n    \n    # add a dimension so that we have one sample\n    image = expand_dims(image, 0)\n    return image, width, height","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pafy\n!pip install youtube-dl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/opt/conda/lib/python3.7/site-packages/pafy/backend_youtube_dl.py', 'r') as file:\n    data = file.readlines()\n\ndata[52] = '        self._likes = 0\\n'\ndata[53] = '        self._dislikes = 0\\n'\n\nwith open('/opt/conda/lib/python3.7/site-packages/pafy/backend_youtube_dl.py', 'w') as file:\n    file.writelines(data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pafy\nimport cv2\nimport numpy as np\nfrom matplotlib import pyplot as plt\n\nurl = \"https://www.youtube.com/watch?v=5_XSYlAfJZM\" #\"https://youtu.be/fuuBpBQElv4\" #\nvideo = pafy.new(url)\nbest = video.getbest(preftype=\"mp4\")\n\nvideoStream = cv2.VideoCapture(best.url) # inputVideoPath\nvideo_width = int(videoStream.get(cv2.CAP_PROP_FRAME_WIDTH))\nvideo_height = int(videoStream.get(cv2.CAP_PROP_FRAME_HEIGHT))\n\nwhile videoStream.isOpened():\n    print(\"================NEW FRAME================\")\n    (grabbed, frame) = videoStream.read()\n\n    if not grabbed:\n        break\n    \n    img = cv2.resize(frame, (416,416))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = img.astype(np.float32)/255.\n    img = np.reshape(img,(1,416,416,3))\n\n    yhat = model.predict(img)\n    \n    # Create boxes\n    boxes = list()\n    for i in range(len(yhat)):\n        # decode the output of the network\n        boxes += decode_netout(yhat[i][0], anchors[i], class_threshold, HEIGHT, WIDTH)\n\n    # correct the sizes of the bounding boxes for the shape of the image\n    correct_yolo_boxes(boxes, frame.shape[0], frame.shape[1], HEIGHT, WIDTH)\n\n    # suppress non-maximal boxes\n    boxes = do_nms(boxes, 0.5)\n\n    # define the labels (Filtered only the ones relevant for this task, which were used in pretraining the YOLOv3 model)\n    labels = [\"bicycle\", \"car\", \"motorbike\", \"bus\", \"truck\"]\n\n    # get the details of the detected objects\n    v_boxes, v_labels, v_scores = get_boxes(boxes, labels, class_threshold)\n\n    # summarize what we found\n    for i in range(len(v_boxes)):\n        print(v_labels[i], v_scores[i])\n\n    # draw what we found\n    #draw_boxes(frame, v_boxes, v_labels, v_scores)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}