{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Environment Setup"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --upgrade tensorflow\n!mkdir data\n!wget wget https://pjreddie.com/media/files/yolov3.weights -O data/yolov3.weights\n!wget https://pjreddie.com/media/files/yolov3-tiny.weights -O data/yolov3-tiny.weights","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Library Imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2, os, glob\nimport xml.etree.ElementTree as ET\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.layers import (\n    Add, Concatenate, Conv2D,\n    Input, Lambda, LeakyReLU,\n    MaxPool2D, UpSampling2D, ZeroPadding2D\n)\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras.losses import (\n    binary_crossentropy,\n    sparse_categorical_crossentropy\n)\nfrom tensorflow.keras.utils import plot_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Configurations"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"YOLOV3_LAYER_LIST = [\n    'yolo_darknet',\n    'yolo_conv_0',\n    'yolo_output_0',\n    'yolo_conv_1',\n    'yolo_output_1',\n    'yolo_conv_2',\n    'yolo_output_2',\n]\n\nYOLOV3_TINY_LAYER_LIST = [\n    'yolo_darknet',\n    'yolo_conv_0',\n    'yolo_output_0',\n    'yolo_conv_1',\n    'yolo_output_1',\n]\n\nyolo_anchors = np.array([\n    (10, 13), (16, 30), (33, 23), (30, 61), (62, 45),\n    (59, 119), (116, 90), (156, 198), (373, 326)],\n    np.float32) / 416\n\nyolo_anchor_masks = np.array([[6, 7, 8], [3, 4, 5], [0, 1, 2]])\n\nyolo_tiny_anchors = np.array([\n    (10, 14), (23, 27), (37, 58),\n    (81, 82), (135, 169), (344, 319)],\n    np.float32) / 416\n\nyolo_tiny_anchor_masks = np.array([[3, 4, 5], [0, 1, 2]])\n\nclass_names = [\n    'person', 'bicycle','car','motorbike','aeroplane','bus','train','truck','boat',\n    'traffic light','fire hydrant','stop sign','parking meter','bench',\n    'bird','cat','dog','horse','sheep','cow','elephant','bear','zebra',\n    'giraffe','backpack','umbrella','handbag','tie','suitcase','frisbee',\n    'skis','snowboard','sports ball','kite','baseball bat','baseball glove',\n    'skateboard','surfboard','tennis racket','bottle','wine glass','cup',\n    'fork','knife','spoon','bowl','banana','apple','sandwich','orange',\n    'broccoli','carrot','hot dog','pizza','donut','cake','chair','sofa',\n    'pottedplant','bed','diningtable','toilet','tvmonitor','laptop','mouse',\n    'remote','keyboard','cell phone','microwave','oven','toaster','sink',\n    'refrigerator','book','clock','vase','scissors','teddy bear',\n    'hair drier','toothbrush'\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utilities"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_darknet_weights(model, weights_file, tiny = False):\n    wf = open(weights_file, 'rb')\n    major, minor, revision, seen, _ = np.fromfile(wf, dtype=np.int32, count=5)\n    if tiny:\n        layers = YOLOV3_TINY_LAYER_LIST\n    else:\n        layers = YOLOV3_LAYER_LIST\n    for layer_name in layers:\n        sub_model = model.get_layer(layer_name)\n        for i, layer in enumerate(sub_model.layers):\n            if not layer.name.startswith('conv2d'):\n                continue\n            batch_norm = None\n            if i + 1 < len(sub_model.layers) and sub_model.layers[i + 1].name.startswith('batch_norm'):\n                batch_norm = sub_model.layers[i + 1]\n            filters = layer.filters\n            size = layer.kernel_size[0]\n            in_dim = layer.input_shape[-1]\n            if batch_norm is None:\n                conv_bias = np.fromfile(wf, dtype=np.float32, count=filters)\n            else:\n                # darknet [beta, gamma, mean, variance]\n                bn_weights = np.fromfile(\n                    wf, dtype=np.float32, count=4 * filters)\n                # tf [gamma, beta, mean, variance]\n                bn_weights = bn_weights.reshape((4, filters))[[1, 0, 2, 3]]\n            # darknet shape (out_dim, in_dim, height, width)\n            conv_shape = (filters, in_dim, size, size)\n            conv_weights = np.fromfile(\n                wf, dtype=np.float32, count=np.product(conv_shape))\n            # tf shape (height, width, in_dim, out_dim)\n            conv_weights = conv_weights.reshape(\n                conv_shape).transpose([2, 3, 1, 0])\n            if batch_norm is None:\n                layer.set_weights([conv_weights, conv_bias])\n            else:\n                layer.set_weights([conv_weights])\n                batch_norm.set_weights(bn_weights)\n    assert len(wf.read()) == 0, 'failed to read all data'\n    wf.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def broadcast_iou(box_1, box_2):\n    '''\n    box_1: (..., (x1, y1, x2, y2))\n    box_2: (N, (x1, y1, x2, y2))\n    '''\n\n    # broadcast boxes\n    box_1 = tf.expand_dims(box_1, -2)\n    box_2 = tf.expand_dims(box_2, 0)\n    # new_shape: (..., N, (x1, y1, x2, y2))\n    new_shape = tf.broadcast_dynamic_shape(tf.shape(box_1), tf.shape(box_2))\n    box_1 = tf.broadcast_to(box_1, new_shape)\n    box_2 = tf.broadcast_to(box_2, new_shape)\n    int_w = tf.maximum(tf.minimum(box_1[..., 2], box_2[..., 2]) - tf.maximum(box_1[..., 0], box_2[..., 0]), 0)\n    int_h = tf.maximum(tf.minimum(box_1[..., 3], box_2[..., 3]) - tf.maximum(box_1[..., 1], box_2[..., 1]), 0)\n    int_area = int_w * int_h\n    box_1_area = (box_1[..., 2] - box_1[..., 0]) * (box_1[..., 3] - box_1[..., 1])\n    box_2_area = (box_2[..., 2] - box_2[..., 0]) * (box_2[..., 3] - box_2[..., 1])\n    return int_area / (box_1_area + box_2_area - int_area)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def freeze_all(model, frozen = True):\n    model.trainable = not frozen\n    if isinstance(model, tf.keras.Model):\n        for l in model.layers:\n            freeze_all(l, frozen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw_outputs(img, outputs, class_names):\n    boxes, objectness, classes, nums = outputs\n    boxes, objectness, classes, nums = boxes[0], objectness[0], classes[0], nums[0]\n    wh = np.flip(img.shape[0:2])\n    for i in range(nums):\n        x1y1 = tuple((np.array(boxes[i][0:2]) * wh).astype(np.int32))\n        x2y2 = tuple((np.array(boxes[i][2:4]) * wh).astype(np.int32))\n        img = cv2.rectangle(img, x1y1, x2y2, (255, 0, 0), 2)\n        img = cv2.putText(img, '{} {:.4f}'.format(\n            class_names[int(classes[i])], objectness[i]),\n            x1y1, cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 0, 255), 2)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw_labels(x, y, class_names):\n    img = x.numpy()\n    boxes, classes = tf.split(y, (4, 1), axis = -1)\n    classes = classes[..., 0]\n    wh = np.flip(img.shape[0 : 2])\n    for i in range(len(boxes)):\n        x1y1 = tuple((np.array(boxes[i][0:2]) * wh).astype(np.int32))\n        x2y2 = tuple((np.array(boxes[i][2:4]) * wh).astype(np.int32))\n        img = cv2.rectangle(img, x1y1, x2y2, (255, 0, 0), 2)\n        img = cv2.putText(\n            img, class_names[classes[i]],\n            x1y1, cv2.FONT_HERSHEY_COMPLEX_SMALL,\n            8, (0, 0, 255), 2\n        )\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform_images(x_train, size):\n    x_train = tf.image.resize(x_train, (size, size))\n    x_train = x_train / 255\n    return x_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef transform_targets_for_output(y_true, grid_size, anchor_idxs, classes):\n    N = tf.shape(y_true)[0]\n    y_true_out = tf.zeros(\n        (N, grid_size, grid_size, tf.shape(anchor_idxs)[0], 6))\n    anchor_idxs = tf.cast(anchor_idxs, tf.int32)\n    indexes = tf.TensorArray(tf.int32, 1, dynamic_size=True)\n    updates = tf.TensorArray(tf.float32, 1, dynamic_size=True)\n    idx = 0\n    for i in tf.range(N):\n        for j in tf.range(tf.shape(y_true)[1]):\n            if tf.equal(y_true[i][j][2], 0):\n                continue\n            anchor_eq = tf.equal(\n                anchor_idxs, tf.cast(y_true[i][j][5], tf.int32))\n            if tf.reduce_any(anchor_eq):\n                box = y_true[i][j][0:4]\n                box_xy = (y_true[i][j][0:2] + y_true[i][j][2:4]) / 2\n                anchor_idx = tf.cast(tf.where(anchor_eq), tf.int32)\n                grid_xy = tf.cast(box_xy // (1/grid_size), tf.int32)\n                indexes = indexes.write(\n                    idx, [i, grid_xy[1], grid_xy[0], anchor_idx[0][0]])\n                updates = updates.write(\n                    idx, [box[0], box[1], box[2], box[3], 1, y_true[i][j][4]])\n                idx += 1\n    return tf.tensor_scatter_nd_update(\n        y_true_out, indexes.stack(), updates.stack())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform_targets(y_train, anchors, anchor_masks, classes):\n    y_outs = []\n    grid_size = 13\n    anchors = tf.cast(anchors, tf.float32)\n    anchor_area = anchors[..., 0] * anchors[..., 1]\n    box_wh = y_train[..., 2:4] - y_train[..., 0:2]\n    box_wh = tf.tile(tf.expand_dims(box_wh, -2), (1, 1, tf.shape(anchors)[0], 1))\n    box_area = box_wh[..., 0] * box_wh[..., 1]\n    intersection = tf.minimum(box_wh[..., 0], anchors[..., 0]) * tf.minimum(box_wh[..., 1], anchors[..., 1])\n    iou = intersection / (box_area + anchor_area - intersection)\n    anchor_idx = tf.cast(tf.argmax(iou, axis=-1), tf.float32)\n    anchor_idx = tf.expand_dims(anchor_idx, axis=-1)\n    y_train = tf.concat([y_train, anchor_idx], axis=-1)\n    for anchor_idxs in anchor_masks:\n        y_outs.append(transform_targets_for_output(\n            y_train, grid_size, anchor_idxs, classes))\n        grid_size *= 2\n    return tuple(y_outs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## YOLO V3 Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class BatchNormalization(tf.keras.layers.BatchNormalization):\n\n    def call(self, x, training = False):\n        if training is None:\n            traininig = tf.constant(False)\n        training = tf.logical_and(training, self.trainable)\n        return super().call(x, training)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def DarknetConv(x, filters, size, strides=1, batch_norm=True):\n    if strides == 1:\n        padding = 'same'\n    else:\n        x = ZeroPadding2D(((1, 0), (1, 0)))(x)  # top left half-padding\n        padding = 'valid'\n    x = Conv2D(filters=filters, kernel_size=size,\n               strides=strides, padding=padding,\n               use_bias=not batch_norm, kernel_regularizer=l2(0.0005))(x)\n    if batch_norm:\n        x = BatchNormalization()(x)\n        x = LeakyReLU(alpha=0.1)(x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def DarknetResidual(x, filters):\n    prev = x\n    x = DarknetConv(x, filters // 2, 1)\n    x = DarknetConv(x, filters, 3)\n    x = Add()([prev, x])\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def DarknetBlock(x, filters, blocks):\n    x = DarknetConv(x, filters, 3, strides=2)\n    for _ in range(blocks):\n        x = DarknetResidual(x, filters)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Darknet(name=None):\n    x = inputs = Input([None, None, 3])\n    x = DarknetConv(x, 32, 3)\n    x = DarknetBlock(x, 64, 1)\n    x = DarknetBlock(x, 128, 2)  # skip connection\n    x = x_36 = DarknetBlock(x, 256, 8)  # skip connection\n    x = x_61 = DarknetBlock(x, 512, 8)\n    x = DarknetBlock(x, 1024, 4)\n    return tf.keras.Model(inputs, (x_36, x_61, x), name=name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def DarknetTiny(name=None):\n    x = inputs = Input([None, None, 3])\n    x = DarknetConv(x, 16, 3)\n    x = MaxPool2D(2, 2, 'same')(x)\n    x = DarknetConv(x, 32, 3)\n    x = MaxPool2D(2, 2, 'same')(x)\n    x = DarknetConv(x, 64, 3)\n    x = MaxPool2D(2, 2, 'same')(x)\n    x = DarknetConv(x, 128, 3)\n    x = MaxPool2D(2, 2, 'same')(x)\n    x = x_8 = DarknetConv(x, 256, 3)  # skip connection\n    x = MaxPool2D(2, 2, 'same')(x)\n    x = DarknetConv(x, 512, 3)\n    x = MaxPool2D(2, 1, 'same')(x)\n    x = DarknetConv(x, 1024, 3)\n    return tf.keras.Model(inputs, (x_8, x), name=name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def YoloConv(filters, name=None):\n    def yolo_conv(x_in):\n        if isinstance(x_in, tuple):\n            inputs = Input(x_in[0].shape[1:]), Input(x_in[1].shape[1:])\n            x, x_skip = inputs\n            # concat with skip connection\n            x = DarknetConv(x, filters, 1)\n            x = UpSampling2D(2)(x)\n            x = Concatenate()([x, x_skip])\n        else:\n            x = inputs = Input(x_in.shape[1:])\n        x = DarknetConv(x, filters, 1)\n        x = DarknetConv(x, filters * 2, 3)\n        x = DarknetConv(x, filters, 1)\n        x = DarknetConv(x, filters * 2, 3)\n        x = DarknetConv(x, filters, 1)\n        return Model(inputs, x, name=name)(x_in)\n    return yolo_conv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def YoloConvTiny(filters, name=None):\n    def yolo_conv(x_in):\n        if isinstance(x_in, tuple):\n            inputs = Input(x_in[0].shape[1:]), Input(x_in[1].shape[1:])\n            x, x_skip = inputs\n            # concat with skip connection\n            x = DarknetConv(x, filters, 1)\n            x = UpSampling2D(2)(x)\n            x = Concatenate()([x, x_skip])\n        else:\n            x = inputs = Input(x_in.shape[1:])\n            x = DarknetConv(x, filters, 1)\n        return Model(inputs, x, name=name)(x_in)\n    return yolo_conv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def YoloOutput(filters, anchors, classes, name=None):\n    def yolo_output(x_in):\n        x = inputs = Input(x_in.shape[1:])\n        x = DarknetConv(x, filters * 2, 3)\n        x = DarknetConv(x, anchors * (classes + 5), 1, batch_norm=False)\n        x = Lambda(lambda x: tf.reshape(x, (-1, tf.shape(x)[1], tf.shape(x)[2], anchors, classes + 5)))(x)\n        return tf.keras.Model(inputs, x, name=name)(x_in)\n    return yolo_output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def yolo_boxes(pred, anchors, classes):\n    '''pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...classes))'''\n    grid_size = tf.shape(pred)[1]\n    box_xy, box_wh, objectness, class_probs = tf.split(\n        pred, (2, 2, 1, classes), axis=-1)\n    box_xy = tf.sigmoid(box_xy)\n    objectness = tf.sigmoid(objectness)\n    class_probs = tf.sigmoid(class_probs)\n    pred_box = tf.concat((box_xy, box_wh), axis=-1)  # original xywh for loss\n    grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size))\n    grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2)  # [gx, gy, 1, 2]\n    box_xy = (box_xy + tf.cast(grid, tf.float32)) / \\\n        tf.cast(grid_size, tf.float32)\n    box_wh = tf.exp(box_wh) * anchors\n    box_x1y1 = box_xy - box_wh / 2\n    box_x2y2 = box_xy + box_wh / 2\n    bbox = tf.concat([box_x1y1, box_x2y2], axis=-1)\n    return bbox, objectness, class_probs, pred_box","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def yolo_nms(outputs, anchors, masks, classes):\n    '''boxes, conf, type'''\n    b, c, t = [], [], []\n    for o in outputs:\n        b.append(tf.reshape(o[0], (tf.shape(o[0])[0], -1, tf.shape(o[0])[-1])))\n        c.append(tf.reshape(o[1], (tf.shape(o[1])[0], -1, tf.shape(o[1])[-1])))\n        t.append(tf.reshape(o[2], (tf.shape(o[2])[0], -1, tf.shape(o[2])[-1])))\n    bbox = tf.concat(b, axis=1)\n    confidence = tf.concat(c, axis=1)\n    class_probs = tf.concat(t, axis=1)\n    scores = confidence * class_probs\n    boxes, scores, classes, valid_detections = tf.image.combined_non_max_suppression(\n        boxes=tf.reshape(bbox, (tf.shape(bbox)[0], -1, 1, 4)),\n        scores=tf.reshape(\n            scores,\n            (tf.shape(scores)[0], -1, tf.shape(scores)[-1])\n        ),\n        max_output_size_per_class=100,\n        max_total_size = 100,\n        iou_threshold = 0.5,\n        score_threshold = 0.5\n    )\n    return boxes, scores, classes, valid_detections","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def YoloV3(size=None, channels=3, anchors=yolo_anchors, masks=yolo_anchor_masks, classes=80, training=False):\n    x = inputs = Input([size, size, channels])\n    x_36, x_61, x = Darknet(name='yolo_darknet')(x)\n    x = YoloConv(512, name='yolo_conv_0')(x)\n    output_0 = YoloOutput(512, len(masks[0]), classes, name='yolo_output_0')(x)\n    x = YoloConv(256, name='yolo_conv_1')((x, x_61))\n    output_1 = YoloOutput(256, len(masks[1]), classes, name='yolo_output_1')(x)\n    x = YoloConv(128, name='yolo_conv_2')((x, x_36))\n    output_2 = YoloOutput(128, len(masks[2]), classes, name='yolo_output_2')(x)\n    if training:\n        return Model(inputs, (output_0, output_1, output_2), name='yolov3')\n    boxes_0 = Lambda(lambda x: yolo_boxes(x, anchors[masks[0]], classes),\n                     name='yolo_boxes_0')(output_0)\n    boxes_1 = Lambda(lambda x: yolo_boxes(x, anchors[masks[1]], classes),\n                     name='yolo_boxes_1')(output_1)\n    boxes_2 = Lambda(lambda x: yolo_boxes(x, anchors[masks[2]], classes),\n                     name='yolo_boxes_2')(output_2)\n    outputs = Lambda(lambda x: yolo_nms(x, anchors, masks, classes),\n                     name='yolo_nms')((boxes_0[:3], boxes_1[:3], boxes_2[:3]))\n    return Model(inputs, outputs, name='yolov3')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def YoloV3Tiny(size=None, channels=3, anchors=yolo_tiny_anchors, masks=yolo_tiny_anchor_masks, classes=80, training=False):\n    x = inputs = Input([size, size, channels])\n    x_8, x = DarknetTiny(name='yolo_darknet')(x)\n    x = YoloConvTiny(256, name='yolo_conv_0')(x)\n    output_0 = YoloOutput(256, len(masks[0]), classes, name='yolo_output_0')(x)\n    x = YoloConvTiny(128, name='yolo_conv_1')((x, x_8))\n    output_1 = YoloOutput(128, len(masks[1]), classes, name='yolo_output_1')(x)\n    if training:\n        return Model(inputs, (output_0, output_1), name='yolov3')\n    boxes_0 = Lambda(lambda x: yolo_boxes(x, anchors[masks[0]], classes),\n                     name='yolo_boxes_0')(output_0)\n    boxes_1 = Lambda(lambda x: yolo_boxes(x, anchors[masks[1]], classes),\n                     name='yolo_boxes_1')(output_1)\n    outputs = Lambda(lambda x: yolo_nms(x, anchors, masks, classes),\n                     name='yolo_nms')((boxes_0[:3], boxes_1[:3]))\n    return Model(inputs, outputs, name='yolov3_tiny')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## YOLO Loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"def YoloLoss(anchors, classes=80, ignore_thresh=0.5):\n    def yolo_loss(y_true, y_pred):\n        # 1. transform all pred outputs\n        # y_pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...cls))\n        pred_box, pred_obj, pred_class, pred_xywh = yolo_boxes(y_pred, anchors, classes)\n        pred_xy = pred_xywh[..., 0:2]\n        pred_wh = pred_xywh[..., 2:4]\n        # 2. transform all true outputs\n        # y_true: (batch_size, grid, grid, anchors, (x1, y1, x2, y2, obj, cls))\n        true_box, true_obj, true_class_idx = tf.split(\n            y_true, (4, 1, 1), axis=-1)\n        true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2\n        true_wh = true_box[..., 2:4] - true_box[..., 0:2]\n        # give higher weights to small boxes\n        box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1]\n        # 3. inverting the pred box equations\n        grid_size = tf.shape(y_true)[1]\n        grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size))\n        grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2)\n        true_xy = true_xy * tf.cast(grid_size, tf.float32) - \\\n            tf.cast(grid, tf.float32)\n        true_wh = tf.math.log(true_wh / anchors)\n        true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh), true_wh)\n        # 4. calculate all masks\n        obj_mask = tf.squeeze(true_obj, -1)\n        # ignore false positive when iou is over threshold\n        true_box_flat = tf.boolean_mask(true_box, tf.cast(obj_mask, tf.bool))\n        best_iou = tf.reduce_max(broadcast_iou(\n            pred_box, true_box_flat), axis=-1)\n        ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32)\n        # 5. calculate all losses\n        xy_loss = obj_mask * box_loss_scale * \\\n            tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1)\n        wh_loss = obj_mask * box_loss_scale * \\\n            tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1)\n        obj_loss = binary_crossentropy(true_obj, pred_obj)\n        obj_loss = obj_mask * obj_loss + \\\n            (1 - obj_mask) * ignore_mask * obj_loss\n        # Could also use binary_crossentropy instead\n        class_loss = obj_mask * sparse_categorical_crossentropy(\n            true_class_idx, pred_class)\n        # 6. sum over (batch, gridx, gridy, anchors) => (batch, 1)\n        xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3))\n        wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3))\n        obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3))\n        class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3))\n        return xy_loss + wh_loss + obj_loss + class_loss\n    return yolo_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tiny YOLO Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"yolo = YoloV3(classes = 80)\nyolo.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_model(\n    yolo, rankdir = 'TB',\n    to_file = 'yolo_model.png',\n    show_shapes = False,\n    show_layer_names = True,\n    expand_nested = True\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"load_darknet_weights(yolo, './data/yolov3.weights', False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(image_file, visualize = True, figsize = (16, 16)):\n    img = tf.image.decode_image(open(image_file, 'rb').read(), channels=3)\n    img = tf.expand_dims(img, 0)\n    img = transform_images(img, 416)\n    boxes, scores, classes, nums = yolo.predict(img)\n    img = cv2.cvtColor(cv2.imread(image_file), cv2.COLOR_BGR2RGB)\n    img = draw_outputs(img, (boxes, scores, classes, nums), class_names)\n    if visualize:\n        fig, axes = plt.subplots(figsize = figsize)\n        plt.imshow(img)\n        plt.show()\n    return boxes, scores, classes, nums","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_file = glob.glob('../input/google-ai-open-images-object-detection-track/test/challenge2018_test/*')\nlen(image_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes, scores, classes, nums = predict(image_file[0], figsize = (20, 20))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes, scores, classes, nums = predict(image_file[1], figsize = (20, 20))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes, scores, classes, nums = predict(image_file[3], figsize = (20, 20))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes, scores, classes, nums = predict(image_file[4], figsize = (20, 20))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes, scores, classes, nums = predict(image_file[8], figsize = (20, 20))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes, scores, classes, nums = predict(image_file[12], figsize = (20, 20))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes, scores, classes, nums = predict(image_file[13], figsize = (20, 20))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes, scores, classes, nums = predict(image_file[15], figsize = (20, 20))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes, scores, classes, nums = predict(image_file[18], figsize = (20, 20))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}