{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Object Detection using YOLOV3\n\n<br>\n\nThis is a starter kernel, mainly for learning purposes and getting started. There is so much to learn more and improve. I am using YOLOv3 model for object classification and detection using a pretrained model.\nReferences: The ideas presented in this notebook came primarily from the two YOLO papers. The implementation here also took significant inspiration and used many components from Allan Zelener's github repository. The pretrained weights used in this exercise came from the official YOLO website.\n\nCheck the speed of detection using YOLO! This detection algorithm will need some fine tuning if you want to run for large number of images. Goal is to demonstrate how YOLO3 performs object detection\n\nJoseph Redmon, Santosh Divvala, Ross Girshick, Ali Farhadi - You Only Look Once: Unified, Real-Time Object Detection (2015)\nJoseph Redmon, Ali Farhadi - YOLO9000: Better, Faster, Stronger (2016)\nAllan Zelener - YAD2K: Yet Another Darknet 2 Keras\nThe official YOLO website (https://pjreddie.com/darknet/yolo/)\n\n","metadata":{"_uuid":"7463e0e93eda3dcf4ddb36d0ea12bb600cd34b49"}},{"cell_type":"code","source":"from __future__ import division\n\nimport torch \nimport torch.nn as nn\nimport torch.nn.functional as F \nfrom torch.autograd import Variable\nimport numpy as np\nimport cv2\nimport tensorflow as tf\n\ndef unique(tensor):\n    tensor_np = tensor.cpu().numpy()\n    unique_np = np.unique(tensor_np)\n    unique_tensor = torch.from_numpy(unique_np)\n    \n    tensor_res = tensor.new(unique_tensor.shape)\n    tensor_res.copy_(unique_tensor)\n    return tensor_res\n\n\ndef bbox_iou(box1, box2):\n    \"\"\"\n    Returns the IoU of two bounding boxes \n    \n    \n    \"\"\"\n    #Get the coordinates of bounding boxes\n    b1_x1, b1_y1, b1_x2, b1_y2 = box1[:,0], box1[:,1], box1[:,2], box1[:,3]\n    b2_x1, b2_y1, b2_x2, b2_y2 = box2[:,0], box2[:,1], box2[:,2], box2[:,3]\n    \n    #get the corrdinates of the intersection rectangle\n    inter_rect_x1 =  torch.max(b1_x1, b2_x1)\n    inter_rect_y1 =  torch.max(b1_y1, b2_y1)\n    inter_rect_x2 =  torch.min(b1_x2, b2_x2)\n    inter_rect_y2 =  torch.min(b1_y2, b2_y2)\n    \n    #Intersection area\n    inter_area = torch.clamp(inter_rect_x2 - inter_rect_x1 + 1, min=0) * torch.clamp(inter_rect_y2 - inter_rect_y1 + 1, min=0)\n\n    #Union Area\n    b1_area = (b1_x2 - b1_x1 + 1)*(b1_y2 - b1_y1 + 1)\n    b2_area = (b2_x2 - b2_x1 + 1)*(b2_y2 - b2_y1 + 1)\n    \n    iou = inter_area / (b1_area + b2_area - inter_area)\n    \n    return iou\n\ndef predict_transform(prediction, inp_dim, anchors, num_classes, CUDA = True):\n\n    \n    batch_size = prediction.size(0)\n    stride =  inp_dim // prediction.size(2)\n    grid_size = inp_dim // stride\n    bbox_attrs = 5 + num_classes\n    num_anchors = len(anchors)\n    #print(\"PRED BEF TR \",prediction[:,:,0],prediction[:,:,1],prediction[:,:,4])\n    prediction = prediction.view(batch_size, bbox_attrs*num_anchors, grid_size*grid_size)\n    prediction = prediction.transpose(1,2).contiguous()\n    prediction = prediction.view(batch_size, grid_size*grid_size*num_anchors, bbox_attrs)\n    anchors = [(a[0]/stride, a[1]/stride) for a in anchors]\n\n    #Sigmoid the  centre_X, centre_Y. and object confidencce\n    prediction[:,:,0] = torch.sigmoid(prediction[:,:,0])\n    prediction[:,:,1] = torch.sigmoid(prediction[:,:,1])\n    prediction[:,:,4] = torch.sigmoid(prediction[:,:,4])\n    \n    #Add the center offsets\n    grid = np.arange(grid_size)\n    a,b = np.meshgrid(grid, grid)\n\n    x_offset = torch.FloatTensor(a).view(-1,1)\n    y_offset = torch.FloatTensor(b).view(-1,1)\n\n    if CUDA:\n        x_offset = x_offset.cuda()\n        y_offset = y_offset.cuda()\n\n    x_y_offset = torch.cat((x_offset, y_offset), 1).repeat(1,num_anchors).view(-1,2).unsqueeze(0)\n    pred_old = prediction.clone() #1\n    prediction[:,:,:2] += x_y_offset\n\n    #log space transform height and the width\n    anchors = torch.FloatTensor(anchors)\n\n    if CUDA:\n        anchors = anchors.cuda()\n    #pred_old = prediction.clone() #1\n    anchors = anchors.repeat(grid_size*grid_size, 1).unsqueeze(0)\n    prediction[:,:,2:4] = torch.exp(prediction[:,:,2:4])*anchors\n    pred_old[:,:,2:4] = torch.exp(pred_old[:,:,2:4])*anchors\n    \n    prediction[:,:,5: 5 + num_classes] = torch.sigmoid((prediction[:,:, 5 : 5 + num_classes]))\n    pred_old[:,:,5: 5 + num_classes] = torch.sigmoid((pred_old[:,:, 5 : 5 + num_classes]))\n    prediction[:,:,:4] *= stride\n    \n    return prediction, pred_old\n\ndef write_results(prediction, confidence, num_classes, nms = True, nms_conf = 0.4):\n    conf_mask = (prediction[:,:,4] > confidence).float().unsqueeze(2)\n    prediction = prediction * conf_mask\n    \n    try:\n        ind_nz = torch.nonzero(prediction[:,:,4]).transpose(0,1).contiguous()\n    except:\n        return 0\n    box_corner = prediction.new(prediction.shape)\n    box_corner[:,:,0] = (prediction[:,:,0] - prediction[:,:,2]/2)\n    box_corner[:,:,1] = (prediction[:,:,1] - prediction[:,:,3]/2)\n    box_corner[:,:,2] = (prediction[:,:,0] + prediction[:,:,2]/2) \n    box_corner[:,:,3] = (prediction[:,:,1] + prediction[:,:,3]/2)\n    prediction[:,:,:4] = box_corner[:,:,:4]\n    \n    batch_size = prediction.size(0)\n    output = prediction.new(1, prediction.size(2) + 1)\n    write = False\n    \n\n    for ind in range(batch_size):\n        image_pred = prediction[ind]          #image Tensor\n       #confidence threshholding \n       #NMS\n    \n        max_conf, max_conf_score = torch.max(image_pred[:,5:5+ num_classes], 1)\n        max_conf = max_conf.float().unsqueeze(1)\n        max_conf_score = max_conf_score.float().unsqueeze(1)\n        seq = (image_pred[:,:5], max_conf, max_conf_score)\n        image_pred = torch.cat(seq, 1)\n        \n        non_zero_ind =  (torch.nonzero(image_pred[:,4]))\n        image_pred_ = image_pred[non_zero_ind.squeeze(),:].view(-1,7)\n        #Get the various classes detected in the image\n        try:\n            img_classes = unique(image_pred_[:,-1]) # -1 index holds the class index\n        except:\n            continue    \n        for cls in img_classes:\n            #perform NMS\n            #get the detections with one particular class\n            cls_mask = image_pred_*(image_pred_[:,-1] == cls).float().unsqueeze(1)\n            class_mask_ind = torch.nonzero(cls_mask[:,-2]).squeeze()\n            image_pred_class = image_pred_[class_mask_ind].view(-1,7)\n            \n            #sort the detections such that the entry with the maximum objectness\n            #confidence is at the top\n            conf_sort_index = torch.sort(image_pred_class[:,4], descending = True )[1]\n            image_pred_class = image_pred_class[conf_sort_index]\n            idx = image_pred_class.size(0)   #Number of detections\n            if nms:\n                for i in range(idx):\n                  #Get the IOUs of all boxes that come after the one we are looking at \n                  #in the loop\n                    try:\n                        ious = bbox_iou(image_pred_class[i].unsqueeze(0), image_pred_class[i+1:])\n                    except ValueError:\n                        break\n                    except IndexError:\n                        break\n            \n                #Zero out all the detections that have IoU > treshhold\n                    iou_mask = (ious < nms_conf).float().unsqueeze(1)\n                    image_pred_class[i+1:] *= iou_mask       \n            \n                #Remove the non-zero entries\n                    non_zero_ind = torch.nonzero(image_pred_class[:,4]).squeeze()\n                    image_pred_class = image_pred_class[non_zero_ind].view(-1,7)\n                \n            batch_ind = image_pred_class.new(image_pred_class.size(0), 1).fill_(ind)      #Repeat the batch_id for as many detections of the class cls in the image\n            seq = batch_ind, image_pred_class\n            \n            if not write:\n                output = torch.cat(seq,1)\n                write = True\n            else:\n                out = torch.cat(seq,1)\n                output = torch.cat((output,out))\n    #batch_id,4 coordinates, max_conf, max_conf_score, object id\n    return output\n    \ndef letterbox_image(img, inp_dim):\n    '''resize image with unchanged aspect ratio using padding'''\n    img_w, img_h = img.shape[1], img.shape[0]\n    w, h = inp_dim\n    new_w = int(img_w * min(w/img_w, h/img_h))\n    new_h = int(img_h * min(w/img_w, h/img_h))\n    resized_image = cv2.resize(img, (new_w,new_h), interpolation = cv2.INTER_CUBIC)\n    \n    canvas = np.full((inp_dim[1], inp_dim[0], 3), 128)\n\n    canvas[(h-new_h)//2:(h-new_h)//2 + new_h,(w-new_w)//2:(w-new_w)//2 + new_w,  :] = resized_image\n    \n    return canvas\n\ndef prep_image(img, inp_dim):\n    \"\"\"\n    Prepare image for inputting to the neural network. \n    \n    Returns a Variable \n    \"\"\"\n    orig_im = cv2.imread(img)\n    dim = orig_im.shape[1], orig_im.shape[0]\n    img = (letterbox_image(orig_im, (inp_dim, inp_dim)))\n    img = img[:,:,::-1].transpose((2,0,1)).copy()\n    img = torch.from_numpy(img).float().div(255.0).unsqueeze(0)\n    return img, orig_im, dim\n\n","metadata":{"_uuid":"f69c3bf99a5b0538d008006185cab42073c4d4c3","execution":{"iopub.status.busy":"2023-06-26T14:31:34.807168Z","iopub.execute_input":"2023-06-26T14:31:34.807766Z","iopub.status.idle":"2023-06-26T14:31:36.613021Z","shell.execute_reply.started":"2023-06-26T14:31:34.807709Z","shell.execute_reply":"2023-06-26T14:31:36.612113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define functions to predict and detect boxes. \nYou could tweak some of the parameters like IOU and box probability to filter boxes","metadata":{"_uuid":"32bb318133522ff3c56fc3c3457a239751c01a15"}},{"cell_type":"markdown","source":"Function to read configuration file and weights file using Torch module","metadata":{"_uuid":"12e5d2189f5efb269766ee36826b0348d08b9965"}},{"cell_type":"code","source":"from __future__ import division\n\nimport torch \nimport torch.nn as nn\nimport torch.nn.functional as F \nfrom torch.autograd import Variable\nimport numpy as np\n#from util import * \n\n\n\ndef get_test_input(img_path,inp_dim):\n    img = cv2.imread(img_path)\n    img = cv2.resize(img, (416,416))          #Resize to the input dimension\n    img_ =  img[:,:,::-1].transpose((2,0,1))  # BGR -> RGB | H X W C -> C X H X W \n    img_ = img_[np.newaxis,:,:,:]/255.0       #Add a channel at 0 (for batch) | Normalise\n    img_ = torch.from_numpy(img_).float()     #Convert to float\n    img_ = Variable(img_)                     # Convert to Variable\n    return img_\n\ndef parse_cfg(cfgfile):\n    \"\"\"\n    Takes a configuration file\n    \n    Returns a list of blocks. Each blocks describes a block in the neural\n    network to be built. Block is represented as a dictionary in the list\n    \n    \"\"\"\n    \n    file = open(cfgfile, 'r')\n    lines = file.read().split('\\n')                        # store the lines in a list\n    lines = [x for x in lines if len(x) > 0]               # get read of the empty lines \n    lines = [x for x in lines if x[0] != '#']              # get rid of comments\n    lines = [x.rstrip().lstrip() for x in lines]           # get rid of fringe whitespaces\n    \n    block = {}\n    blocks = []\n    \n    for line in lines:\n        if line[0] == \"[\":               # This marks the start of a new block\n            if len(block) != 0:          # If block is not empty, implies it is storing values of previous block.\n                blocks.append(block)     # add it the blocks list\n                block = {}               # re-init the block\n            block[\"type\"] = line[1:-1].rstrip()     \n        else:\n            key,value = line.split(\"=\") \n            block[key.rstrip()] = value.lstrip()\n    blocks.append(block)\n    \n    return blocks\n\n\nclass EmptyLayer(nn.Module):\n    def __init__(self):\n        super(EmptyLayer, self).__init__()\n        \n\nclass DetectionLayer(nn.Module):\n    def __init__(self, anchors):\n        super(DetectionLayer, self).__init__()\n        self.anchors = anchors\n\n\n\ndef create_modules(blocks):\n    net_info = blocks[0]     #Captures the information about the input and pre-processing    \n    module_list = nn.ModuleList()\n    prev_filters = 3\n    output_filters = []\n    \n    for index, x in enumerate(blocks[1:]):\n        module = nn.Sequential()\n    \n        #check the type of block\n        #create a new module for the block\n        #append to module_list\n        \n        #If it's a convolutional layer\n        if (x[\"type\"] == \"convolutional\"):\n            #Get the info about the layer\n            activation = x[\"activation\"]\n            try:\n                batch_normalize = int(x[\"batch_normalize\"])\n                bias = False\n            except:\n                batch_normalize = 0\n                bias = True\n        \n            filters= int(x[\"filters\"])\n            padding = int(x[\"pad\"])\n            kernel_size = int(x[\"size\"])\n            stride = int(x[\"stride\"])\n        \n            if padding:\n                pad = (kernel_size - 1) // 2\n            else:\n                pad = 0\n        \n            #Add the convolutional layer\n            conv = nn.Conv2d(prev_filters, filters, kernel_size, stride, pad, bias = bias)\n            module.add_module(\"conv_{0}\".format(index), conv)\n        \n            #Add the Batch Norm Layer\n            if batch_normalize:\n                bn = nn.BatchNorm2d(filters)\n                module.add_module(\"batch_norm_{0}\".format(index), bn)\n        \n            #Check the activation. \n            #It is either Linear or a Leaky ReLU for YOLO\n            if activation == \"leaky\":\n                activn = nn.LeakyReLU(0.1, inplace = True)\n                module.add_module(\"leaky_{0}\".format(index), activn)\n        \n            #If it's an upsampling layer\n            #We use Bilinear2dUpsampling\n        elif (x[\"type\"] == \"upsample\"):\n            stride = int(x[\"stride\"])\n            upsample = nn.Upsample(scale_factor = 2, mode = \"nearest\")\n            module.add_module(\"upsample_{}\".format(index), upsample)\n                \n        #If it is a route layer\n        elif (x[\"type\"] == \"route\"):\n            x[\"layers\"] = x[\"layers\"].split(',')\n            #Start  of a route\n            start = int(x[\"layers\"][0])\n            #end, if there exists one.\n            try:\n                end = int(x[\"layers\"][1])\n            except:\n                end = 0\n            #Positive anotation\n            if start > 0: \n                start = start - index\n            if end > 0:\n                end = end - index\n            route = EmptyLayer()\n            module.add_module(\"route_{0}\".format(index), route)\n            if end < 0:\n                filters = output_filters[index + start] + output_filters[index + end]\n            else:\n                filters= output_filters[index + start]\n    \n        #shortcut corresponds to skip connection\n        elif x[\"type\"] == \"shortcut\":\n            shortcut = EmptyLayer()\n            module.add_module(\"shortcut_{}\".format(index), shortcut)\n            \n        #Yolo is the detection layer\n        elif x[\"type\"] == \"yolo\":\n            mask = x[\"mask\"].split(\",\")\n            mask = [int(x) for x in mask]\n    \n            anchors = x[\"anchors\"].split(\",\")\n            anchors = [int(a) for a in anchors]\n            anchors = [(anchors[i], anchors[i+1]) for i in range(0, len(anchors),2)]\n            anchors = [anchors[i] for i in mask]\n    \n            detection = DetectionLayer(anchors)\n            module.add_module(\"Detection_{}\".format(index), detection)\n                              \n        module_list.append(module)\n        prev_filters = filters\n        output_filters.append(filters)\n        \n    return (net_info, module_list)\n\nclass Darknet(nn.Module):\n    def __init__(self, cfgfile):\n        super(Darknet, self).__init__()\n        self.blocks = parse_cfg(cfgfile)\n        self.net_info, self.module_list = create_modules(self.blocks)\n        \n    def forward(self, x, CUDA):\n        modules = self.blocks[1:]\n        outputs = {}   #We cache the outputs for the route layer\n        \n        write = 0\n        for i, module in enumerate(modules):        \n            module_type = (module[\"type\"])\n            \n            if module_type == \"convolutional\" or module_type == \"upsample\":\n                x = self.module_list[i](x)\n    \n            elif module_type == \"route\":\n                layers = module[\"layers\"]\n                layers = [int(a) for a in layers]\n    \n                if (layers[0]) > 0:\n                    layers[0] = layers[0] - i\n    \n                if len(layers) == 1:\n                    x = outputs[i + (layers[0])]\n    \n                else:\n                    if (layers[1]) > 0:\n                        layers[1] = layers[1] - i\n    \n                    map1 = outputs[i + layers[0]]\n                    map2 = outputs[i + layers[1]]\n                    x = torch.cat((map1, map2), 1)\n                \n    \n            elif  module_type == \"shortcut\":\n                from_ = int(module[\"from\"])\n                x = outputs[i-1] + outputs[i+from_]\n    \n            elif module_type == 'yolo':        \n                anchors = self.module_list[i][0].anchors\n                #Get the input dimensions\n                inp_dim = int (self.net_info[\"height\"])\n        \n                #Get the number of classes\n                num_classes = int (module[\"classes\"])\n        \n                #Transform \n                if CUDA:\n                    x = x.data.cuda()\n                else:\n                    x = x.data\n                x, pred_old = predict_transform(x, inp_dim, anchors, num_classes, CUDA)\n                if not write:              #if no collector has been intialised. \n                    detections = x\n                    detections1 = pred_old\n                    write = 1\n        \n                else:       \n                    detections = torch.cat((detections, x), 1)\n                    detections1 = torch.cat((detections1, pred_old), 1)\n        \n            outputs[i] = x\n        \n        return detections, detections1\n\n\n    def load_weights(self, weightfile):\n        #Open the weights file\n        fp = open(weightfile, \"rb\")\n    \n        #The first 5 values are header information \n        # 1. Major version number\n        # 2. Minor Version Number\n        # 3. Subversion number \n        # 4,5. Images seen by the network (during training)\n        header = np.fromfile(fp, dtype = np.int32, count = 5)\n        self.header = torch.from_numpy(header)\n        self.seen = self.header[3]   \n        \n        weights = np.fromfile(fp, dtype = np.float32)\n        \n        ptr = 0\n        for i in range(len(self.module_list)):\n            module_type = self.blocks[i + 1][\"type\"]\n    \n            #If module_type is convolutional load weights\n            #Otherwise ignore.\n            \n            if module_type == \"convolutional\":\n                model = self.module_list[i]\n                try:\n                    batch_normalize = int(self.blocks[i+1][\"batch_normalize\"])\n                except:\n                    batch_normalize = 0\n            \n                conv = model[0]\n                \n                \n                if (batch_normalize):\n                    bn = model[1]\n        \n                    #Get the number of weights of Batch Norm Layer\n                    num_bn_biases = bn.bias.numel()\n        \n                    #Load the weights\n                    bn_biases = torch.from_numpy(weights[ptr:ptr + num_bn_biases])\n                    ptr += num_bn_biases\n        \n                    bn_weights = torch.from_numpy(weights[ptr: ptr + num_bn_biases])\n                    ptr  += num_bn_biases\n        \n                    bn_running_mean = torch.from_numpy(weights[ptr: ptr + num_bn_biases])\n                    ptr  += num_bn_biases\n        \n                    bn_running_var = torch.from_numpy(weights[ptr: ptr + num_bn_biases])\n                    ptr  += num_bn_biases\n        \n                    #Cast the loaded weights into dims of model weights. \n                    bn_biases = bn_biases.view_as(bn.bias.data)\n                    bn_weights = bn_weights.view_as(bn.weight.data)\n                    bn_running_mean = bn_running_mean.view_as(bn.running_mean)\n                    bn_running_var = bn_running_var.view_as(bn.running_var)\n        \n                    #Copy the data to model\n                    bn.bias.data.copy_(bn_biases)\n                    bn.weight.data.copy_(bn_weights)\n                    bn.running_mean.copy_(bn_running_mean)\n                    bn.running_var.copy_(bn_running_var)\n                \n                else:\n                    #Number of biases\n                    num_biases = conv.bias.numel()\n                \n                    #Load the weights\n                    conv_biases = torch.from_numpy(weights[ptr: ptr + num_biases])\n                    ptr = ptr + num_biases\n                \n                    #reshape the loaded weights according to the dims of the model weights\n                    conv_biases = conv_biases.view_as(conv.bias.data)\n                \n                    #Finally copy the data\n                    conv.bias.data.copy_(conv_biases)\n                    \n                #Let us load the weights for the Convolutional layers\n                num_weights = conv.weight.numel()\n                \n                #Do the same as above for weights\n                conv_weights = torch.from_numpy(weights[ptr:ptr+num_weights])\n                ptr = ptr + num_weights\n                \n                conv_weights = conv_weights.view_as(conv.weight.data)\n                conv.weight.data.copy_(conv_weights)\n                   \n","metadata":{"_uuid":"6844ecea24ea1b7c429966bc08b156035fa89842","_kg_hide-output":true,"_kg_hide-input":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = open(\"../input/class-labels-500/class-descriptions-500.csv\",\"r\")\nd_class_label = {}\nfor rec in f:\n    line = rec.split(\",\")\n    d_class_label[line[1][:-1].lower()] = line[0]\nd_class_label[\"remote control\"] = \"/m/0qjjc\"\nd_class_label[\"frisbee\"] = \"/m/0df_n8\"\nprint(d_class_label)","metadata":{"_uuid":"cbe1feec3f3328075d7f6ec860554328e1c8974a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Darknet(\"../input/yolov3cfg/yolov3.cfg\")\ninp = get_test_input(\"../input/google-ai-open-images-object-detection-track/test/challenge2018_test/00001a21632de752.jpg\",416)\n#print(inp)\npred = model(inp, torch.cuda.is_available())\nprint (\"Pred..\",pred)\n#f = open(\"ouput.csv\",\"w\")\n#print (pred)\n\nmodel = Darknet(\"../input/yolov3cfg/yolov3.cfg\")\nmodel.load_weights(\"../input/yolov3-weights/yolov3.weights\")","metadata":{"_uuid":"794230167c268518877a01fdf3aef68112d1e55d","collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.python.keras import backend as K\nsess = K.get_session()","metadata":{"_uuid":"ff80db05035509037f6ea01e9c198e358c08706d","collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from __future__ import division\nimport time\nimport torch \nimport torch.nn as nn\nfrom torch.autograd import Variable\nimport numpy as np\nimport cv2 \n#from util import *\nimport argparse\nimport os \nimport os.path as osp\n#from darknet import Darknet\nimport pickle as pkl\nimport pandas as pd\nimport random\nfrom PIL import Image, ImageDraw, ImageFont #sajin\ndef arg_parse():\n    \"\"\"\n    Parse arguements to the detect module\n    \n    \"\"\"\n    \n    parser = argparse.ArgumentParser(description='YOLO v3 Detection Module')\n   \n    parser.add_argument(\"--images\", dest = 'images', help = \n                        \"Image / Directory containing images to perform detection upon\",\n                        default = \"../input/google-ai-open-images-object-detection-track/test/challenge2018_test/00001a21632de752.jpg\", type = str)\n    parser.add_argument(\"--det\", dest = 'det', help = \n                        \"Image / Directory to store detections to\",\n                        default = \"det\", type = str)\n    parser.add_argument(\"--bs\", dest = \"bs\", help = \"Batch size\", default = 1)\n    parser.add_argument(\"--confidence\", dest = \"confidence\", help = \"Object Confidence to filter predictions\", default = 0.5)\n    parser.add_argument(\"--nms_thresh\", dest = \"nms_thresh\", help = \"NMS Threshhold\", default = 0.4)\n    parser.add_argument(\"--cfg\", dest = 'cfgfile', help = \n                        \"Config file\",\n                        default = \"../input/yolov3cfg/yolov3.cfg\", type = str)\n    parser.add_argument(\"--weights\", dest = 'weightsfile', help = \n                        \"weightsfile\",\n                        default = \"../input/yolov3-weights/yolov3.weights\", type = str)\n    parser.add_argument(\"--reso\", dest = 'reso', help = \n                        \"Input resolution of the network. Increase to increase accuracy. Decrease to increase speed\",\n                        default = \"416\", type = str)\n    \n    return parser.parse_args()\n\ndef get_test_input(image, input_dim):\n    img = cv2.imread(image)\n    img = cv2.resize(img, (input_dim, input_dim)) \n    img_ =  img[:,:,::-1].transpose((2,0,1))\n    img_ = img_[np.newaxis,:,:,:]/255.0\n    img_ = torch.from_numpy(img_).float()\n    img_ = Variable(img_) \ndef getmodel(cfgfile, weightsfile):\n        #Set up the neural network\n    print(\"Loading network.....\")\n    model = Darknet(cfgfile)\n    model.load_weights(weightsfile)\n    print(\"Network successfully loaded\")\n    return model\ndef load_classes(namesfile):\n    fp = open(namesfile, \"r\")\n    names = fp.read().split(\"\\n\")[:-1]\n    f = []\n    #names = [x if x==\"tvmonitor\": \"television\" else: x for x in names]\n    for x in names:\n        if x == \"tvmonitor\":\n            f.append(\"television\")\n        elif x == \"aeroplane\":\n            f.append(\"airplane\")\n        elif x == \"pottedplant\":\n            f.append(\"houseplant\")\n        elif x == \"cell phone\":\n            f.append(\"mobile phone\")\n        elif x == \"cup\":\n            f.append(\"coffee cup\")\n        elif x == \"diningtable\":\n            f.append(\"kitchen & dining room table\")\n        elif x == \"sofa\":\n            f.append(\"sofa bed\")\n        elif x == \"motorbike\":\n            f.append(\"motorcycle\")\n        elif x == \"cow\":\n            f.append(\"cattle\")\n        elif x == \"microwave\":\n            f.append(\"microwave oven\")\n        elif x == \"remote\":\n            f.append(\"remote control\")\n        elif x == \"sports ball\":\n            f.append(\"ball\")\n        else:\n            f.append(x)\n    names = f\n    return names    \ndef detect(model, images_i, bs_i, confidence_i, nms_thresh_i, class_path, weightsfile, cfgfile, reso):\n    #args = arg_parse()\n\n    images = images_i\n    batch_size = int(bs_i)\n    confidence = float(confidence_i)\n    nms_thesh = float(nms_thresh_i)\n    #scales = scales_i\n    start = 0\n    CUDA = torch.cuda.is_available()\n    print(\"CUDA:\",str(CUDA))\n    num_classes = 80\n    classes = load_classes(class_path)\n    '''\n    #Set up the neural network\n    print(\"Loading network.....\")\n    model = Darknet(cfgfile)\n    model.load_weights(weightsfile)\n    print(\"Network successfully loaded\")\n    '''\n    model.net_info[\"height\"] = reso\n    inp_dim = int(model.net_info[\"height\"])\n    assert inp_dim % 32 == 0 \n    assert inp_dim > 32\n\n    #If there's a GPU availible, put the model on GPU\n    if CUDA:\n        model.cuda()\n    #Set the model in evaluation mode\n    model.eval()\n\n    read_dir = time.time()\n    #Detection phase\n    try:\n        imlist = [osp.join(osp.realpath('.'), images, img) for img in os.listdir(images)]\n    except NotADirectoryError:\n        imlist = []\n        imlist.append(osp.join(osp.realpath('.'), images))\n        #print(\"Not a directory...\")\n    except FileNotFoundError:\n        print (\"No file or directory with the name {}\".format(images))\n        exit()\n    \n    #if not os.path.exists(args.det):\n        #os.makedirs(args.det)\n\n    load_batch = time.time()\n    #loaded_ims = [cv2.imread(x) for x in imlist]\n\n    #im_batches = list(map(prep_image, loaded_ims, [inp_dim for x in range(len(imlist))]))\n    batches = list(map(prep_image, imlist, [inp_dim for x in range(len(imlist))]))\n    im_batches = [x[0] for x in batches]\n    orig_ims = [x[1] for x in batches]\n    im_dim_list = [x[2] for x in batches]\n    #im_dim_list = [(x.shape[1], x.shape[0]) for x in loaded_ims]\n    im_dim_list = torch.FloatTensor(im_dim_list).repeat(1,2)\n    if CUDA:\n        im_dim_list = im_dim_list.cuda()\n\n    leftover = 0\n    if (len(im_dim_list) % batch_size):\n        leftover = 1\n\n    if batch_size != 1:\n        num_batches = len(imlist) // batch_size + leftover            \n        im_batches = [torch.cat((im_batches[i*batch_size : min((i +  1)*batch_size,\n                            len(im_batches))]))  for i in range(num_batches)]  \n\n    i=0\n    write = False\n    #model(get_test_input(inp_dim, CUDA), CUDA)\n    start_det_loop = time.time()\n    if CUDA:\n        im_dim_list = im_dim_list.cuda()\n    \n    start_det_loop = time.time()\n    objs = {}\n    \n    for batch in im_batches:\n    #load the image \n        start = time.time()\n        if CUDA:\n            batch = batch.cuda()\n        with torch.no_grad():\n            prediction, pred_old = model(Variable(batch), CUDA)\n        #print(\"PREDICTION:\",prediction)\n        #prediction_old = prediction[0]\n        #prediction = prediction[1]\n        prediction = write_results(prediction, confidence, num_classes, nms_conf = nms_thesh)\n        #pred_old = write_results(pred_old, confidence, num_classes, nms_conf = nms_thesh)\n        #print(\"PREDICTION: \",prediction)\n        #print(\"PREDICTION_OLD: \",pred_old)\n        end = time.time()\n\n        if type(prediction) == int:\n            i += 1\n            continue\n        end = time.time()\n        prediction[:,0] += i*batch_size\n\n        if not write:                      #If we have't initialised output\n            output = prediction  \n            write = 1\n        else:\n            output = torch.cat((output,prediction))\n            \n        for im_num, image in enumerate(imlist[i*batch_size: min((i +  1)*batch_size, len(imlist))]):\n            im_id = i*batch_size + im_num\n            #print(\"{0:20s} predicted in {1:6.3f} seconds\".format(image.split(\"/\")[-1], (end - start)/batch_size))\n            print(\"{0:20s} {1:s}\".format(\"Objects Detected:\", \"\"))\n            print(\"----------------------------------------------------------\")\n            image_id = image.split(\"/\")[-1].split(\".\")[0]\n\n        i += 1\n        if CUDA:\n            torch.cuda.synchronize()       \n    try:\n        output\n    except NameError:\n        print (\"No detections were made\")\n        return ['']#exit()\n\n    im_dim_list = torch.index_select(im_dim_list, 0, output[:,0].long())\n\n    scaling_factor = torch.min(inp_dim/im_dim_list,1)[0].view(-1,1)\n\n\n    output[:,[1,3]] -= (inp_dim - scaling_factor*im_dim_list[:,0].view(-1,1))/2\n    output[:,[2,4]] -= (inp_dim - scaling_factor*im_dim_list[:,1].view(-1,1))/2\n\n    output[:,1:5] /= scaling_factor\n\n    for i in range(output.shape[0]):\n        output[i, [1,3]] = torch.clamp(output[i, [1,3]], 0.0, im_dim_list[i,0])\n        output[i, [2,4]] = torch.clamp(output[i, [2,4]], 0.0, im_dim_list[i,1])\n    \n    output_recast = time.time()\n    class_load = time.time()\n    colors = pkl.load(open(\"../input/pallete/pallete\", \"rb\"))\n\n    draw = time.time()\n    sub_str = \" \" #initialise to a space value\n    def write(x, batches, results):\n        c1 = tuple(x[1:3].int())\n        c2 = tuple(x[3:5].int())\n        img = results[int(x[0])]\n        cls = int(x[-1])\n        color = random.choice(colors)\n        label = \"{0}\".format(classes[cls])\n        cv2.rectangle(img, c1, c2,color, 1)\n        #sajin\n        #d_class_label[classes[cls]]\n        height = img.shape[0]\n        width = img.shape[1]\n        x_min = np.around(c1[0].data.numpy()/width,decimals = 2)\n        y_min = np.around(c1[1].data.numpy()/height,decimals = 2)\n        x_max = np.around(c2[0].data.numpy()/width,decimals = 2)\n        y_max = np.around(c2[1].data.numpy()/height,decimals = 2)\n        confi = np.around(x[-3].data.numpy(), decimals = 2)\n        #print(\"iiii..\",t,tf.divide(t ,img.shape[0] ) )\n        #print(\"Box \",image_id,c1, c2, classes[cls],d_class_label[classes[cls].lower()],img.shape)\n        #print(\"Box1 \",image_id,d_class_label[classes[cls].lower()],confi,x_min,y_min,x_max,y_max)\n        sub_str = d_class_label[classes[cls].lower()]+' '+str(confi)+' '+str(x_min)+' '+str(y_min)+' '+str(x_max)+' '+str(y_max)\n        #font = ImageFont.truetype(font='../input/firamonomedium/FiraMono-Medium.otf',size=np.floor(3e-2 * img.size[1] + 0.5).astype('int32'))\n        #t_size = cv2.getTextSize(label, font, 2 , 1)[0]\n        #end\n        t_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_COMPLEX_SMALL, 1 , 1)[0] # 1 to 2 Sajin\n        c2 = c1[0] + t_size[0] + 3, c1[1] + t_size[1] + 4\n        cv2.rectangle(img, c1, c2,color, -1)\n        cv2.putText(img, label, (c1[0], c1[1] + t_size[1] + 4), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, [225,255,255], 1);\n        #cv2.putText(img, label, (c1[0], c1[1] + t_size[1] + 4), font, 1, [225,255,255], 1); #sajin\n        return sub_str\n\n\n    sub_str =  list(map(lambda x: write(x, im_batches, orig_ims), output))\n    #print(\"Final...\",sub_str)\n    #det_names = pd.Series(imlist).apply(lambda x: \"{}/det_{}\".format(args.det,x.split(\"/\")[-1]))\n    #print(\"Det names: \",det_names)\n    det_names = \"output_image.png\"\n    #list(map(cv2.imwrite, det_names, loaded_ims))\n    #list(map(cv2.imwrite, \"im.png\", loaded_ims))\n    #list(map(cv2.imshow, det_names, loaded_ims))\n    #cv2.waitKey(0)\n    #cv2.destroyAllWindows()\n    #\n    '''\n    import matplotlib.pyplot as plt\n    for img in orig_ims:\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        print(\"Img width:\",img.shape[0])\n        #plt.xticks(np.arange(img.shape[0], img.shape[0]+1, 100))\n        plt.figure(figsize=(12,12))\n        plt.imshow(img)\n        plt.show()\n    \n    end = time.time()\n\n    print(\"SUMMARY\")\n    print(\"----------------------------------------------------------\")\n    print(\"{:25s}: {}\".format(\"Task\", \"Time Taken (in seconds)\"))\n    print()\n    print(\"{:25s}: {:2.3f}\".format(\"Reading addresses\", load_batch - read_dir))\n    print(\"{:25s}: {:2.3f}\".format(\"Loading batch\", start_det_loop - load_batch))\n    print(\"{:25s}: {:2.3f}\".format(\"Detection (\" + str(len(imlist)) +  \" images)\", output_recast - start_det_loop))\n    print(\"{:25s}: {:2.3f}\".format(\"Output Processing\", class_load - output_recast))\n    print(\"{:25s}: {:2.3f}\".format(\"Drawing Boxes\", end - draw))\n    print(\"{:25s}: {:2.3f}\".format(\"Average time_per_img\", (end - load_batch)/len(imlist)))\n    print(\"----------------------------------------------------------\")\n    '''\n\n    torch.cuda.empty_cache()\n    return sub_str","metadata":{"_uuid":"2187735948f9b73b3f17798ff33b0ae0189af681","collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coco_classes = \"../input/coco-classesv2/coco_classesv2.txt\"\nweightsfile = \"../input/yolov3-weights/yolov3.weights\"\ncfgfile = \"../input/yolov3cfg/yolov3.cfg\"\nimg_path = \"../input/google-ai-open-images-object-detection-track/test/challenge2018_test/00000b4dcff7f799.jpg\"\n#00001a21632de752.jpg\"\n'''\nbatches = 1\nmodel = getmodel(cfgfile, weightsfile)\nsub_str1 = detect(model,img_path,batches,0.5,0.4, coco_classes, weightsfile, cfgfile, \"416\")\nprint(sub_str1)\n'''\n\n","metadata":{"_uuid":"90fc7c6920bfabfda1f18d811c3075a2a73bfbac","_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model = Darknet(\"../input/yolov3cfg/yolov3.cfg\")\n#inp = get_test_input(\"../input/google-ai-open-images-object-detection-track/test/challenge2018_test/00001a21632de752.jpg\",416)\n#print(inp)\n#pred = model(inp, torch.cuda.is_available())\n#print (pred)\n#print (pred.shape)","metadata":{"_uuid":"4302997bd7d24ae2318fd82bce8cd579c4b85a49","collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{"_uuid":"c801f034dcd39c057641167ed73620416de3346a"}},{"cell_type":"markdown","source":"Lets view some predictions and detected objects ","metadata":{"_uuid":"0387b6fa4ab3b4d1e1d8b2285687e2003836441e"}},{"cell_type":"code","source":"import torch\nprint(torch.__version__)\n","metadata":{"_uuid":"bf28cb6b08f2c13a17a6ff26f1315e304f32e8bd","collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nimport math, os\nimage_path = \"../input/google-ai-open-images-object-detection-track/test/\"\n\nbatch_size = 1\nimg_generator = ImageDataGenerator().flow_from_directory(image_path, shuffle=False, batch_size = batch_size)\nn_rounds = math.ceil(img_generator.samples / img_generator.batch_size)\nfilenames = img_generator.filenames\nprint(len(filenames))\n#print(filenames)","metadata":{"_uuid":"93a60648ec0ccbec8f57db75a892e9ffcf036d59","collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batches = 1\nconfidence = 0.5\nnms_thresh = 0.4\nmodel = getmodel(cfgfile, weightsfile)\nf = open(\"od_sub1.csv\",\"w\")\nf.write('ImageId'+','+'PredictionString'+'\\n')\ndet_cnt = 0\nfor i in range(len(filenames)):\n    image = filenames[i]\n    print(image)\n    sub_str1 = detect(model, image_path + image,batches, confidence, nms_thresh, coco_classes, weightsfile, cfgfile, 416)\n    #print(image.split('/')[1]+','+' '.join(sub_str1))\n    f.write(image.split('/')[1]+','+' '.join(sub_str1)+'\\n')\n    if len(sub_str1) > 1:\n        det_cnt = det_cnt + 1\n    if i == 10:\n        print(\"i is :\",i,det_cnt)\n        print(\"% of detection \",det_cnt * 100/i)\n        break\nprint('Closing file....')\nf.close()","metadata":{"_uuid":"5e3ad3186294e2b9713c7410e9919718a5ad562e","_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"714f08288a726685148e7560c5975735ff27e1fc","collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"a1a69867abd6b989c5bb473cd18f78e60d025609","collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"6a3bc265553799df2ab9e9468106da4a924670a0","collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]}]}