{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torchvision\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision import transforms\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport cv2\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if(torch.cuda.is_available()):\n    device = torch.device(\"cuda\")\n    print(device,torch.cuda.get_device_name(0))\nelse:\n    device = torch.device(\"cpu\")\n    print(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **READ A BATCH OF TRAINING IMAGES ALONG WITH BOUNDRING BOXES AND LABELS**\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# input image could be of any size\nimg0 = cv2.imread('../input/faster-rcnn/Profile_Pic.jpeg')\nimg0 = cv2.cvtColor(img0, cv2.COLOR_BGR2RGB) \nprint(img0.shape)\nplt.imshow(img0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Object information\nbbox0 = np.array([[140,500,500,750]]) #a set of bounding boxes [ymin, xmin, ymax, xmax] and their labels \nlabels = np.array([1,1]) #0:background,1:face","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# display bounding box and labels\nimg0_clone = np.copy(img0)\nfor i in range(len(bbox0)):\n    cv2.rectangle(img0_clone, (bbox0[i][1], bbox0[i][0]), (bbox0[i][3], bbox0[i][2]), color=(0, 255, 0), thickness=3) \n    cv2.putText(img0_clone, str(int(labels[i])), (bbox0[i][3], bbox0[i][2]), cv2.FONT_HERSHEY_SIMPLEX, 3, (0,0,255),thickness=3) \nplt.imshow(img0_clone)\nplt.show()    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n# **Resize the input images to (h=800, w=800)¶**"},{"metadata":{"trusted":true},"cell_type":"code","source":"img = cv2.resize(img0, dsize=(800, 800), interpolation=cv2.INTER_CUBIC)\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# change the bounding box coordinates \nWratio = 800/img0.shape[1]\nHratio = 800/img0.shape[0]\nratioLst = [Hratio, Wratio, Hratio, Wratio]\nbbox = []\nfor box in bbox0:\n    box = [int(a * b) for a, b in zip(box, ratioLst)] \n    bbox.append(box)\nbbox = np.array(bbox)\nprint(bbox)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# display bounding box and labels\nimg_clone = np.copy(img)\nbbox_clone = bbox.astype(int)\nfor i in range(len(bbox)):\n    cv2.rectangle(img_clone, (bbox[i][1], bbox[i][0]), (bbox[i][3], bbox[i][2]), color=(0, 255, 0), thickness=3) # Draw Rectangle\n    cv2.putText(img_clone, str(int(labels[i])), (bbox[i][3], bbox[i][2]), cv2.FONT_HERSHEY_SIMPLEX, 3, (0,0,255),thickness=3) # Write the prediction class\nplt.imshow(img_clone)\nplt.show()    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Use VGG16 to extract features from input images**\n\nInput images (batch_size, H=800, W=800, d=3), Features: (batch_size, H= 50, W=50, d=512)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# List all the layers of VGG16\nmodel = torchvision.models.vgg16(pretrained=True).to(device)\nfe = list(model.features)\nprint(len(fe))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# collect layers with output feature map size (W, H) < 50\ndummy_img = torch.zeros((1, 3, 800, 800)).float() # test image array [1, 3, 800, 800] \nprint(dummy_img.shape)\n\nreq_features = []\nk = dummy_img.clone().to(device)\nfor i in fe:\n    k = i(k)\n    if k.size()[2] < 800//16:   #800/16=50\n        break\n    req_features.append(i)\n    out_channels = k.size()[1]\nprint(len(req_features)) #30\nprint(out_channels) # 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert this list into a Sequential module\nfaster_rcnn_fe_extractor = nn.Sequential(*req_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transform = transforms.Compose([transforms.ToTensor()]) # Defing PyTorch Transform\nimgTensor = transform(img).to(device) \nimgTensor = imgTensor.unsqueeze(0)\nout_map = faster_rcnn_fe_extractor(imgTensor)\nprint(out_map.size())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# visualize the first 5 channels of the 50*50*512 feature maps\nimgArray=out_map.data.cpu().numpy().squeeze(0)\nfig=plt.figure(figsize=(12, 4))\nfigNo = 1\nfor i in range(5): \n    fig.add_subplot(1, 5, figNo) \n    plt.imshow(imgArray[i], cmap='gray')\n    figNo +=1\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Generate 22,500 anchor boxes on each input image¶**\n\n50x50=2500 anchors, each anchor generate 9 anchor boxes, Total = 50x50x9=22,500"},{"metadata":{"trusted":true},"cell_type":"code","source":"# x, y intervals to generate anchor box center\nfe_size = (800//16)\nctr_x = np.arange(16, (fe_size+1) * 16, 16)\nctr_y = np.arange(16, (fe_size+1) * 16, 16)\nprint(len(ctr_x), ctr_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# coordinates of the 2500 center points to generate anchor boxes\nindex = 0\nctr = np.zeros((2500, 2))\nfor x in range(len(ctr_x)):\n    for y in range(len(ctr_y)):\n        ctr[index, 1] = ctr_x[x] - 8\n        ctr[index, 0] = ctr_y[y] - 8\n        index +=1\nprint(ctr.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# display the 2500 anchors\nimg_clone = np.copy(img)\nplt.figure(figsize=(9, 6))\nfor i in range(ctr.shape[0]):\n    cv2.circle(img_clone, (int(ctr[i][0]), int(ctr[i][1])), radius=1, color=(255, 0, 0), thickness=1) \nplt.imshow(img_clone)\nplt.show()    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for each of the 2500 anchors, generate 9 anchor boxes\n# 2500*9 = 22500 anchor boxes\nratios = [0.5, 1, 2]\nscales = [8, 16, 32]\nsub_sample = 16\nanchor_boxes = np.zeros( ((fe_size * fe_size * 9), 4))\nindex = 0\nfor c in ctr:\n    ctr_y, ctr_x = c\n    for i in range(len(ratios)):\n        for j in range(len(scales)):\n            h = sub_sample * scales[j] * np.sqrt(ratios[i])\n            w = sub_sample * scales[j] * np.sqrt(1./ ratios[i])\n            anchor_boxes[index, 0] = ctr_y - h / 2.\n            anchor_boxes[index, 1] = ctr_x - w / 2.\n            anchor_boxes[index, 2] = ctr_y + h / 2.\n            anchor_boxes[index, 3] = ctr_x + w / 2.\n            index += 1\nprint(anchor_boxes.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# display the 9 anchor boxes of one anchor and the ground trugh bbox\nimg_clone = np.copy(img)\nfor i in range(11025, 11034):  #9*1225=11025\n    x0 = int(anchor_boxes[i][1])\n    y0 = int(anchor_boxes[i][0])\n    x1 = int(anchor_boxes[i][3])\n    y1 = int(anchor_boxes[i][2])\n    cv2.rectangle(img_clone, (x0, y0), (x1, y1), color=(255, 255, 2550), thickness=3) \n\nfor i in range(len(bbox)):\n    cv2.rectangle(img_clone, (bbox[i][1], bbox[i][0]), (bbox[i][3], bbox[i][2]), color=(0, 255, 0), thickness=3) # Draw Rectangle\n    \nplt.imshow(img_clone)\nplt.show()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Ignore cross-boundary anchor boxes\n# valid anchor boxes with (y1, x1)>0 and (y2, x2)<=800\nindex_inside = np.where(\n        (anchor_boxes[:, 0] >= 0) &\n        (anchor_boxes[:, 1] >= 0) &\n        (anchor_boxes[:, 2] <= 800) &\n        (anchor_boxes[:, 3] <= 800)\n    )[0]\nprint(index_inside.shape)\n\nvalid_anchor_boxes = anchor_boxes[index_inside]\nprint(valid_anchor_boxes.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate iou of the valid anchor boxes \n# Since we have 8940 anchor boxes and 2 ground truth objects, we should get an array with (8490, 2) as the output. \nious = np.empty((len(valid_anchor_boxes), 2), dtype=np.float32)\nious.fill(0)\nfor num1, i in enumerate(valid_anchor_boxes):\n    ya1, xa1, ya2, xa2 = i  \n    anchor_area = (ya2 - ya1) * (xa2 - xa1)\n    for num2, j in enumerate(bbox):\n        yb1, xb1, yb2, xb2 = j\n        box_area = (yb2- yb1) * (xb2 - xb1)\n        inter_x1 = max([xb1, xa1])\n        inter_y1 = max([yb1, ya1])\n        inter_x2 = min([xb2, xa2])\n        inter_y2 = min([yb2, ya2])\n        if (inter_x1 < inter_x2) and (inter_y1 < inter_y2):\n            iter_area = (inter_y2 - inter_y1) * (inter_x2 - inter_x1)\n            iou = iter_area / (anchor_area+ box_area - iter_area)            \n        else:\n            iou = 0.\n        ious[num1, num2] = iou\nprint(ious.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# What anchor box has max iou with the ground truth bbox  \ngt_argmax_ious = ious.argmax(axis=0)\nprint(gt_argmax_ious)\n\ngt_max_ious = ious[gt_argmax_ious, np.arange(ious.shape[1])]\nprint(gt_max_ious)\n\ngt_argmax_ious = np.where(ious == gt_max_ious)[0]\nprint(gt_argmax_ious)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# What ground truth bbox is associated with each anchor box \nargmax_ious = ious.argmax(axis=1)\nprint(argmax_ious.shape)\nprint(argmax_ious)\nmax_ious = ious[np.arange(len(index_inside)), argmax_ious]\nprint(max_ious)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label = np.empty((len(index_inside), ), dtype=np.int32)\nlabel.fill(-1)\nprint(label.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use iou to assign 1 (objects) to two kind of anchors \n# a) The anchors with the highest iou overlap with a ground-truth-box\n# b) An anchor that has an IoU overlap higher than 0.7 with ground-truth box\n\n# Assign 0 (background) to an anchor if its IoU ratio is lower than 0.3 for all ground-truth boxes\npos_iou_threshold  = 0.7\nneg_iou_threshold = 0.3\nlabel[gt_argmax_ious] = 1\nlabel[max_ious >= pos_iou_threshold] = 1\nlabel[max_ious < neg_iou_threshold] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_sample = 256\npos_ratio = 0.5\nn_pos = pos_ratio * n_sample\n\npos_index = np.where(label == 1)[0]\nif len(pos_index) > n_pos:\n    disable_index = np.random.choice(pos_index, size=(len(pos_index) - n_pos), replace=False)\n    label[disable_index] = -1\n    \nn_neg = n_sample * np.sum(label == 1)\nneg_index = np.where(label == 0)[0]\nif len(neg_index) > n_neg:\n    disable_index = np.random.choice(neg_index, size=(len(neg_index) - n_neg), replace = False)\n    label[disable_index] = -1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For each valid anchor box, find the groundtruth object which has max_iou \nmax_iou_bbox = bbox[argmax_ious]\nprint(max_iou_bbox.shape)\n\n# valid anchor boxes 的 h, w, cx, cy \nheight = valid_anchor_boxes[:, 2] - valid_anchor_boxes[:, 0]\nwidth = valid_anchor_boxes[:, 3] - valid_anchor_boxes[:, 1]\nctr_y = valid_anchor_boxes[:, 0] + 0.5 * height\nctr_x = valid_anchor_boxes[:, 1] + 0.5 * width\n\n# valid anchor box 的 max iou 的 bbox 的 h, w, cx, cy \nbase_height = max_iou_bbox[:, 2] - max_iou_bbox[:, 0]\nbase_width = max_iou_bbox[:, 3] - max_iou_bbox[:, 1]\nbase_ctr_y = max_iou_bbox[:, 0] + 0.5 * base_height\nbase_ctr_x = max_iou_bbox[:, 1] + 0.5 * base_width\n\n# valid anchor boxes 的 loc = (y-ya/ha), (x-xa/wa), log(h/ha), log(w/wa)\neps = np.finfo(height.dtype).eps\nheight = np.maximum(height, eps) #讓 height !=0, 最小值為 eps\nwidth = np.maximum(width, eps)\ndy = (base_ctr_y - ctr_y) / height\ndx = (base_ctr_x - ctr_x) / width\ndh = np.log(base_height / height)\ndw = np.log(base_width / width)\nanchor_locs = np.vstack((dy, dx, dh, dw)).transpose()\nprint(anchor_locs.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"anchor_labels = np.empty((len(anchor_boxes),), dtype=label.dtype)\nanchor_labels.fill(-1)\nanchor_labels[index_inside] = label\nprint(anchor_labels.shape)\n\nanchor_locations = np.empty((len(anchor_boxes),) + anchor_boxes.shape[1:], dtype=anchor_locs.dtype)\nanchor_locations.fill(0)\nanchor_locations[index_inside, :] = anchor_locs\nprint(anchor_locations.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"in_channels = 512 # depends on the output feature map. in vgg 16 it is equal to 512\nmid_channels = 512\nn_anchor = 9  # Number of anchors at each location\n\nconv1 = nn.Conv2d(in_channels, mid_channels, 3, 1, 1).to(device)\nconv1.weight.data.normal_(0, 0.01)\nconv1.bias.data.zero_()\n\nreg_layer = nn.Conv2d(mid_channels, n_anchor *4, 1, 1, 0).to(device)\nreg_layer.weight.data.normal_(0, 0.01)\nreg_layer.bias.data.zero_()\n\ncls_layer = nn.Conv2d(mid_channels, n_anchor *2, 1, 1, 0).to(device) ## I will be going to use softmax here. you can equally use sigmoid if u replace 2 with 1.\ncls_layer.weight.data.normal_(0, 0.01)\ncls_layer.bias.data.zero_()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = conv1(out_map.to(device)) # out_map = faster_rcnn_fe_extractor(imgTensor)\npred_anchor_locs = reg_layer(x)\npred_cls_scores = cls_layer(x)\nprint(pred_anchor_locs.shape, pred_cls_scores.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_anchor_locs = pred_anchor_locs.permute(0, 2, 3, 1).contiguous().view(1, -1, 4)\nprint(pred_anchor_locs.shape)\n\npred_cls_scores = pred_cls_scores.permute(0, 2, 3, 1).contiguous()\nprint(pred_cls_scores.shape)\n\nobjectness_score = pred_cls_scores.view(1, 50, 50, 9, 2)[:, :, :, :, 1].contiguous().view(1, -1)\nprint(objectness_score.shape)\n\npred_cls_scores  = pred_cls_scores.view(1, -1, 2)\nprint(pred_cls_scores.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(pred_anchor_locs.shape)\nprint(pred_cls_scores.shape)\nprint(anchor_locations.shape)\nprint(anchor_labels.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rpn_loc = pred_anchor_locs[0]\nrpn_score = pred_cls_scores[0]\n\ngt_rpn_loc = torch.from_numpy(anchor_locations)\ngt_rpn_score = torch.from_numpy(anchor_labels)\n\nprint(rpn_loc.shape, rpn_score.shape, gt_rpn_loc.shape, gt_rpn_score.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For classification we use cross-entropy loss\nrpn_cls_loss = F.cross_entropy(rpn_score, gt_rpn_score.long().to(device), ignore_index = -1)\nprint(rpn_cls_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#For Regression we use smooth L1 loss as defined in the Fast RCNN paper\npos = gt_rpn_score > 0\nmask = pos.unsqueeze(1).expand_as(rpn_loc)\nprint(mask.shape)\n\n# take those bounding boxes which have positve labels\nmask_loc_preds = rpn_loc[mask].view(-1, 4)\nmask_loc_targets = gt_rpn_loc[mask].view(-1, 4)\nprint(mask_loc_preds.shape, mask_loc_targets.shape)\n\nx = torch.abs(mask_loc_targets.cpu() - mask_loc_preds.cpu())\nrpn_loc_loss = ((x < 1).float() * 0.5 * x**2) + ((x >= 1).float() * (x-0.5))\nprint(rpn_loc_loss.sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Combining both the rpn_cls_loss and rpn_reg_loss\nrpn_lambda = 10.\nN_reg = (gt_rpn_score >0).float().sum()\nrpn_loc_loss = rpn_loc_loss.sum() / N_reg\nrpn_loss = rpn_cls_loss + (rpn_lambda * rpn_loc_loss)\nprint(rpn_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nms_thresh = 0.7  # non-maximum supression (NMS) \nn_train_pre_nms = 12000 # no. of train pre-NMS\nn_train_post_nms = 2000 # after nms, training Fast R-CNN using 2000 RPN proposals\nn_test_pre_nms = 6000\nn_test_post_nms = 300 # During testing we evaluate 300 proposals,\nmin_size = 16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The labelled 22500 anchor boxes \n# format converted from [y1, x1, y2, x2] to [ctr_x, ctr_y, h, w]\nanc_height = anchor_boxes[:, 2] - anchor_boxes[:, 0]\nanc_width = anchor_boxes[:, 3] - anchor_boxes[:, 1]\nanc_ctr_y = anchor_boxes[:, 0] + 0.5 * anc_height\nanc_ctr_x = anchor_boxes[:, 1] + 0.5 * anc_width\nprint(anc_ctr_x.shape)\n\n# The 22500 anchor boxes location and labels predicted by RPN (convert to numpy)\n# format = (dy, dx, dh, dw)\npred_anchor_locs_numpy = pred_anchor_locs[0].cpu().data.numpy()\nobjectness_score_numpy = objectness_score[0].cpu().data.numpy()\ndy = pred_anchor_locs_numpy[:, 0::4] #每個 anchor box 的 dy\ndx = pred_anchor_locs_numpy[:, 1::4] # dx\ndh = pred_anchor_locs_numpy[:, 2::4] # dh\ndw = pred_anchor_locs_numpy[:, 3::4] # dw\nprint(dy.shape)\n\n# ctr_y = dy predicted by RPN * anchor_h + anchor_cy\n# ctr_x similar\n# h = exp(dh predicted by RPN) * anchor_h\n# w similar\nctr_y = dy * anc_height[:, np.newaxis] + anc_ctr_y[:, np.newaxis]\nctr_x = dx * anc_width[:, np.newaxis] + anc_ctr_x[:, np.newaxis]\nh = np.exp(dh) * anc_height[:, np.newaxis]\nw = np.exp(dw) * anc_width[:, np.newaxis]\nprint(w.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roi = np.zeros(pred_anchor_locs_numpy.shape, dtype=anchor_locs.dtype)\nroi[:, 0::4] = ctr_y - 0.5 * h\nroi[:, 1::4] = ctr_x - 0.5 * w\nroi[:, 2::4] = ctr_y + 0.5 * h\nroi[:, 3::4] = ctr_x + 0.5 * w\nprint(roi.shape)\n\n# clip the predicted boxes to the image\nimg_size = (800, 800) #Image size\nroi[:, slice(0, 4, 2)] = np.clip(roi[:, slice(0, 4, 2)], 0, img_size[0])\nroi[:, slice(1, 4, 2)] = np.clip(roi[:, slice(1, 4, 2)], 0, img_size[1])\nprint(roi.shape, np.max(roi), np.min(roi))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Remove predicted boxes with either height or width < threshold.\nhs = roi[:, 2] - roi[:, 0]\nws = roi[:, 3] - roi[:, 1]\nkeep = np.where((hs >= min_size) & (ws >= min_size))[0] #min_size=16\nroi = roi[keep, :]\nscore = objectness_score_numpy[keep]\nprint(keep.shape, roi.shape, score.shape)\n\n# Sort all (proposal, score) pairs by score from highest to lowest\norder = score.ravel().argsort()[::-1]\nprint(order.shape)\n\n#Take top pre_nms_topN (e.g. 12000 while training and 300 while testing)\norder = order[:n_train_pre_nms]\nroi = roi[order, :]\nprint(order.shape, roi.shape, roi.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Take all the roi boxes [roi_array]\ny1 = roi[:, 0]\nx1 = roi[:, 1]\ny2 = roi[:, 2]\nx2 = roi[:, 3]\n\n# Find the areas of all the boxes [roi_area]\nareas = (x2 - x1 + 1) * (y2 - y1 + 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Take the indexes of order the probability score in descending order \norder = order.argsort()[::-1]\nkeep = []\nwhile (order.size > 0):\n    i = order[0] #take the 1st elt in order and append to keep \n    keep.append(i)\n    xx1 = np.maximum(x1[i], x1[order[1:]]) \n    yy1 = np.maximum(y1[i], y1[order[1:]])\n    xx2 = np.minimum(x2[i], x2[order[1:]])\n    yy2 = np.minimum(y2[i], y2[order[1:]])\n    w = np.maximum(0.0, xx2 - xx1 + 1)\n    h = np.maximum(0.0, yy2 - yy1 + 1)\n    inter = w * h\n    ovr = inter / (areas[i] + areas[order[1:]] - inter)\n    inds = np.where(ovr <= nms_thresh)[0]\n    order = order[inds + 1]\nkeep = keep[:n_train_post_nms] # while training/testing , use accordingly\nroi = roi[keep] # the final region proposals\nprint(len(keep), roi.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_sample = 128  # Number of samples from roi \npos_ratio = 0.25 # Number of positive examples out of the n_samples\npos_iou_thresh = 0.5  # Min iou of region proposal with any groundtruth object to consider it as positive label\nneg_iou_thresh_hi = 0.5  # iou 0~0.5 is considered as negative (0, background)\nneg_iou_thresh_lo = 0.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find the iou of each ground truth object with the region proposals, \nious = np.empty((len(roi), 2), dtype=np.float32)\nious.fill(0)\nfor num1, i in enumerate(roi):\n    ya1, xa1, ya2, xa2 = i  \n    anchor_area = (ya2 - ya1) * (xa2 - xa1)\n    for num2, j in enumerate(bbox):\n        yb1, xb1, yb2, xb2 = j\n        box_area = (yb2- yb1) * (xb2 - xb1)\n        inter_x1 = max([xb1, xa1])\n        inter_y1 = max([yb1, ya1])\n        inter_x2 = min([xb2, xa2])\n        inter_y2 = min([yb2, ya2])\n        if (inter_x1 < inter_x2) and (inter_y1 < inter_y2):\n            iter_area = (inter_y2 - inter_y1) * (inter_x2 - inter_x1)\n            iou = iter_area / (anchor_area+ box_area - iter_area)            \n        else:\n            iou = 0.\n        ious[num1, num2] = iou\nprint(ious.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find out which ground truth has high IoU for each region proposal, Also find the maximum IoU\ngt_assignment = ious.argmax(axis=1)\nmax_iou = ious.max(axis=1)\nprint(gt_assignment)\nprint(max_iou)\n\n# Assign the labels to each proposal\ngt_roi_label = labels[gt_assignment]\nprint(gt_roi_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n# Select the foreground rois as per the pos_iou_thesh and \n# n_sample x pos_ratio (128 x 0.25 = 32) foreground samples.\npos_roi_per_image = 32 \npos_index = np.where(max_iou >= pos_iou_thresh)[0]\npos_roi_per_this_image = int(min(pos_roi_per_image, pos_index.size))\nif pos_index.size > 0:\n    pos_index = np.random.choice(\n        pos_index, size=pos_roi_per_this_image, replace=False)\nprint(pos_roi_per_this_image)\nprint(pos_index)\n\n# Similarly we do for negitive (background) region proposals\nneg_index = np.where((max_iou < neg_iou_thresh_hi) &\n                             (max_iou >= neg_iou_thresh_lo))[0]\nneg_roi_per_this_image = n_sample - pos_roi_per_this_image\nneg_roi_per_this_image = int(min(neg_roi_per_this_image, neg_index.size))\nif  neg_index.size > 0 :\n    neg_index = np.random.choice(\n        neg_index, size=neg_roi_per_this_image, replace=False)\nprint(neg_roi_per_this_image)\nprint(neg_index)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# display ROI samples with postive \nimg_clone = np.copy(img)\nfor i in range(pos_roi_per_this_image):\n    y0, x0, y1, x1 = roi[pos_index[i]].astype(int)\n    cv2.rectangle(img_clone, (x0, y0), (x1, y1), color=(255, 255, 2550), thickness=3) \n\nfor i in range(len(bbox)):\n    cv2.rectangle(img_clone, (bbox[i][1], bbox[i][0]), (bbox[i][3], bbox[i][2]), color=(0, 255, 0), thickness=3) # Draw Rectangle\n    \nplt.imshow(img_clone)\nplt.show()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n# display ROI samples with negative \nimg_clone = np.copy(img)\nplt.figure(figsize=(9, 6))\nfor i in range(neg_roi_per_this_image):\n    y0, x0, y1, x1 = roi[neg_index[i]].astype(int)\n    cv2.rectangle(img_clone, (x0, y0), (x1, y1), color=(255, 255, 2550), thickness=3) \n\nfor i in range(len(bbox)):\n    cv2.rectangle(img_clone, (bbox[i][1], bbox[i][0]), (bbox[i][3], bbox[i][2]), color=(0, 255, 0), thickness=3) # Draw Rectangle\n    \nplt.imshow(img_clone)\nplt.show()  \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Now we gather positve samples index and negitive samples index, \n# their respective labels and region proposals\n\nkeep_index = np.append(pos_index, neg_index)\ngt_roi_labels = gt_roi_label[keep_index]\ngt_roi_labels[pos_roi_per_this_image:] = 0  # negative labels --> 0\nsample_roi = roi[keep_index]\nprint(sample_roi.shape)\n\n# Pick the ground truth objects for these sample_roi and \n# later parameterize as we have done while assigning locations to anchor boxes in section 2.\nbbox_for_sampled_roi = bbox[gt_assignment[keep_index]]\nprint(bbox_for_sampled_roi.shape)\n\nheight = sample_roi[:, 2] - sample_roi[:, 0]\nwidth = sample_roi[:, 3] - sample_roi[:, 1]\nctr_y = sample_roi[:, 0] + 0.5 * height\nctr_x = sample_roi[:, 1] + 0.5 * width\n\nbase_height = bbox_for_sampled_roi[:, 2] - bbox_for_sampled_roi[:, 0]\nbase_width = bbox_for_sampled_roi[:, 3] - bbox_for_sampled_roi[:, 1]\nbase_ctr_y = bbox_for_sampled_roi[:, 0] + 0.5 * base_height\nbase_ctr_x = bbox_for_sampled_roi[:, 1] + 0.5 * base_width","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"eps = np.finfo(height.dtype).eps\nheight = np.maximum(height, eps)\nwidth = np.maximum(width, eps)\n\ndy = (base_ctr_y - ctr_y) / height\ndx = (base_ctr_x - ctr_x) / width\ndh = np.log(base_height / height)\ndw = np.log(base_width / width)\n\ngt_roi_locs = np.vstack((dy, dx, dh, dw)).transpose()\nprint(gt_roi_locs.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rois = torch.from_numpy(sample_roi).float()\nroi_indices = 0 * np.ones((len(rois),), dtype=np.int32)\nroi_indices = torch.from_numpy(roi_indices).float()\nprint(rois.shape, roi_indices.shape)\n\nindices_and_rois = torch.cat([roi_indices[:, None], rois], dim=1)\nxy_indices_and_rois = indices_and_rois[:, [0, 2, 1, 4, 3]]\nindices_and_rois = xy_indices_and_rois.contiguous()\nprint(xy_indices_and_rois.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"size = (7, 7)\nadaptive_max_pool = nn.AdaptiveMaxPool2d(size[0], size[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output = []\nrois = indices_and_rois.data.float()\nrois[:, 1:].mul_(1/16.0) # Subsampling ratio\nrois = rois.long()\nnum_rois = rois.size(0)\nfor i in range(num_rois):\n    roi = rois[i]\n    im_idx = roi[0]\n    im = out_map.narrow(0, im_idx, 1)[..., roi[2]:(roi[4]+1), roi[1]:(roi[3]+1)]\n    tmp = adaptive_max_pool(im)\n    output.append(tmp[0])\noutput = torch.cat(output, 0)\nprint(output.size())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualize the first 5 ROI's feature map (for each feature map, only show the 1st channel of d=512)\nfig=plt.figure(figsize=(12, 4))\nfigNo = 1\nfor i in range(5):\n    roi = rois[i]\n    im_idx = roi[0]\n    im = out_map.narrow(0, im_idx, 1)[..., roi[2]:(roi[4]+1), roi[1]:(roi[3]+1)]\n    tmp = im[0][0].detach().cpu().numpy()\n    fig.add_subplot(1, 5, figNo) \n    plt.imshow(tmp, cmap='gray')\n    figNo +=1\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualize the first 5 ROI's feature maps after ROI pooling (for each feature map, only show the 1st channel of d=512)\nfig=plt.figure(figsize=(12, 4))\nfigNo = 1\nfor i in range(5):\n    roi = rois[i]\n    im_idx = roi[0]\n    im = out_map.narrow(0, im_idx, 1)[..., roi[2]:(roi[4]+1), roi[1]:(roi[3]+1)]\n    tmp = adaptive_max_pool(im)[0]\n    tmp = tmp[0][0].detach().cpu().numpy()\n    fig.add_subplot(1, 5, figNo) \n    plt.imshow(tmp, cmap='gray')\n    figNo +=1\nplt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reshape the tensor so that we can pass it through the feed forward layer.\nk = output.view(output.size(0), -1)\nprint(k.shape) # 25088 = 7*7*512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roi_head_classifier = nn.Sequential(*[nn.Linear(25088, 4096), nn.Linear(4096, 4096)]).to(device)\ncls_loc = nn.Linear(4096, 2 * 4).to(device)\ncls_loc.weight.data.normal_(0, 0.01)\ncls_loc.bias.data.zero_()\nscore = nn.Linear(4096, 2).to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# passing the output of roi-pooling to ROI head \nk = roi_head_classifier(k.to(device))\nroi_cls_loc = cls_loc(k)\nroi_cls_score = score(k)\nprint(roi_cls_loc.shape, roi_cls_score.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predicted\nprint(roi_cls_loc.shape)\nprint(roi_cls_score.shape)\n\n#actual\nprint(gt_roi_locs.shape)\nprint(gt_roi_labels.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gt_roi_labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Converting ground truth to torch variable\ngt_roi_loc = torch.from_numpy(gt_roi_locs)\ngt_roi_label = torch.from_numpy(np.float32(gt_roi_labels)).long()\nprint(gt_roi_loc.shape, gt_roi_label.shape)\n\n#Classification loss\nroi_cls_loss = F.cross_entropy(roi_cls_score.cpu(), gt_roi_label.cpu(), ignore_index=-1)\nprint(roi_cls_loss.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Regression loss\nn_sample = roi_cls_loc.shape[0]\nroi_loc = roi_cls_loc.view(n_sample, -1, 4)\nprint(roi_loc.shape)\n\nroi_loc = roi_loc[torch.arange(0, n_sample).long(), gt_roi_label]\nprint(roi_loc.shape)\n\n# For Regression we use smooth L1 loss as defined in the Fast RCNN paper\npos = gt_roi_label > 0\nmask = pos.unsqueeze(1).expand_as(roi_loc)\nprint(mask.shape)\n\n# take those bounding boxes which have positve labels\nmask_loc_preds = roi_loc[mask].view(-1, 4)\nmask_loc_targets = gt_roi_loc[mask].view(-1, 4)\nprint(mask_loc_preds.shape, mask_loc_targets.shape)\n\nx = torch.abs(mask_loc_targets.cpu() - mask_loc_preds.cpu())\nroi_loc_loss = ((x < 1).float() * 0.5 * x**2) + ((x >= 1).float() * (x-0.5))\nprint(roi_loc_loss.sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roi_lambda = 10.\nroi_loss = roi_cls_loss + (roi_lambda * roi_loc_loss)\nprint(roi_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ntotal_loss = rpn_loss + roi_loss\nprint(total_loss)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}