{"cells":[{"metadata":{},"cell_type":"markdown","source":"**V3: added Matching detected objects with dataset**\n**V4: added Bounding box form yaw/pitch/roll/x/y/z vusualization based on @zstusnoopy code**\n\n**References:**\n1. https://www.kaggle.com/hocop1/centernet-baseline\n2. https://github.com/OlafenwaMoses/ImageAI/\n3. https://www.kaggle.com/zstusnoopy/visualize-the-location-and-3d-bounding-box-of-car"},{"metadata":{},"cell_type":"markdown","source":"**Import modules**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport os\nfrom math import sin, cos","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Read initial data**"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DATASET_DIR = '/kaggle/input/pku-autonomous-driving/'\nCAM_INTRINSICS = os.path.join(DATASET_DIR, 'camera/camera_intrinsic.txt')\n\nwith open(CAM_INTRINSICS, 'r') as f:\n cam_intr = f.readlines()\nprint(cam_intr)\n\ncamera_matrix = np.array([[2304.5479, 0, 1686.2379],\n [0, 2305.8757, 1354.9849],\n [0, 0, 1]], dtype=np.float32)\n\ndf_train = pd.read_csv(os.path.join(DATASET_DIR, 'train.csv'))\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **Utility functions**"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# @hocop1 function\ndef str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):\n '''\n Input:\n s: PredictionString (e.g. from train dataframe)\n names: array of what to extract from the string\n Output:\n list of dicts with keys from `names`\n '''\n coords = []\n for l in np.array(s.split()).reshape([-1, 7]):\n coords.append(dict(zip(names, l.astype('float'))))\n if 'id' in coords[-1]:\n coords[-1]['id'] = int(coords[-1]['id'])\n return coords\n\n# @hocop1 function\ndef get_img_coords(s):\n '''\n Input is a PredictionString (e.g. from train dataframe)\n Output is two arrays:\n xs: x coordinates in the image\n ys: y coordinates in the image\n '''\n coords = str2coords(s)\n xs = [c['x'] for c in coords]\n ys = [c['y'] for c in coords]\n zs = [c['z'] for c in coords]\n P = np.array(list(zip(xs, ys, zs))).T\n img_p = np.dot(camera_matrix, P).T\n img_p[:, 0] /= img_p[:, 2]\n img_p[:, 1] /= img_p[:, 2]\n img_xs = img_p[:, 0]\n img_ys = img_p[:, 1]\n img_zs = img_p[:, 2] # z = Distance from the camera\n return img_xs, img_ys\n\ndef hide_masked_area(img, mask, th=32):\n mask[mask >= 32] = 255\n mask[mask < 32] = 0\n\n img_acc = img.astype(np.int32) + mask\n img[img_acc > 255] = 255\n return img\n\ndef visualize_image(img, mask, str_coord, mask_overlay='blend'):\n \n img = visualize_bb(img, str2coords(str_coord))\n \n if mask is None:\n mask = np.zeros(img.shape, dtype=np.uint8)\n if mask_overlay=='blend':\n plt.imshow(img)\n plt.imshow(mask, alpha=0.5, cmap='gray')\n elif mask_overlay=='draw':\n img = hide_masked_area(img, mask)\n plt.imshow(img)\n elif mask_overlay=='none':\n plt.imshow(img)\n \n x, y = get_img_coords(str_coord)\n for i in range(len(x)):\n plt.text(x[i],y[i], str(i), color = 'red', fontweight = 'bold', bbox=dict(fill=False, edgecolor='red', linewidth=1))\n \ndef find_closest_center(xcm, ycm, xcd, ycd):\n dist_min = 100000\n dist_th = 250\n id_min = -1\n for i in range(len(xcm)):\n dist = np.sqrt((xcm[i]-xcd)**2 + (ycm[i]-ycd)**2)\n if dist < dist_min:\n dist_min = dist\n if dist_min < dist_th:\n id_min = i\n return id_min \n\n# convert euler angle to rotation matrix\ndef euler_to_Rot(yaw, pitch, roll):\n Y = np.array([[cos(yaw), 0, sin(yaw)],\n [0, 1, 0],\n [-sin(yaw), 0, cos(yaw)]])\n P = np.array([[1, 0, 0],\n [0, cos(pitch), -sin(pitch)],\n [0, sin(pitch), cos(pitch)]])\n R = np.array([[cos(roll), -sin(roll), 0],\n [sin(roll), cos(roll), 0],\n [0, 0, 1]])\n return np.dot(Y, np.dot(P, R))\n\ndef visualize_bb(img, coords):\n x_l = 1.02\n y_l = 0.80\n z_l = 2.31\n \n img = img.copy()\n for point in coords:\n # Get values\n x, y, z = point['x'], point['y'], point['z']\n yaw, pitch, roll = -point['pitch'], -point['yaw'], -point['roll']\n # Math\n Rt = np.eye(4)\n t = np.array([x, y, z])\n Rt[:3, 3] = t\n Rt[:3, :3] = euler_to_Rot(yaw, pitch, roll).T\n Rt = Rt[:3, :]\n \n P = np.array([[x_l, y_l, -z_l, 1],\n [x_l, y_l, z_l, 1],\n [-x_l, y_l, z_l, 1],\n [-x_l, y_l, -z_l, 1],\n [x_l, -y_l, -z_l, 1],\n [x_l, -y_l, z_l, 1],\n [-x_l, -y_l, z_l, 1],\n [-x_l, -y_l, -z_l, 1]]).T\n \n img_cor_points = np.dot(camera_matrix, np.dot(Rt, P))\n img_cor_points = img_cor_points.T\n img_cor_points[:, 0] /= img_cor_points[:, 2]\n img_cor_points[:, 1] /= img_cor_points[:, 2]\n \n x_min = np.min(img_cor_points[:,0])\n x_max = np.max(img_cor_points[:,0])\n y_min = np.min(img_cor_points[:,1])\n y_max = np.max(img_cor_points[:,1])\n \n cv2.rectangle(img, (int(x_min), int(y_min)), (int(x_max), int(y_max)), (0, 0, 255), 7)\n \n return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Visualization with mask and car positions from dataframe**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# visualize 4 examples\nidl = [4, 5, 6, 0]\nplt.figure(figsize=[20, 15])\nfor i,id in enumerate(idl):\n img_path = os.path.join(DATASET_DIR, 'train_images', '{}.{}'.format(df_train.loc[id,'ImageId'], 'jpg'))\n tmp_im = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)\n mask_path = os.path.join(DATASET_DIR, 'train_masks', '{}.{}'.format(df_train.loc[id,'ImageId'], 'jpg'))\n mask = cv2.imread(mask_path)\n plt.subplot(2,2,i+1)\n visualize_image(tmp_im, mask, df_train['PredictionString'][id], mask_overlay='draw')\nplt.show()\nim_shape = tmp_im.shape\nprint(im_shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Car positions distribution (*based on @hocop1 EDA*) with extreme samples visualization**\n\nOn xmax visualization unadequate mask can be observed!"},{"metadata":{"trusted":true},"cell_type":"code","source":"xs, ys = [], []\n\nx_min = im_shape[1]\ny_min = im_shape[0]\nx_max = 0\ny_max = 0\n\nfor i, ps in enumerate(df_train['PredictionString']):\n x, y = get_img_coords(ps)\n xs += list(x)\n ys += list(y)\n if np.min(x) < x_min:\n x_min = np.min(x)\n idx_xmin = i\n if np.min(y) < y_min:\n y_min = np.min(y)\n idx_ymin = i\n if np.max(x) > x_max:\n x_max = np.max(x)\n idx_xmax = i\n if np.max(y) > y_max:\n y_max = np.max(y)\n idx_ymax = i\n\nprint([idx_xmin, idx_xmax, idx_ymin, idx_ymax])\nprint([x_min, x_max, y_min, y_max])\n\nplt.figure(figsize=(10,10))\nplt.imshow(cv2.imread(DATASET_DIR + 'train_images/' + df_train['ImageId'][0] + '.jpg'), alpha=0.3)\nplt.scatter(xs, ys, color='red', s=10, alpha=0.2);\n\nidl = [idx_xmin, idx_xmax, idx_ymin, idx_ymax]\nplt.figure(figsize=[20, 15])\nfor i,id in enumerate(idl):\n img_path = os.path.join(DATASET_DIR, 'train_images', '{}.{}'.format(df_train.loc[id,'ImageId'], 'jpg'))\n tmp_im = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)\n mask_path = os.path.join(DATASET_DIR, 'train_masks', '{}.{}'.format(df_train.loc[id,'ImageId'], 'jpg'))\n mask = cv2.imread(mask_path)\n plt.subplot(2,2,i+1)\n visualize_image(tmp_im, mask, df_train['PredictionString'][id], mask_overlay='draw')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**ImageAI installation**"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install imageai --quiet\n!pip install tensorflow==1.14.0 --quiet\n!pip install tensorflow-gpu==1.14.0 --quiet","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Initialize RetinaNet and YoloV3**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from imageai.Detection import ObjectDetection\n\n\ndetector_ret = ObjectDetection()\ndetector_ret.setModelTypeAsRetinaNet()\ndetector_ret.setModelPath('/kaggle/input/imageaiweighs/resnet50_coco_best_v2.0.1.h5')\ndetector_ret.loadModel()\ncustom_ret = detector_ret.CustomObjects(car=True)\n\ndetector_yolo = ObjectDetection()\ndetector_yolo.setModelTypeAsYOLOv3()\ndetector_yolo.setModelPath('/kaggle/input/imageaiweighs/yolo.h5')\ndetector_yolo.loadModel()\ncustom_yolo = detector_yolo.CustomObjects(car=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Visualization with object detection (*default probability = 50 %*)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# visualize 4 examples with object detection\nidl = [4, 5, 6, 0]\nfig = plt.figure(figsize=[20, 15])\nfor i,id in enumerate(idl):\n img_path = os.path.join(DATASET_DIR, 'train_images', '{}.{}'.format(df_train.loc[id,'ImageId'], 'jpg'))\n tmp_im = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)\n \n mask_path = os.path.join(DATASET_DIR, 'train_masks', '{}.{}'.format(df_train.loc[id,'ImageId'], 'jpg'))\n mask = cv2.imread(mask_path)\n img = hide_masked_area(tmp_im, mask)\n returned_image, detections = detector_ret.detectCustomObjectsFromImage(custom_objects=custom_ret, input_image=img, input_type=\"array\", output_type=\"array\")\n ax = fig.add_subplot(2,2,i+1)\n visualize_image(img, None, df_train['PredictionString'][id], mask_overlay='none')\n for eachObject in detections:\n box = eachObject[\"box_points\"]\n rect = patches.Rectangle((box[0],box[1]),box[2]-box[0],box[3]-box[1],linewidth=2,edgecolor='g',facecolor='none')\n ax.add_patch(rect)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Yolo can be affected by part of car on which camera is installed\npoly_to_hide_car = np.array([[1100,2400],[3000,2480], [3384,2640],[3384,2710], [800,2710]])\n\nextracted_img = []\n# visualize 4 examples with object detection\nidl = [4, 5, 6, 0]\nfig = plt.figure(figsize=[20, 15])\nfor i,id in enumerate(idl):\n img_path = os.path.join(DATASET_DIR, 'train_images', '{}.{}'.format(df_train.loc[id,'ImageId'], 'jpg'))\n tmp_im = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)\n tmp_im = cv2.fillConvexPoly(tmp_im, poly_to_hide_car, [0,0,0])\n mask_path = os.path.join(DATASET_DIR, 'train_masks', '{}.{}'.format(df_train.loc[id,'ImageId'], 'jpg'))\n mask = cv2.imread(mask_path)\n img = hide_masked_area(tmp_im, mask)\n returned_image, detections, extracted_images = detector_yolo.detectCustomObjectsFromImage(custom_objects=custom_yolo, input_image=img, input_type=\"array\", \n output_type=\"array\", extract_detected_objects=True)\n ax = fig.add_subplot(2,2,i+1)\n visualize_image(img, None, df_train['PredictionString'][id], mask_overlay='none')\n for eachObject in detections:\n box = eachObject[\"box_points\"]\n rect = patches.Rectangle((box[0],box[1]),box[2]-box[0],box[3]-box[1],linewidth=2,edgecolor='g',facecolor='none')\n ax.add_patch(rect)\n extracted_img.append(extracted_images)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for extracted_images in extracted_img:\n print('*')\n plt.figure(figsize=[50, 15])\n n_cars = len(extracted_images)\n for i, img in enumerate(extracted_images):\n print(img.shape)\n plt.subplot(1, n_cars, i+1)\n plt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Matching detected objects with dataset**"},{"metadata":{"trusted":true},"cell_type":"code","source":"id = 2\n\nimg_path = os.path.join(DATASET_DIR, 'train_images', '{}.{}'.format(df_train.loc[id,'ImageId'], 'jpg'))\ntmp_im = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)\ntmp_im = cv2.fillConvexPoly(tmp_im, poly_to_hide_car, [0,0,0])\nmask_path = os.path.join(DATASET_DIR, 'train_masks', '{}.{}'.format(df_train.loc[id,'ImageId'], 'jpg'))\nmask = cv2.imread(mask_path)\nimg = hide_masked_area(tmp_im, mask)\nreturned_image, detections, extracted_images = detector_yolo.detectCustomObjectsFromImage(custom_objects=custom_yolo, input_image=img, input_type=\"array\", \n output_type=\"array\", extract_detected_objects=True)\nxcm, ycm = get_img_coords(df_train['PredictionString'][id])\nn_det_cars = len(detections)\nfig = plt.figure(figsize=[10, 20])\nfor i, det in enumerate(detections):\n xcd = (det['box_points'][0]+det['box_points'][2])/2\n ycd = (det['box_points'][1]+det['box_points'][3])/2\n w = (det['box_points'][2]-det['box_points'][0])/2\n h = (det['box_points'][3]-det['box_points'][1])/2\n match_id = find_closest_center(xcm, ycm, xcd, ycd)\n plt.subplot(n_det_cars,2,2*i+1)\n plt.imshow(extracted_images[i])\n plt.subplot(n_det_cars,2,2*i+2)\n if match_id != -1:\n xx = np.array(range(int(xcm[match_id]-w),int(xcm[match_id]+w)))\n xx = xx[(xx>=0) & (xx