{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":9993,"databundleVersionId":868308,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import json\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom keras.preprocessing.image import load_img\nfrom math import sin, cos\nfrom PIL import ImageDraw, Image\nimport cv2","metadata":{"execution":{"iopub.status.busy":"2024-09-21T15:16:33.399984Z","iopub.execute_input":"2024-09-21T15:16:33.400370Z","iopub.status.idle":"2024-09-21T15:16:38.092860Z","shell.execute_reply.started":"2024-09-21T15:16:33.400327Z","shell.execute_reply":"2024-09-21T15:16:38.091351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/pku-autonomous-driving/train.csv')\n\n#Intrinsic Camera Matrix\nk = np.array([[2304.5479, 0, 1686.2379],       \n              [0, 2305.8757, 1354.9849],\n              [0, 0, 1]], dtype = np.float32)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T15:16:38.094536Z","iopub.execute_input":"2024-09-21T15:16:38.095222Z","iopub.status.idle":"2024-09-21T15:16:38.151246Z","shell.execute_reply.started":"2024-09-21T15:16:38.095175Z","shell.execute_reply":"2024-09-21T15:16:38.150158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams['axes.grid'] = False\nimg_name = train.loc[4002]['ImageId']\npred_string = train.loc[4002]['PredictionString']\nfig, ax = plt.subplots(figsize = (8, 8))\nimg = load_img(f'/kaggle/input/pku-autonomous-driving/train_images/{img_name}.jpg')\nimgg = load_img(f'/kaggle/input/pku-autonomous-driving/train_images/{img_name}.jpg')\nplt.imshow(img)\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-21T15:16:38.152654Z","iopub.execute_input":"2024-09-21T15:16:38.153023Z","iopub.status.idle":"2024-09-21T15:16:39.941440Z","shell.execute_reply.started":"2024-09-21T15:16:38.152984Z","shell.execute_reply":"2024-09-21T15:16:39.939663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"items = pred_string.split(' ') #splitting a string (pred_string) into a list of items\nmodel_types, yaws, pitches, rolls, xs, ys, zs = [items[i::7] for i in range(7)]","metadata":{"execution":{"iopub.status.busy":"2024-09-21T15:16:39.945229Z","iopub.execute_input":"2024-09-21T15:16:39.945766Z","iopub.status.idle":"2024-09-21T15:16:39.952670Z","shell.execute_reply.started":"2024-09-21T15:16:39.945711Z","shell.execute_reply":"2024-09-21T15:16:39.951139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def euler_to_rot(yaw, pitch, roll):\n    y = np.array([[cos(yaw), 0, sin(yaw)],\n                  [0, 1, 0],\n                  [-sin(yaw), 0, cos(yaw)]])\n    p = np.array([[1, 0, 0],\n                  [0, cos(pitch), -sin(pitch)],\n                  [0, sin(pitch), cos(pitch)]])\n    r = np.array([[cos(roll), -sin(roll), 0],\n                  [sin(roll), cos(roll), 0],\n                  [0, 0, 1]])\n    return np.dot(y, np.dot(p, r))","metadata":{"execution":{"iopub.status.busy":"2024-09-21T15:16:39.954237Z","iopub.execute_input":"2024-09-21T15:16:39.954686Z","iopub.status.idle":"2024-09-21T15:16:39.968058Z","shell.execute_reply.started":"2024-09-21T15:16:39.954642Z","shell.execute_reply":"2024-09-21T15:16:39.966610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_lines(image, points):\n    color = (255, 0, 255)\n    cv2.line(image, tuple(points[1][:2]), tuple(points[2][:2]), color, 15)\n    cv2.line(image, tuple(points[1][:2]), tuple(points[4][:2]), color, 15)\n    cv2.line(image, tuple(points[1][:2]), tuple(points[5][:2]), color, 15)\n    \n    cv2.line(image, tuple(points[2][:2]), tuple(points[3][:2]), color, 15)\n    cv2.line(image, tuple(points[2][:2]), tuple(points[6][:2]), color, 15)\n    \n    cv2.line(image, tuple(points[3][:2]), tuple(points[4][:2]), color, 15)\n    cv2.line(image, tuple(points[3][:2]), tuple(points[7][:2]), color, 15)\n    \n    cv2.line(image, tuple(points[4][:2]), tuple(points[8][:2]), color, 15)\n    \n    cv2.line(image, tuple(points[5][:2]), tuple(points[8][:2]), color, 15)\n    cv2.line(image, tuple(points[5][:2]), tuple(points[6][:2]), color, 15)\n    \n    cv2.line(image, tuple(points[6][:2]), tuple(points[7][:2]), color, 15)\n    \n    cv2.line(image, tuple(points[7][:2]), tuple(points[8][:2]), color, 15)\n    return image\n\ndef draw_points(image, points):\n    image = np.array(image)\n    for (p_x, p_y, p_z) in points:\n        #print(p_x, p_y, p_z)\n        cv2.circle(image, (p_x, p_y), 5,  (255,0, 255),-1)\n    return image    ","metadata":{"execution":{"iopub.status.busy":"2024-09-21T15:16:39.969673Z","iopub.execute_input":"2024-09-21T15:16:39.970139Z","iopub.status.idle":"2024-09-21T15:16:39.990267Z","shell.execute_reply.started":"2024-09-21T15:16:39.970096Z","shell.execute_reply":"2024-09-21T15:16:39.988800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def img_cor_2_world_cor():       # convert image coordinates into world coordinates\n    x_img, y_img, z_img = img_cor_points[0]\n    xc, yc, zc = x_img*z_img, y_img*z_img, z_img\n    p_cam = np.array([xc, yc, zc])\n    xw, yw, zw = np.dot(np.linalg.inv(k), p_cam)\n    #print(xw, yw, zw)\n    #print(x, y, z)","metadata":{"execution":{"iopub.status.busy":"2024-09-21T15:16:39.991909Z","iopub.execute_input":"2024-09-21T15:16:39.992338Z","iopub.status.idle":"2024-09-21T15:16:40.004070Z","shell.execute_reply.started":"2024-09-21T15:16:39.992296Z","shell.execute_reply":"2024-09-21T15:16:40.002599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3D Bounding box","metadata":{}},{"cell_type":"code","source":"x_l = 0.9\ny_l = 0.8\nz_l = 2.0\n\nfor yaw, pitch, roll, x, y, z in zip(yaws, pitches, rolls, xs, ys, zs):\n    yaw, pitch, roll, x, y, z = [float(x) for x in [yaw, pitch, roll, x, y, z]]\n    yaw, pitch, roll = -pitch, -yaw, -roll\n    rt = np.eye(4)\n    t = np.array([x, y, z])\n    rt[:3, 3] = t\n    rt[:3, :3] = euler_to_rot(yaw, pitch, roll).T\n    rt = rt[:3, :]\n    p = np.array([[0, 0, 0, 1],\n                  [x_l, y_l, -z_l, 1], \n                  [x_l, y_l, z_l, 1],\n                  [-x_l, y_l, z_l, 1],\n                  [-x_l, y_l, -z_l, 1],\n                  [x_l, -y_l, -z_l, 1],\n                  [x_l, -y_l, z_l, 1],\n                  [-x_l, -y_l, z_l, 1],\n                  [-x_l, -y_l, -z_l, 1]]).T\n    img_cor_points = np.dot(k, np.dot(rt, p)).T\n    img_cor_points[:, 0] /= img_cor_points[:, 2]\n    img_cor_points[:, 1] /= img_cor_points[:, 2]\n    img_cor_2_world_cor()\n    img_cor_points = img_cor_points.astype(int)\n    img = draw_points(img, img_cor_points)\n    img = draw_lines(img, img_cor_points)\nimg = Image.fromarray(img)\nplt.figure(figsize = (8, 8))\nplt.imshow(img)\nplt.axis('off')\nplt.show()\n    ","metadata":{"execution":{"iopub.status.busy":"2024-09-21T15:16:40.005509Z","iopub.execute_input":"2024-09-21T15:16:40.005921Z","iopub.status.idle":"2024-09-21T15:16:41.732412Z","shell.execute_reply.started":"2024-09-21T15:16:40.005879Z","shell.execute_reply":"2024-09-21T15:16:41.731138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3D Modeling\n      \n #####    ✨ Extract any model from car_models_json and apply, in this example I used                 dazhongmaiteng model.","metadata":{}},{"cell_type":"code","source":"with open('/kaggle/input/pku-autonomous-driving/car_models_json/dazhongmaiteng.json') as json_file:\n    data = json.load(json_file)\n    vertices = np.array(data['vertices'])\n    triangles = np.array(data['faces']) - 1\n    plt.figure(figsize= (10, 5))\n    ax = plt.axes(projection= '3d')\n    ax.set_title('Car Type: '+ data['car_type'])\n    ax.set_xlim([-4, 4])\n    ax.set_ylim([-4, 4])\n    ax.set_zlim([0, 4])\n    ax.plot_trisurf(vertices[:, 0], vertices[:,2], triangles, -vertices[:,1],\n                   shade=True, color='grey')","metadata":{"execution":{"iopub.status.busy":"2024-09-21T15:16:41.734005Z","iopub.execute_input":"2024-09-21T15:16:41.734452Z","iopub.status.idle":"2024-09-21T15:16:42.881522Z","shell.execute_reply.started":"2024-09-21T15:16:41.734406Z","shell.execute_reply":"2024-09-21T15:16:42.880257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vertices[:, 1] = -vertices[:, 1]  # Flip the model to can apply it ","metadata":{"execution":{"iopub.status.busy":"2024-09-21T15:16:42.883212Z","iopub.execute_input":"2024-09-21T15:16:42.883763Z","iopub.status.idle":"2024-09-21T15:16:42.890808Z","shell.execute_reply.started":"2024-09-21T15:16:42.883702Z","shell.execute_reply":"2024-09-21T15:16:42.889522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_obj(image, vertices, triangles):\n    for t in triangles:\n        coord = np.array([vertices[t[0]][:2], vertices[t[1]][:2], vertices[t[2]][:2]], dtype=np.int32)\n        cv2.polylines(image, np.int32([coord]), 1, (0,0,255))","metadata":{"execution":{"iopub.status.busy":"2024-09-21T15:16:42.892265Z","iopub.execute_input":"2024-09-21T15:16:42.892654Z","iopub.status.idle":"2024-09-21T15:16:42.906177Z","shell.execute_reply.started":"2024-09-21T15:16:42.892615Z","shell.execute_reply":"2024-09-21T15:16:42.904694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Augmented Reality ","metadata":{}},{"cell_type":"code","source":"overlay = np.zeros_like(imgg)\nfor yaw, pitch, roll, x, y, z in zip(yaws, pitches, rolls, xs, ys, zs):\n    yaw, pitch, roll, x, y, z = [float(x) for x in [yaw, pitch, roll, x, y, z]]\n    yaw, pitch, roll = -pitch, -yaw, -roll\n    Rt = np.eye(4)\n    t = np.array([x, y, z])\n    Rt[:3, 3] = t\n    Rt[:3, :3] = euler_to_rot(yaw, pitch, roll).T\n    Rt = Rt[:3, :]\n    P = np.ones((vertices.shape[0],vertices.shape[1]+1))\n    P[:, :-1] = vertices\n    P = P.T\n    img_cor_points = np.dot(k, np.dot(Rt, P))\n    img_cor_points = img_cor_points.T\n    img_cor_points[:, 0] /= img_cor_points[:, 2]\n    img_cor_points[:, 1] /= img_cor_points[:, 2]\n    draw_obj(overlay, img_cor_points, triangles)\n\nalpha = .5\nimgg = np.array(imgg)\ncv2.addWeighted(overlay, alpha, imgg, 1 - alpha, 0, imgg)\nplt.figure(figsize=(8,8))\nplt.imshow(imgg);\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-21T15:16:42.907643Z","iopub.execute_input":"2024-09-21T15:16:42.908049Z","iopub.status.idle":"2024-09-21T15:16:45.290292Z","shell.execute_reply.started":"2024-09-21T15:16:42.908008Z","shell.execute_reply":"2024-09-21T15:16:45.288971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### But take care, the output does not match all the types of cars for AR, it only matches the same car model. ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}