{"cells":[{"metadata":{},"cell_type":"markdown","source":"# PKU Autonomous Driving Data Understanding and EDA"},{"metadata":{},"cell_type":"markdown","source":"## Let us load and analyse the data"},{"metadata":{},"cell_type":"markdown","source":"Import Modules"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2\nimport json\nimport os\nimport matplotlib\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"List File folders"},{"metadata":{"trusted":true},"cell_type":"code","source":"Input = '../input/pku-autonomous-driving/'\nfile_list = os.listdir(Input)\nprint(file_list)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Load Train Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(Input+'train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### What is Prediction String? "},{"metadata":{},"cell_type":"markdown","source":"As per the data: \n    The primary data is images of cars and related pose information. The pose information is formatted as strings, as follows: **model type, yaw, pitch, roll, x, y, z**\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Annotations for Image ',train['ImageId'][0],' are ',train['PredictionString'][0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### How many cars are there in Image ID_8a6e65317?"},{"metadata":{"trusted":true},"cell_type":"code","source":"annotations = train['PredictionString'][0].split(' ')\nprint(len(annotations) / 7)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"####  Annotations for cars"},{"metadata":{},"cell_type":"markdown","source":"#### Seperate the list of annotations to each car's pose"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nposes = []\n\nfor index in range(0,int(len(annotations)/7)):\n    i = index*7\n    poses.append(annotations[i:i+7])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(poses)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualise the x,y annotations on Image ID_8a6e65317  "},{"metadata":{},"cell_type":"markdown","source":"   #### 2D Visualisation"},{"metadata":{},"cell_type":"markdown","source":"The points are in World Coordinates. They have to be converted to camera coordinate for visualisation"},{"metadata":{},"cell_type":"markdown","source":"#### Conversion to camera coordinate"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Read camera parameters \nfx = 2304.5479\nfy = 2305.8757\ncx = 1686.2379\ncy = 1354.9849\ncamera_matrix = np.array([[fx, 0,  cx],\n           [0, fy, cy],\n           [0, 0, 1]], dtype=np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert2camera(worldCoord):\n    x,y,z = worldCoord[0],worldCoord[1],worldCoord[2]\n    return x * fx / z + cx, y * fy / z + cy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot inline\n%matplotlib inline\n\nx = []\ny = []\nimg = cv2.imread(Input+'train_images/ID_8a6e65317.jpg')\n\nfor item in poses:\n    coord = (float(item[4]),float(item[5]),float(item[6]))\n    x.append(convert2camera(coord)[0])\n    y.append(convert2camera(coord)[1])\n   \nplt.scatter(x,y, color='red', s=100);\n\n\nplt.imshow(img)\nplt.title('2D visualisation')\nplt.show()\n   ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 3D Visualisation"},{"metadata":{},"cell_type":"markdown","source":"Adapted version of code from https://www.kaggle.com/zstusnoopy/visualize-the-location-and-3d-bounding-box-of-car"},{"metadata":{"trusted":true},"cell_type":"code","source":"from math import sin, cos\n\n# convert euler angle to rotation matrix\ndef euler_to_Rot(yaw, pitch, roll):\n    Y = np.array([[cos(yaw), 0, sin(yaw)],\n                  [0, 1, 0],\n                  [-sin(yaw), 0, cos(yaw)]])\n    P = np.array([[1, 0, 0],\n                  [0, cos(pitch), -sin(pitch)],\n                  [0, sin(pitch), cos(pitch)]])\n    R = np.array([[cos(roll), -sin(roll), 0],\n                  [sin(roll), cos(roll), 0],\n                  [0, 0, 1]])\n    return np.dot(Y, np.dot(P, R))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw_line(image, points):\n    color = (255, 0, 0)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[3][:2]), color, 16)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[1][:2]), color, 16)\n    cv2.line(image, tuple(points[1][:2]), tuple(points[2][:2]), color, 16)\n    cv2.line(image, tuple(points[2][:2]), tuple(points[3][:2]), color, 16)\n    return image\n\n\ndef draw_points(image, points):\n    for (p_x, p_y, p_z) in points:\n        cv2.circle(image, (p_x, p_y), int(1000 / p_z), (0, 255, 0), -1)\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize(img, coords):\n    #Just the mean value of all type of car's dimension \n    x_l = 1.02\n    y_l = 0.80\n    z_l = 2.31\n    \n    img = img.copy()\n    for point in coords:\n        # Get values\n        x, y, z = float(point[4]), float(point[5]), float(point[6])\n        yaw, pitch, roll = -float(point[1]), -float(point[2]), -float(point[3])\n        # Math\n        Rt = np.eye(4)\n        t = np.array([x, y, z])\n        Rt[:3, 3] = t\n        Rt[:3, :3] = euler_to_Rot(yaw, pitch, roll).T\n        Rt = Rt[:3, :]\n        P = np.array([[x_l, -y_l, -z_l, 1],\n                      [x_l, -y_l, z_l, 1],\n                      [-x_l, -y_l, z_l, 1],\n                      [-x_l, -y_l, -z_l, 1],\n                      [0, 0, 0, 1]]).T\n        img_cor_points = np.dot(camera_matrix, np.dot(Rt, P))\n        img_cor_points = img_cor_points.T\n        img_cor_points[:, 0] /= img_cor_points[:, 2]\n        img_cor_points[:, 1] /= img_cor_points[:, 2]\n        img_cor_points = img_cor_points.astype(int)\n        # Drawing\n        img = draw_line(img, img_cor_points)\n        img = draw_points(img, img_cor_points[-1:])\n    \n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = cv2.imread(Input+'train_images/ID_8a6e65317.jpg')\n\nbbox = visualize(img,poses)\n\nplt.imshow(bbox)\nplt.title('3D visualisation')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#As we can see there're some inaccuracies in boxes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3D car model json files"},{"metadata":{},"cell_type":"markdown","source":"### How many models  do we have?"},{"metadata":{"trusted":true},"cell_type":"code","source":"json_files = os.listdir(Input+'car_models_json')\nprint(len(json_files))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Lets examine one model"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Load the model\nwith open(Input+'car_models_json/fengtian-SUV-gai.json') as json_file:\n    car_model_data = json.load(json_file)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The File contains the car type, vertices and faces."},{"metadata":{"trusted":true},"cell_type":"code","source":"for keys in enumerate(car_model_data):\n    print(keys)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Plot the 3D model \nFrom https://www.kaggle.com/robikscube/autonomous-driving-introduction-data-review#Training-Set,-First-Car-Stats"},{"metadata":{"trusted":true},"cell_type":"code","source":"from mpl_toolkits.mplot3d import Axes3D\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_3d_car(model_json_file):\n    with open(Input+f'car_models_json/{model_json_file}') as json_file:\n        car_model_data = json.load(json_file)\n\n    vertices = np.array(car_model_data['vertices'])\n    faces = np.array(car_model_data['faces']) - 1\n    car_type = car_model_data['car_type']\n    x, y, z = vertices[:,0], vertices[:,2], -vertices[:,1]\n    fig = plt.figure(figsize=(30, 10))\n    ax = plt.axes(projection='3d')\n    ax.plot_trisurf(x, y, faces, z,\n                    cmap='viridis', edgecolor='none')\n    ax.set_title(car_type)\n    ax.view_init(30, 0)\n    plt.show()\n    fig = plt.figure(figsize=(30, 10))\n    ax = plt.axes(projection='3d')\n    ax.plot_trisurf(x, y, faces, z,\n                    cmap='viridis', edgecolor='none')\n    ax.set_title(car_type)\n    ax.view_init(60, 0)\n    plt.show()\n    fig = plt.figure(figsize=(30, 10))\n    ax = plt.axes(projection='3d')\n    ax.plot_trisurf(x, y, faces, z,\n                    cmap='viridis', edgecolor='none')\n    ax.set_title(car_type)\n    ax.view_init(-20, 180)\n    plt.show()\n    return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_3d_car('fengtian-SUV-gai.json')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Masks"},{"metadata":{},"cell_type":"markdown","source":"From Documentation :                                                                                                        \n**Some cars in the images are not of interest (too far away, etc.). Binary masks are provided to allow competitors to remove them from consideration.**"},{"metadata":{},"cell_type":"markdown","source":"### Load the masks"},{"metadata":{"trusted":true},"cell_type":"code","source":"mask_files = os.listdir(Input+'train_masks')\nprint('Number of mask images : ',len(mask_files))\ntrain_images = os.listdir(Input+'train_images')\nprint('Number of train images : ',len(train_images))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**There are different number of masks and images**"},{"metadata":{},"cell_type":"markdown","source":"#### Read mask image"},{"metadata":{"trusted":true},"cell_type":"code","source":"mask = cv2.imread(Input+'train_masks/ID_8a6e65317.jpg')\nimage = cv2.imread(Input+'train_images/ID_8a6e65317.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(mask)\nplt.title('Mask')\nplt.show()\n\nplt.imshow(image)\nplt.title('Image')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Which cars are out of scope?"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(20, 25))\nplt.imshow(image)\nplt.imshow(mask, cmap=plt.cm.viridis, interpolation='none', alpha=0.5)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The cars shown inside white boxes are out of scope"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}