{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In this notebook is based entirely on the centernet baseline notebook of Ruslan Baynazarov (https://www.kaggle.com/hocop1/centernet-baseline). I am only trying to understand Ruslan's code in this notebook.\n\n* The CenterNet paper (\"Objects as Points) https://arxiv.org/pdf/1904.07850.pdf"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\nfrom tqdm import tqdm#_notebook as tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom functools import reduce\nimport os\nfrom scipy.optimize import minimize\nimport plotly.express as px\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nfrom torchvision import transforms, utils","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"PATH = '../input/pku-autonomous-driving/'\nos.listdir(PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"import pandas as pd\n#sample_submission = pd.read_csv(\"../input/pku-autonomous-driving/sample_submission.csv\")\ntrain = pd.read_csv(\"../input/pku-autonomous-driving/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"camera_matrix = np.array([[2304.5479, 0,  1686.2379],\n                          [0, 2305.8757, 1354.9849],\n                          [0, 0, 1]], dtype=np.float32)\ncamera_matrix_inv = np.linalg.inv(camera_matrix)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def imread(path, fast_mode=False):\n    img = cv2.imread(path)\n    if not fast_mode and img is not None and len(img.shape) == 3:\n        img = np.array(img[:, :, ::-1])\n    return img\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = imread(PATH + 'train_images/ID_8a6e65317' + '.jpg')\n\nIMG_SHAPE = img.shape\nIMG_SHAPE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,8))\nplt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Extract pose data from prediction string:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):\n    '''\n    Input:\n        s: PredictionString (e.g. from train dataframe)\n        names: array of what to extract from the string\n    Output:\n        list of dicts with keys from `names`\n    '''\n    coords = []\n    for l in np.array(s.split()).reshape([-1, 7]):\n        coords.append(dict(zip(names, l.astype('float'))))\n        if 'id' in coords[-1]:\n            coords[-1]['id'] = int(coords[-1]['id'])\n    return coords","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"PredictionString\"][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inp = train['PredictionString'][0]\nprint('Example input:\\n', inp)\nprint()\nprint('Output:\\n', str2coords(inp))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2D Visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img_coords(s):\n    '''\n    Input is a PredictionString (e.g. from train dataframe)\n    Output is two arrays:\n        xs: x coordinates in the image (row)\n        ys: y coordinates in the image (column)\n    '''\n    coords = str2coords(s)\n    xs = [c['x'] for c in coords]\n    ys = [c['y'] for c in coords]\n    zs = [c['z'] for c in coords]\n    P = np.array(list(zip(xs, ys, zs))).T\n    img_p = np.dot(camera_matrix, P).T\n    img_p[:, 0] /= img_p[:, 2]\n    img_p[:, 1] /= img_p[:, 2]\n    img_xs = img_p[:, 0]\n    img_ys = img_p[:, 1]\n    img_zs = img_p[:, 2] # z = Distance from the camera\n    return img_xs, img_ys\n\nplt.figure(figsize=(14,14))\nplt.imshow(imread(PATH + 'train_images/' + train['ImageId'][2217] + '.jpg'))\nplt.scatter(*get_img_coords(train['PredictionString'][2217]), color='yellow', s=100);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_img_coords(train['PredictionString'][2217])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# 3D Visualization"},{"metadata":{},"cell_type":"markdown","source":"Apparently used code from here: https://www.kaggle.com/zstusnoopy/visualize-the-location-and-3d-bounding-box-of-car"},{"metadata":{"trusted":true},"cell_type":"code","source":"from math import sin, cos\n\n# convert euler angle to rotation matrix\ndef euler_to_Rot(yaw, pitch, roll):\n    Y = np.array([[cos(yaw), 0, sin(yaw)],\n                  [0, 1, 0],\n                  [-sin(yaw), 0, cos(yaw)]])\n    P = np.array([[1, 0, 0],\n                  [0, cos(pitch), -sin(pitch)],\n                  [0, sin(pitch), cos(pitch)]])\n    R = np.array([[cos(roll), -sin(roll), 0],\n                  [sin(roll), cos(roll), 0],\n                  [0, 0, 1]])\n    return np.dot(Y, np.dot(P, R))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw_line(image, points):\n    color = (255, 0, 0)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[3][:2]), color, 16)\n    cv2.line(image, tuple(points[0][:2]), tuple(points[1][:2]), color, 16)\n    cv2.line(image, tuple(points[1][:2]), tuple(points[2][:2]), color, 16)\n    cv2.line(image, tuple(points[2][:2]), tuple(points[3][:2]), color, 16)\n    return image\n\n\ndef draw_points(image, points):\n    for (p_x, p_y, p_z) in points:\n        cv2.circle(image, (p_x, p_y), int(1000 / p_z), (0, 255, 0), -1)\n#         if p_x > image.shape[1] or p_y > image.shape[0]:\n#             print('Point', p_x, p_y, 'is out of image with shape', image.shape)\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize(img, coords):\n    # You will also need functions from the previous cells\n    # coords is a list of dictionaries.\n    x_l = 1.02\n    y_l = 0.80\n    z_l = 2.31\n    \n    img = img.copy()\n    for point in coords:\n        # Get values\n        x, y, z = point['x'], point['y'], point['z']\n        yaw, pitch, roll = -point['pitch'], -point['yaw'], -point['roll']\n        # Math\n        Rt = np.eye(4)\n        t = np.array([x, y, z])\n        Rt[:3, 3] = t\n        Rt[:3, :3] = euler_to_Rot(yaw, pitch, roll).T\n        Rt = Rt[:3, :]\n        P = np.array([[x_l, -y_l, -z_l, 1],\n                      [x_l, -y_l, z_l, 1],\n                      [-x_l, -y_l, z_l, 1],\n                      [-x_l, -y_l, -z_l, 1],\n                      [0, 0, 0, 1]]).T\n        img_cor_points = np.dot(camera_matrix, np.dot(Rt, P))\n        img_cor_points = img_cor_points.T\n        img_cor_points[:, 0] /= img_cor_points[:, 2]\n        img_cor_points[:, 1] /= img_cor_points[:, 2]\n        img_cor_points = img_cor_points.astype(int)\n        # Drawing\n        img = draw_line(img, img_cor_points)\n        img = draw_points(img, img_cor_points[-1:])\n    \n    return img\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_rows = 6\n\nfor idx in range(n_rows):\n    fig, axes = plt.subplots(1, 2, figsize=(20,20))\n    img = imread(PATH + 'train_images/' + train['ImageId'].iloc[idx] + '.jpg')\n    axes[0].imshow(img)\n    img_vis = visualize(img, str2coords(train['PredictionString'].iloc[idx]))\n    axes[1].imshow(img_vis)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = 0\n#fig, axes = plt.subplots(1, 2, figsize=(20,20))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH + 'train_images/' + train['ImageId'].iloc[idx] + '.jpg'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['PredictionString'][idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['ImageId'].iloc[idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"str2coords(train['PredictionString'].iloc[idx])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As far as I understand, in the function `visualize`, he has taken a fixed size for the bounding box."},{"metadata":{},"cell_type":"markdown","source":"# 3D visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_WIDTH = 1024\nIMG_HEIGHT = IMG_WIDTH // 16 * 5\nMODEL_SCALE = 8\n\ndef preprocess_image(img, flip=False):\n    img = img[img.shape[0] // 2:]\n    bg = np.ones_like(img) * img.mean(1, keepdims=True).astype(img.dtype)\n    bg = bg[:, :img.shape[1] // 6]\n    img = np.concatenate([bg, img, bg], 1)\n    img = cv2.resize(img, (IMG_WIDTH, IMG_HEIGHT))\n    if flip:\n        img = img[:,::-1]\n    return (img / 255).astype('float32')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img0 = imread(PATH + 'train_images/' + train['ImageId'][22] + '.jpg')\nimg = preprocess_image(img0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img0.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(img0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Image Preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"def rotate(x, angle): #adds angle to x, and the sum is made to lie in [-pi,pi). 180 is changed to -180.\n    x = x + angle\n    x = x - (x + np.pi) // (2 * np.pi) * 2 * np.pi\n    return x\n\n\n\ndef _regr_preprocess(regr_dict, flip=False):\n    #basically in regr (40x128x7), the pitch is replaced by its sin and cos, and id is eliminated\n    if flip:\n        for k in ['x', 'pitch', 'roll']:\n            regr_dict[k] = -regr_dict[k]\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] / 100 #why divide by 100?\n    regr_dict['roll'] = rotate(regr_dict['roll'], np.pi)\n    regr_dict['pitch_sin'] = sin(regr_dict['pitch'])\n    regr_dict['pitch_cos'] = cos(regr_dict['pitch'])\n    regr_dict.pop('pitch')\n    regr_dict.pop('id')\n    return regr_dict\n\n\n\ndef get_mask_and_regr(img, labels, flip=False):\n    #labels is the PresdictionString column of the training data and is a single string.\n    #the only thing used of img is its shape for transformation purposes, not img itself.\n    mask = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE], dtype='float32')\n    regr_names = ['x', 'y', 'z', 'yaw', 'pitch', 'roll']\n    regr = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE, 7], dtype='float32')\n    coords = str2coords(labels) #coords is a list of dictionaries\n    xs, ys = get_img_coords(labels) #image coordinates of the labeled cars.\n    for x, y, regr_dict in zip(xs, ys, coords):\n        x, y = y, x\n        x = (x - img.shape[0] // 2) * IMG_HEIGHT / (img.shape[0] // 2) / MODEL_SCALE\n        x = np.round(x).astype('int')\n        y = (y + img.shape[1] // 6) * IMG_WIDTH / (img.shape[1] * 4/3) / MODEL_SCALE\n        y = np.round(y).astype('int')\n        if x >= 0 and x < IMG_HEIGHT // MODEL_SCALE and y >= 0 and y < IMG_WIDTH // MODEL_SCALE:\n            mask[x, y] = 1\n            regr_dict = _regr_preprocess(regr_dict, flip)\n            regr[x, y] = [regr_dict[n] for n in sorted(regr_dict)] # WHAT IS THIS SORTED THING??? This returns a list of the keys sorted alphabetically\n    if flip:\n        mask = np.array(mask[:,::-1])\n        regr = np.array(regr[:,::-1])\n    return mask, regr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DISTANCE_THRESH_CLEAR = 2\n\ndef convert_3d_to_2d(x, y, z, fx = 2304.5479, fy = 2305.8757, cx = 1686.2379, cy = 1354.9849):\n    # stolen from https://www.kaggle.com/theshockwaverider/eda-visualization-baseline\n    return x * fx / z + cx, y * fy / z + cy\n\ndef clear_duplicates(coords):\n    for c1 in coords:\n        xyz1 = np.array([c1['x'], c1['y'], c1['z']])\n        for c2 in coords:\n            xyz2 = np.array([c2['x'], c2['y'], c2['z']])\n            distance = np.sqrt(((xyz1 - xyz2)**2).sum())\n            if distance < DISTANCE_THRESH_CLEAR:\n                if c1['confidence'] < c2['confidence']:\n                    c1['confidence'] = -1\n    return [c for c in coords if c['confidence'] > 0]\n\n\ndef optimize_xy(r, c, x0, y0, z0, flipped=False):\n    def distance_fn(xyz):\n        x, y, z = xyz\n        xx = -x if flipped else x\n        slope_err = 0.0#(xzy_slope.predict([[xx,z]])[0] - y)**2 #ACHTUNG! I CHANGED IT TO ZERO...\n        x, y = convert_3d_to_2d(x, y, z)\n        y, x = x, y\n        x = (x - IMG_SHAPE[0] // 2) * IMG_HEIGHT / (IMG_SHAPE[0] // 2) / MODEL_SCALE\n        y = (y + IMG_SHAPE[1] // 6) * IMG_WIDTH / (IMG_SHAPE[1] * 4 / 3) / MODEL_SCALE\n        return max(0.2, (x-r)**2 + (y-c)**2) + max(0.4, slope_err)\n    \n    res = minimize(distance_fn, [x0, y0, z0], method='Powell')\n    x_new, y_new, z_new = res.x\n    return x_new, y_new, z_new\n\ndef _regr_back(regr_dict):\n    for name in ['x', 'y', 'z']:\n        regr_dict[name] = regr_dict[name] * 100\n    regr_dict['roll'] = rotate(regr_dict['roll'], -np.pi)\n    \n    pitch_sin = regr_dict['pitch_sin'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    pitch_cos = regr_dict['pitch_cos'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)\n    regr_dict['pitch'] = np.arccos(pitch_cos) * np.sign(pitch_sin)\n    return regr_dict\n\ndef extract_coords(prediction, flipped=False): #This extracts coordinates from the prediction\n    #\"prediction\" is a 8x40x128 matrix\n    logits = prediction[0] #shape is 40x128, contains the probabilities measures\n    regr_output = prediction[1:] #shape is 7x40x128\n    points = np.argwhere(logits > 0) #choose only those points where prediction[0] is positive. pixel positions of those points where probability measures are positive\n    col_names = sorted(['x', 'y', 'z', 'yaw', 'pitch_sin', 'pitch_cos', 'roll'])\n    coords = []\n    for r, c in points: #pixel positions of only those points where prediction[0] is positive\n        regr_dict = dict(zip(col_names, regr_output[:, r, c]))\n        coords.append(_regr_back(regr_dict))\n        coords[-1]['confidence'] = 1 / (1 + np.exp(-logits[r, c]))\n        coords[-1]['x'], coords[-1]['y'], coords[-1]['z'] = \\\n                optimize_xy(r, c,\n                            coords[-1]['x'],\n                            coords[-1]['y'],\n                            coords[-1]['z'], flipped)\n    coords = clear_duplicates(coords)\n    return coords","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ax_i = 0\nimg0 = imread(PATH + 'train_images/' + train['ImageId'].iloc[idx] + '.jpg')\n#img = preprocess_image(img0, ax_i==1)\nmask, regr = get_mask_and_regr(img0, train['PredictionString'][idx], ax_i==1)\n#mask is a matrix of the same dimensions as the preprocessed image. (40 x 128)\n#regr has one more dimension which has length 7 (40 x 128 x 7) to account for the regressed values.\nregr = np.rollaxis(regr, 2, 0) #brings the 3rd (ie 2+1) to the start (0) position. Has now shape (7 x 40 x 128).\n#this rolled form of regr has to go to extract_coordinates.\nmrconc = np.concatenate([mask[None], regr], 0) #regr: (7x40x128). mask[None]: 1x40x128. Remember that mask shape was 40x128. None adds an additional dimension.\ncoords = extract_coords(mrconc, ax_i==1) # mrconc IS THE PREDICTION!!!","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mask[None].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mask.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mrconc.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mrconc = np.concatenate([mask[None], regr], 0)\n#mask[None].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#this function is only for understanding purpoes.\ndef rotate_degrees(x, angle):\n    x = x + angle\n    x = x - (x + 180.0) // (2 * 180.0) * 2 * 180.0\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rotate_degrees(50,180) #adds angle to x, and the sum is made to lie in [-pi,pi). 180 is changed to -180.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(20,20))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}