{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -qqq lyft-dataset-sdk","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This code shows how to get points from .bin files in Dataset to train your own neural network."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport math\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\n\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport matplotlib as mpl\n\nfrom lyft_dataset_sdk.lyftdataset import LyftDataset\nfrom lyft_dataset_sdk.utils.data_classes import LidarPointCloud","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"INP_DIR = '/kaggle/input/3d-object-detection-for-autonomous-vehicles/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load the dataset\n# Adjust the dataroot parameter below to point to your local dataset path.\n# The correct dataset path contains at least the following four folders (or similar): images, lidar, maps, v1.0.1-train\n!ln -s {INP_DIR}/train_images images\n!ln -s {INP_DIR}/train_maps maps\n!ln -s {INP_DIR}/train_lidar lidar","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"level5data = LyftDataset(\n    data_path='.',\n    json_path=os.path.join(INP_DIR + 'train_data'),\n    verbose=False\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_scene = level5data.scene[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_sample_token = my_scene[\"first_sample_token\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_sample = level5data.get('sample', my_sample_token)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lidar_top_data_token = my_sample['data']['LIDAR_TOP']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lidar_top_data = level5data.get('sample_data', lidar_top_data_token)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# lidar_top_ego_pose_token = lidar_top_data['ego_pose_token']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # car / lidar top coords\n# lidar_top_ego_pose_data = level5data.get('ego_pose', lidar_top_ego_pose_token)\n# lidar_top_coords = np.array(lidar_top_ego_pose_data['translation'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_coords_from_ann_idx(ann_idx):\n    return np.array(level5data.get('sample_annotation', my_sample['anns'][ann_idx])['translation'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"anns_inds_to_show = [13, 27, 41, 42, 56] # i select several cars near lyft's pod\nann_tokens = []\nfor ind in anns_inds_to_show:\n    my_annotation_token = my_sample['anns'][ind]\n    print(f'{ind}: {my_annotation_token}')\n    ann_tokens.append(my_annotation_token)\n    level5data.render_annotation(my_annotation_token)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_sample['data'].keys() # available sensors for sampling","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# here we get all objects (bboxes) for selected sensor and filter by selected cars (above)\nret_sampled = level5data.get_sample_data(lidar_top_data_token, selected_anntokens=ann_tokens)[1]\n# we can not pass selected_anntokens and get full info of the bboxes around \n# car - literally a complete set of data for training a neural network","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data_from_sample(chanel_to_get):\n    return level5data.get('sample_data', my_sample['data'][chanel_to_get])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_img_from_data(data):\n    plt.imshow(\n        cv2.cvtColor(\n            cv2.imread(data['filename']),\n            cv2.COLOR_BGR2RGB\n        )\n    );","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lidar_channel = 'LIDAR_TOP'\ncamera1_chanel = 'CAM_BACK'\ncamera2_chanel = 'CAM_BACK_LEFT'\ncamera3_chanel = 'CAM_FRONT'\nlidar_data = get_data_from_sample(lidar_channel)\ncamera1_data = get_data_from_sample(camera1_chanel)\ncamera2_data = get_data_from_sample(camera2_chanel)\ncamera3_data = get_data_from_sample(camera3_chanel)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Let's take a look at the cars around us"},{"metadata":{"trusted":true},"cell_type":"code","source":"# car 1 on back side\nshow_img_from_data(camera1_data);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# car 2 on back side\nshow_img_from_data(camera2_data);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# car 3 on front side\nshow_img_from_data(camera3_data);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pc = LidarPointCloud.from_file(Path(lidar_data['filename']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_offset = 15\nplt.xlim(-plot_offset, plot_offset)\nplt.ylim(-plot_offset, plot_offset)\n\n# plot PointCloud\nplt.scatter(pc.points[0, :], pc.points[1, :], c=pc.points[2, :], s=0.1);\n# plot center of cars / bboxes\nfor cur_point_idx in range(len(ret_sampled)):\n    crds = ret_sampled[cur_point_idx].center\n    plt.scatter(crds[0], crds[1], c='red');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here we see car silhouette."},{"metadata":{},"cell_type":"markdown","source":"Dive into data.\nPoints have x, y and z coords and intensity - always 100."},{"metadata":{"trusted":true},"cell_type":"code","source":"pc.points[:, 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(pc.points[3, :]==100).all()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here we get ego centered coordinates of bbox, rotation and etc:"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_to_explore = ret_sampled[4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_to_explore","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# rotation in radians of bbox: x, y and z. For degrees see example above (next MPL Figure)\nsample_to_explore.orientation.yaw_pitch_roll","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Width, leght and height of bbox: x, y and z\nsample_to_explore.wlh","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# and of course coords of bbox \nsample_to_explore.center","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class of object\nsample_to_explore.name","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we have all ingridients!"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure()\nax = fig.add_subplot(111)\n\nplot_offset = 15\nplt.xlim(-plot_offset, plot_offset)\nplt.ylim(-plot_offset, plot_offset)\n\n# plot PointCloud\nax.scatter(pc.points[0, :], pc.points[1, :], c=pc.points[2, :], s=0.1);\n\n# plot center of cars / bboxes\ncrds = sample_to_explore.center\nax.scatter(crds[0], crds[1], c='red');\n\nw, l, h  = sample_to_explore.wlh\nangles_to_rotate = sample_to_explore.orientation.yaw_pitch_roll\n\nrect_angle = math.degrees(angles_to_rotate[0]) # radians to degrees\nmpl_rotate = (\n    mpl\n    .transforms\n    .Affine2D()\n    .rotate_deg_around(crds[0], crds[1], rect_angle)\n    + ax.transData\n)\n\nrect = patches.Rectangle(\n    (crds[0] - l / 2, crds[1] - w / 2), l, w, fill=False,\n)\n\nrect.set_transform(mpl_rotate)\n\nax.add_patch(rect);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Later we create pytorch dataloader, which provide all data for NN!"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}