{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"raw_csv = pd.read_csv(\"../input/3d-object-detection-for-autonomous-vehicles/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"raw_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"raw_csv.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"raw_csv[['PredictionString']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"raw_csv.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image=plt.imread(\"/kaggle/input/3d-object-detection-for-autonomous-vehicles/train_images/host-a101_cam3_1242749262599200006.jpeg\", format=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image=plt.imread(\"/kaggle/input/3d-object-detection-for-autonomous-vehicles/test_images/host-a011_cam3_1232841055800995006.jpeg\", format=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(image,cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(test_image,cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install -U lyft_dataset_sdk","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install -U git+https://github.com/lyft/nuscenes-devkit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U git+https://github.com/lyft/nuscenes-devkit moviepy >> /dev/tmp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\nwarnings.filterwarnings(\"ignore\", category=UserWarning)\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)\nfrom IPython.display import HTML","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pdb\nimport cv2\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image\nfrom pathlib import Path\nfrom matplotlib import pyplot as plt\nfrom mpl_toolkits.mplot3d import axes3d, Axes3D\n\n# Load the SDK\nfrom lyft_dataset_sdk.lyftdataset import LyftDataset, LyftDatasetExplorer, Quaternion, view_points\nfrom lyft_dataset_sdk.utils.data_classes import LidarPointCloud\n\nfrom moviepy.editor import ImageSequenceClip\nfrom tqdm import tqdm_notebook as tqdm\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# gotta do this for LyftDataset SDK, it expects folders to be named as `images`, `maps`, `lidar`\n\n!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/train_images images\n!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/train_maps maps\n!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/train_lidar lidar\n!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/train_data data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lyftdata = LyftDataset(data_path='.', json_path='data/', verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lyftdata.category[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"car_token = lyftdata.category[0]['token']\ncar_token","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lyftdata.get('category',car_token)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lyftdata.sample_annotation[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sample_annotation\n#sample_annotation refers to any bounding box defining the position of an object seen in a sample. \n#All location data is given with respect to the global coordinate system. Let's examine an example from our sample above.\nmy_annotation_token = my_sample['anns'][16]\nmy_annotation =  my_sample_data.get('sample_annotation', my_annotation_token)\nmy_annotation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#We can also render an annotation to have a closer look.\nlyftdata.render_annotation(my_annotation_token)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#The attribute record indicates about what was the state of the concerned object when it was annotated\nlyftdata.get('attribute', lyftdata.sample_annotation[0]['attribute_tokens'][0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Scenes**\nA scene is a 25-45s long sequence of consecutive frames extracted from a log. A frame (also called a sample) is a collection of sensor outputs (images, lidar points) at a given timestamp\n\nscene {\n   \"token\":                   <str> -- Unique record identifier.\n   \"name\":                    <str> -- Short string identifier.\n   \"description\":             <str> -- Longer description of the scene.\n   \"log_token\":               <str> -- Foreign key. Points to log from where the data was extracted.\n   \"nbr_samples\":             <int> -- Number of samples in this scene.\n   \"first_sample_token\":      <str> -- Foreign key. Points to the first sample in scene.\n   \"last_sample_token\":       <str> -- Foreign key. Points to the last sample in scene.\n}"},{"metadata":{"trusted":true},"cell_type":"code","source":"lyftdata.scene[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Each scene provides the first sample token and the last sample token, we can see there are 126 sample records (nbr_samples) in between these two."},{"metadata":{"trusted":true},"cell_type":"code","source":"my_scene = lyftdata.scene[1]\nmy_sample_token= my_scene['first_sample_token']\nlyftdata.render_sample(my_sample_token)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/3d-object-detection-for-autonomous-vehicles/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"train dataframe's Id column contains tokens (unique identifiers) of train sample records present in sample table and PredictionString contains corresponding ground truth annotations (bounding boxes) for different object categories"},{"metadata":{"trusted":true},"cell_type":"code","source":"#We'll be using token0 to as our reference sample token\ntoken0 = train.iloc[0]['Id']\ntoken0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Sample**\nA sample is defined as an annotated keyframe of a scene at a given timestamp. A sample is data collected at (approximately) the same timestamp as part of a single LIDAR sweep.\n\nsample {\n   \"token\":                   <str> -- Unique record identifier.\n   \"timestamp\":               <int> -- Unix time stamp.\n   \"scene_token\":             <str> -- Foreign key pointing to the scene.\n   \"next\":                    <str> -- Foreign key. Sample that follows this in time. Empty if end of scene.\n   \"prev\":                    <str> -- Foreign key. Sample that precedes this in time. Empty if start of scene.\n}\n    \n  Remember, token0 is a token to a particular sample record in sample data table (sample.json), let's look at that sample using lyft SDK's inbuilt .get function"},{"metadata":{"trusted":true},"cell_type":"code","source":"my_sample = lyftdata.get('sample', my_sample_token)\nmy_sample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#A useful method is list_sample() which lists all related sample_data keyframes and sample_annotation associated with a sample which we will discuss in detail in the subsequent parts.\n\nlyftdata.list_sample(my_sample['token'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**3D interactive visualization of a sample**\n\n\nWe can visualize a sample interactively using lyft SDK's inbuilt render_sample_3d_interactive functionality"},{"metadata":{"trusted":true},"cell_type":"code","source":"lyftdata.render_sample_3d_interactive(my_sample['token'], render_sample=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Instead of looking at camera and lidar data separately, we can also project the lidar pointcloud into camera images\nlyftdata.render_pointcloud_in_image(sample_token = my_sample[\"token\"],\n                                      dot_size = 1,\n                                      camera_channel = 'CAM_FRONT')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#The dataset contains data that is collected from a full sensor suite. Hence, for each snapshot of a scene, we provide references to a family of data that is collected from these sensors.\n\n#We provide a data key to access these:\nmy_sample['data']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Notice that the keys are referring to the different sensors that form our sensor suite. Let's take a look at the metadata of a sample_data taken from CAM_FRONT.\nsensor_channel = 'CAM_FRONT' \nmy_sample_data = lyftdata.get('sample_data', my_sample['data'][sensor_channel])\nmy_sample_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" # also try this e.g. with 'LIDAR_TOP'\nsensor_channel = 'LIDAR_TOP'  \nmy_sample_data_lidar = lyftdata.get('sample_data', my_sample['data'][sensor_channel])\nmy_sample_data_lidar","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}