{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Lyft Prediction Exploration\n\nDescription: TODO","metadata":{}},{"cell_type":"markdown","source":"## Explore the data folders\n\nLet's fisrt familiarize what are provided in the input folder.","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-28T03:22:34.262024Z","iopub.execute_input":"2022-02-28T03:22:34.262527Z","iopub.status.idle":"2022-02-28T03:22:34.287494Z","shell.execute_reply.started":"2022-02-28T03:22:34.262444Z","shell.execute_reply":"2022-02-28T03:22:34.286836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Add Lyft utility script\n\nThis is only needed once.\n\n> File -> Add utility script -> Searach with \"kaggle-l5kit\". (Cancel the filter if there is any. In my case, I see the filter \"My Work\" there in default.) -> Pick \"Lyft - l5kit (unofficial fix)\" and Click \"Add\". (Tried with \"kaggle_l5kit\" but always have issuse in saving versions.)","metadata":{}},{"cell_type":"markdown","source":"## Attach additional dataset source: for config files\n\nWe are using some config files when we want to load/visualize Lyft level5 dataset.<br/>\n@jpbremer already uploaded config files as Kaggle Dataset platform: [lyft-config-files](https://www.kaggle.com/jpbremer/lyft-config-files).<br/>\nThis is originally from official github [lyft/l5kit example page](https://github.com/lyft/l5kit/tree/master/examples).\n\nClick \"Add data\" button and press \"Search by URL\". Typing \"https://www.kaggle.com/jpbremer/lyft-config-files\" shows the dataset.<br/>\nOnce the dataset is successfully added you can see \"lyft-config-files\" as dataset on left side bar.","metadata":{}},{"cell_type":"markdown","source":"## Explore and prepare the python environment\n\n- Check the kernel\n- Import the libararies","metadata":{}},{"cell_type":"code","source":"import sys\nsys.executable","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:34.288995Z","iopub.execute_input":"2022-02-28T03:22:34.289311Z","iopub.status.idle":"2022-02-28T03:22:34.297212Z","shell.execute_reply.started":"2022-02-28T03:22:34.289276Z","shell.execute_reply":"2022-02-28T03:22:34.296463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip list","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:34.298382Z","iopub.execute_input":"2022-02-28T03:22:34.30052Z","iopub.status.idle":"2022-02-28T03:22:34.307696Z","shell.execute_reply.started":"2022-02-28T03:22:34.300482Z","shell.execute_reply":"2022-02-28T03:22:34.306682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import l5kit\nl5kit.__version__","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:34.309209Z","iopub.execute_input":"2022-02-28T03:22:34.309651Z","iopub.status.idle":"2022-02-28T03:22:34.329261Z","shell.execute_reply.started":"2022-02-28T03:22:34.309626Z","shell.execute_reply":"2022-02-28T03:22:34.328646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\ntorch.__version__","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:34.33044Z","iopub.execute_input":"2022-02-28T03:22:34.330909Z","iopub.status.idle":"2022-02-28T03:22:35.777232Z","shell.execute_reply.started":"2022-02-28T03:22:34.330877Z","shell.execute_reply":"2022-02-28T03:22:35.776455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check if GPU available\ntorch.cuda.is_available()","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:35.778546Z","iopub.execute_input":"2022-02-28T03:22:35.778761Z","iopub.status.idle":"2022-02-28T03:22:35.826024Z","shell.execute_reply.started":"2022-02-28T03:22:35.778731Z","shell.execute_reply":"2022-02-28T03:22:35.825347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import libaries\n\nWe don't have to find all the needed libaries in the beginning. As we are completing each step, we can add in the relevant libaries in here.","metadata":{}},{"cell_type":"code","source":"# common\nimport os\nimport time\nimport random\n\n# data processing and visualization\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom collections import Counter\nfrom prettytable import PrettyTable\n\n# torch\nimport torch\n\n# l5kit\nimport l5kit\nfrom l5kit.data import ChunkedDataset, LocalDataManager\nfrom l5kit.data import PERCEPTION_LABELS\nfrom l5kit.dataset import EgoDataset, AgentDataset\nfrom l5kit.rasterization import build_rasterizer\nfrom l5kit.configs import load_config_data\nfrom l5kit.visualization import draw_trajectory, TARGET_POINTS_COLOR\nfrom l5kit.geometry import transform_points\n\n\n\nfrom matplotlib import animation, rc\nfrom IPython.display import HTML\n\nrc('animation', html='jshtml')","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:34:11.188229Z","iopub.execute_input":"2022-02-28T03:34:11.188496Z","iopub.status.idle":"2022-02-28T03:34:11.199292Z","shell.execute_reply.started":"2022-02-28T03:34:11.188467Z","shell.execute_reply":"2022-02-28T03:34:11.19858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the seed to make the results reproducable.\ndef set_seed(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n\nset_seed(42)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:40.702533Z","iopub.execute_input":"2022-02-28T03:22:40.703034Z","iopub.status.idle":"2022-02-28T03:22:40.710811Z","shell.execute_reply.started":"2022-02-28T03:22:40.702998Z","shell.execute_reply":"2022-02-28T03:22:40.710094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore the dataset","metadata":{}},{"cell_type":"markdown","source":"### Overview of the dataset\n\nThe dataset consists of 170,000 scenes capturing the environment around the autonomous vehicle. Each scene encodes the state of the vehicle’s surroundings at a given point in time.\n\n<p align=\"center\">\n<img src=\"https://self-driving.lyft.com/wp-content/uploads/2020/06/motion_dataset_lrg_redux.gif\" style=\"width:45%\"/>\n<img src=\"https://self-driving.lyft.com/wp-content/uploads/2020/06/motion_dataset_2-1.png\" style=\"width:45%\"/>\n</p>\n\n<br/>\n\n<p>\nSource: https://level-5.global/data/prediction/\n</p>\n\n### Goal\n\nThe goal of this competition is to predict other car/cyclist/pedestrian (called \"agent\")'s motion.\n\n<p align=\"center\">\n<img src=\"https://self-driving.lyft.com/wp-content/uploads/2020/06/diagram-prediction-1.jpg\" style=\"width:50%;center\"/>\n</p>","metadata":{}},{"cell_type":"markdown","source":"### numpy structured array\n\nLyft dataset uses numpy's [structured array](https://docs.scipy.org/doc/numpy/user/basics.rec.html) functionality to store various kinds of features together.<br/>\n\nLet's see example to how it works","metadata":{}},{"cell_type":"code","source":"my_arr = np.zeros(3, dtype=[(\"color\", (np.uint8, 3)), (\"label\", np.bool)])","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:40.712232Z","iopub.execute_input":"2022-02-28T03:22:40.712532Z","iopub.status.idle":"2022-02-28T03:22:40.71922Z","shell.execute_reply.started":"2022-02-28T03:22:40.712496Z","shell.execute_reply":"2022-02-28T03:22:40.718509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_arr[0][\"color\"] = [0, 218, 130]\nmy_arr[0][\"label\"] = True\nmy_arr[1][\"color\"] = [245, 59, 255]\nmy_arr[1][\"label\"] = True","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:40.720657Z","iopub.execute_input":"2022-02-28T03:22:40.72091Z","iopub.status.idle":"2022-02-28T03:22:40.730143Z","shell.execute_reply.started":"2022-02-28T03:22:40.720877Z","shell.execute_reply":"2022-02-28T03:22:40.729437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_arr","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:40.731059Z","iopub.execute_input":"2022-02-28T03:22:40.731602Z","iopub.status.idle":"2022-02-28T03:22:40.741106Z","shell.execute_reply.started":"2022-02-28T03:22:40.731568Z","shell.execute_reply":"2022-02-28T03:22:40.740361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Summary:**\n\nWe've defined a length=3 array. Usually each element of array consists of only integer or float, but we can define custom structured format by specifying `dtype` as list of \"fields\" which consists of name and structure.\n\nAbove example contains 2 fields. 1. 8byte uint with length 3 array, 2. single element boolean array.\n\nAs you can see, `my_arr[i][\"name\"]` will access i-th element's \"name\" field.\n\nUsually when we train neural network, we would like to access all the field of random i-th element.\n\nAccording to [Data Format page](https://github.com/woven-planet/l5kit/blob/master/docs/data_format.rst), \"Structured arrays are a great fit to group this data together in memory and on disk.\", rather than preparing \"color\" array and \"label\" array separately and access each array's i-th element, especially when the number of field glow. ","metadata":{}},{"cell_type":"markdown","source":"### zarr\n\nZarr data format is used to store and read these numpy structured arrays from disk.\n\nZarr allows us to write very large (structured) arrays to disk in n-dimensional compressed chunks.\n\nHere is a short tutorial:","metadata":{}},{"cell_type":"code","source":"import zarr\n\nz = zarr.open(\"./dataset.zarr\", mode=\"w\", shape=(500,), dtype=np.float32, chunks=(100,))","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:40.742423Z","iopub.execute_input":"2022-02-28T03:22:40.742661Z","iopub.status.idle":"2022-02-28T03:22:40.749711Z","shell.execute_reply.started":"2022-02-28T03:22:40.742627Z","shell.execute_reply":"2022-02-28T03:22:40.749023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We can write to it by assigning to it. This gets persisted on disk.\nz[0:150] = np.arange(150)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:40.751249Z","iopub.execute_input":"2022-02-28T03:22:40.751544Z","iopub.status.idle":"2022-02-28T03:22:40.757986Z","shell.execute_reply.started":"2022-02-28T03:22:40.751501Z","shell.execute_reply":"2022-02-28T03:22:40.757284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.arange(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:40.759363Z","iopub.execute_input":"2022-02-28T03:22:40.760007Z","iopub.status.idle":"2022-02-28T03:22:40.767473Z","shell.execute_reply.started":"2022-02-28T03:22:40.759968Z","shell.execute_reply":"2022-02-28T03:22:40.766713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Summary:**\n\nAs we specified chunks to be of size 100, we just wrote to two separate chunks. On your filesystem in the dataset.zarr folder you will now find these two chunks. As we didn't completely fill the second chunk, those missing values will be set to the fill value (defaults to 0). The chunks are actually compressed on disk too!\n\nWe can print some info: by not doing much work at all we saved almost 75% in disk space!","metadata":{}},{"cell_type":"code","source":"print(z.info)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:40.769122Z","iopub.execute_input":"2022-02-28T03:22:40.769462Z","iopub.status.idle":"2022-02-28T03:22:40.776049Z","shell.execute_reply.started":"2022-02-28T03:22:40.769428Z","shell.execute_reply":"2022-02-28T03:22:40.775032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"When we check filesystem, `dataset.zarr` directory is created and there are 2 files \"0\" and \"1\" which are chunks currently created by just assigning value to zarr array.","metadata":{}},{"cell_type":"code","source":"!ls -l ./*","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:40.777539Z","iopub.execute_input":"2022-02-28T03:22:40.778054Z","iopub.status.idle":"2022-02-28T03:22:41.452769Z","shell.execute_reply.started":"2022-02-28T03:22:40.778019Z","shell.execute_reply":"2022-02-28T03:22:41.45199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reading from a zarr array is as easy as slicing from it like you would for any numpy array. The return value is an ordinary numpy array. Zarr takes care of determining which chunks to read from.","metadata":{}},{"cell_type":"code","source":"print(z[::20])  # print every 20th element","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:41.454589Z","iopub.execute_input":"2022-02-28T03:22:41.454906Z","iopub.status.idle":"2022-02-28T03:22:41.462203Z","shell.execute_reply.started":"2022-02-28T03:22:41.454866Z","shell.execute_reply":"2022-02-28T03:22:41.461336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Use 'l5kit' to access the data\n\nThe dataset structure is a bit complicated since the Level 5 dataset contains various kinds of information. 'l5kit' is provided as a useful library to deal with the data. It is worthywhile to spend a little time to familiarize the the tools we can utilize.\n\nSource: [l5kit visualize_data.ipynb](https://github.com/woven-planet/l5kit/blob/master/examples/visualisation/visualise_data.ipynb).\n\n#### Terminology\n\n - **\"Ego\"** is the host car which is recording/measuring the dataset.\n - **\"Agent\"** is the surronding car except \"Ego\" car.\n - **\"Frame\"** is the 1 image snapshot, where **\"Scene\"** is made of multiple frames of contious-time (video).\n\n#### Class diagram\n<img src=\"https://storage.googleapis.com/kaggle-forum-message-attachments/987047/16744/l5kit_class.png\" width=\"600\" />\n\n<br/>","metadata":{}},{"cell_type":"markdown","source":"#### Initial setup","metadata":{}},{"cell_type":"code","source":"# set env variable for data\nos.environ[\"L5KIT_DATA_FOLDER\"] = \"/kaggle/input/lyft-motion-prediction-autonomous-vehicles\"\n# get config\ncfg = load_config_data(\"/kaggle/input/lyft-config-files/visualisation_config.yaml\")\nprint(cfg)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:41.464061Z","iopub.execute_input":"2022-02-28T03:22:41.464725Z","iopub.status.idle":"2022-02-28T03:22:41.47924Z","shell.execute_reply.started":"2022-02-28T03:22:41.46468Z","shell.execute_reply":"2022-02-28T03:22:41.478579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load the sample data\n\nHere we will only use the first dataset from the sample set.\n\nWe're building a `LocalDataManager` object. This will resolve relative paths from the config using the `L5KIT_DATA_FOLDER` env variable we have just set.\n\nHere sample.zarr data is used for visualization, please use train.zarr / validate.zarr / test.zarr for actual model training/validation/prediction.\n","metadata":{}},{"cell_type":"code","source":"dm = LocalDataManager()\ndataset_path = dm.require('scenes/sample.zarr')\nzarr_dataset = ChunkedDataset(dataset_path)\nzarr_dataset.open()\nprint(zarr_dataset)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:41.482295Z","iopub.execute_input":"2022-02-28T03:22:41.482604Z","iopub.status.idle":"2022-02-28T03:22:41.56479Z","shell.execute_reply.started":"2022-02-28T03:22:41.482578Z","shell.execute_reply":"2022-02-28T03:22:41.564058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_path","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:41.565939Z","iopub.execute_input":"2022-02-28T03:22:41.566198Z","iopub.status.idle":"2022-02-28T03:22:41.574034Z","shell.execute_reply.started":"2022-02-28T03:22:41.566149Z","shell.execute_reply":"2022-02-28T03:22:41.573231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Working with the raw data\n\n`zarr_dataset` contains **scenes, frames, agents, tl_faces** attributes, which are the raw structured array data.\n\nEach data structure definition can be checked at [here](https://github.com/lyft/l5kit/blob/master/data_format.md#2020-lyft-competition-dataset-format).\n\n#### scenes\n\n```\nSCENE_DTYPE = [\n    (\"frame_index_interval\", np.int64, (2,)),\n    (\"host\", \"<U16\"),  # Unicode string up to 16 chars\n    (\"start_time\", np.int64),\n    (\"end_time\", np.int64),\n]\n```\n\n#### frames\n\n```\nFRAME_DTYPE = [\n    (\"timestamp\", np.int64),\n    (\"agent_index_interval\", np.int64, (2,)),\n    (\"traffic_light_faces_index_interval\", np.int64, (2,)),\n    (\"ego_translation\", np.float64, (3,)),\n    (\"ego_rotation\", np.float64, (3, 3)),\n]\n```\n\n#### agents\n\n```\nAGENT_DTYPE = [\n    (\"centroid\", np.float64, (2,)),\n    (\"extent\", np.float32, (3,)),\n    (\"yaw\", np.float32),\n    (\"velocity\", np.float32, (2,)),\n    (\"track_id\", np.uint64),\n    (\"label_probabilities\", np.float32, (len(LABELS),)),\n]\n```\n\n#### traffic_light_faces\n\n```\nTL_FACE_DTYPE = [\n    (\"face_id\", \"<U16\"),\n    (\"traffic_light_id\", \"<U16\"),\n    (\"traffic_light_face_status\", np.float32, (len(TL_FACE_LABELS,))),\n]\n```","metadata":{}},{"cell_type":"markdown","source":"As an example, we will try scatter plot using **frames \"ego_translation\"** data. This is the movement of ego car.","metadata":{}},{"cell_type":"code","source":"frames = zarr_dataset.frames\n\n## This is slow.\n# coords = np.zeros((len(frames), 2))\n# for idx_coord, idx_data in enumerate(tqdm(range(len(frames)), desc=\"getting centroid to plot trajectory\")):\n#     frame = zarr_dataset.frames[idx_data]\n#     coords[idx_coord] = frame[\"ego_translation\"][:2]\n\n# This is much faster!\ncoords = frames[\"ego_translation\"][:, :2]\n\nplt.scatter(coords[:, 0], coords[:, 1], marker='.')\naxes = plt.gca()\naxes.set_xlim([-2500, 1600])\naxes.set_ylim([-2500, 1600])\nplt.title(\"ego_translation of frames\")","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:41.575548Z","iopub.execute_input":"2022-02-28T03:22:41.576041Z","iopub.status.idle":"2022-02-28T03:22:41.910254Z","shell.execute_reply.started":"2022-02-28T03:22:41.576006Z","shell.execute_reply":"2022-02-28T03:22:41.909603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Note:** `frames[\"ego_translation\"]` is same as `frames[:][\"ego_translation\"]`","metadata":{}},{"cell_type":"markdown","source":"### pytorch Dataset class\n\nInstead of working with raw data, L5Kit provides PyTorch ready datasets.\nIt's much easier to use this wrapped dataset class to access data.\n\n2 dataset class is implemented.\n\n - **EgoDataset**: this dataset iterates over the AV (Autonomous Vehicle) annotations\n - **AgentDataset**: this dataset iterates over other agents annotations","metadata":{}},{"cell_type":"code","source":"# 'map_type': 'py_semantic' for cfg.\n# Rasterizer is in charge of visualizing the data, as we will see next.\nsemantic_rasterizer = build_rasterizer(cfg, dm)\nsemantic_dataset = EgoDataset(cfg, zarr_dataset, semantic_rasterizer)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:22:41.911644Z","iopub.execute_input":"2022-02-28T03:22:41.911888Z","iopub.status.idle":"2022-02-28T03:22:46.862283Z","shell.execute_reply.started":"2022-02-28T03:22:41.911854Z","shell.execute_reply":"2022-02-28T03:22:46.861555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization example\n\nLyft l5kit also provides visualization functionalities.\n\nWe will visualize data to understand what kind of information is stored in this dataset.\n\nBugfix: https://github.com/woven-planet/l5kit/issues/266","metadata":{}},{"cell_type":"code","source":"def visualize_trajectory(dataset, index, title=\"target_positions movement with draw_trajectory\"):\n    data = dataset[index]\n    im = data[\"image\"].transpose(1, 2, 0)\n    im = dataset.rasterizer.to_rgb(im)\n    target_positions_pixels = transform_points(data[\"target_positions\"] + data[\"centroid\"][:2], data[\"world_to_image\"])\n    draw_trajectory(im, target_positions_pixels, TARGET_POINTS_COLOR, yaws=data[\"target_yaws\"])\n\n    plt.title(title)\n    plt.imshow(im[::-1])\n    plt.show()\n\nvisualize_trajectory(semantic_dataset, index=0)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:25:14.562703Z","iopub.execute_input":"2022-02-28T03:25:14.563518Z","iopub.status.idle":"2022-02-28T03:25:14.809981Z","shell.execute_reply.started":"2022-02-28T03:25:14.563467Z","shell.execute_reply":"2022-02-28T03:25:14.809291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can switch rasterizer to visualize satellite image easily!","metadata":{}},{"cell_type":"code","source":"# map_type was changed from 'py_semantic' to 'py_satellite'.\ncfg[\"raster_params\"][\"map_type\"] = \"py_satellite\"\nsatellite_rasterizer = build_rasterizer(cfg, dm)\nsatellite_dataset = EgoDataset(cfg, zarr_dataset, satellite_rasterizer)\n\nvisualize_trajectory(satellite_dataset, index=0)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:29:34.203159Z","iopub.execute_input":"2022-02-28T03:29:34.203438Z","iopub.status.idle":"2022-02-28T03:29:37.938541Z","shell.execute_reply.started":"2022-02-28T03:29:34.203402Z","shell.execute_reply":"2022-02-28T03:29:37.937864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(satellite_rasterizer), type(semantic_rasterizer)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:27:02.742134Z","iopub.execute_input":"2022-02-28T03:27:02.742642Z","iopub.status.idle":"2022-02-28T03:27:02.748701Z","shell.execute_reply.started":"2022-02-28T03:27:02.742605Z","shell.execute_reply":"2022-02-28T03:27:02.747861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we visualized **EgoDataset**.\n\n**AgentDataset** can be used to visualize an agent. This dataset iterates over agents and not the AV anymore, and the first one happens to be the pace car (you will see this one around a lot in the dataset).","metadata":{}},{"cell_type":"code","source":"agent_dataset = AgentDataset(cfg, zarr_dataset, satellite_rasterizer)\nvisualize_trajectory(agent_dataset, index=0)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:30:30.510743Z","iopub.execute_input":"2022-02-28T03:30:30.510997Z","iopub.status.idle":"2022-02-28T03:30:32.204866Z","shell.execute_reply.started":"2022-02-28T03:30:30.51097Z","shell.execute_reply":"2022-02-28T03:30:32.204215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**System Origin and Orientation**\n\nAt this point you may have noticed that we flip the image on the Y-axis before plotting it.\n\nWhen moving from 3D to 2D we stick to a right-hand system, where the origin is in the bottom-left corner with positive x-values going right and positive y-values going up the image plane. The camera is facing down the negative z axis.\n\nHowever, both opencv and pyplot place the origin in the top-left corner with positive x going right and positive y going down in the image plane. The camera is facing down the positive z-axis.\n\nThe flip done on the resulting image is for visualisation purposes to accommodate the difference in the two coordinate frames.\n\nFurther, all our rotations are counter-clockwise for positive value of the angle.","metadata":{}},{"cell_type":"markdown","source":"## Scene handling\n\nBoth EgoDataset and AgentDataset provide 2 methods for getting interesting indices:\n\n - **get_frame_indices** returns the indices for a given frame. For the `EgoDataset` this matches a single observation, while more than one index could be available for the `AgentDataset`, as that given frame may contain more than one valid agent\n - **get_scene_indices** returns indices for a given scene. For both datasets, these might return more than one index","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:31:51.133942Z","iopub.execute_input":"2022-02-28T03:31:51.134263Z","iopub.status.idle":"2022-02-28T03:31:51.140467Z","shell.execute_reply.started":"2022-02-28T03:31:51.134234Z","shell.execute_reply":"2022-02-28T03:31:51.139572Z"}}},{"cell_type":"code","source":"from IPython.display import display, clear_output\nimport PIL\n \ndataset = semantic_dataset\nscene_idx = 34\nindexes = dataset.get_scene_indices(scene_idx)\nimages = []\n\nfor idx in indexes:\n    data = dataset[idx]\n    im = data[\"image\"].transpose(1, 2, 0)\n    im = dataset.rasterizer.to_rgb(im)\n    target_positions_pixels = transform_points(data[\"target_positions\"] + data[\"centroid\"][:2], data[\"world_to_image\"])\n    center_in_pixels = np.asarray(cfg[\"raster_params\"][\"ego_center\"]) * cfg[\"raster_params\"][\"raster_size\"]\n    draw_trajectory(im, target_positions_pixels, TARGET_POINTS_COLOR, yaws=data[\"target_yaws\"])\n    clear_output(wait=True)\n    images.append(PIL.Image.fromarray(im[::-1]))","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:32:35.793256Z","iopub.execute_input":"2022-02-28T03:32:35.793779Z","iopub.status.idle":"2022-02-28T03:32:46.678947Z","shell.execute_reply.started":"2022-02-28T03:32:35.793742Z","shell.execute_reply":"2022-02-28T03:32:46.677981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n# From https://www.kaggle.com/jpbremer/lyft-scene-visualisations by @jpbremer\ndef animate_solution(images):\n\n    def animate(i):\n        im.set_data(images[i])\n        return (im,)\n \n    fig, ax = plt.subplots()\n    im = ax.imshow(images[0])\n    def init():\n        im.set_data(images[0])\n        return (im,)\n    \n    return animation.FuncAnimation(fig, animate, init_func=init, frames=len(images), interval=60, blit=True)\n\nanim = animate_solution(images)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:34:17.05355Z","iopub.execute_input":"2022-02-28T03:34:17.053809Z","iopub.status.idle":"2022-02-28T03:34:17.289407Z","shell.execute_reply.started":"2022-02-28T03:34:17.053781Z","shell.execute_reply":"2022-02-28T03:34:17.28869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HTML(anim.to_jshtml())","metadata":{"execution":{"iopub.status.busy":"2022-02-28T03:35:36.968118Z","iopub.execute_input":"2022-02-28T03:35:36.968378Z","iopub.status.idle":"2022-02-28T03:35:51.633407Z","shell.execute_reply.started":"2022-02-28T03:35:36.968351Z","shell.execute_reply":"2022-02-28T03:35:51.63263Z"},"trusted":true},"execution_count":null,"outputs":[]}]}