{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Lyft Chopped Dataset Creation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from typing import Dict\nimport numpy as np\n\nfrom l5kit.data import LocalDataManager, ChunkedDataset\nfrom l5kit.dataset import AgentDataset\nfrom l5kit.rasterization import build_rasterizer\nfrom l5kit.evaluation import create_chopped_dataset\nfrom l5kit.evaluation.chop_dataset import MIN_FUTURE_STEPS\nfrom pathlib import Path\n\nimport os\nimport yaml\nfrom io import StringIO\nimport shutil","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AGENT_MOTION_CONFIG = \"\"\"\n# Config format schema number\nformat_version: 4\n\n###################\n## Model options\nmodel_params:\n  model_architecture: \"resnet50\"\n\n  history_num_frames: 0\n  history_step_size: 1\n  history_delta_time: 0.1\n\n  future_num_frames: 50\n  future_step_size: 1\n  future_delta_time: 0.1\n\n###################\n## Input raster parameters\nraster_params:\n  # raster image size [pixels]\n  raster_size:\n    - 224\n    - 224\n  # raster's spatial resolution [meters per pixel]: the size in the real world one pixel corresponds to.\n  pixel_size:\n    - 0.5\n    - 0.5\n  # From 0 to 1 per axis, [0.5,0.5] would show the ego centered in the image.\n  ego_center:\n    - 0.25\n    - 0.5\n  map_type: \"py_semantic\"\n\n  # the keys are relative to the dataset environment variable\n  satellite_map_key: \"aerial_map/aerial_map.png\"\n  semantic_map_key: \"semantic_map/semantic_map.pb\"\n  dataset_meta_key: \"meta.json\"\n\n  # e.g. 0.0 include every obstacle, 0.5 show those obstacles with >0.5 probability of being\n  # one of the classes we care about (cars, bikes, peds, etc.), >=1.0 filter all other agents.\n  filter_agents_threshold: 0.5\n\n###################\n## Data loader options\ntrain_data_loader:\n  key: \"scenes/sample.zarr\"\n  batch_size: 12\n  shuffle: True\n  num_workers: 16\n\nval_data_loader:\n  key: \"scenes/sample.zarr\"\n  batch_size: 12\n  shuffle: False\n  num_workers: 16\n\n###################\n## Train params\ntrain_params:\n  checkpoint_every_n_steps: 10000\n  max_num_steps: 5\n  eval_every_n_steps: 10000\n\"\"\"\n\ncfg: dict = yaml.load(StringIO(AGENT_MOTION_CONFIG), Loader=yaml.FullLoader)\nprint(cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dm = LocalDataManager(\"/kaggle/input/lyft-motion-prediction-autonomous-vehicles\")\nrasterizer = build_rasterizer(cfg, dm)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Original Unchopped Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"eval_zarr = ChunkedDataset(dm.require(cfg[\"val_data_loader\"][\"key\"])).open()\nprint(AgentDataset(cfg, eval_zarr, rasterizer))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Note average 248 frmes per scene.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Chop Evaluation Data (Copy to /tmp)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# ===== GENERATE AND LOAD CHOPPED DATASET\nnum_frames_to_chop = 100\neval_cfg = cfg[\"val_data_loader\"]\n\n# As the /kaggle/input directory is not writeable as required to chop,\n# copy the sample set to /tmp\n!rm -rf /tmp/lyft\neval_dir = shutil.copytree(dm.require(eval_cfg[\"key\"]), '/tmp/lyft/sample.zarr')\n\neval_base_path = create_chopped_dataset(eval_dir, cfg[\"raster_params\"][\"filter_agents_threshold\"], \n                              num_frames_to_chop, cfg[\"model_params\"][\"future_num_frames\"], MIN_FUTURE_STEPS)\n!ls {eval_base_path}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Chopped data mask and ground truth created.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Load","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"eval_zarr_path = str(Path(eval_base_path) / Path(dm.require(eval_cfg[\"key\"])).name)\neval_mask_path = str(Path(eval_base_path) / \"mask.npz\")\neval_gt_path = str(Path(eval_base_path) / \"gt.csv\")\n\neval_zarr = ChunkedDataset(eval_zarr_path).open()\neval_mask = np.load(eval_mask_path)[\"arr_0\"]\n# ===== INIT DATASET AND LOAD MASK\neval_dataset = AgentDataset(cfg, eval_zarr, rasterizer, agents_mask=eval_mask)\nprint(eval_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Note average 100 frames per scene.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf /tmp/lyft","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Chop Evaluation Data (Symlink to /tmp)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"eval_dir = dm.require(eval_cfg[\"key\"])\n!mkdir /tmp/lyft && ln -s {eval_dir} /tmp/lyft\neval_dir = \"/tmp/lyft/\" + Path(eval_dir).name\n!ls -la  {eval_dir}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"eval_base_path = create_chopped_dataset(eval_dir, cfg[\"raster_params\"][\"filter_agents_threshold\"], \n                              num_frames_to_chop, cfg[\"model_params\"][\"future_num_frames\"], MIN_FUTURE_STEPS)\n!ls {eval_base_path}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"eval_zarr_path = str(Path(eval_base_path) / Path(dm.require(eval_cfg[\"key\"])).name)\neval_mask_path = str(Path(eval_base_path) / \"mask.npz\")\neval_gt_path = str(Path(eval_base_path) / \"gt.csv\")\n\neval_zarr = ChunkedDataset(eval_zarr_path).open()\neval_mask = np.load(eval_mask_path)[\"arr_0\"]\n# ===== INIT DATASET AND LOAD MASK\neval_dataset = AgentDataset(cfg, eval_zarr, rasterizer, agents_mask=eval_mask)\nprint(eval_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Again looks fine.","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}