{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Make masks for sub-sampling the training dataset\n\nThis notebook will make a mask that returns only agents between `start_frame` and `end_frame` in steps of `interval`.  \nThis mask can be used as input to an AgentDataset.  \n\nThe rationale behind this is that each frame gets sampled at a rate of 10Hz, and if you look at an agent in frame `n`, the scene won't look much different if you look at the same agent in frame `n+1`, therefore we can sample the dataset more coarsly and still get a good set of training data.  \n\nAdditionally, in the test set we get asked to predict agents at frame 100, for another 50 frames, so selecting only frames for which there are at least 100 preceding frames and at least 50 more frames to come, we will select a subset of the training data that looks more like the test set.  \n\n**NOTE:** I view this as a prototyping tool, i.e. as a useful way of reducing training time without reducing the data quality by much, but this does reduce the quality of the data, and you probably shoudn't use this for your final training. "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install --no-index -f ../input/kaggle-l5kit pip==20.2.2 >/dev/nul\n!pip install --no-index -f ../input/kaggle-l5kit -U l5kit > /dev/nul\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from l5kit.data import LocalDataManager, ChunkedDataset\nfrom l5kit.dataset import AgentDataset\nfrom l5kit.rasterization import build_rasterizer\nfrom l5kit.evaluation import write_pred_csv\nfrom l5kit.data.filter import get_agents_slice_from_frames\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import DataLoader\n\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.environ[\"L5KIT_DATA_FOLDER\"] = \"/kaggle/input/lyft-motion-prediction-autonomous-vehicles\"\n# local data manager\ndm = LocalDataManager()\n# set dataset path\ndataset_path = dm.require('scenes/train.zarr')\n# load the dataset; this is a zarr format, chunked dataset\nchunked_dataset = ChunkedDataset(dataset_path)\n# open the dataset\nchunked_dataset.open()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"cfg = {\n    'format_version': 4,\n    'model_params': {\n        'history_num_frames': 99,\n        'history_step_size': 1,\n        'history_delta_time': 0.1,\n        'future_num_frames': 50,\n        'future_step_size': 1,\n        'future_delta_time': 0.1\n    },\n    \n    'raster_params': {\n        'raster_size': [1, 1],\n        'pixel_size': [0.5, 0.5],\n        'ego_center': [0.5, 0.5],\n        'map_type': 'box_debug',\n        'satellite_map_key': 'aerial_map/aerial_map.png',\n        'semantic_map_key': 'semantic_map/semantic_map.pb',\n        'dataset_meta_key': 'meta.json',\n        'filter_agents_threshold': 0.5,\n        'disable_traffic_light_faces' : False\n\n    },\n    \n    'sample_data_loader': {\n        'key': 'scenes/sample.zarr',\n        'batch_size': 4,\n        'shuffle': False,\n        'num_workers': 8\n    }\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## First we make a basic mask by selecting only the agents between `start_frame` and `end_frame` in steps of `interval`"},{"metadata":{"trusted":true},"cell_type":"code","source":"n_frames = len(chunked_dataset.frames)\nframe_mask = np.zeros((n_frames,))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"interval = 10\nstart_frame = 100\nend_frame = 200\n\n\nfor scene in chunked_dataset.scenes:\n    f1, _ = scene['frame_index_interval']\n    for frame_no in np.arange(f1 + start_frame, f1 + end_frame + 1, interval):\n        #ag_s = get_agents_slice_from_frames(chunked_dataset.frames[frame_no])\n        #print(frame_no)\n        frame_mask[frame_no] = 1\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create the name of the oputput file\noutfile = \"frame_mask_\" + str(start_frame) + \"_\" + str(end_frame) + \"_\" + str(interval)\n\n# Save the mask\nnp.savez(outfile, frame_mask.astype(bool))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}