{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Submission Visualization\n\nI simplified the visualization codes from @jpbremer and @corochann"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.patches as mpatches\n\nimport numpy as np\nimport pandas as pd\n\nfrom l5kit.data import ChunkedDataset, LocalDataManager\nfrom l5kit.dataset import EgoDataset, AgentDataset\n\nfrom l5kit.rasterization import build_rasterizer\nfrom l5kit.configs import load_config_data\nfrom l5kit.visualization import draw_trajectory, TARGET_POINTS_COLOR\nfrom l5kit.geometry import transform_points\nfrom tqdm import tqdm\nfrom collections import Counter\nfrom l5kit.data import PERCEPTION_LABELS\nfrom prettytable import PrettyTable\n\nimport os\n\nfrom matplotlib import animation, rc\nfrom IPython.display import HTML, display\n\nrc('animation', html='jshtml')\n\ndef set_seed(seed):\n    # random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    # torch.manual_seed(seed)\n    # torch.cuda.manual_seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"set_seed(42)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# set env variable for data\ndata_path = '../input/lyft-motion-prediction-autonomous-vehicles'\nos.environ[\"L5KIT_DATA_FOLDER\"] = data_path\n# get config\ncfg = load_config_data(\"../input/lyft-config-files/visualisation_config.yaml\")\ncfg['model_params']['history_num_frames'] = 10  # note when training model, we set this to 10\ncfg['raster_params']['disable_traffic_light_faces'] = False\ncfg['test_data_loader'] = {\n    'key': 'scenes/test.zarr',\n    'batch_size': 128,\n    'shuffle': False,\n    'num_workers': 4,\n}\nprint(cfg)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load the test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"dm = LocalDataManager()\nzarr_dataset = ChunkedDataset(dm.require(cfg['test_data_loader']['key'])).open()\nprint(zarr_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build the Rasterizers"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Semantic view\ncfg['raster_params']['map_type'] = 'py_semantic'\nsemantic_rasterizer = build_rasterizer(cfg, dm)\n\n# Satellite view\ncfg['raster_params']['map_type'] = 'py_satellite'\nsatellite_rasterizer = build_rasterizer(cfg, dm)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read submission.csv\nAdd output from your submission notebook as data on the right. Then enter the path to the csv below"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub = pd.read_csv('../input/lyft-complete-train-and-prediction-pipeline/submission.csv')\ndf_sub = df_sub.set_index(['timestamp', 'track_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(df_sub)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def row_to_confs(row):\n    return [row[f'conf_{i}'] for i in range(3)]\ndef row_to_coords(row):\n    return row[3:].values.reshape(3, 50, 2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Frame"},{"metadata":{},"cell_type":"markdown","source":"## Autonomous Vehicle (Ego)"},{"metadata":{"trusted":true},"cell_type":"code","source":"semantic_dataset = EgoDataset(cfg, zarr_dataset, semantic_rasterizer)\nsatellite_dataset = EgoDataset(cfg, zarr_dataset, satellite_rasterizer)\ntest_mask = np.load(f\"{data_path}/scenes/mask.npz\")[\"arr_0\"]\nagent_semantic_dataset = AgentDataset(cfg, zarr_dataset, semantic_rasterizer, agents_mask=test_mask)\nagent_satellite_dataset = AgentDataset(cfg, zarr_dataset, satellite_rasterizer, agents_mask=test_mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# here I use matplotlib default colors\ncmap = plt.get_cmap(\"tab10\")\nmatplotlib_colors_in_rgb_int = [\n    [int(255 * x) for x in cmap(i)[:3]] for i in range(10)\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# note raster_from_agent is actually a constant matrix for each raster once you fix the raster params\nraster_params = cfg['raster_params']\nraster_from_agent = np.array([\n    [2., 0.,  56.],\n    [0., 2., 112.],\n    [0., 0.,   1.],\n]) if (\n    raster_params['raster_size'] == [224, 224] and\n    raster_params['pixel_size'] == [0.5, 0.5] and\n    raster_params['ego_center'] == [0.25, 0.5]\n) else None\n    \ndef generate_image_trajectory(dataset, index):\n    data = dataset[index]\n    im = data['image'].transpose(1, 2, 0)\n    im = dataset.rasterizer.to_rgb(im)\n    rfg = raster_from_agent if raster_from_agent is not None else data['raster_from_agent']\n    target_positions_pixels = transform_points(data['target_positions'], rfg)\n    draw_trajectory(im, target_positions_pixels, yaws=data['target_yaws'], rgb_color=TARGET_POINTS_COLOR)\n    return im\n\ndef plot_trajectory(dataset, indices, width=12, height=4, n_cols=3, title=''):\n    if not isinstance(indices, (list, np.ndarray)):\n        indices = [indices]\n    n_rows = len(indices) // n_cols + len(indices) % n_cols\n    plt.figure(figsize=(width, height*n_rows))\n    for k, index in enumerate(indices):\n        plt.subplot(n_rows, n_cols, 1+k).set_title(str(index))\n        im = generate_image_trajectory(dataset, index)\n        plt.imshow(im, origin='lower')\n    if title:\n        plt.suptitle(title)\n    plt.show()\n\ndef generate_image_predicted_trajectory(dataset, df_sub, index):\n    data = dataset[index]\n    im = data['image'].transpose(1, 2, 0)\n    im = dataset.rasterizer.to_rgb(im)\n    row = df_sub.loc[(data['timestamp'], data['track_id'])]\n    # note submission coordinate system = world - centroid\n    predicted_target_positions_in_sub = row_to_coords(row)\n    predicted_target_positions_in_world = predicted_target_positions_in_sub + data['centroid']\n    for i, coords in enumerate(predicted_target_positions_in_world):\n        target_positions_pixels = transform_points(coords, data['raster_from_world'])\n        draw_trajectory(im, target_positions_pixels, rgb_color=matplotlib_colors_in_rgb_int[i])\n    return im, row_to_confs(row)\n\ndef plot_predicted_trajectory(dataset, df_sub, indices, width=12, height=4, n_cols=3, title=''):\n    if not isinstance(indices, (list, np.ndarray)):\n        indices = [indices]\n    n_rows = len(indices) // n_cols + len(indices) % n_cols\n    plt.figure(figsize=(width, height*n_rows))\n    for k, index in enumerate(indices):\n        plt.subplot(n_rows, n_cols, 1+k).set_title(str(index))\n        im, confs = generate_image_predicted_trajectory(dataset, df_sub, index)\n        patches = [mpatches.Patch(color=cmap(m), label='%.3f'%conf) for m, conf in enumerate(confs)]\n        plt.imshow(im, origin='lower')\n        plt.legend(handles=patches)\n    if title:\n        plt.suptitle(title)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"3 curves corresponding to 3 mode of predictions. The legend indicates the confidence scores. The bright green is history."},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_predicted_trajectory(agent_semantic_dataset, df_sub, [18431], width=6, height=6, n_cols=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i_plots = np.random.randint(len(agent_semantic_dataset), size=9)\n# i_plots = (i_plots - i_plots[0] + 18552) % len(agent_semantic_dataset)\ni_plots","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_predicted_trajectory(agent_semantic_dataset, df_sub, i_plots)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_predicted_trajectory(agent_satellite_dataset, df_sub, i_plots)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# note in the test set the each agent only have very few frames.\nplot_predicted_trajectory(agent_semantic_dataset, df_sub, list(range(5311, 5311+9)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# weird case\nplot_predicted_trajectory(agent_semantic_dataset, df_sub, [25658], width=6, height=6, n_cols=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# weird case\nplot_predicted_trajectory(agent_satellite_dataset, df_sub, [25658], width=6, height=6, n_cols=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#  Entire Scene\nA scene is just a lot of frames"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"def animate(images):\n    fig = plt.figure()\n    ims = [(plt.imshow(im, animated=True, origin='lower'),) for im in images]\n    anim = animation.ArtistAnimation(fig, ims, interval=60, blit=True, repeat_delay=1000)\n    plt.close()\n    return HTML(anim.to_jshtml())\n\ndef plot_scene(dataset, scene_id):\n    indices = dataset.get_scene_indices(scene_id)\n    print('scene', scene_id, ':', indices[0], '-', indices[-1])\n    images = [generate_image_trajectory(dataset, i) for i in indices]\n    return animate(images)\n\nimport bisect\ndef get_scene_index_from_frame_id(dataset, frame_id):\n    return bisect.bisect_right(dataset.cumulative_sizes, frame_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scene_id = get_scene_index_from_frame_id(semantic_dataset, i_plots[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_scene(semantic_dataset, scene_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_scene(satellite_dataset, scene_id)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Agent\nNote one agent scene contains lots of agents. So you will see the image center jump around at each agent"},{"metadata":{"trusted":true},"cell_type":"code","source":"agent_scene_id = get_scene_index_from_frame_id(agent_semantic_dataset, i_plots[4])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_scene(agent_semantic_dataset, agent_scene_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_scene(agent_satellite_dataset, agent_scene_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# see what each agent were doing\nplot_trajectory(agent_semantic_dataset, list(range(47930, 47930+9)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### References\n* https://www.kaggle.com/corochann/lyft-comprehensive-guide-to-start-competition/\n* https://www.kaggle.com/jpbremer/lyft-scene-visualisations"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}