{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os, gc\nimport zarr\nimport numpy as np \nimport pandas as pd \nfrom tqdm import tqdm\nfrom typing import Dict\nfrom collections import Counter\nfrom prettytable import PrettyTable\n\nfrom l5kit.data import PERCEPTION_LABELS\nfrom l5kit.dataset import EgoDataset, AgentDataset\nfrom l5kit.data import ChunkedDataset, LocalDataManager\n\nfrom l5kit.configs import load_config_data\nfrom l5kit.geometry import transform_points\nfrom l5kit.rasterization import build_rasterizer\nfrom l5kit.visualization import draw_trajectory, draw_reference_trajectory, TARGET_POINTS_COLOR\nfrom l5kit.evaluation import write_pred_csv, compute_metrics_csv, read_gt_csv, create_chopped_dataset\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom matplotlib import animation\nfrom colorama import Fore, Back, Style\n\nprint((os.listdir('../input/lyft-motion-prediction-autonomous-vehicles/')))\n\nplt.rc('animation', html='jshtml')\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# animation for scene\ndef animate_solution(images):\n\n    def animate(i):\n        im.set_data(images[i])\n \n    fig, ax = plt.subplots()\n    im = ax.imshow(images[0])\n    \n    return animation.FuncAnimation(fig, animate, frames=len(images), interval=80)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nmy_arr = np.zeros(3, dtype=[(\"color\", (np.uint8, 3)), (\"label\", np.bool)])\n\nprint(my_arr[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_arr[0][\"color\"] = [0, 218, 130]\nmy_arr[0][\"label\"] = True\nmy_arr[1][\"color\"] = [245, 59, 255]\nmy_arr[1][\"label\"] = True\nmy_arr[1][\"color\"] = [7, 6, 97]\nmy_arr[1][\"label\"] = True\n\nprint(my_arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = zarr.open(\"../input/lyft-motion-prediction-autonomous-vehicles/scenes/train.zarr\")\nvalidation = zarr.open(\"../input/lyft-motion-prediction-autonomous-vehicles/scenes/validate.zarr\")\ntest = zarr.open(\"../input/lyft-motion-prediction-autonomous-vehicles/scenes/test.zarr/\")\ntrain.info","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(f'We have {len(train.agents)} agents, {len(train.scenes)} scenes, {len(train.frames)} frames and {len(train.traffic_light_faces)} traffic light faces in train.zarr.')\nprint(f'We have {len(validation.agents)} agents, {len(validation.scenes)} scenes, {len(validation.frames)} frames and {len(validation.traffic_light_faces)} traffic light faces in validation.zarr.')\nprint(f'We have {len(test.agents)} agents, {len(test.scenes)} scenes, {len(test.frames)} frames and {len(test.traffic_light_faces)} traffic light faces in test.zarr.')\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"# set env variable for data\nos.environ[\"L5KIT_DATA_FOLDER\"] = \"../input/lyft-motion-prediction-autonomous-vehicles\"\n\n# get configuration yaml\ncfg = load_config_data(\"../input/visualization-config/visualisation_config.yaml\")\nprint(cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Raster Parameters\nprint(f'current raster_param:\\n')\nfor k,v in cfg[\"raster_params\"].items():\n    print(f\"{k}:{v}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dm = LocalDataManager()\ndataset_path = dm.require(cfg[\"val_data_loader\"][\"key\"])\nzarr_dataset = ChunkedDataset(dataset_path)\nzarr_dataset.open()\nprint(zarr_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(dataset_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"agents = pd.DataFrame.from_records(zarr_dataset.agents, columns = ['centroid', 'extent', 'yaw', 'velocity', 'track_id', 'label_probabilities'])\nagents.head()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"agents[['centroid_x','centroid_y']] = agents['centroid'].to_list()\nagents = agents.drop('centroid', axis=1)\nagents_new = agents[[\"centroid_x\", \"centroid_y\", \"extent\", \"yaw\", \"velocity\", \"track_id\", \"label_probabilities\"]]\ndel agents\nagents_new","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,1,figsize=(8,8))\nplt.scatter(agents_new['centroid_x'], agents_new['centroid_y'], marker='+')\nplt.xlabel('x', fontsize=11); plt.ylabel('y', fontsize=11)\nplt.title(\"Centroids distribution (sample.zarr)\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"agents_new[['extent_x','extent_y', 'extent_z']] = agents_new['extent'].to_list()\nagents_new = agents_new.drop('extent', axis=1)\nagents = agents_new[[\"centroid_x\", \"centroid_y\", 'extent_x', 'extent_y', 'extent_z', \"yaw\", \"velocity\", \"track_id\", \"label_probabilities\"]]\ndel agents_new\nagents","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"sns.axes_style(\"white\")\n\nfig, ax = plt.subplots(1,1,figsize=(16,5))\n\n# plt.subplot(1,3,1)\n# sns.kdeplot(agents['extent_x'], shade=True, color='red');\n# plt.title(\"Extent_x distribution\")\n\nplt.subplot(1,3,1)\nsns.kdeplot(agents['extent_y'], shade=True, color='steelblue');\nplt.title(\"Extent_y distribution\")\n\n# plt.subplot(1,3,1)\n# sns.kdeplot(agents['extent_z'], shade=True, color='green');\n# plt.title(\"Extent_z distribution\")\n\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"sns.set_style('whitegrid')\n\nfig, ax = plt.subplots(1,3,figsize=(16,5))\nplt.subplot(1,3,1)\nplt.scatter(agents['extent_x'], agents['extent_y'], marker='*')\nplt.xlabel('ex', fontsize=11); plt.ylabel('ey', fontsize=11)\nplt.title(\"Extent: ex-ey\")\n\nplt.subplot(1,3,2)\nplt.scatter(agents['extent_y'], agents['extent_z'], marker='*', color=\"red\")\nplt.xlabel('ey', fontsize=11); plt.ylabel('ez', fontsize=11)\nplt.title(\"Extent: ey-ez\")\n\nplt.subplot(1,3,3)\nplt.scatter(agents['extent_z'], agents['extent_x'], marker='*', color=\"green\")\nplt.xlabel('ez', fontsize=11); plt.ylabel('ex', fontsize=11)\nplt.title(\"Extent: ez-ex\")\n\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,1,figsize=(10,8))\nsns.distplot(agents['yaw'])\nplt.title(\"Yaw Distribution\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"agents[['velocity_x','velocity_y']] = agents['velocity'].to_list()\nagents_vel = agents.drop('velocity', axis=1)\nagents_v = agents_vel[[\"centroid_x\", \"centroid_y\", 'extent_x', 'extent_y', 'extent_z', \"yaw\", \"velocity_x\", \"velocity_y\", \"track_id\", \"label_probabilities\"]]\ndel agents\nagents_v","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,1,figsize=(10,8))\n\nwith sns.axes_style(\"whitegrid\"):\n    sns.scatterplot(x=agents_v[\"velocity_x\"], y=agents_v[\"velocity_y\"], color='k');\n    plt.title('Velocity Distribution')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"agents = zarr_dataset.agents\nprobabilities = agents[\"label_probabilities\"]\nlabels_indexes = np.argmax(probabilities, axis=1)\ncounts = []\nfor idx_label, label in enumerate(PERCEPTION_LABELS):\n    counts.append(np.sum(labels_indexes == idx_label))\n    \ntable = PrettyTable(field_names=[\"label\", \"counts\"])\nfor count, label in zip(counts, PERCEPTION_LABELS):\n    table.add_row([label, count])\nprint(table)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(f'{Fore.YELLOW}Total number of agents in sample .zarr files is {Style.RESET_ALL}{len(zarr_dataset.agents)}. {Fore.BLUE}\\nAfter summing up the elements in count column we can see we have {Style.RESET_ALL}{(1324481 + 519385 + 6688 + 43182)} {Fore.BLUE}agents in total.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"scenes = pd.DataFrame.from_records(zarr_dataset.scenes, columns = ['frame_index_interval', 'host', 'start_time', 'end_time'])\nscenes.head()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"scenes[['frame_start_index','frame_end_index']] = scenes['frame_index_interval'].to_list()\nscenes_new = scenes.drop('frame_index_interval', axis=1)\nscenes_new = scenes_new[[\"frame_start_index\", \"frame_end_index\", 'host', 'start_time', 'end_time']]\ndel scenes\nscenes_new.head()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"f = plt.figure(figsize=(10, 8))\ngs = f.add_gridspec(1, 1)\n\n# with sns.axes_style(\"whitegrid\"):\n#     ax = f.add_subplot(gs[0,0])\n#     sns.scatterplot(scenes_new['frame_start_index'], scenes_new['frame_end_index'])\n#     plt.title('Frame Index Interval Distribution')\n    \nwith sns.axes_style(\"whitegrid\"):\n    ax = f.add_subplot(gs[0,0])\n    sns.scatterplot(scenes_new['frame_start_index'], scenes_new['frame_end_index'], hue=scenes_new['host'])\n    plt.title('Frame Index Interval Distribution (Grouped per host)')\n    \nf.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"f = plt.figure(figsize=(10, 8))\n\nwith sns.axes_style(\"white\"):\n    sns.countplot(scenes_new['host']);\n    plt.title(\"Host Count\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"frames = pd.DataFrame.from_records(zarr_dataset.frames, columns = ['timestamp', 'agent_index_interval', 'traffic_light_faces_index_interval', 'ego_translation','ego_rotation'])\nframes.head()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"frames[['ego_translation_x', 'ego_translation_y', 'ego_translation_z']] = frames['ego_translation'].to_list()\nframes_new = frames.drop('ego_translation', axis=1)\nframes_new = frames_new[['timestamp', 'agent_index_interval', 'traffic_light_faces_index_interval',\n                         'ego_translation_x', 'ego_translation_y', 'ego_translation_z', 'ego_rotation']]\ndel frames\nframes_new.head()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"f = plt.figure(figsize=(16, 8))\ngs = f.add_gridspec(1, 3)\n\nwith sns.axes_style(\"whitegrid\"):\n    ax = f.add_subplot(gs[0,0])\n    sns.distplot(frames_new['ego_translation_x'], color='Orange')\n    plt.title('Ego Translation Distribution X')\n    \nwith sns.axes_style(\"whitegrid\"):\n    ax = f.add_subplot(gs[0,1])\n    sns.distplot(frames_new['ego_translation_y'], color='Red')\n    plt.title('Ego Translation Distribution Y')\n    \nwith sns.axes_style(\"whitegrid\"):\n    ax = f.add_subplot(gs[0,2])\n    sns.distplot(frames_new['ego_translation_z'], color='Green')\n    plt.title('Ego Translation Distribution Z')\n    \nf.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"f = plt.figure(figsize=(16, 6))\ngs = f.add_gridspec(1, 3)\n\nwith sns.axes_style(\"darkgrid\"):\n    ax = f.add_subplot(gs[0,0])\n    plt.scatter(frames_new['ego_translation_x'], frames_new['ego_translation_y'],\n                    color='darkkhaki', marker='+')\n    plt.title('Ego Translation X-Y')\n    plt.xlabel('ego_translation_x')\n    plt.ylabel('ego_translation_y')\n    \nwith sns.axes_style(\"darkgrid\"):\n    ax = f.add_subplot(gs[0,1])\n    plt.scatter(frames_new['ego_translation_y'], frames_new['ego_translation_z'],\n                    color='slateblue', marker='*')\n    plt.title('Ego Translation Distribution Y-Z')\n    plt.xlabel('ego_translation_y')\n    plt.ylabel('ego_translation_z')\n    \nwith sns.axes_style(\"darkgrid\"):\n    ax = f.add_subplot(gs[0,2])\n    plt.scatter(frames_new['ego_translation_z'], frames_new['ego_translation_x'],\n                    color='turquoise', marker='^')\n    plt.title('Ego Translation Distribution Z-X')\n    plt.xlabel('ego_translation_z')\n    plt.ylabel('ego_translation_x')\n    \nf.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(3,3,figsize=(16,16))\ncolors = ['red', 'blue', 'green', 'magenta', 'orange', 'darkblue', 'black', 'cyan', 'darkgreen']\nfor i in range(0,3):\n    for j in range(0,3):\n        df = frames_new['ego_rotation'].apply(lambda x: x[i][j])\n        plt.subplot(3,3,i * 3 + j + 1)\n        sns.distplot(df, hist=False, color = colors[ i * 3 + j  ])\n        plt.xlabel(f'r[ {i + 1} ][ {j + 1} ]')\nfig.suptitle(\"Ego rotation angles distribution\", size=14)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"traffic_light_faces = pd.DataFrame.from_records(zarr_dataset.tl_faces, columns = ['face_id', 'traffic_light_id', 'traffic_light_face_status'])\ntraffic_light_faces.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}