{"cells":[{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# !conda install -c plotly plotly-orca\nimport os\nos.system('pip install --target=/kaggle/working pymap3d==2.1.0')\nos.system('pip install --target=/kaggle/working protobuf==3.12.2')\nos.system('pip install --target=/kaggle/working transforms3d')\nos.system('pip install --target=/kaggle/working zarr')\nos.system('pip install --target=/kaggle/working ptable')\n\nos.system('pip install --no-dependencies --target=/kaggle/working l5kit')\n!pip install -U kaleido","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"%matplotlib inline\nimport pandas as pd\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\nfrom plotly.offline import init_notebook_mode\ninit_notebook_mode(connected=True)\nimport datetime\nimport time\nimport numpy as np\nimport plotly.graph_objects as go\nimport plotly.express as px\nfrom tqdm import tqdm\n\nfrom l5kit.data import ChunkedDataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = '/kaggle/input/lyft-motion-prediction-autonomous-vehicles'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dt = ChunkedDataset(DATA_PATH+'/scenes/train.zarr').open(cached=False)\nvalid_dt = ChunkedDataset(DATA_PATH+'/scenes/validate.zarr').open(cached=False)\ntest_dt = ChunkedDataset(DATA_PATH+'/scenes/test.zarr').open(cached=False)\n\n\npd.DataFrame([\n    ['train', len(train_dt.frames), len(train_dt.agents), len(train_dt.scenes), len(train_dt.tl_faces)],\n    ['valid', len(valid_dt.frames), len(valid_dt.agents), len(valid_dt.scenes), len(valid_dt.tl_faces)],\n    ['test', len(test_dt.frames), len(test_dt.agents), len(test_dt.scenes), len(test_dt.tl_faces)],\n], columns=['set', 'frames', 'agents', 'scenes', 'tl_faces'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def ts_to_dt(ts):\n    return datetime.datetime.fromtimestamp(ts // 10**9)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Scenes\nEach scene is 25 sec long."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_scenes = pd.DataFrame(\n    [[f['host'], f['start_time'], f['end_time']] for f in train_dt.scenes],\n    columns=['host', 'start', 'end']\n)\nvalid_scenes = pd.DataFrame(\n    [[f['host'], f['start_time'], f['end_time']] for f in valid_dt.scenes],\n    columns=['host', 'start', 'end']\n)\ntest_scenes = pd.DataFrame(\n    [[f['host'], f['start_time'], f['end_time']] for f in test_dt.scenes],\n    columns=['host', 'start', 'end']\n)\ntrain_scenes['set'] = 'train'\ntest_scenes['set'] = 'test'\nvalid_scenes['set'] = 'valid'\nscenes = pd.concat([train_scenes, test_scenes, valid_scenes])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scenes['start_time'] = scenes.start.apply(ts_to_dt)\nscenes['end_time'] = scenes.end.apply(ts_to_dt)\nscenes['duration'] = (scenes.end - scenes.start) / 10**9\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'duration', scenes.duration.unique()\npd.concat([\n    scenes.groupby(['set', 'host']).start_time.min(),\n    scenes.groupby(['set', 'host']).end_time.max()\n], axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Frame times"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_times = pd.DataFrame({'t': [ts_to_dt(f['timestamp']) for f in train_dt.frames]})\ntest_times = pd.DataFrame({'t': [ts_to_dt(f['timestamp']) for f in test_dt.frames]})\nvalid_times = pd.DataFrame({'t': [ts_to_dt(f['timestamp']) for f in valid_dt.frames]})\n\n\ntrain_times['set'] = 'train'\ntest_times['set'] = 'test'\nvalid_times['set'] = 'valid'\ntimes = pd.concat([train_times, test_times, valid_times])\n\ntimes['cnt'] = 1\ntimes['h'] = times.t.dt.round(\"H\")\ntimes['day'] = times.t.dt.round(\"D\")\n\n\ntimes\n\ndf = times.groupby(['set', 'day']).sum().reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.bar(df, x='day', y='cnt', color='set', title='Train-Valid-Test split')\nfig.write_image('train-test-split.png')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Frame locations"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_locs = pd.DataFrame(\n    [f['ego_translation'] for f in train_dt.frames], columns=['x', 'y', 'z'])\ntest_locs = pd.DataFrame(\n    [f['ego_translation'] for f in test_dt.frames], columns=['x', 'y', 'z'])\nvalid_locs = pd.DataFrame(\n    [f['ego_translation'] for f in valid_dt.frames], columns=['x', 'y', 'z'])\n\ntrain_locs = train_locs.round()\n\ntrain_locs['set'] = 'train'\ntest_locs['set'] = 'test'\nvalid_locs['set'] = 'valid'\nlocs = pd.concat([train_locs, test_locs, valid_locs])\nlocs = locs.round()\nlocs['cnt'] = 1\n\ndf = locs.groupby(['set', 'x', 'y']).sum().reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f1 = px.scatter(df[df.set == 'train'], x='x', y='y', size='cnt', title='Train - Locations', opacity=0.5)\nf1.update_traces(marker=dict(color='red', line_width=0))\nf1.write_image('train-locations.png')\nf2 = px.scatter(df[df.set == 'test'], x='x', y='y', size='cnt', title='Test - Locations', opacity=0.5)\nf2.update_traces(marker=dict(color='blue', line_width=0))\nf2.write_image('test-locations.png')\nf3 = px.scatter(df[df.set == 'valid'], x='x', y='y', size='cnt', title='Valid - Locations', opacity=0.5)\nf3.update_traces(marker=dict(color='green', line_width=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}