{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        os.path.join(dirname, filename)\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import l5kit\nl5kit.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport pathlib as path\nimport random\nimport sys\n\nfrom tqdm import tqdm\nimport numpy as np\nimport pandas as pd\nimport scipy\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom IPython.display import HTML, display\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('/kaggle/input/lyft-motion-prediction-autonomous-vehicles/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from l5kit.rasterization import build_rasterizer\nfrom l5kit.configs import load_config_data\nfrom l5kit.visualization import draw_trajectory, TARGET_POINTS_COLOR\nfrom l5kit.geometry import transform_points\nfrom collections import Counter\nfrom l5kit.data import PERCEPTION_LABELS\nfrom prettytable import PrettyTable\nfrom l5kit.dataset import EgoDataset, AgentDataset\nfrom l5kit.evaluation import write_pred_csv\n\nos.environ['L5KIT_DATA_FOLDER'] = '/kaggle/input/lyft-motion-prediction-autonomous-vehicles/'\n# cfg = load_config_data('/kaggle/input/lyft-config-files/agent_motion_config.yaml')\nDEBUG = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg = {\n    'format_version': 4,\n    'model_params': {\n        'model_architecture': 'resnet50',\n        'history_num_frames': 10,\n        'history_step_size': 1,\n        'history_delta_time': 0.1,\n        'future_num_frames': 50,\n        'future_step_size': 1,\n        'future_delta_time': 0.1\n    },\n    \n    'raster_params': {\n        'raster_size': [224, 224],\n        'pixel_size': [0.5, 0.5],\n        'ego_center': [0.25, 0.5],\n        'map_type': 'py_semantic',\n        'satellite_map_key': 'aerial_map/aerial_map.png',\n        'semantic_map_key': 'semantic_map/semantic_map.pb',\n        'dataset_meta_key': 'meta.json',\n        'filter_agents_threshold': 0.5\n    },\n    \n    'train_data_loader': {\n        'key': 'scenes/train.zarr',\n        'batch_size': 32,\n        'shuffle': True,\n        'num_workers': 4\n    },\n    \n    'train_params': {\n        'max_num_steps': 12000 if DEBUG else 10000,\n        'checkpoint_every_n_steps': 5000,\n        \n        # 'eval_every_n_steps': -1\n    }\n\n\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(f'current raster param:\\n')\n# for k, v in cfg['raster_params'].items():\n#     print(f'{k}:{v}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from l5kit.data import LocalDataManager, ChunkedDataset\ndm = LocalDataManager()\ndataset_path = dm.require('/kaggle/input/lyft-motion-prediction-autonomous-vehicles/scenes/train.zarr/')\nzarr_dataset = ChunkedDataset(dataset_path)\nzarr_dataset.open()\nprint(zarr_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualizing the positons of Autonomus Vehicles(AV)\nThere are far too many frames to look through to plot the AVs location which is why I've limited the range to 39k"},{"metadata":{"trusted":true},"cell_type":"code","source":"frames = zarr_dataset.frames\ncoords = np.zeros((len(frames), 2))\nfor idx_coord, idx_data in enumerate(tqdm(range(len(frames)-4000000), desc = 'getting centroid to plot trajectory')):\n    frame = zarr_dataset.frames[idx_data]\n    coords[idx_coord] = frame['ego_translation'][:2]\nsns.set_style('white')\nplt.figure(figsize = (13, 8))\nax = sns.scatterplot(coords[:, 0], coords[:, 1], marker = '*', s = 150)\nax.set_xlim([-2500, 1600]);\nax.set_ylim([-2500, 1600]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(zarr_dataset.agents)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"agent = zarr_dataset.agents[:10000000]\n\nprobabilities = agent['label_probabilities']\nlabel_indexes = np.argmax(probabilities, axis = 1)\ncounts = []\nfor idx_label, label in enumerate(PERCEPTION_LABELS):\n    counts.append(np.sum(label_indexes == idx_label))\n    \ntable = PrettyTable(field_names=['labels', 'counts'])\nfor count, label in zip(counts, PERCEPTION_LABELS):\n    table.add_row([label, count])\nprint(table)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Using EgoDataset to get dataset containing AV positions"},{"metadata":{"trusted":true},"cell_type":"code","source":"rast = build_rasterizer(cfg, dm)\ndataset = EgoDataset(cfg, zarr_dataset, rast)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualizing an AV"},{"metadata":{"trusted":true},"cell_type":"code","source":"data = dataset[1000]\n\nim = dataset.rasterizer.to_rgb(data['image'].T)\nsns.set_style('white')\nplt.figure(figsize = (10, 7))\nplt.imshow(im)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plotting the trajectories"},{"metadata":{"trusted":true},"cell_type":"code","source":"target_positon_pixels = transform_points(data['target_positions']+data['centroid'][:2], data['world_to_image'])\ndraw_trajectory(cv2.UMat(im), target_positon_pixels, data['target_yaws'], (120, 160, 189))\n\nplt.figure(figsize = (10, 7))\nplt.imshow(im)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Changing the map type to Satellite"},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg['raster_params']['map_type'] = 'py_satellite'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build a new rast object and a dataset object with new cfg"},{"metadata":{"trusted":true},"cell_type":"code","source":"sat_rast = build_rasterizer(cfg, dm)\nsat_dataset = EgoDataset(cfg, zarr_dataset, sat_rast)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sat_data = sat_dataset[1000]\nsat_im = sat_rast.to_rgb(sat_data['image'].T)\nplt.figure(figsize = (10, 7))\nplt.imshow(sat_im)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sat_target_positon_pixels = transform_points(sat_data['target_positions']+sat_data['centroid'], sat_data['world_to_image'])\ndraw_trajectory(cv2.UMat(sat_im), sat_target_positon_pixels, sat_data['target_yaws'], TARGET_POINTS_COLOR)\nplt.figure(figsize = (10, 7))\nplt.imshow(sat_im)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualizing an Agent"},{"metadata":{"trusted":true},"cell_type":"code","source":"agent_dataset = AgentDataset(cfg, zarr_dataset, rast)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (10, 7))\nagent_data = agent_dataset[1000]\nplt.imshow(rast.to_rgb(agent_data['image'].T))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_positon_pixels = transform_points(agent_data['target_positions']+agent_data['centroid'], agent_data['world_to_image'])\ndraw_trajectory(cv2.UMat(rast.to_rgb(agent_data['image'].T)), target_positon_pixels, agent_data['target_yaws'], [120, 122, 221])\nplt.figure(figsize = (10, 7))\nplt.imshow(rast.to_rgb(agent_data['image'].T))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sat_agent_dataset = AgentDataset(cfg, zarr_dataset, sat_rast)\nsat_agent_data = sat_agent_dataset[1000]\nsat_im = sat_rast.to_rgb(sat_agent_data['image'].T)\nplt.figure(figsize = (10, 7))\nplt.imshow(sat_im)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import IPython\nfrom IPython.display import display, clear_output\nimport PIL\n\nscene_idx = 2\nindexes = dataset.get_scene_indices(2)\nimages = []\n\nfor idx in indexes:\n    data = dataset[idx]\n    im = rast.to_rgb(data['image'].T)\n    clear_output(wait=True)\n    display(PIL.Image.fromarray(im))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images = []\nfrom matplotlib.animation import FuncAnimation\ndef animate_sol(images):\n    def animate(i):\n        im.set_data(images[i])\n    \n    fig, ax = plt.subplots()\n    im = ax.imshow(images[0])\n    fig.show()\n    return FuncAnimation(fig, animate, frames=len(images), interval=100)\n\nscene_idx = 2\nindexes = sat_dataset.get_scene_indices(scene_idx)\nimages = []\nfor idx in indexes:\n    data = sat_dataset[idx]\n    im = rast.to_rgb(data['image'].T)\n    clear_output(wait = True)\n    images.append(PIL.Image.fromarray(im))\nanim = animate_sol(images)\nHTML(anim.to_jshtml())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from torch.utils.data import DataLoader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dm = LocalDataManager(None)\n# train_cfg = cfg['train_data_loader']\n# rasterizer = build_rasterizer(cfg, dm)\n# train_zarr = ChunkedDataset(dm.require(cfg['train_data_loader']['key'])).open()\n# # train_mask = np.load('/kaggle/input/lyft-motion-prediction-autonomous-vehicles/scenes/mask.npz')['arr_0']\n# train_dataset = AgentDataset(cfg, train_zarr, rasterizer)\n# train_dataloader = DataLoader(train_dataset, shuffle=train_cfg[\"shuffle\"], batch_size=train_cfg[\"batch_size\"], \n#                              num_workers=train_cfg[\"num_workers\"])\n# print(train_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import torch\n# import torchvision\n# from torchvision import datasets, transforms\n# import torch.nn as nn\n# import torch.nn.functional as F\n# import torch.optim as optim\n# from torchvision.models.resnet import resnet18, resnet34","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class Net(nn.Module):\n    \n#     def __init__(self, cfg):\n#         super().__init__()\n        \n#         self.backbone = resnet18(pretrained=True, progress=True)\n        \n#         num_history_channels = (cfg[\"model_params\"][\"history_num_frames\"] + 1) * 2\n#         num_in_channels = 3 + num_history_channels\n\n#         self.backbone.conv1 = nn.Conv2d(\n#             num_in_channels,\n#             self.backbone.conv1.out_channels,\n#             kernel_size=self.backbone.conv1.kernel_size,\n#             stride=self.backbone.conv1.stride,\n#             padding=self.backbone.conv1.padding,\n#             bias=False,\n#         )\n        \n#         # This is 512 for resnet18 and resnet34;\n#         # And it is 2048 for the other resnets\n#         backbone_out_features = 512\n\n#         # X, Y coords for the future positions (output shape: Bx50x2)\n#         num_targets = 2 * cfg[\"model_params\"][\"future_num_frames\"]\n\n#         # You can add more layers here.\n#         self.head = nn.Sequential(\n#             # nn.Dropout(0.2),\n#             nn.Linear(in_features=backbone_out_features, out_features=4096),\n#         )\n\n#         self.logit = nn.Linear(4096, out_features=num_targets)\n        \n#     def forward(self, x):\n#         x = self.backbone.conv1(x)\n#         x = self.backbone.bn1(x)\n#         x = self.backbone.relu(x)\n#         x = self.backbone.maxpool(x)\n\n#         x = self.backbone.layer1(x)\n#         x = self.backbone.layer2(x)\n#         x = self.backbone.layer3(x)\n#         x = self.backbone.layer4(x)\n\n#         x = self.backbone.avgpool(x)\n#         x = torch.flatten(x, 1)\n        \n#         x = self.head(x)\n#         x = self.logit(x)\n        \n#         return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# net = Net(cfg)\n# net = net.to(device)\n# optimizer = optim.Adam(net.parameters(), lr=1e-3)\n\n# # Later we have to filter the invalid steps.\n# criterion = nn.MSELoss(reduction=\"none\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tr_it = iter(train_dataloader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_dataloader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# progress_bar = tqdm(range(cfg[\"train_params\"][\"max_num_steps\"]))\n# losses_train = []\n\n# for itr in progress_bar:\n\n#     try:\n#         data = next(tr_it)\n#     except StopIteration:\n#         tr_it = iter(train_dataloader)\n#         data = next(tr_it)\n\n#     net.train()\n#     torch.set_grad_enabled(True)\n    \n#     # Forward pass\n#     inputs = data[\"image\"].to(device)\n#     target_availabilities = data[\"target_availabilities\"].unsqueeze(-1).to(device)\n#     targets = data[\"target_positions\"].to(device)\n    \n#     outputs = net(inputs).reshape(targets.shape)\n#     loss = criterion(outputs, targets)\n\n#     # not all the output steps are valid, but we can filter them out from the loss using availabilities\n#     loss = loss * target_availabilities\n#     loss = loss.mean()\n\n#     # Backward pass\n#     optimizer.zero_grad()\n#     loss.backward()\n#     optimizer.step()\n\n#     losses_train.append(loss.item())\n\n#     if (itr+1) % cfg['train_params']['checkpoint_every_n_steps'] == 0 and not DEBUG:\n#         torch.save(model.state_dict(), f'model_state_{itr}.pth')\n    \n#     progress_bar.set_description(f\"loss: {loss.item()} loss(avg): {np.mean(losses_train[-100:])}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# torch.save(net.state_dict(), f'model_state_last.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DIR_INPUT = \"/kaggle/input/lyft-motion-prediction-autonomous-vehicles\"\n\n# SINGLE_MODE_SUBMISSION = f\"{DIR_INPUT}/single_mode_sample_submission.csv\"\n# MULTI_MODE_SUBMISSION = f\"{DIR_INPUT}/multi_mode_sample_submission.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cfg = {\n#     'format_version': 4,\n#     'model_params': {\n#         'history_num_frames': 10,\n#         'history_step_size': 1,\n#         'history_delta_time': 0.1,\n#         'future_num_frames': 50,\n#         'future_step_size': 1,\n#         'future_delta_time': 0.1\n#     },\n    \n#     'raster_params': {\n#         'raster_size': [224, 224],\n#         'pixel_size': [0.5, 0.5],\n#         'ego_center': [0.25, 0.5],\n#         'map_type': 'py_semantic',\n#         'satellite_map_key': 'aerial_map/aerial_map.png',\n#         'semantic_map_key': 'semantic_map/semantic_map.pb',\n#         'dataset_meta_key': 'meta.json',\n#         'filter_agents_threshold': 0.5\n#     },\n    \n#     'test_data_loader': {\n#         'key': 'scenes/test.zarr',\n#         'batch_size': 16,\n#         'shuffle': False,\n#         'num_workers': 4\n#     }\n\n# }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# os.environ[\"L5KIT_DATA_FOLDER\"] = DIR_INPUT\n# dm = LocalDataManager(None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_cfg = cfg[\"test_data_loader\"]\n\n# # Rasterizer\n# rasterizer = build_rasterizer(cfg, dm)\n\n# # Test dataset/dataloader\n# test_zarr = ChunkedDataset(dm.require(test_cfg[\"key\"])).open()\n# test_mask = np.load(\"/kaggle/input/lyft-motion-prediction-autonomous-vehicles/scenes/mask.npz\")[\"arr_0\"]\n# test_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)\n# test_dataloader = DataLoader(test_dataset,\n#                              shuffle=test_cfg[\"shuffle\"],\n#                              batch_size=test_cfg[\"batch_size\"],\n#                              num_workers=test_cfg[\"num_workers\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# net.eval()\n\n# future_coords_offsets_pd = []\n# timestamps = []\n# agent_ids = []\n\n# with torch.no_grad():\n#     dataiter = tqdm(test_dataloader )\n    \n#     for data in dataiter:\n\n#         inputs = data[\"image\"].to(device)\n#         target_availabilities = data[\"target_availabilities\"].unsqueeze(-1).to(device)\n#         targets = data[\"target_positions\"].to(device)\n\n#         outputs = net(inputs).reshape(targets.shape)\n        \n#         future_coords_offsets_pd.append(outputs.cpu().numpy().copy())\n#         timestamps.append(data[\"timestamp\"].numpy().copy())\n#         agent_ids.append(data[\"track_id\"].numpy().copy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# device","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# write_pred_csv('submission.csv',\n#                timestamps=np.concatenate(timestamps),\n#                track_ids=np.concatenate(agent_ids),\n#                coords=np.concatenate(future_coords_offsets_pd))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}