{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install --upgrade pip\n!pip install pymap3d==2.1.0\n!pip install -U l5kit\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nimport numpy as np\n\nfrom l5kit.data import ChunkedDataset, LocalDataManager\nfrom l5kit.dataset import EgoDataset, AgentDataset\n\nfrom l5kit.rasterization import build_rasterizer\nfrom l5kit.configs import load_config_data\nfrom l5kit.visualization import draw_trajectory, TARGET_POINTS_COLOR\nfrom l5kit.geometry import transform_points\nfrom tqdm import tqdm\nfrom collections import Counter\nfrom l5kit.data import PERCEPTION_LABELS\nfrom prettytable import PrettyTable\nimport os\nPATH_TO_DATA = \"../input/lyft-motion-prediction-autonomous-vehicles\"\nos.environ[\"L5KIT_DATA_FOLDER\"] = PATH_TO_DATA","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg = {\n    'format_version': 4,\n    'model_params': {\n        'model_architecture': 'resnet50',\n        \n        'history_num_frames': 0,\n        'history_step_size': 1,\n        'history_delta_time': 0.1,\n        \n        'future_num_frames': 50,\n        'future_step_size': 1,\n        'future_delta_time': 0.1\n    },\n    \n    'raster_params': {\n        'raster_size': [224, 224],\n        'pixel_size': [0.5, 0.5],\n        'ego_center': [0.25, 0.5],\n        'map_type': 'py_semantic',\n        'satellite_map_key': 'aerial_map/aerial_map.png',\n        'semantic_map_key': 'semantic_map/semantic_map.pb',\n        'dataset_meta_key': 'meta.json',\n        'filter_agents_threshold': 0.5\n    },\n    \n    'train_data_loader': {\n        'key': 'scenes/train.zarr',\n        'batch_size': 32,\n        'shuffle': True,\n        'num_workers': 4\n    },\n    \n    'val_data_loader': {\n        'key': 'scenes/validate.zarr',\n        'batch_size': 12,\n        'shuffle': False,\n        'num_workers': 4\n    },\n    \n    'test_data_loader': {\n        'key': 'scenes/test.zarr',\n        'batch_size': 8,\n        'shuffle': False,\n        'num_workers': 4\n    },\n    \n    'train_params': {\n        'checkpoint_every_n_steps': 5000,\n        'max_num_steps': 1000,\n        \n    }\n}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"zarr_loc = cfg[\"train_data_loader\"][\"key\"]\n\ndm = LocalDataManager()\ntrain_zarr = ChunkedDataset(dm.require(zarr_loc)).open()\n\n#let's see what one of the objects looks like\nprint(train_zarr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Let us visualize the EGO DATASET","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rast = build_rasterizer(cfg, dm)\n#Rasterisation is one of the typical techniques of rendering 3D models\ndataset = EgoDataset(cfg, train_zarr , rast)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#For visualizing the Av. Av stands for Autonomous Vehicle\n#Let's get a sample from the dataset and use our rasterizer to get an RGB image we can plot","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = dataset[70]\n\nim = data['image'].transpose(1, 2, 0)\nim = dataset.rasterizer.to_rgb(im)\ntarget_positions_pixels = transform_points(data[\"target_positions\"] + data[\"centroid\"][:2], data[\"world_to_image\"])\ndraw_trajectory(im, target_positions_pixels, data[\"target_yaws\"], TARGET_POINTS_COLOR)\n\nplt.imshow(im[::-1])\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# FROM WHAT I CAN DEDUCE THE RED BOX YOU SEE IS THE intersection entry wait line for ego car\n#If it is Yellow means an intersection entry wait line for other cars while Green is intersection exit line. ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Let Visualize other images|","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = dataset[30]\n\nim = data['image'].transpose(1, 2, 0)\nim = dataset.rasterizer.to_rgb(im)\ntarget_positions_pixels = transform_points(data[\"target_positions\"] + data[\"centroid\"][:2], data[\"world_to_image\"])\ndraw_trajectory(im, target_positions_pixels, data[\"target_yaws\"], TARGET_POINTS_COLOR)\n\nplt.imshow(im[::-1])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Let me try visualizing image through another rasterizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg[\"raster_params\"][\"map_type\"] = \"py_satellite\"\nrast = build_rasterizer(cfg, dm)\ndataset = EgoDataset(cfg, train_zarr, rast)\ndata = dataset[2]\n\nim = data[\"image\"].transpose(1, 2, 0)\nim = dataset.rasterizer.to_rgb(im)\ntarget_positions_pixels = transform_points(data[\"target_positions\"] + data[\"centroid\"][:2], data[\"world_to_image\"])\ndraw_trajectory(im, target_positions_pixels, data[\"target_yaws\"], TARGET_POINTS_COLOR)\n\nplt.imshow(im[::-1])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#There are two ways in which our self driving car could go it can either continue going,\n# straight or it can make a turn to the right hand side","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Visualizing an agent\ndataset = AgentDataset(cfg, train_zarr, rast)\ndata = dataset[0]\n\nim = data[\"image\"].transpose(1, 2, 0)\nim = dataset.rasterizer.to_rgb(im)\ntarget_positions_pixels = transform_points(data[\"target_positions\"] + data[\"centroid\"][:2], data[\"world_to_image\"])\ndraw_trajectory(im, target_positions_pixels, data[\"target_yaws\"], TARGET_POINTS_COLOR)\n\nplt.imshow(im[::-1])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import display, clear_output\nfrom IPython.display import HTML\n\nimport PIL\nimport matplotlib.pyplot as plt\nfrom matplotlib import animation, rc\ndef animate_solution(images):\n\n    def animate(i):\n        im.set_data(images[i])\n \n    fig, ax = plt.subplots()\n    im = ax.imshow(images[0])\n    \n    return animation.FuncAnimation(fig, animate, frames=len(images), interval=60)\ncfg[\"raster_params\"][\"map_type\"] = \"py_satellite\"\nrast = build_rasterizer(cfg, dm)\ndataset = EgoDataset(cfg, train_zarr, rast)\nscene_idx = 34\nindexes = dataset.get_scene_indices(scene_idx)\nimages = []\n\nfor idx in indexes:\n    \n    data = dataset[idx]\n    im = data[\"image\"].transpose(1, 2, 0)\n    im = dataset.rasterizer.to_rgb(im)\n    target_positions_pixels = transform_points(data[\"target_positions\"] + data[\"centroid\"][:2], data[\"world_to_image\"])\n    center_in_pixels = np.asarray(cfg[\"raster_params\"][\"ego_center\"]) * cfg[\"raster_params\"][\"raster_size\"]\n    draw_trajectory(im, target_positions_pixels, data[\"target_yaws\"], TARGET_POINTS_COLOR)\n    clear_output(wait=True)\n    images.append(PIL.Image.fromarray(im[::-1]))\nanim = animate_solution(images)\nHTML(anim.to_jshtml())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#STILL WORKING ON THIS NOTEBOOK. NOTEBOOK YET TO BE COMPLETED PLEASE STAY TUNE FOR MORE \n#THANK YOU","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import display, clear_output\nfrom IPython.display import HTML\n\nimport PIL\nimport matplotlib.pyplot as plt\nfrom matplotlib import animation, rc\ndef animate_solution(images):\n\n    def animate(i):\n        im.set_data(images[i])\n \n    fig, ax = plt.subplots()\n    im = ax.imshow(images[0])\n    \n    return animation.FuncAnimation(fig, animate, frames=len(images), interval=60)\ncfg[\"raster_params\"][\"map_type\"] = \"py_satellite\"\nrast = build_rasterizer(cfg, dm)\ndataset = EgoDataset(cfg, train_zarr, rast)\nscene_idx = 60\nindexes = dataset.get_scene_indices(scene_idx)\nimages = []\n\nfor idx in indexes:\n    \n    data = dataset[idx]\n    im = data[\"image\"].transpose(1, 2, 0)\n    im = dataset.rasterizer.to_rgb(im)\n    target_positions_pixels = transform_points(data[\"target_positions\"] + data[\"centroid\"][:2], data[\"world_to_image\"])\n    center_in_pixels = np.asarray(cfg[\"raster_params\"][\"ego_center\"]) * cfg[\"raster_params\"][\"raster_size\"]\n    draw_trajectory(im, target_positions_pixels, data[\"target_yaws\"], TARGET_POINTS_COLOR)\n    clear_output(wait=True)\n    images.append(PIL.Image.fromarray(im[::-1]))\nanim = animate_solution(images)\nHTML(anim.to_jshtml())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Loading the train dataset which for me is the train_zarr\n#we will load the train_zarr into a chunkdataset object.\n#The object has a reference to the different arrays into the zarr e.g the agent and the traffic light\n#we will pass the agent dataset into the torchdataLoader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import DataLoader\ntrain_cfg = cfg['train_data_loader']\nrasterizer = build_rasterizer(cfg, dm)\ntrain_zarr = ChunkedDataset(dm.require(train_cfg['key'])).open()\ntrain_dataset = AgentDataset(cfg, train_zarr, rasterizer)\ntrain_dataloader = DataLoader(train_dataset, shuffle=train_cfg['shuffle'], batch_size=train_cfg['batch_size'], \n                             num_workers=train_cfg['num_workers'])\nprint(train_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch import nn\nfrom torchvision.models.resnet import resnet18\nfrom typing import Dict\n\nclass LyftModel(nn.Module):\n    \n    def __init__(self, cfg: Dict):\n        super().__init__()\n        \n        self.backbone = resnet18(pretrained=True, progress=True)\n        \n        num_history_channels = (cfg[\"model_params\"][\"history_num_frames\"] + 1) * 2\n        num_in_channels = 3 + num_history_channels\n\n        self.backbone.conv1 = nn.Conv2d(\n            num_in_channels,\n            self.backbone.conv1.out_channels,\n            kernel_size=self.backbone.conv1.kernel_size,\n            stride=self.backbone.conv1.stride,\n            padding=self.backbone.conv1.padding,\n            bias=False,\n        )\n        \n        # This is 512 for resnet18 and resnet34;\n        # And it is 2048 for the other resnets\n        backbone_out_features = 512\n\n        # X, Y coords for the future positions (output shape: Bx50x2)\n        num_targets = 2 * cfg[\"model_params\"][\"future_num_frames\"]\n\n        # You can add more layers here.\n        self.head = nn.Sequential(\n            # nn.Dropout(0.2),\n            nn.Linear(in_features=backbone_out_features, out_features=4096),\n        )\n\n        self.logit = nn.Linear(4096, out_features=num_targets)\n        \n    def forward(self, x):\n        x = self.backbone.conv1(x)\n        x = self.backbone.bn1(x)\n        x = self.backbone.relu(x)\n        x = self.backbone.maxpool(x)\n\n        x = self.backbone.layer1(x)\n        x = self.backbone.layer2(x)\n        x = self.backbone.layer3(x)\n        x = self.backbone.layer4(x)\n\n        x = self.backbone.avgpool(x)\n        x = torch.flatten(x, 1)\n        \n        x = self.head(x)\n        x = self.logit(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#I WILL INITIALIZE THE MODEL HERE AND WE WILL USE THE ADAM OPTIMIZER\n#WE WILL USE THE ADAM OPTIMIZER BECAUSE IT IS GOOD FOR LARGE DATASET","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch import optim\nfrom tqdm import tqdm\n\nimport torch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\nmodel = LyftModel(cfg).to(device)\noptimizer = optim.Adam(model.parameters(), lr=1e-3)\ncriterion = nn.MSELoss(reduction=\"none\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#TRAIN LOOP","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_it = iter(train_dataloader)\nprogress_bar = tqdm(range(cfg['train_params']['max_num_steps']))\nlosses_train = []\nfor _ in progress_bar:\n    try:\n        data = next(tr_it)\n    except StopIteration:\n        tr_it = iter(train_dataloader)\n        data = next(tr_it)\n        \n    model.train()\n    torch.set_grad_enabled(True)\n    #THE INPUTS IMAGES AND THE TARGET POSITIONS     \n    inputs = data[\"image\"].to(device)\n    target_availabilities = data[\"target_availabilities\"].unsqueeze(-1).to(device)\n    targets = data[\"target_positions\"].to(device)\n    loss = criterion(outputs, targets)\n    \n    optimizer.zero_grad()\n    loss.backward()\n    optimizer.step()\n\n    losses_train.append(loss.item())\n    progress_bar.set_description(f\"loss: {loss.item()} loss(avg): {np.mean(losses_train)}\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#STILL WORKING ON THIS NOTEBOOK. WILL BE COMPLETED SOON, STAY TUNED. THANKS FOR VIEWING ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}