{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"pip install zarr","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\n\n## =====================================================================================\n## This is a temporarly fix for the freezing and the cuda issues. You can add this\n## utility script instead of kaggle_l5kit until Kaggle resolve these issues.\n## \n## You will be able to train and submit your results, but not all the functionality of\n## l5kit will work properly.\n\n## More details here:\n## https://www.kaggle.com/c/lyft-motion-prediction-autonomous-vehicles/discussion/177125\n\n## this script transports l5kit and dependencies\nos.system('pip install --target=/kaggle/working pymap3d==2.1.0')\nos.system('pip install --target=/kaggle/working protobuf==3.12.2')\nos.system('pip install --target=/kaggle/working transforms3d')\nos.system('pip install --target=/kaggle/working zarr')\nos.system('pip install --target=/kaggle/working ptable')\n\nos.system('pip install --no-dependencies --target=/kaggle/working l5kit')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import packages\nimport os, gc\nimport zarr\nimport numpy as np \nimport pandas as pd \nfrom tqdm import tqdm\nfrom typing import Dict\nfrom collections import Counter\nfrom prettytable import PrettyTable\n\n#level5 toolkit\nfrom l5kit.data import PERCEPTION_LABELS\nfrom l5kit.dataset import EgoDataset, AgentDataset\nfrom l5kit.data import ChunkedDataset, LocalDataManager\n\n# level5 toolkit \nfrom l5kit.configs import load_config_data\nfrom l5kit.geometry import transform_points\nfrom l5kit.rasterization import build_rasterizer\nfrom l5kit.visualization import draw_trajectory, draw_reference_trajectory, TARGET_POINTS_COLOR\nfrom l5kit.evaluation import write_pred_csv, compute_metrics_csv, read_gt_csv, create_chopped_dataset\n\n# visualization\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom matplotlib import animation\nfrom colorama import Fore, Back, Style\n\n# deep learning\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader\nfrom torchvision.models.resnet import resnet18, resnet50, resnet34\n\nimport time\n\n# check files in directory\nprint((os.listdir('../input/lyft-motion-prediction-autonomous-vehicles/')))\n\nplt.rc('animation', html='jshtml')\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = zarr.open(\"../input/lyft-motion-prediction-autonomous-vehicles/scenes/train.zarr\")\n\nvalidation = zarr.open(\"../input/lyft-motion-prediction-autonomous-vehicles/scenes/validate.zarr\")\n\ntest = zarr.open(\"../input/lyft-motion-prediction-autonomous-vehicles/scenes/test.zarr/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# set env variable for data\nos.environ[\"L5KIT_DATA_FOLDER\"] = \"../input/lyft-motion-prediction-autonomous-vehicles\"\n\n# get configuration yaml\ncfg = load_config_data(\"../input/lyft-config-files/visualisation_config.yaml\")\nprint(cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- Function utils ---\n# Original code from https://github.com/lyft/l5kit/blob/20ab033c01610d711c3d36e1963ecec86e8b85b6/l5kit/l5kit/evaluation/metrics.py\nimport numpy as np\n\nimport torch\nfrom torch import Tensor\n\n\ndef pytorch_neg_multi_log_likelihood_batch(\n    gt: Tensor, pred: Tensor, confidences: Tensor, avails: Tensor\n) -> Tensor:\n    \"\"\"\n    Compute a negative log-likelihood for the multi-modal scenario.\n    log-sum-exp trick is used here to avoid underflow and overflow, For more information about it see:\n    https://en.wikipedia.org/wiki/LogSumExp#log-sum-exp_trick_for_log-domain_calculations\n    https://timvieira.github.io/blog/post/2014/02/11/exp-normalize-trick/\n    https://leimao.github.io/blog/LogSumExp/\n    Args:\n        gt (Tensor): array of shape (bs)x(time)x(2D coords)\n        pred (Tensor): array of shape (bs)x(modes)x(time)x(2D coords)\n        confidences (Tensor): array of shape (bs)x(modes) with a confidence for each mode in each sample\n        avails (Tensor): array of shape (bs)x(time) with the availability for each gt timestep\n    Returns:\n        Tensor: negative log-likelihood for this example, a single float number\n    \"\"\"\n    assert len(pred.shape) == 4, f\"expected 3D (MxTxC) array for pred, got {pred.shape}\"\n    batch_size, num_modes, future_len, num_coords = pred.shape\n\n    assert gt.shape == (batch_size, future_len, num_coords), f\"expected 2D (Time x Coords) array for gt, got {gt.shape}\"\n    assert confidences.shape == (batch_size, num_modes), f\"expected 1D (Modes) array for gt, got {confidences.shape}\"\n    assert torch.allclose(torch.sum(confidences, dim=1), confidences.new_ones((batch_size,))), \"confidences should sum to 1\"\n    assert avails.shape == (batch_size, future_len), f\"expected 1D (Time) array for gt, got {avails.shape}\"\n    # assert all data are valid\n    assert torch.isfinite(pred).all(), \"invalid value found in pred\"\n    assert torch.isfinite(gt).all(), \"invalid value found in gt\"\n    assert torch.isfinite(confidences).all(), \"invalid value found in confidences\"\n    assert torch.isfinite(avails).all(), \"invalid value found in avails\"\n\n    # convert to (batch_size, num_modes, future_len, num_coords)\n    gt = torch.unsqueeze(gt, 1)  # add modes\n    avails = avails[:, None, :, None]  # add modes and cords\n\n    # error (batch_size, num_modes, future_len)\n    error = torch.sum(((gt - pred) * avails) ** 2, dim=-1)  # reduce coords and use availability\n\n    with np.errstate(divide=\"ignore\"):  # when confidence is 0 log goes to -inf, but we're fine with it\n        # error (batch_size, num_modes)\n        error = torch.log(confidences) - 0.5 * torch.sum(error, dim=-1)  # reduce time\n\n    # use max aggregator on modes for numerical stability\n    # error (batch_size, num_modes)\n    max_value, _ = error.max(dim=1, keepdim=True)  # error are negative at this point, so max() gives the minimum one\n    error = -torch.log(torch.sum(torch.exp(error - max_value), dim=-1, keepdim=True)) - max_value  # reduce modes\n    # print(\"error\", error)\n    return torch.mean(error)\n\n\ndef pytorch_neg_multi_log_likelihood_single(\n    gt: Tensor, pred: Tensor, avails: Tensor\n) -> Tensor:\n    \"\"\"\n\n    Args:\n        gt (Tensor): array of shape (bs)x(time)x(2D coords)\n        pred (Tensor): array of shape (bs)x(time)x(2D coords)\n        avails (Tensor): array of shape (bs)x(time) with the availability for each gt timestep\n    Returns:\n        Tensor: negative log-likelihood for this example, a single float number\n    \"\"\"\n    # pred (bs)x(time)x(2D coords) --> (bs)x(mode=1)x(time)x(2D coords)\n    # create confidence (bs)x(mode=1)\n    batch_size, future_len, num_coords = pred.shape\n    confidences = pred.new_ones((batch_size, 1))\n    return pytorch_neg_multi_log_likelihood_batch(gt, pred.unsqueeze(1), confidences, avails)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dm = LocalDataManager()\ndataset_path = dm.require(cfg[\"val_data_loader\"][\"key\"])\nzarr_dataset = ChunkedDataset(dataset_path)\nzarr_dataset.open()\nprint(zarr_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = True\n\n# training cfg\ntraining_cfg = {\n    \n    'format_version': 4,\n    'data_path': \"/kaggle/input/lyft-motion-prediction-autonomous-vehicles\",\n    \n     ## Model options\n    'model_params': {\n        'model_architecture': 'resnet34',\n        'history_num_frames': 10,\n        'history_step_size': 1,\n        'history_delta_time': 0.1,\n        'future_num_frames': 50,\n        'future_step_size': 1,\n        'future_delta_time': 0.1,\n        'model_name': \"model_resnet34_output\",\n        'lr': 1e-3,\n        'train': False,\n        'predict': True\n    },\n\n    ## Input raster parameters\n    'raster_params': {\n        \n        'raster_size': [224, 224], # raster's spatial resolution [meters per pixel]: the size in the real world one pixel corresponds to.\n        'pixel_size': [0.5, 0.5], # From 0 to 1 per axis, [0.5,0.5] would show the ego centered in the image.\n        'ego_center': [0.25, 0.5],\n        'map_type': \"py_semantic\",\n        \n        # the keys are relative to the dataset environment variable\n        'satellite_map_key': \"aerial_map/aerial_map.png\",\n        'semantic_map_key': \"semantic_map/semantic_map.pb\",\n        'dataset_meta_key': \"meta.json\",\n\n        # e.g. 0.0 include every obstacle, 0.5 show those obstacles with >0.5 probability of being\n        # one of the classes we care about (cars, bikes, peds, etc.), >=1.0 filter all other agents.\n        'filter_agents_threshold': 0.5\n    },\n\n    ## Data loader options\n    'train_data_loader': {\n        'key': \"scenes/train.zarr\",\n        'batch_size': 12,\n        'shuffle': True,\n        'num_workers': 4\n    },\n\n    ## Train params\n    'train_params': {\n        'checkpoint_every_n_steps': 5000,\n        'max_num_steps': 100 if DEBUG else 10000\n    }\n}\n\n# inference cfg\ninference_cfg = {\n    \n    'format_version': 4,\n    'model_params': {\n        'history_num_frames': 10,\n        'history_step_size': 1,\n        'history_delta_time': 0.1,\n        'future_num_frames': 50,\n        'future_step_size': 1,\n        'future_delta_time': 0.1\n    },\n    \n    'raster_params': {\n        'raster_size': [300, 300],\n        'pixel_size': [0.5, 0.5],\n        'ego_center': [0.25, 0.5],\n        'map_type': 'py_semantic',\n        'satellite_map_key': 'aerial_map/aerial_map.png',\n        'semantic_map_key': 'semantic_map/semantic_map.pb',\n        'dataset_meta_key': 'meta.json',\n        'filter_agents_threshold': 0.5\n    },\n    \n        'test_data_loader': {\n        'key': 'scenes/test.zarr',\n        'batch_size': 8,\n        'shuffle': False,\n        'num_workers': 4\n    }\n\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# root directory\nDIR_INPUT = \"/kaggle/input/lyft-motion-prediction-autonomous-vehicles\"\n\n#submission\nSINGLE_MODE_SUBMISSION = f\"{DIR_INPUT}/single_mode_sample_submission.csv\"\nMULTI_MODE_SUBMISSION = f\"{DIR_INPUT}/multi_mode_sample_submission.csv\"\n\n# set env variable for data\nos.environ[\"L5KIT_DATA_FOLDER\"] = DIR_INPUT\ndm = LocalDataManager(None)\nprint(training_cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# training cfg\ntrain_cfg = training_cfg[\"train_data_loader\"]\n\n# rasterizer\nrasterizer = build_rasterizer(training_cfg, dm)\n\n# dataloader\ntrain_zarr = ChunkedDataset(dm.require(train_cfg[\"key\"])).open()\ntrain_dataset = AgentDataset(training_cfg, train_zarr, rasterizer)\ntrain_dataloader = DataLoader(train_dataset, shuffle=train_cfg[\"shuffle\"], batch_size=train_cfg[\"batch_size\"], \n                             num_workers=train_cfg[\"num_workers\"])\nprint(train_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#====== INIT TEST DATASET=============================================================\ntest_cfg = inference_cfg[\"test_data_loader\"]\nrasterizer = build_rasterizer(cfg, dm)\ntest_zarr = ChunkedDataset(dm.require(test_cfg[\"key\"])).open()\ntest_mask = np.load(f\"{DIR_INPUT}/scenes/mask.npz\")[\"arr_0\"]\ntest_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)\ntest_dataloader = DataLoader(test_dataset,shuffle=test_cfg[\"shuffle\"],batch_size=test_cfg[\"batch_size\"],\n                             num_workers=test_cfg[\"num_workers\"])\nprint(\"==================================TEST DATA==================================\")\nprint(test_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LyftModel(nn.Module):\n    \n    def __init__(self, cfg):\n        super().__init__()\n        \n        # set pretrained=True while training\n        self.backbone = resnet34(pretrained=False) \n        \n        num_history_channels = (cfg[\"model_params\"][\"history_num_frames\"] + 1) * 2\n        num_in_channels = 3 + num_history_channels\n\n        self.backbone.conv1 = nn.Conv2d(\n            num_in_channels,\n            self.backbone.conv1.out_channels,\n            kernel_size=self.backbone.conv1.kernel_size,\n            stride=self.backbone.conv1.stride,\n            padding=self.backbone.conv1.padding,\n            bias=False,\n        )\n        \n        # This is 512 for resnet18 and resnet34;\n        # And it is 2048 for the other resnets\n        backbone_out_features = 512\n        \n        # X, Y coords for the future positions (output shape: Bx50x2)\n        num_targets = 2 * cfg[\"model_params\"][\"future_num_frames\"]\n\n        # You can add more layers here.\n        self.head = nn.Sequential(\n            # nn.Dropout(0.2),\n            nn.Linear(in_features=backbone_out_features, out_features=4096),\n        )\n\n        self.logit = nn.Linear(4096, out_features=num_targets)\n        \n    def forward(self, x):\n        x = self.backbone.conv1(x)\n        x = self.backbone.bn1(x)\n        x = self.backbone.relu(x)\n        x = self.backbone.maxpool(x)\n\n        x = self.backbone.layer1(x)\n        x = self.backbone.layer2(x)\n        x = self.backbone.layer3(x)\n        x = self.backbone.layer4(x)\n\n        x = self.backbone.avgpool(x)\n        x = torch.flatten(x, 1)\n        \n        x = self.head(x)\n        x = self.logit(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# compiling model\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel = LyftModel(training_cfg).to(device)\noptimizer = optim.Adam(model.parameters(), lr=1e-3)\ncriterion = nn.MSELoss(reduction=\"none\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_it = iter(train_dataloader)\nprogress_bar = tqdm(range(training_cfg[\"train_params\"][\"max_num_steps\"]))\n\nlosses_train = []\n\nfor _ in progress_bar:\n    try:\n        data = next(tr_it)\n    except StopIteration:\n        tr_it = iter(train_dataloader)\n        data = next(tr_it)\n    model.train()\n    torch.set_grad_enabled(True)\n    \n    # forward pass\n    inputs = data[\"image\"].to(device)\n    target_availabilities = data[\"target_availabilities\"].unsqueeze(-1).to(device)\n    targets = data[\"target_positions\"].to(device)\n    \n    outputs = model(inputs).reshape(targets.shape)\n    loss = criterion(outputs, targets)\n\n    # not all the output steps are valid, but we can filter them out from the loss using availabilities\n    loss = loss * target_availabilities\n    loss = loss.mean()\n    # Backward pass\n    optimizer.zero_grad()\n    loss.backward()\n    optimizer.step()\n\n    losses_train.append(loss.item())\n\nif (itr+1) % cfg['train_params']['checkpoint_every_n_steps'] == 0:\n    torch.save({'model_state_dict': model.state_dict()},\n               f'resnet18_300x300_model_state_{itr}.pth')\n\nprogress_bar.set_description(f\"loss: {loss.item()} loss(avg): {np.mean(losses_train[-100:])}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f1 = plt.figure(1)\nplt.plot(losses_train, label = 'Training Loss')\nplt.title('Loss vs. Epochs Graph')\nplt.xlabel('Epochs')\nplt.ylabel('Loss & Accuracy')\nplt.legend(loc= 'upper right')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# save full trained model\ntorch.save(model.state_dict(), f'model_state_last.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_cfg = inference_cfg[\"test_data_loader\"]\n\n# Rasterizer\nrasterizer = build_rasterizer(inference_cfg, dm)\n\n# Test dataset/dataloader\ntest_zarr = ChunkedDataset(dm.require(test_cfg[\"key\"])).open()\ntest_mask = np.load(f\"{DIR_INPUT}/scenes/mask.npz\")[\"arr_0\"]\ntest_dataset = AgentDataset(inference_cfg, test_zarr, rasterizer, agents_mask=test_mask)\ntest_dataloader = DataLoader(test_dataset,\n                             shuffle=test_cfg[\"shuffle\"],\n                             batch_size=test_cfg[\"batch_size\"],\n                             num_workers=test_cfg[\"num_workers\"])\n\n\nprint(test_dataloader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"WEIGHT_FILE = '/kaggle/input/lyft-l5-weights/model_state_last.pth'\nmodel_state = torch.load(WEIGHT_FILE, map_location=device)\nmodel.load_state_dict(model_state)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}