{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\n\n## =====================================================================================\n## This is a temporarly fix for the freezing and the cuda issues. You can add this\n## utility script instead of kaggle_l5kit until Kaggle resolve these issues.\n## \n## You will be able to train and submit your results, but not all the functionality of\n## l5kit will work properly.\n\n## More details here:\n## https://www.kaggle.com/c/lyft-motion-prediction-autonomous-vehicles/discussion/177125\n\n## this script transports l5kit and dependencies\nos.system('pip install --target=/kaggle/working pymap3d==2.1.0')\nos.system('pip install --target=/kaggle/working protobuf==3.12.2')\nos.system('pip install --target=/kaggle/working transforms3d')\nos.system('pip install --target=/kaggle/working zarr')\nos.system('pip install --target=/kaggle/working ptable')\n\nos.system('pip install --no-dependencies --target=/kaggle/working l5kit')\nos.system('pip install timm')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# import packages\nimport os, gc\nimport zarr\nimport numpy as np \nimport pandas as pd \nimport timm\nfrom tqdm import tqdm\nfrom typing import Dict\nfrom collections import Counter\nfrom prettytable import PrettyTable\nfrom collections import OrderedDict\nimport math\nimport pickle\n\n#level5 toolkit\nfrom l5kit.data import PERCEPTION_LABELS\nfrom l5kit.dataset import EgoDataset, AgentDataset\nfrom l5kit.data import ChunkedDataset, LocalDataManager\n\n# level5 toolkit \nfrom l5kit.configs import load_config_data\nfrom l5kit.geometry import transform_points\nfrom l5kit.rasterization import build_rasterizer\nfrom l5kit.visualization import draw_trajectory, draw_reference_trajectory, TARGET_POINTS_COLOR, PREDICTED_POINTS_COLOR, write_gif\nfrom l5kit.evaluation import write_pred_csv, compute_metrics_csv, read_gt_csv, create_chopped_dataset, export_zarr_to_csv, write_gt_csv\nfrom l5kit.evaluation.metrics import neg_multi_log_likelihood, time_displace\n\n# visualization\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom matplotlib import animation\nfrom colorama import Fore, Back, Style\n\n# deep learning\nimport torch\nfrom torch import nn, optim, Tensor\nfrom torch.utils.data import DataLoader\nfrom torchvision.models.resnet import resnet18, resnet50, resnet34\nfrom torchvision.models.mobilenet import mobilenet_v2\nfrom torch.nn import functional as F\n# check files in directory\nprint((os.listdir('../input/lyft-motion-prediction-autonomous-vehicles/')))\n\nplt.rc('animation', html='jshtml')\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = False\n\n# training cfg\ntraining_cfg = {\n    \n    'format_version': 4,\n    \n     ## Model options\n    'model_params': {\n        'model_architecture': 'resnet34',\n        'history_num_frames': 10,\n        'history_step_size': 1,\n        'history_delta_time': 0.1,\n        'future_num_frames': 50,\n        'future_step_size': 1,\n        'future_delta_time': 0.1,\n    },\n\n    ## Input raster parameters\n    'raster_params': {\n        \n        'raster_size': [300, 300], # raster's spatial resolution [meters per pixel]: the size in the real world one pixel corresponds to.\n        'pixel_size': [0.5, 0.5], # From 0 to 1 per axis, [0.5,0.5] would show the ego centered in the image.\n        'ego_center': [0.25, 0.5],\n        'map_type': \"py_semantic\",\n        \n        # the keys are relative to the dataset environment variable\n        'satellite_map_key': \"aerial_map/aerial_map.png\",\n        'semantic_map_key': \"semantic_map/semantic_map.pb\",\n        'dataset_meta_key': \"meta.json\",\n\n        # e.g. 0.0 include every obstacle, 0.5 show those obstacles with >0.5 probability of being\n        # one of the classes we care about (cars, bikes, peds, etc.), >=1.0 filter all other agents.\n        'filter_agents_threshold': 0.5,\n        'disable_traffic_light_faces': False\n    },\n\n    ## Data loader options\n    'train_data_loader': {\n        'key': \"scenes/train.zarr\",\n        'batch_size': 12,\n        'shuffle': True,\n        'num_workers': 4\n    },\n\n    ## Train params\n    'train_params': {\n        'checkpoint_every_n_steps': 5000,\n        'max_num_steps': 100 if DEBUG else 12500\n    }\n}\n\n# inference cfg\ninference_cfg = {\n    \n    'format_version': 4,\n    'model_params': {\n        'history_num_frames': 10,\n        'history_step_size': 1,\n        'history_delta_time': 0.1,\n        'future_num_frames': 50,\n        'future_step_size': 1,\n        'future_delta_time': 0.1\n    },\n    \n    'raster_params': {\n        'raster_size': [300, 300],\n        'pixel_size': [0.5, 0.5],\n        'ego_center': [0.25, 0.5],\n        'map_type': 'py_semantic',\n        'satellite_map_key': 'aerial_map/aerial_map.png',\n        'semantic_map_key': 'semantic_map/semantic_map.pb',\n        'dataset_meta_key': 'meta.json',\n        'filter_agents_threshold': 0.5\n    },\n    \n        'test_data_loader': {\n        'key': 'scenes/test.zarr',\n        'batch_size': 8,\n        'shuffle': False,\n        'num_workers': 4\n    }\n\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class LyftModel(nn.Module):\n    \n#     def __init__(self, cfg):\n#         super().__init__()\n        \n#         # set pretrained=True while training\n#         self.backbone = resnet50(pretrained=True) \n        \n#         num_history_channels = (cfg[\"model_params\"][\"history_num_frames\"] + 1) * 2\n#         num_in_channels = 3 + num_history_channels\n\n#         self.backbone.conv1 = nn.Conv2d(\n#             num_in_channels,\n#             self.backbone.conv1.out_channels,\n#             kernel_size=self.backbone.conv1.kernel_size,\n#             stride=self.backbone.conv1.stride,\n#             padding=self.backbone.conv1.padding,\n#             bias=False,\n#         )\n        \n#         # This is 512 for resnet18 and resnet34;\n#         # And it is 2048 for the other resnets\n#         backbone_out_features = 2048\n        \n#         # X, Y coords for the future positions (output shape: Bx50x2)\n#         num_targets = 2 * cfg[\"model_params\"][\"future_num_frames\"]\n\n#         # You can add more layers here.\n#         self.head = nn.Sequential(\n#             # nn.Dropout(0.2),\n#             nn.Linear(in_features=backbone_out_features, out_features=4096),\n#         )\n\n#         self.logit = nn.Linear(4096, out_features=num_targets)\n        \n#     def forward(self, x):\n#         x = self.backbone.conv1(x)\n#         x = self.backbone.bn1(x)\n#         x = self.backbone.relu(x)\n#         x = self.backbone.maxpool(x)\n\n#         x = self.backbone.layer1(x)\n#         x = self.backbone.layer2(x)\n#         x = self.backbone.layer3(x)\n#         x = self.backbone.layer4(x)\n\n#         x = self.backbone.avgpool(x)\n#         x = torch.flatten(x, 1)\n        \n#         x = self.head(x)\n#         x = self.logit(x)\n        \n#         return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LyftModel(nn.Module):\n    \n    def __init__(self, cfg):\n        super().__init__()\n        \n        # set pretrained=True while training\n        self.backbone = timm.create_model('mixnet_xl',pretrained=True)\n        \n        num_history_channels = (cfg[\"model_params\"][\"history_num_frames\"] + 1) * 2\n        num_in_channels = 3 + num_history_channels\n\n        self.backbone.conv_stem = nn.Conv2d(\n            num_in_channels,\n            self.backbone.conv_stem.out_channels,\n            kernel_size=self.backbone.conv_stem.kernel_size,\n            stride=self.backbone.conv_stem.stride,\n            padding=self.backbone.conv_stem.padding,\n            bias=False,\n        )\n        \n        # This is 512 for resnet18 and resnet34;\n        # And it is 2048 for the other resnets\n        backbone_out_features = 1536\n        \n        # X, Y coords for the future positions (output shape: Bx50x2)\n        num_targets = 2 * cfg[\"model_params\"][\"future_num_frames\"]\n\n        # You can add more layers here.\n        self.head = nn.Sequential(\n            # nn.Dropout(0.2),\n            nn.Linear(in_features=backbone_out_features, out_features=4096),\n        )\n\n        self.logit = nn.Linear(4096, out_features=num_targets)\n        \n    def forward(self, x):\n        x = self.backbone.conv_stem(x)\n        x = self.backbone.bn1(x)\n        x = self.backbone.act1(x)\n        \n        x = self.backbone.blocks(x)\n\n        x = self.backbone.conv_head(x)\n        x = self.backbone.bn2(x)\n        x = self.backbone.act2(x)\n        x = self.backbone.global_pool(x)\n        x = torch.flatten(x, 1)\n        \n        x = self.head(x)\n        x = self.logit(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# compiling model\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel = LyftModel(training_cfg).to(device)\noptimizer = optim.Adam(model.parameters(), lr=1e-3)\n# criterion = nn.MSELoss(reduction=\"none\")\ncriterion = nn.SmoothL1Loss(reduction=\"none\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# validate cfg\nvalidate_cfg = {\n    \n    'format_version': 4,\n    'model_params': {\n        'history_num_frames': 10,\n        'history_step_size': 1,\n        'history_delta_time': 0.1,\n        'future_num_frames': 50,\n        'future_step_size': 1,\n        'future_delta_time': 0.1\n    },\n    \n    'raster_params': {\n        'raster_size': [300, 300],\n        'pixel_size': [0.5, 0.5],\n        'ego_center': [0.25, 0.5],\n        'map_type': 'py_semantic',\n        'satellite_map_key': 'aerial_map/aerial_map.png',\n        'semantic_map_key': 'semantic_map/semantic_map.pb',\n        'dataset_meta_key': 'meta.json',\n        'filter_agents_threshold': 0.5,\n        'disable_traffic_light_faces': False\n    },\n    \n    'validate_data_loader': {\n    'key': 'scenes/validate.zarr',\n    'batch_size': 8,\n    'shuffle': False,\n    'num_workers': 4\n    }\n\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# root directory\nDIR_INPUT = \"/kaggle/input/lyft-motion-prediction-autonomous-vehicles\"\n\n#submission\nSINGLE_MODE_SUBMISSION = f\"{DIR_INPUT}/single_mode_sample_submission.csv\"\nMULTI_MODE_SUBMISSION = f\"{DIR_INPUT}/multi_mode_sample_submission.csv\"\n\n# set env variable for data\nos.environ[\"L5KIT_DATA_FOLDER\"] = DIR_INPUT\ndm = LocalDataManager(None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# validation configuration\nvalid_cfg = validate_cfg[\"validate_data_loader\"]\n\n# Rasterizer\nrasterizer = build_rasterizer(validate_cfg, dm)\n\n# Validation dataset/dataloader\nvalid_zarr = ChunkedDataset(dm.require(valid_cfg[\"key\"])).open()\nvalid_dataset = AgentDataset(validate_cfg, valid_zarr, rasterizer)\nwhole_size = valid_dataset.__len__()\nvalid_dataset_use, valid_dataset_valid, _ = torch.utils.data.random_split(valid_dataset, [70000, 5000, whole_size-75000], generator=torch.Generator().manual_seed(42))\n\nvalid_dataloader = DataLoader(valid_dataset_use,\n                             shuffle=valid_cfg[\"shuffle\"],\n                             batch_size=valid_cfg[\"batch_size\"],\n                             num_workers=valid_cfg[\"num_workers\"])\n\nvalid_dataloader_valid = DataLoader(valid_dataset_valid,\n                             shuffle=valid_cfg[\"shuffle\"],\n                             batch_size=valid_cfg[\"batch_size\"],\n                             num_workers=valid_cfg[\"num_workers\"])\n\nprint(valid_dataloader.dataset.__len__())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"WEIGHT_FILE = '../input/mixnet-pth/model_state_mixnet_xl_12500.pth'\nmodel_state = torch.load(WEIGHT_FILE, map_location=device)\nmodel.load_state_dict(model_state)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Final - Evaluate Validation Dataset \nmodel.eval()\ntorch.set_grad_enabled(False)\n\n# store information for evaluation\nfuture_coords_offsets_pd = []\ntimestamps = []\n# coordinates ground truth\nvalid_coords_gts = []\n# target avalabilities\ntarget_avail_pd = []\nagent_ids = []\nprogress_bar = tqdm(valid_dataloader)\nfor data in progress_bar:\n    \n    inputs = data[\"image\"].to(device)\n    target_availabilities = data[\"target_availabilities\"].unsqueeze(-1).to(device)\n    targets = data[\"target_positions\"].to(device)\n    \n    outputs = model(inputs).reshape(targets.shape)\n    \n    future_coords_offsets_pd.append(outputs.cpu().numpy().copy())\n    timestamps.append(data[\"timestamp\"].numpy().copy())\n    agent_ids.append(data[\"track_id\"].numpy().copy())\n    valid_coords_gts.append(data[\"target_positions\"].numpy().copy())\n    target_avail_pd.append(target_availabilities.cpu().numpy().copy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"timestamps_concat = np.concatenate(timestamps)\ntrack_ids_concat = np.concatenate(agent_ids)\ncoords_concat = np.concatenate(future_coords_offsets_pd)\ngt_valid_final = np.concatenate(valid_coords_gts)\ntarget_avail_concat = np.concatenate(target_avail_pd)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission.csv\nwrite_gt_csv(\n    csv_path=\"mixnet_xl_12500_valid_gt_rand.csv\", \n    timestamps=timestamps_concat, \n    track_ids=track_ids_concat, \n    coords=gt_valid_final, \n    avails=target_avail_concat.squeeze(-1)\n)\n\nwrite_pred_csv('submission_mixnet_xl_12500_val.csv',\n               timestamps=timestamps_concat,\n               track_ids=track_ids_concat,\n               coords=coords_concat,\n              )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.chdir(r'/kaggle/working')\nfrom IPython.display import FileLink\nFileLink(r'submission_mixnet_xl_12500_val.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FileLink(r'mixnet_xl_12500_valid_gt_rand.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Negative Log Likelihood Metrics\neval_gt_path = \"./mixnet_xl_12500_valid_gt_rand.csv\"\npred_path = \"./submission_mixnet_xl_12500_val.csv\"\n\nmetrics = compute_metrics_csv(eval_gt_path, pred_path, [neg_multi_log_likelihood, time_displace])\nfor metric_name, metric_mean in metrics.items():\n    print(metric_name, metric_mean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.eval()\ntorch.set_grad_enabled(False)\n\n# Uncomment to choose satelliter or semantic rasterizer\n# validate_cfg[\"raster_params\"][\"map_type\"] = \"py_satellite\"\nvalidate_cfg[\"raster_params\"][\"map_type\"] = \"py_semantic\"\n\n\nrast = build_rasterizer(validate_cfg, dm)\n\neval_ego_dataset = EgoDataset(validate_cfg, valid_dataset.dataset, rast)\nnum_frames = 2 # randomly pick _ frames\nrandom_frames = np.random.randint(0,len(eval_ego_dataset)-1, (num_frames,))\n\nfor frame_number in random_frames:  \n    agent_indices = valid_dataset.get_frame_indices(frame_number) \n    if not len(agent_indices):\n        continue\n\n    # get AV point-of-view frame\n    data_ego = eval_ego_dataset[frame_number]\n    im_ego = rasterizer.to_rgb(data_ego[\"image\"].transpose(1, 2, 0))\n    center = np.asarray(validate_cfg[\"raster_params\"][\"ego_center\"]) * validate_cfg[\"raster_params\"][\"raster_size\"]\n    \n    predicted_positions = []\n    target_positions = []\n\n    for v_index in agent_indices:\n        data_agent = valid_dataset[v_index]\n\n        out_net = model(torch.from_numpy(data_agent[\"image\"]).unsqueeze(0).to(device))\n        out_pos = out_net[0].reshape(-1, 2).detach().cpu().numpy()\n        # store absolute world coordinates\n        predicted_positions.append(transform_points(out_pos, data_agent[\"world_from_agent\"]))\n        # retrieve target positions from the GT and store as absolute coordinates\n        track_id, timestamp = data_agent[\"track_id\"], data_agent[\"timestamp\"]\n        target_positions.append(transform_points(data_agent[\"target_positions\"], data_agent[\"world_from_agent\"]) )\n\n    # convert coordinates to AV point-of-view so we can draw them\n    predicted_positions = transform_points(np.concatenate(predicted_positions), data_ego[\"raster_from_world\"])\n    target_positions = transform_points(np.concatenate(target_positions), data_ego[\"raster_from_world\"])\n    \n    # make sure ground truth and prediction have the same data size\n    assert len(target_positions) == len(predicted_positions)\n    \n    # draw_trajectory(im_ego, predicted_positions, PREDICTED_POINTS_COLOR)\n    draw_trajectory(im_ego, target_positions, TARGET_POINTS_COLOR)\n    \n    plt.rcParams['figure.figsize'] = 6, 6\n    plt.imshow(im_ego[::-1])\n#     plt.show()\n    \n    draw_trajectory(im_ego, predicted_positions, PREDICTED_POINTS_COLOR)\n    \n    plt.rcParams['figure.figsize'] = 6, 6\n    plt.imshow(im_ego[::-1])\n    plt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}