{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pytorch-pfn-extras==0.3.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport os\nfrom pathlib import Path\nimport random\nimport sys\nfrom io import StringIO\nimport shutil\n\nfrom tqdm.notebook import tqdm\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\n\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom IPython.core.display import display, HTML\n\n# --- plotly ---\nfrom plotly import tools, subplots\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.express as px\nimport plotly.figure_factory as ff\nimport plotly.io as pio\npio.templates.default = \"plotly_dark\"\n\n# --- models ---\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import KFold\nimport lightgbm as lgb\nimport xgboost as xgb\nimport catboost as cb\n\n# --- setup ---\npd.set_option('max_columns', 50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom pathlib import Path\n\nimport pytorch_pfn_extras as ppe\nfrom math import ceil\nfrom pytorch_pfn_extras.training import IgniteExtensionsManager\nfrom pytorch_pfn_extras.training.triggers import MinValueTrigger\n\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data.dataset import Subset\nimport pytorch_pfn_extras.training.extensions as E","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import zarr\n\nimport l5kit\nfrom l5kit.data import ChunkedDataset, LocalDataManager\nfrom l5kit.dataset import EgoDataset, AgentDataset\n\nfrom l5kit.rasterization import build_rasterizer\nfrom l5kit.configs import load_config_data\nfrom l5kit.visualization import draw_trajectory, TARGET_POINTS_COLOR, PREDICTED_POINTS_COLOR\nfrom l5kit.geometry import transform_points\nfrom tqdm import tqdm\nfrom collections import Counter\nfrom l5kit.data import PERCEPTION_LABELS\nfrom prettytable import PrettyTable\nfrom l5kit.evaluation import write_pred_csv, compute_metrics_csv, read_gt_csv, create_chopped_dataset\nfrom l5kit.evaluation.chop_dataset import MIN_FUTURE_STEPS\nfrom l5kit.evaluation.metrics import neg_multi_log_likelihood, time_displace\n\nfrom matplotlib import animation, rc\nfrom IPython.display import HTML\n\nrc('animation', html='jshtml')\nprint(\"l5kit version:\", l5kit.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- Utils ---\nimport yaml\n\n\ndef save_yaml(filepath, content, width=120):\n    with open(filepath, 'w') as f:\n        yaml.dump(content, f, width=width)\n\n\ndef load_yaml(filepath):\n    with open(filepath, 'r') as f:\n        content = yaml.safe_load(f)\n    return content\n\n\nclass DotDict(dict):\n    \"\"\"dot.notation access to dictionary attributes\n\n    Refer: https://stackoverflow.com/questions/2352181/how-to-use-a-dot-to-access-members-of-dictionary/23689767#23689767\n    \"\"\"  # NOQA\n\n    __getattr__ = dict.get\n    __setattr__ = dict.__setitem__\n    __delattr__ = dict.__delitem__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- Model utils ---\nimport torch\nfrom torchvision.models.resnet import resnet50\nfrom torch import nn\nfrom typing import Dict\n\n\nclass LyftMultiModel(nn.Module):\n\n    def __init__(self, cfg: Dict, num_modes=3):\n        super().__init__()\n\n        # TODO: support other than resnet18?\n        backbone = resnet50(pretrained=True, progress=True)\n        self.backbone = backbone\n\n        num_history_channels = (cfg[\"model_params\"][\"history_num_frames\"] + 1) * 2\n        num_in_channels = 3 + num_history_channels\n\n        self.backbone.conv1 = nn.Conv2d(\n            num_in_channels,\n            self.backbone.conv1.out_channels,\n            kernel_size=self.backbone.conv1.kernel_size,\n            stride=self.backbone.conv1.stride,\n            padding=self.backbone.conv1.padding,\n            bias=False,\n        )\n\n        # This is 512 for resnet18 and resnet34;\n        # And it is 2048 for the other resnets\n        backbone_out_features = 2048\n\n        # X, Y coords for the future positions (output shape: Bx50x2)\n        self.future_len = cfg[\"model_params\"][\"future_num_frames\"]\n        num_targets = 2 * self.future_len\n\n        # You can add more layers here.\n        self.head = nn.Sequential(\n            # nn.Dropout(0.2),\n            nn.Linear(in_features=backbone_out_features, out_features=4096),\n        )\n\n        self.num_preds = num_targets * num_modes\n        self.num_modes = num_modes\n\n        self.logit = nn.Linear(4096, out_features=self.num_preds + num_modes)\n\n    def forward(self, x):\n        x = self.backbone.conv1(x)\n        x = self.backbone.bn1(x)\n        x = self.backbone.relu(x)\n        x = self.backbone.maxpool(x)\n\n        x = self.backbone.layer1(x)\n        x = self.backbone.layer2(x)\n        x = self.backbone.layer3(x)\n        x = self.backbone.layer4(x)\n\n        x = self.backbone.avgpool(x)\n        x = torch.flatten(x, 1)\n\n        x = self.head(x)\n        x = self.logit(x)\n\n        # pred (bs)x(modes)x(time)x(2D coords)\n        # confidences (bs)x(modes)\n        bs, _ = x.shape\n        pred, confidences = torch.split(x, self.num_preds, dim=1)\n        pred = pred.view(bs, self.num_modes, self.future_len, 2)\n        assert confidences.shape == (bs, self.num_modes)\n        confidences = torch.softmax(confidences, dim=1)\n        return pred, confidences\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Referred https://www.kaggle.com/pestipeti/pytorch-baseline-inference\ndef run_prediction(predictor, data_loader):\n    predictor.eval()\n\n    pred_coords_list = []\n    confidences_list = []\n    timestamps_list = []\n    track_id_list = []\n\n    with torch.no_grad():\n        dataiter = tqdm(data_loader)\n        for data in dataiter:\n            image = data[\"image\"].to(device)\n            # target_availabilities = data[\"target_availabilities\"].to(device)\n            # targets = data[\"target_positions\"].to(device)\n            pred, confidences = predictor(image)\n\n            pred_coords_list.append(pred.cpu().numpy().copy())\n            confidences_list.append(confidences.cpu().numpy().copy())\n            timestamps_list.append(data[\"timestamp\"].numpy().copy())\n            track_id_list.append(data[\"track_id\"].numpy().copy())\n    timestamps = np.concatenate(timestamps_list)\n    track_ids = np.concatenate(track_id_list)\n    coords = np.concatenate(pred_coords_list)\n    confs = np.concatenate(confidences_list)\n    return timestamps, track_ids, coords, confs\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Configs**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- Lyft configs ---\ncfg = {\n    'format_version': 4,\n    'model_params': {\n        'model_architecture': 'resnet50',\n        'history_num_frames': 5, # fading \n        'history_step_size': 1,\n        'history_delta_time': 0.1,\n        'future_num_frames': 50,\n        'future_step_size': 1,\n        'future_delta_time': 0.1\n    },\n\n    'raster_params': {\n        'raster_size': [300, 300],\n        'pixel_size': [0.1, 0.1],\n        'ego_center': [0.5, 0.25], #actor is positioned at pixel 150,75 in 300x300 raster\n        'map_type': 'py_semantic',\n        'satellite_map_key': 'aerial_map/aerial_map.png',\n        'semantic_map_key': 'semantic_map/semantic_map.pb',\n        'dataset_meta_key': 'meta.json',\n        'filter_agents_threshold': 0.5\n    },\n\n    'train_data_loader': {\n        'key': 'scenes/train.zarr',\n        'batch_size': 12,\n        'shuffle': True,\n        'num_workers': 4\n    },\n\n    'val_data_loader': {\n        'key': 'scenes/validate.zarr',\n        'batch_size': 32,\n        'shuffle': False,\n        'num_workers': 4\n    },\n    \n    'test_data_loader': {\n        'key': 'scenes/test.zarr',\n        'batch_size': 8,\n        'shuffle': False,\n        'num_workers': 4\n    },\n\n    'train_params': {\n        'max_num_steps': 10000,\n        'checkpoint_every_n_steps': 5000\n        #'eval_every_n_steps': -1\n    }\n}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"flags_dict = {\n    \"debug\": False,\n    # --- Data configs ---\n    \"l5kit_data_folder\": \"/kaggle/input/lyft-motion-prediction-autonomous-vehicles\",\n    # --- Model configs ---\n    \"pred_mode\": \"multi\",\n    # --- Training configs ---\n    \"device\": \"cuda:0\",\n    \"out_dir\": \"results/multi_train\",\n    \"epoch\": 2,\n    \"snapshot_freq\": 50,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"flags = DotDict(flags_dict)\nout_dir = Path(flags.out_dir)\nos.makedirs(str(out_dir), exist_ok=True)\nprint(f\"flags: {flags_dict}\")\nsave_yaml(out_dir / 'flags.yaml', flags_dict)\nsave_yaml(out_dir / 'cfg.yaml', cfg)\ndebug = flags.debug","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# set env variable for data\nl5kit_data_folder = \"/kaggle/input/lyft-motion-prediction-autonomous-vehicles\"\nos.environ[\"L5KIT_DATA_FOLDER\"] = l5kit_data_folder\ndm = LocalDataManager(None)\n\nprint(\"Load dataset...\")\ndefault_test_cfg = {\n    'key': 'scenes/test.zarr',\n    'batch_size': 32,\n    'shuffle': False,\n    'num_workers': 4\n}\ntest_cfg = cfg.get(\"test_data_loader\", default_test_cfg)\n\n# Rasterizer\nrasterizer = build_rasterizer(cfg, dm)\n\ntest_path = test_cfg[\"key\"]\nprint(f\"Loading from {test_path}\")\ntest_zarr = ChunkedDataset(dm.require(test_path)).open()\nprint(\"test_zarr\", type(test_zarr))\ntest_mask = np.load(f\"{l5kit_data_folder}/scenes/mask.npz\")[\"arr_0\"]\ntest_agent_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)\ntest_dataset = test_agent_dataset\nif debug:\n    # Only use 100 dataset for fast check...\n    test_dataset = Subset(test_dataset, np.arange(100))\ntest_loader = DataLoader(\n    test_dataset,\n    shuffle=test_cfg[\"shuffle\"],\n    batch_size=test_cfg[\"batch_size\"],\n    num_workers=test_cfg[\"num_workers\"],\n    pin_memory=True,\n)\n\nprint(test_agent_dataset)\nprint(\"# AgentDataset test:\", len(test_agent_dataset))\nprint(\"# ActualDataset test:\", len(test_dataset))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(flags.device)\nprint(device)\n\nif flags.pred_mode == \"multi\":\n    predictor = LyftMultiModel(cfg)\nelse:\n    raise ValueError(f\"[ERROR] Unexpected value flags.pred_mode={flags.pred_mode}\")\n\npt_path = \"../input/resnet50-training/results/multi_train/predictor.pt\"\nprint(f\"Loading from {pt_path}\")\npredictor.load_state_dict(torch.load(pt_path))\npredictor.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- Inference ---\ntimestamps, track_ids, coords, confs = run_prediction(predictor, test_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"csv_path = \"submission.csv\"\nwrite_pred_csv(\n    csv_path,\n    timestamps=timestamps,\n    track_ids=track_ids,\n    coords=coords,\n    confs=confs)\nprint(f\"Saved to {csv_path}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ===== GENERATE AND LOAD CHOPPED DATASET\nnum_frames_to_chop = 100\neval_cfg = cfg[\"val_data_loader\"]\n\neval_cfg[\"key\"] = \"scenes/sample.zarr\"\n\n# As the /kaggle/input directory is not writeable as required to chop,\n# copy the sample set to /tmp\n!rm -rf /tmp/lyft\neval_dir = shutil.copytree(dm.require(eval_cfg[\"key\"]), '/tmp/lyft/sample.zarr')\n\neval_base_path = create_chopped_dataset(eval_dir, cfg[\"raster_params\"][\"filter_agents_threshold\"], \n                              num_frames_to_chop, cfg[\"model_params\"][\"future_num_frames\"], MIN_FUTURE_STEPS)\n!ls {eval_base_path}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Chopped data mask and ground truth created."},{"metadata":{},"cell_type":"markdown","source":"**Load**"},{"metadata":{"trusted":true},"cell_type":"code","source":"eval_zarr_path = str(Path(eval_base_path) / Path(dm.require(eval_cfg[\"key\"])).name)\neval_mask_path = str(Path(eval_base_path) / \"mask.npz\")\neval_gt_path = str(Path(eval_base_path) / \"gt.csv\")\n\neval_zarr = ChunkedDataset(eval_zarr_path).open()\neval_mask = np.load(eval_mask_path)[\"arr_0\"]\n# ===== INIT DATASET AND LOAD MASK\neval_dataset = AgentDataset(cfg, eval_zarr, rasterizer, agents_mask=eval_mask)\nprint(eval_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Note average 100 frames per scene.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf /tmp/lyft","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Chop Evaluation Data (Symlink to /tmp)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"eval_dir = dm.require(eval_cfg[\"key\"])\n!mkdir /tmp/lyft && ln -s {eval_dir} /tmp/lyft\neval_dir = \"/tmp/lyft/\" + Path(eval_dir).name\n!ls -la  {eval_dir}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"eval_base_path = create_chopped_dataset(eval_dir, cfg[\"raster_params\"][\"filter_agents_threshold\"], \n                              num_frames_to_chop, cfg[\"model_params\"][\"future_num_frames\"], MIN_FUTURE_STEPS)\n!ls {eval_base_path}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The result is that **each scene has been reduced to only 100 frames, and only valid agents in the 100th frame will be used to compute the metrics**. Because following frames in the scene have been chopped off, we can't just look ahead to get the future of those agents.\n\nIn this example, we simulate this pipeline by running chop_dataset on the validation set. The function stores:\n\n* a new chopped .zarr dataset, in which each scene has only the first 100 frames;\n* a numpy mask array where only valid agents in the 100th frame are True;\n* a ground-truth file with the future coordinates of those agents;\n\nPlease note how the total number of frames is now equal to the number of scenes multipled by num_frames_to_chop.\n\nThe remaining frames in the scene have been sucessfully chopped off from the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"eval_zarr_path = str(Path(eval_base_path) / Path(dm.require(eval_cfg[\"key\"])).name)\neval_mask_path = str(Path(eval_base_path) / \"mask.npz\")\neval_gt_path = str(Path(eval_base_path) / \"gt.csv\")\n\neval_zarr = ChunkedDataset(eval_zarr_path).open()\neval_mask = np.load(eval_mask_path)[\"arr_0\"]\n# ===== INIT DATASET AND LOAD MASK\neval_dataset = AgentDataset(cfg, eval_zarr, rasterizer, agents_mask=eval_mask)\nprint(eval_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Perform Evaluation**"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(pred_path)\nprint(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictor.eval()\ntorch.set_grad_enabled(False)\n\n# build a dict to retrieve future trajectories from GT\ngt_rows = {}\nfor row in read_gt_csv(eval_gt_path):\n    gt_rows[row[\"track_id\"] + row[\"timestamp\"]] = row[\"coord\"]\n\neval_ego_dataset = EgoDataset(cfg, eval_dataset.dataset, rasterizer)\n\nfor frame_number in range(99, len(eval_zarr.frames), 100):  # start from last frame of scene_0 and increase by 100\n    agent_indices = eval_dataset.get_frame_indices(frame_number) \n    if not len(agent_indices):\n        continue\n\n    # get AV point-of-view frame\n    data_ego = eval_ego_dataset[frame_number]\n    im_ego = rasterizer.to_rgb(data_ego[\"image\"].transpose(1, 2, 0))\n    center = np.asarray(cfg[\"raster_params\"][\"ego_center\"]) * cfg[\"raster_params\"][\"raster_size\"]\n    \n    most_likely_positions = []\n    other_positions = []\n    target_positions = []\n\n    for v_index in agent_indices:\n        data_agent = eval_dataset[v_index]\n\n        out_net = predictor(torch.from_numpy(data_agent[\"image\"]).unsqueeze(0).to(device))\n        \n        # get index of most likely trajectory from 2nd tensor (confs)\n        i = torch.argmax(out_net[1])\n        most_likely_arr = out_net[0][0][i].detach().cpu().numpy()\n        most_likely_positions.append(transform_points(most_likely_arr, data_agent[\"world_from_agent\"]))\n        \n        # filter max index out\n        out_net = torch.cat([out_net[0][0][0:i,:,:], out_net[0][0][i+1:,:,:]])\n        #print(out_net.shape)\n        out_pos = out_net.reshape(-1, 2).detach().cpu().numpy()\n        # store absolute world coordinates\n        other_positions.append(transform_points(out_pos, data_agent[\"world_from_agent\"]))\n        # retrieve target positions from the GT and store as absolute coordinates\n        track_id, timestamp = data_agent[\"track_id\"], data_agent[\"timestamp\"]\n        target_positions.append(gt_rows[str(track_id) + str(timestamp)] + data_agent[\"centroid\"][:2])\n\n\n    # convert coordinates to AV point-of-view so we can draw them\n    most_likely_positions = transform_points(np.concatenate(most_likely_positions), data_ego[\"raster_from_world\"])\n    other_positions = transform_points(np.concatenate(other_positions), data_ego[\"raster_from_world\"])\n    target_positions = transform_points(np.concatenate(target_positions), data_ego[\"raster_from_world\"])\n    \n    #print(predicted_positions.shape)\n    #print(target_positions.shape)\n    \n    # Red dots represent most likely future positions\n    draw_trajectory(im_ego, most_likely_positions, (255,0,0))\n    # Blue dots represent other possible future positions\n    draw_trajectory(im_ego, other_positions, (0,0,255))\n    # Green dots represent ground truth positions\n    draw_trajectory(im_ego, target_positions, (0,255,0))\n\n    plt.imshow(im_ego[::-1])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}