{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\n\"\"\"\nMax time for:\n    - GPU: 30h\n    - TPU: 20h\n\"\"\"\nimport time\n\nimport os\nos.system(\"mkdir /kaggle/working/results\")\nimport timer_app_ras as timer\nMAX_TRAIN_TIME = 60*60*11 #60*60*28 # Default 24hours \nGPU_START = time.perf_counter()\n\nimport installer_app_ras\nfrom dataset_wrapper_app_ras import DatasetWrapper\nfrom model_app_ras import build_resnet50, build_efficientnetb4, build_regnet, dump_model\nfrom rasterizer_app_ras import build_custom_rasterizer\nimport evaluation_app_ras as eval_util_functions\nimport logger_app_ras as logging\n\n\nfrom sys import stdin\nfrom signal import SIGINT, signal, getsignal\nfrom tempfile import gettempdir\nfrom datetime import datetime\n\nimport numpy as np\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader\nfrom adabound import AdaBound\nfrom l5kit.configs import load_config_data\nfrom l5kit.data import LocalDataManager, ChunkedDataset\nfrom l5kit.dataset import AgentDataset, EgoDataset\nfrom l5kit.evaluation.metrics import *\nfrom l5kit.evaluation import write_pred_csv, compute_metrics_csv, read_gt_csv, create_chopped_dataset\nfrom l5kit.geometry import transform_points\nfrom l5kit.visualization import PREDICTED_POINTS_COLOR, TARGET_POINTS_COLOR, draw_trajectory\n\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport datetime\nimport traceback\nfrom pathlib import Path\n\n\n\n# TPU imports\n#from torch_xla.core import xla_model","metadata":{"_uuid":"dd544d84-c311-4ff5-9973-a932897d3ce8","_cell_guid":"b1cf48c6-4dce-43c1-a624-bf375a3ff2c8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-23T08:36:45.010118Z","iopub.execute_input":"2022-02-23T08:36:45.010432Z","iopub.status.idle":"2022-02-23T08:37:02.523302Z","shell.execute_reply.started":"2022-02-23T08:36:45.010363Z","shell.execute_reply":"2022-02-23T08:37:02.522567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble Class","metadata":{"_uuid":"cf953db7-bb18-4481-9456-33504b9c673d","_cell_guid":"9269ea63-cf99-4154-93ee-216e912e960c","trusted":true}},{"cell_type":"code","source":"# class EnsemblingModel(nn.Module):\n#     def __init__(self, device, out_dim, models):\n#         '''\n#             Create an instance of the ensembling model\n        \n#             Params:\n#                 device        PyTorch device\n#                 out_dim       Output features of each model in `models`\n#                 models        List of models assumed to have same input and output dimensions\n#         '''\n#         super().__init__()\n\n#         self.device = device\n\n#         # NOTE: Move all models to the given device\n#         self.models = models\n#         for model in models:\n#             model.to(device)\n\n#         self.relu = nn.ReLU()\n#         self.output = nn.Linear(in_features=len(models)*out_dim, out_features=out_dim)\n        \n        \n#     def forward(self, X):\n#         predictions = torch.flatten(\n#             torch.cat(tuple(model(X)[:, None, :] for model in self.models), dim=1),\n#             start_dim=1,\n#         )\n#         return self.output(self.relu(predictions))\n\nclass EnsemblingModel(nn.Module):\n    def __init__(self, device, out_dim, models):\n        '''\n            Create an instance of the ensembling model\n        \n            Params:\n                device        PyTorch device\n                out_dim       Output features of each model in `models`\n                models        List of models assumed to have same input and output dimensions\n        '''\n        super().__init__()\n\n        self.device = device\n\n        # NOTE: Move all models to the given device\n        self.models = models\n        for model in models:\n            model.to(device)\n\n        self.relu = nn.ReLU()\n        self.mixture = nn.Linear(in_features=len(models)*out_dim, out_features=out_dim)\n        self.output = nn.Linear(in_features=(len(models)+1)*out_dim, out_features=out_dim)\n        \n        \n    def forward(self, X):\n        with torch.no_grad():\n            predictions = [model(X)[:, None, :] for model in self.models]\n            \n        # NOTE: Append mixture to predictions\n        predictions.append(\n            self.mixture(\n                torch.flatten(\n                    torch.cat(predictions, dim=1),\n                    start_dim=1,\n                )\n            )[:, None, :],\n        )\n        \n        return self.output(\n            self.relu(\n                torch.flatten(\n                    torch.cat(predictions, dim=1),\n                    start_dim=1,\n                )\n            ),\n        )\n        \n","metadata":{"_uuid":"95111448-f1b7-4f19-acce-cd5cacf9f512","_cell_guid":"cfa62cfd-9c3f-45a5-9179-8e35fa090ea9","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-23T08:37:02.527483Z","iopub.execute_input":"2022-02-23T08:37:02.527886Z","iopub.status.idle":"2022-02-23T08:37:02.539612Z","shell.execute_reply.started":"2022-02-23T08:37:02.527846Z","shell.execute_reply":"2022-02-23T08:37:02.53874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Model (with model_app-ras.py)","metadata":{"_uuid":"8f839d36-3e82-43d7-97c1-4e06f78e8c0d","_cell_guid":"bce70a2c-bd88-4ae7-be5f-b31f85c2f444","trusted":true}},{"cell_type":"code","source":"def build_model(cfg, build_model, file=None):\n    '''\n    Builds regression model using CNN.\n\n    Model inputs:   selected history frames(box) + present frame(semantic)\n    Model outputs:  target coords(x, y) for each predicted future frame\n\n    Params:\n        cfg     config dict\n        file    a tuple(path to model, in_channels, out_features),\n                where 'in_channels' and 'out_features' are referring to\n                the model that is being loaded\n    Returns:\n        the configured model\n    '''\n    # NOTE: Calculate number of input channels\n    # NOTE: We multiply by 2 since each frame consists of an agent and an ego\n    #       image\n    history_box_frames = 2 * len(cfg['model_params']['history_box_frames'])\n    # NOTE: We add 3 since the semantic rasterizer images always have 3 channels\n    num_in_channels = 3 + history_box_frames\n\n    # NOTE: Calculate output dimensions\n    # NOTE: We multiply by 2 since we're predicting x and y coords for eac\n    num_targets = 2 * cfg['model_params']['future_num_frames']\n\n    return build_model((num_in_channels, num_targets), file)","metadata":{"_uuid":"11e32234-3ca8-42ff-8925-f00339b62ee1","_cell_guid":"f6f05438-8420-4a2e-a16f-6a40a00b5fbc","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-23T08:37:02.541079Z","iopub.execute_input":"2022-02-23T08:37:02.54196Z","iopub.status.idle":"2022-02-23T08:37:02.555045Z","shell.execute_reply.started":"2022-02-23T08:37:02.541917Z","shell.execute_reply":"2022-02-23T08:37:02.554265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def forward(data, model, device, criterion):\n    inputs = data[\"image\"].to(device)\n    target_availabilities = data[\"target_availabilities\"].unsqueeze(-1).to(device)\n    targets = data[\"target_positions\"].to(device)\n    # Forward pass\n    outputs = model(inputs).reshape(targets.shape)\n    loss = criterion(outputs, targets)\n    # not all the output steps are valid, but we can filter them out from the loss using availabilities\n    loss = loss * target_availabilities\n    loss = loss.mean()\n    return loss, outputs","metadata":{"_uuid":"65b02508-c0d4-4c99-933c-d16b6e7e9717","_cell_guid":"36b87c21-6c99-494a-8522-65c30a396257","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-23T08:37:02.557889Z","iopub.execute_input":"2022-02-23T08:37:02.558205Z","iopub.status.idle":"2022-02-23T08:37:02.567902Z","shell.execute_reply.started":"2022-02-23T08:37:02.558172Z","shell.execute_reply":"2022-02-23T08:37:02.567031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_agent_motion_config():\n    amc = {\n        'format_version': 4,\n        'model_params': {\n            'model_architecture': 'efficientnet_b4',\n            'history_num_frames': 5,\n            'history_box_frames': [0, 1, 2, 4, 8],\n            'future_num_frames': 50,\n            'step_time': 0.1,\n            'render_ego_history': True\n        },\n        'raster_params': {\n            'raster_size': [384, 192],\n            'pixel_size': [0.25, 0.25],\n            'ego_center': [0.25, 0.5],\n            'map_type': 'box_semantic_fast',\n            'satellite_map_key': 'lyft-motion-prediction-autonomous-vehicles/aerial_map/aerial_map.png',\n            'semantic_map_key': 'lyft-motion-prediction-autonomous-vehicles/semantic_map/semantic_map.pb',\n            'dataset_meta_key': 'lyft-motion-prediction-autonomous-vehicles/meta.json',\n            'filter_agents_threshold': 0.5,\n            'disable_traffic_light_faces': False,\n            'set_origin_to_bottom': True,\n        },\n        'train_data_loader': {\n            'key': \"lyft-full-chopped\", #\"lyft-full-training-set/train_full.zarr\",# TODO: set to directory with all chunks\n            'batch_size': 4,\n            'shuffle': True,\n            'num_workers': 0,\n            'split_offset': 0               # TODO: CHANGE IF TRAINING SHOULD BE RESUMED FROM ANOTHER CHUNK\n        },\n        'val_data_loader': {\n            'key': \"chopped-val-dataset/validate_chopped_100/validate.zarr\", #'lyft-motion-prediction-autonomous-vehicles/scenes/validate.zarr',\n            'batch_size': 18,\n            'shuffle': False,\n            'num_workers': 2,\n        },\n        'train_params': {\n            'checkpoint_every_n_steps': 1000,\n            'max_num_steps': 1000,\n            'eval_every_n_steps': 10000,\n        },\n    }\n    return amc","metadata":{"_uuid":"6bda1808-33f1-4214-a7fd-3396a7a22ebc","_cell_guid":"0a2ab1cd-2bfd-4894-b985-f247daf83239","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-23T08:37:02.569058Z","iopub.execute_input":"2022-02-23T08:37:02.569724Z","iopub.status.idle":"2022-02-23T08:37:02.580521Z","shell.execute_reply.started":"2022-02-23T08:37:02.569685Z","shell.execute_reply":"2022-02-23T08:37:02.579712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rasterizer = build_rasterizer(cfg, dm)\ndm = LocalDataManager('/kaggle/input/')\ncfg = get_agent_motion_config()\nrasterizer = build_custom_rasterizer(cfg, dm)","metadata":{"_uuid":"4a006ec4-2f77-4fe8-9971-aef9f8b12f33","_cell_guid":"51738cf3-8524-47fd-b4a0-d0b19ecd7b1c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-23T08:37:02.581847Z","iopub.execute_input":"2022-02-23T08:37:02.58233Z","iopub.status.idle":"2022-02-23T08:37:13.022428Z","shell.execute_reply.started":"2022-02-23T08:37:02.58229Z","shell.execute_reply":"2022-02-23T08:37:13.021669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build trained models for ensembling","metadata":{"_uuid":"43a4c182-d53b-4415-93b9-2ab610044fe3","_cell_guid":"d2d80602-9442-4f91-9cd1-1d42b59e6a75","trusted":true}},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# model list for ensemble\nmodels = []\n\n# # resnet50\n# res_50 = build_model(\n#         cfg,\n#         build_resnet50,\n#         # TODO: Handle dimensions better\n#         ('../input/ensemble-models/resnet50_9h40m.pt', 13, 100),    # TODO: Change for new model\n#     ).to(device)\n# models.append(res_50)\n\n\n\n# efficient-net_b4\neff_b4 = build_model(\n        cfg,\n        build_efficientnetb4,\n        # TODO: Handle dimensions better\n        ('../input/ensemble-models/efficientnet_b4_22hours.pt', 13, 100),    # TODO: Change for new model\n    ).to(device)\nmodels.append(eff_b4)\n\n\nreg_ = build_model(\n        cfg,\n        build_regnet,\n        ('../input/ensemble-models/model_reg.pt', 13, 100)\n    ).to(device)\nmodels.append(reg_)\n\n# regnet (from Marko)\n# reg_ = build_model(\n#         cfg,\n#         build_regnet,\n#         # TODO: Handle dimensions better\n#         ('PATH', 13, 100),    # TODO: Change for new model\n#     ).to(device)\n# models.append(reg_)","metadata":{"_uuid":"95e98f7b-8222-4b10-8251-a898dc2993c9","_cell_guid":"b31de0fa-e88d-4623-ac46-5c322bd2b862","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-23T08:37:13.023672Z","iopub.execute_input":"2022-02-23T08:37:13.023942Z","iopub.status.idle":"2022-02-23T08:37:17.654348Z","shell.execute_reply.started":"2022-02-23T08:37:13.023906Z","shell.execute_reply":"2022-02-23T08:37:17.653559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set ensemble model and opt. and crit.","metadata":{"_uuid":"6cb396a3-1a40-4cba-9bbc-c5163883107a","_cell_guid":"017d05d5-32e2-434c-b8de-0dd0f6a4f3c1","trusted":true}},{"cell_type":"code","source":"ensemble_model = EnsemblingModel(device, 100, models).to(device)\noptimizer = optim.Adam(ensemble_model.parameters(), lr=1e-3)\ncriterion = nn.MSELoss(reduction=\"none\")","metadata":{"_uuid":"919fbae8-0832-4619-94d7-82a47be9fcaf","_cell_guid":"c10b4fcd-07cb-47f7-9c8e-de3ec42e2e67","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-23T08:37:17.655795Z","iopub.execute_input":"2022-02-23T08:37:17.656158Z","iopub.status.idle":"2022-02-23T08:37:17.674048Z","shell.execute_reply.started":"2022-02-23T08:37:17.656117Z","shell.execute_reply":"2022-02-23T08:37:17.673381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the Train Data and Training Ensemble model","metadata":{"_uuid":"f7741509-8505-4a36-94bd-0f4854de79ce","_cell_guid":"19c57ad7-4b43-4286-91d4-00c36757041e","trusted":true}},{"cell_type":"code","source":"import timer_app_ras as timer\n# Initialize logger\nnow = datetime.datetime.now()\nnow = now.strftime('%Y-%m-%d_%H:%M:%S')\nVERSION = f\"{now}\"\nlogging.create_logger(VERSION)\nlogger = logging.get_logger(VERSION) \nlogger.info(f\"Initialized Logger {VERSION}\")\n\n# initialize timer\ntimer = timer.Timer(logger)\nlogger.info(\"Initialized timer\")\n\n# start training\nlogger.info(\"************** Ensemble Training **************\")\ncfg = get_agent_motion_config()\n\n\ntrain_cfg = cfg['train_data_loader']\nnum_workers = train_cfg['num_workers']\n\n# NOTE: Setup signal handler to manual interruption\noriginal_handler = getsignal(SIGINT)\nstate = { 'interrupted': False}\ndef handler(_signum, _frame):\n    state['interrupted'] = True\n    signal(SIGINT, original_handler)\nsignal(SIGINT, handler)\n\n\ndirs = [*map(lambda d: f\"{train_cfg['key']}/{d}\", next(os.walk(f\"/kaggle/input/{train_cfg['key']}\"))[1])]\nlogger.info(f\"all zarrs: {dirs}\")\nto_skip = train_cfg[\"split_offset\"]\n\n# SET EPOCHS\ntotal_epochs = 1 #5\nnum_epochs = 0\n#while not state['interrupted']: # for epochs to be specified(or not)\nfor _ in range(total_epochs):\n    logger.info(\"Start Epoch \" + str(num_epochs+1))\n\n    # ==== Train for all chopped subsets\n    for train_path in dirs:\n        if to_skip > 0:\n            to_skip -= 1\n            continue\n        logger.info(f\"Start training with {train_path}\")\n        # NOTE: Load training dataset\n        dataset = ChunkedDataset(dm.require(train_path)).open()\n        dataset = AgentDataset(\n            cfg,\n            dataset,\n            build_custom_rasterizer(cfg, dm),\n            min_frame_history=0,\n            min_frame_future=10,\n        ) if num_workers <= 0 else DatasetWrapper(\n            cfg,\n            dm,\n            dataset,\n            min_frame_history=0,\n            min_frame_future=10,\n        )\n        num_max_batches = np.ceil(\n            len(dataset) / train_cfg['batch_size'],\n        ).astype(int)\n        dataloader_opts = dict(\n            shuffle=train_cfg['shuffle'],\n            batch_size=train_cfg['batch_size'],\n            num_workers=num_workers,\n        )\n        if num_workers > 0:\n            dataloader_opts['prefetch_factor'] = 2 * num_workers\n        train_dataloader = DataLoader(\n            dataset,\n            **dataloader_opts\n        )\n\n        # ==== TRAIN in a subset\n        tr_it = iter(train_dataloader)\n        progress_bar = tqdm(range(cfg[\"train_params\"][\"max_num_steps\"]))\n        losses_train = []\n\n        # max_num_steps\n        for _ in progress_bar:\n            try:\n                data = next(tr_it)\n            except StopIteration:\n                tr_it = iter(train_dataloader)\n                data = next(tr_it)\n            ensemble_model.train()\n            torch.set_grad_enabled(True)\n            data['image'] = data['image'].float() / 255\n            loss, outputs = forward(data, ensemble_model, device, criterion) \n\n            # Backward pass\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            losses_train.append(loss.item())\n            progress_bar.set_description(f\"loss: {loss.item()} loss(avg): {np.mean(losses_train)}\")\n\n\n\n            # EVALUATION\n            gt_train_path = \"/kaggle/working/results/gt_train.csv\"\n            pred_path = \"/kaggle/working/results/pred_train.csv\"\n            eval_util_functions.write_csvs(data, gt_train_path, pred_path, device, outputs)\n\n            # calculate metrics\n            metrics = eval_util_functions.compute_metrics_csv(gt_train_path, pred_path, [\n                                                                neg_multi_log_likelihood,\n                                                                rmse,\n                                                                average_displacement_error_oracle,\n                                                                average_displacement_error_mean,\n                                                                final_displacement_error_oracle,\n                                                                final_displacement_error_mean])       \n            # create csv file\n            eval_util_functions.save_metrics('results/metrics.csv', loss.item(), np.mean(losses_train), metrics) \n\n        # dump model\n        dump_model(ensemble_model, './model_ensemble.pt')\n        \n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()\n    num_epochs += 1\nprint(f'Stopped at epoch {num_epochs+1}')\n#print(f'Training took {datetime.datetime.now() - start_time}')\n\nif torch.cuda.is_available():\n    torch.cuda.empty_cache()\n\n# NOTE: Release memory after training\n\n# # NOTE: Save model to disk","metadata":{"_uuid":"f529c83c-dbc5-42e0-ac92-88e0121b191c","_cell_guid":"b41e7390-565a-430e-bd33-2c8bf5e1e06a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-23T08:37:17.675474Z","iopub.execute_input":"2022-02-23T08:37:17.675752Z","iopub.status.idle":"2022-02-23T10:08:48.719603Z","shell.execute_reply.started":"2022-02-23T08:37:17.675717Z","shell.execute_reply":"2022-02-23T10:08:48.718127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dirs","metadata":{"execution":{"iopub.status.busy":"2022-02-23T10:08:48.721295Z","iopub.execute_input":"2022-02-23T10:08:48.721594Z","iopub.status.idle":"2022-02-23T10:08:48.734163Z","shell.execute_reply.started":"2022-02-23T10:08:48.721555Z","shell.execute_reply":"2022-02-23T10:08:48.732668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation","metadata":{"_uuid":"f9c12066-64fe-4387-aaa9-734d136dce3c","_cell_guid":"ecacf4df-7fd4-4f23-826a-9d0b67584519","trusted":true}},{"cell_type":"code","source":"# ===== GENERATE AND LOAD CHOPPED DATASET\n#num_frames_to_chop = 100\ncfg = get_agent_motion_config()\neval_cfg = cfg[\"val_data_loader\"]\neval_base_path = '/kaggle/input/chopped-val-dataset/validate_chopped_100'\n#eval_base_path = create_chopped_dataset(dm.require(eval_cfg[\"key\"]), cfg[\"raster_params\"][\"filter_agents_threshold\"], num_frames_to_chop, cfg[\"model_params\"][\"future_num_frames\"], MIN_FUTURE_STEPS)","metadata":{"_uuid":"94366913-1aec-48e9-ae02-beb5267f2573","_cell_guid":"f587c67e-8fef-404c-9225-ce5264c2699d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-23T10:08:48.735565Z","iopub.execute_input":"2022-02-23T10:08:48.736209Z","iopub.status.idle":"2022-02-23T10:08:48.746115Z","shell.execute_reply.started":"2022-02-23T10:08:48.736169Z","shell.execute_reply":"2022-02-23T10:08:48.745352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\neval_zarr_path = str(Path(eval_base_path) / Path(dm.require(eval_cfg[\"key\"])).name)\neval_mask_path = str(Path(eval_base_path) / \"mask.npz\")\neval_gt_path = str(Path(eval_base_path) / \"gt.csv\")\n\neval_zarr = ChunkedDataset(dm.require(eval_zarr_path)).open()\neval_mask = np.load(eval_mask_path)[\"arr_0\"]\n# ===== INIT DATASET AND LOAD MASK\neval_dataset = DatasetWrapper(cfg, dm, eval_zarr, agents_mask=eval_mask)\neval_dataloader = DataLoader(\n    eval_dataset,\n    shuffle=eval_cfg[\"shuffle\"],\n    batch_size=eval_cfg[\"batch_size\"],\n    num_workers=eval_cfg[\"num_workers\"],\n    prefetch_factor=2*eval_cfg[\"num_workers\"],\n)\nprint(eval_dataset)","metadata":{"_uuid":"bb35999d-8e77-4327-9522-03c0d1d452ac","_cell_guid":"6cf1adfc-e88c-400d-a228-f4032929061b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-23T10:08:48.747084Z","iopub.execute_input":"2022-02-23T10:08:48.748119Z","iopub.status.idle":"2022-02-23T10:09:28.396254Z","shell.execute_reply.started":"2022-02-23T10:08:48.748083Z","shell.execute_reply":"2022-02-23T10:09:28.395517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualise Results","metadata":{"_uuid":"bf5d5b15-d27f-4ccc-9e25-b3985e3bd1b0","_cell_guid":"a3823cef-843b-49d0-89e1-76376f06a682","trusted":true}},{"cell_type":"code","source":"\n\nensemble_model.eval() # not training anymore\ntorch.set_grad_enabled(False)\n\n# build a dict to retrieve future trajectories from GT\ngt_rows = {}\nfor row in read_gt_csv(eval_gt_path):\n    gt_rows[row[\"track_id\"] + row[\"timestamp\"]] = row[\"coord\"]\n\neval_ego_dataset = EgoDataset(cfg, eval_dataset.dataset, rasterizer)\n\nfor frame_number in range(99, len(eval_zarr.frames), 1000):  # start from last frame of scene_0 and increase by 100\n    agent_indices = eval_dataset.get_frame_indices(frame_number) \n    if not len(agent_indices):\n        continue\n\n    # get AV point-of-view frame\n    data_ego = eval_ego_dataset[frame_number]\n    im_ego = rasterizer.to_rgb(data_ego[\"image\"].transpose(1, 2, 0)).copy()\n    center = np.asarray(cfg[\"raster_params\"][\"ego_center\"]) * cfg[\"raster_params\"][\"raster_size\"]\n    \n    predicted_positions = []\n    target_positions = []\n\n    for v_index in agent_indices:\n        data_agent = eval_dataset[v_index]\n        data = torch.from_numpy(data_agent[\"image\"]).float() / 255\n\n        out_net = ensemble_model(data.unsqueeze(0).to(device))\n        out_pos = out_net[0].reshape(-1, 2).detach().cpu().numpy() # output -> converts into (x,y)\n        # store absolute world coordinates\n        predicted_positions.append(transform_points(out_pos, data_agent[\"world_from_agent\"]))\n        # retrieve target positions from the GT and store as absolute coordinates\n        track_id, timestamp = data_agent[\"track_id\"], data_agent[\"timestamp\"]\n        target_positions.append(gt_rows[str(int(track_id)) + str(timestamp)] + data_agent[\"centroid\"][:2])\n\n\n    # convert coordinates to AV point-of-view so we can draw them\n    predicted_positions = transform_points(np.concatenate(predicted_positions), data_ego[\"raster_from_world\"])\n    target_positions = transform_points(np.concatenate(target_positions), data_ego[\"raster_from_world\"])\n\n    plt.figure(figsize=(16, 4))\n\n    plt.subplot(1, 2, 1)\n    plt.title('Predicted')\n    im_ego_pred = im_ego.copy()\n    draw_trajectory(im_ego_pred, predicted_positions, PREDICTED_POINTS_COLOR)\n    plt.imshow(im_ego_pred)\n\n    plt.subplot(1, 2, 2)\n    plt.title('Ground truth')\n    im_ego_target = im_ego.copy()\n    draw_trajectory(im_ego_target, target_positions, TARGET_POINTS_COLOR)\n    plt.imshow(im_ego_target)\n\n    plt.show()","metadata":{"_uuid":"dbdaed20-6d3d-4235-97a4-262709207d66","_cell_guid":"dc924ce8-dcf6-4733-be4f-e33932c34f1b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-02-23T10:09:28.399667Z","iopub.execute_input":"2022-02-23T10:09:28.399877Z","iopub.status.idle":"2022-02-23T10:37:49.389081Z","shell.execute_reply.started":"2022-02-23T10:09:28.399851Z","shell.execute_reply":"2022-02-23T10:37:49.388359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}