{"cells":[{"metadata":{"papermill":{"duration":0.039748,"end_time":"2020-11-05T04:08:49.658552","exception":false,"start_time":"2020-11-05T04:08:49.618804","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Lyft - Expt 30 - Resnet34 - 1cycleLR - 64 - lr=1e-4, SGD - l5kit 20201121-rot no_rot, 24M\n\nSet to 24M for total steps in the 1cycleLR\n\nNo rotated or mv\n\nModified from [Lyft: Complete train and prediction pipeline](https://www.kaggle.com/huanvo/lyft-complete-train-and-prediction-pipeline)"},{"metadata":{"papermill":{"duration":0.040896,"end_time":"2020-11-05T04:08:49.738534","exception":false,"start_time":"2020-11-05T04:08:49.697638","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Environment setup"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-11-05T04:08:49.829165Z","iopub.status.busy":"2020-11-05T04:08:49.828384Z","iopub.status.idle":"2020-11-05T04:08:55.758120Z","shell.execute_reply":"2020-11-05T04:08:55.758707Z"},"papermill":{"duration":5.982032,"end_time":"2020-11-05T04:08:55.758883","exception":false,"start_time":"2020-11-05T04:08:49.776851","status":"completed"},"scrolled":true,"tags":[],"trusted":false},"cell_type":"code","source":"from typing import Dict\n\nfrom tempfile import gettempdir\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch import nn, optim\nfrom torch.optim.lr_scheduler import OneCycleLR\nfrom torch.utils.data import DataLoader\nimport torchvision\nfrom torchvision.models.resnet import resnet50, resnet18, resnet34, resnet101\nfrom tqdm import tqdm\n\nimport l5kit\nfrom l5kit.configs import load_config_data\nfrom l5kit.data import LocalDataManager, ChunkedDataset\nfrom l5kit.dataset import AgentDataset, EgoDataset\nfrom l5kit.rasterization import build_rasterizer\nfrom l5kit.evaluation import write_pred_csv, compute_metrics_csv, read_gt_csv, create_chopped_dataset\nfrom l5kit.evaluation.chop_dataset import MIN_FUTURE_STEPS\nfrom l5kit.evaluation.metrics import neg_multi_log_likelihood, time_displace\nfrom l5kit.geometry import transform_points\nfrom l5kit.visualization import PREDICTED_POINTS_COLOR, TARGET_POINTS_COLOR, draw_trajectory\nfrom prettytable import PrettyTable\nfrom pathlib import Path\n\nimport matplotlib.pyplot as plt\n\nimport os\nimport random\nimport time\n\nfrom IPython.display import display\nfrom tqdm import tqdm_notebook\nimport gc, psutil\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nprint(l5kit.__version__)\nprint(l5kit.__file__)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:08:55.845396Z","iopub.status.busy":"2020-11-05T04:08:55.843237Z","iopub.status.idle":"2020-11-05T04:08:55.846191Z","shell.execute_reply":"2020-11-05T04:08:55.846751Z"},"papermill":{"duration":0.049652,"end_time":"2020-11-05T04:08:55.846902","exception":false,"start_time":"2020-11-05T04:08:55.797250","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    \n# set_seed(42)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:08:55.935853Z","iopub.status.busy":"2020-11-05T04:08:55.934788Z","iopub.status.idle":"2020-11-05T04:08:55.943793Z","shell.execute_reply":"2020-11-05T04:08:55.944341Z"},"papermill":{"duration":0.057756,"end_time":"2020-11-05T04:08:55.944511","exception":false,"start_time":"2020-11-05T04:08:55.886755","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"# Memory measurement\ndef memory(verbose=True):\n    mem = psutil.virtual_memory()\n    gb = 1024*1024*1024\n    if verbose:\n        print('Physical memory:',\n              '%.2f GB (used),'%((mem.total - mem.available) / gb),\n              '%.2f GB (available)'%((mem.available) / gb), '/',\n              '%.2f GB'%(mem.total / gb))\n    return (mem.total - mem.available) / gb\n\ndef gc_memory(verbose=True):\n    m = gc.collect()\n    if verbose:\n        print('GC:', m, end=' | ')\n        memory()\n\nmemory();","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.038876,"end_time":"2020-11-05T04:08:56.024321","exception":false,"start_time":"2020-11-05T04:08:55.985445","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Configs"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:08:56.108767Z","iopub.status.busy":"2020-11-05T04:08:56.107910Z","iopub.status.idle":"2020-11-05T04:08:56.113041Z","shell.execute_reply":"2020-11-05T04:08:56.112350Z"},"papermill":{"duration":0.049574,"end_time":"2020-11-05T04:08:56.113174","exception":false,"start_time":"2020-11-05T04:08:56.063600","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"on_kaggle = False\nfolder = '/kaggle' if on_kaggle else '..'\noriginal_model_weight_path = f'{folder}/input/lyft-pretrained-model-hv/model_multi_update_lyft_public.pth'\n\nmodel_name = 'model-30-resnet34-1cycleLR-64-sgd-24M'\noriginal_model_name = model_name  # when using from other model\n\ntest_run = False\n\ntrain_round = 1  # start from 1\n\nset_seed(41+train_round)  # so that we don't go through the same train set for every round","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:08:56.387672Z","iopub.status.busy":"2020-11-05T04:08:56.386544Z","iopub.status.idle":"2020-11-05T04:08:56.390267Z","shell.execute_reply":"2020-11-05T04:08:56.389646Z"},"papermill":{"duration":0.056688,"end_time":"2020-11-05T04:08:56.390400","exception":false,"start_time":"2020-11-05T04:08:56.333712","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"# --- Lyft configs ---\ncfg = {\n    'format_version': 4,\n    'data_path': f'{folder}/input/lyft-motion-prediction-autonomous-vehicles',\n    'model_params': {\n        'train': True,\n        'predict': True,\n        'validate': True,\n        'model_architecture': 'resnet34',\n        'history_num_frames': 10,\n        'history_step_size': 1,\n        'history_delta_time': 0.1,\n        'future_num_frames': 50,\n        'future_step_size': 1,\n        'future_delta_time': 0.1,\n        'velocity_corrected_yaw': False,\n        'num_modes': 3,\n        'lr': 1e-4,                 # (1e-3) change learning rate\n        'lr_reduce': 0.667,         # not used when using lr scheduler\n        'lr_reduce_steps': 300_000, # not used when using lr scheduler\n        'lr_scheduler': True,\n        'max_lr': 0.001,              # 0.01 only work when using lr_scheduler\n        'lr_scheduler_expect_rounds': 5,\n        'load_pretrain_weight': True if train_round > 1 else False,\n        'weight_path': f'{folder}/{\"input/lyftmodels\" if on_kaggle else \"models\"}/{original_model_name}-{train_round - 1}-final.pth',\n        'load_pretrain_optimizer': True if train_round > 1 else False,\n        'optimizer_path': f'{folder}/{\"input/lyftmodels\" if on_kaggle else \"models\"}/{original_model_name}_optimizer-{train_round - 1}-final.pth',\n        'load_pretrain_scheduler': True if train_round > 1 else False,  # expect to train 5 rounds\n#         'load_pretrain_scheduler': False,\n        'scheduler_path': f'{folder}/{\"input/lyftmodels\" if on_kaggle else \"models\"}/{original_model_name}_scheduler-{train_round - 1}-final.pth',\n    },\n    'raster_params': {\n        'raster_size': [224, 224],\n        'pixel_size': [0.5, 0.5],\n        'ego_center': [0.25, 0.5],\n        'map_type': 'py_semantic',\n        'satellite_map_key': 'aerial_map/aerial_map.png',\n        'semantic_map_key': 'semantic_map/semantic_map.pb',\n        'dataset_meta_key': 'meta.json',\n        'filter_agents_threshold': 0.5,\n    },\n    'train_data_loader': {\n        'key': 'scenes/train.zarr',\n        'batch_size': 64,  # 16\n        'shuffle': True,\n        'num_workers': 4 if on_kaggle else 6,  # test parallelization\n        'prefetch_factor': 16,  # 32\n        'min_frame_future': 1,   # default 1, new in v26\n    },\n    'val_data_loader': {\n        'key': 'scenes/validate.zarr',\n        'batch_size': 128,\n        'shuffle': True,\n        'num_workers': 4 if on_kaggle else 6,\n        'prefetch_factor': 4,\n        'num_batches': 1000,\n    },\n    'val_100_data_loader': {\n        'key': 'scenes/validate_chopped_100/validate.zarr',\n        'batch_size': 128,\n        'shuffle': False,\n        'num_workers': 4 if on_kaggle else 6,\n        'prefetch_factor': 4,\n        'num_batches': 1000,\n        'mask_path': f'{folder}/input/lyft-motion-prediction-autonomous-vehicles/scenes/validate_chopped_100/mask.npz',\n        'truth_path': f'{folder}/input/lyft-motion-prediction-autonomous-vehicles/scenes/validate_chopped_100/gt.csv',\n    },\n    'test_data_loader': {\n        'key': 'scenes/test.zarr',\n        'batch_size': 128,\n        'shuffle': False,\n        'num_workers': 4 if on_kaggle else 6,\n        'prefetch_factor': 4,\n        'mask_path': f'{folder}/input/lyft-motion-prediction-autonomous-vehicles/scenes/mask.npz',\n    },\n    'train_params': {\n        'steps': 100,\n        'update_steps': 10,\n        'checkpoint_steps': 50,\n        'replay_steps': 0,\n        'replay_cache': 50,\n    } if test_run else {\n        'steps': 16000 if on_kaggle else 75_000,  # 300_000, 75_000, 50_000, 12_500\n        'update_steps': 200,  # change in v12\n        'checkpoint_steps': 50000,\n        'replay_steps': 0,\n        'replay_cache': 50,\n    }}","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"if on_kaggle and cfg['train_data_loader']['num_workers'] > 0:\n    os.environ[\"BLOSC_NOLOCK\"] = \"1\"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.039084,"end_time":"2020-11-05T04:08:56.469111","exception":false,"start_time":"2020-11-05T04:08:56.430027","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Couple of things to note:\n\n - **model_architecture:** you can put 'resnet18', 'resnet34' or 'resnet50'. For the pretrained model we use resnet18 so we need to use 'resnet18' in the config.\n - **weight_path:** path to the pretrained model. If you don't have a pretrained model and want to train from scratch, put **weight_path** = False. \n - **model_name:** the name of the model that will be saved as output, this is only when **train**= True.\n - **train:** True if you want to continue to train the model. Unfortunately due to Kaggle memory constraint if **train**=True then you should put **predict** = False.\n - **predict:** True if you want to predict and submit to Kaggle. Unfortunately due to Kaggle memory constraint if you want to predict then you need  to put **train** = False.\n - **lr:** learning rate of the model, feel free to change as you see fit. In the future I also plan to implement learning rate decay. \n - **raster_size:** specify the size of the image, the default is [224,224]. Increase **raster_size** can improve the score. However the training time will be significantly longer. \n - **batch_size:** number of inputs for one forward pass, again one of the parameters to tune. \n - **max_num_steps:** the number of iterations to train, i.e. number of epochs.\n - **checkpoint_every_n_steps:** the model will be saved at every n steps, again change this number as to how you want to keep track of the model."},{"metadata":{"papermill":{"duration":0.038818,"end_time":"2020-11-05T04:08:56.547353","exception":false,"start_time":"2020-11-05T04:08:56.508535","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Load the train and test datasets"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:08:56.633646Z","iopub.status.busy":"2020-11-05T04:08:56.632652Z","iopub.status.idle":"2020-11-05T04:08:56.636912Z","shell.execute_reply":"2020-11-05T04:08:56.637593Z"},"papermill":{"duration":0.050569,"end_time":"2020-11-05T04:08:56.637739","exception":false,"start_time":"2020-11-05T04:08:56.587170","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"%%time\nDIR_INPUT = cfg[\"data_path\"]\nos.environ[\"L5KIT_DATA_FOLDER\"] = DIR_INPUT\ndm = LocalDataManager()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:09:03.993346Z","iopub.status.busy":"2020-11-05T04:09:03.992218Z","iopub.status.idle":"2020-11-05T04:09:53.405469Z","shell.execute_reply":"2020-11-05T04:09:53.406389Z"},"papermill":{"duration":49.46674,"end_time":"2020-11-05T04:09:53.406620","exception":false,"start_time":"2020-11-05T04:09:03.939880","status":"completed"},"scrolled":true,"tags":[],"trusted":false},"cell_type":"code","source":"%%time\n# Train dataset\nif cfg[\"model_params\"][\"train\"]:\n    from parallelized_lyft_dataset6 import LyftDataset, lyft_dataset_worker_init_func\n    train_cfg = cfg[\"train_data_loader\"]\n    # train_zarr = ChunkedDataset(dm.require(train_cfg[\"key\"])).open(cached=False)  # try to turn off cache\n    # train_dataset = AgentDataset(cfg, train_zarr, rasterizer)\n    train_dataset = LyftDataset(cfg, dm, 'train_data_loader', fast=True, min_frame_future=train_cfg['min_frame_future'])\n    train_dataloader = DataLoader(\n        train_dataset, shuffle=train_cfg[\"shuffle\"], batch_size=train_cfg[\"batch_size\"],\n        num_workers=train_cfg[\"num_workers\"], persistent_workers=True,\n        worker_init_fn=lyft_dataset_worker_init_func, pin_memory=True,\n        prefetch_factor=train_cfg['prefetch_factor'],  # update in v11\n    )\n    # print(train_dataset)\n    print('train set size:', len(train_dataset))","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true,"papermill":{"duration":0.042178,"end_time":"2020-11-05T04:09:54.272735","exception":false,"start_time":"2020-11-05T04:09:54.230557","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Simple visualization\n\nLet us visualize how an input to the model looks like."},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:09:54.368166Z","iopub.status.busy":"2020-11-05T04:09:54.365991Z","iopub.status.idle":"2020-11-05T04:09:54.368940Z","shell.execute_reply":"2020-11-05T04:09:54.369554Z"},"hidden":true,"papermill":{"duration":0.054638,"end_time":"2020-11-05T04:09:54.369701","exception":false,"start_time":"2020-11-05T04:09:54.315063","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"def visualize_trajectory(dataset, index, title=\"target_positions movement with draw_trajectory\"):\n    data = dataset[index]\n    im = dataset.rasterizer.to_rgb(data[\"image\"].transpose(1, 2, 0))\n    target_positions_pixels = transform_points(data[\"target_positions\"], data[\"raster_from_agent\"])\n    draw_trajectory(im, target_positions_pixels, TARGET_POINTS_COLOR, radius=1, yaws=data[\"target_yaws\"])\n\n    plt.title(title)\n    plt.imshow(im, origin='lower')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:09:54.706748Z","iopub.status.busy":"2020-11-05T04:09:54.705904Z","iopub.status.idle":"2020-11-05T04:10:02.263773Z","shell.execute_reply":"2020-11-05T04:10:02.264446Z"},"hidden":true,"papermill":{"duration":7.631745,"end_time":"2020-11-05T04:10:02.264615","exception":false,"start_time":"2020-11-05T04:09:54.632870","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"# i_plot = 66652\n\n# plt.figure(figsize=(8, 6))\n# visualize_trajectory(train_dataset, index=i_plot)\n\n# plt.figure(figsize=(15, 15))\n# for i in range(25):\n#     plt.subplot(5, 5, i+1).set_title(f'{i}')\n#     plt.imshow(train_dataset[i_plot]['image'][i])\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true,"papermill":{"duration":0.046184,"end_time":"2020-11-05T04:10:02.363193","exception":false,"start_time":"2020-11-05T04:10:02.317009","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Loss function\nFor this competition it is important to use the correct loss function when train the model. Our goal is to predict three possible paths together with the confidence score, so we need to use the loss function that takes that into account, simply using RMSE will not lead to an accurate model. More information about the loss function can be found here [negative log likelihood](https://github.com/lyft/l5kit/blob/master/competition.md)."},{"metadata":{"code_folding":[],"execution":{"iopub.execute_input":"2020-11-05T04:10:02.467515Z","iopub.status.busy":"2020-11-05T04:10:02.466415Z","iopub.status.idle":"2020-11-05T04:10:02.483840Z","shell.execute_reply":"2020-11-05T04:10:02.483232Z"},"hidden":true,"papermill":{"duration":0.07257,"end_time":"2020-11-05T04:10:02.483987","exception":false,"start_time":"2020-11-05T04:10:02.411417","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"# --- Function utils ---\n# Original code from https://github.com/lyft/l5kit/blob/20ab033c01610d711c3d36e1963ecec86e8b85b6/l5kit/l5kit/evaluation/metrics.py\nimport numpy as np\n\nimport torch\nfrom torch import Tensor\n\n\ndef pytorch_neg_multi_log_likelihood_batch(\n    gt: Tensor, pred: Tensor, confidences: Tensor, avails: Tensor\n) -> Tensor:\n    \"\"\"\n    Compute a negative log-likelihood for the multi-modal scenario.\n    log-sum-exp trick is used here to avoid underflow and overflow, For more information about it see:\n    https://en.wikipedia.org/wiki/LogSumExp#log-sum-exp_trick_for_log-domain_calculations\n    https://timvieira.github.io/blog/post/2014/02/11/exp-normalize-trick/\n    https://leimao.github.io/blog/LogSumExp/\n    Args:\n        gt (Tensor): array of shape (bs)x(time)x(2D coords)\n        pred (Tensor): array of shape (bs)x(modes)x(time)x(2D coords)\n        confidences (Tensor): array of shape (bs)x(modes) with a confidence for each mode in each sample\n        avails (Tensor): array of shape (bs)x(time) with the availability for each gt timestep\n    Returns:\n        Tensor: negative log-likelihood for this example, a single float number\n    \"\"\"\n    assert len(pred.shape) == 4, f\"expected 3D (MxTxC) array for pred, got {pred.shape}\"\n    batch_size, num_modes, future_len, num_coords = pred.shape\n\n    assert gt.shape == (batch_size, future_len, num_coords), f\"expected 2D (Time x Coords) array for gt, got {gt.shape}\"\n    assert confidences.shape == (batch_size, num_modes), f\"expected 1D (Modes) array for gt, got {confidences.shape}\"\n    assert torch.allclose(torch.sum(confidences, dim=1), confidences.new_ones((batch_size,))), \"confidences should sum to 1\"\n    assert avails.shape == (batch_size, future_len), f\"expected 1D (Time) array for gt, got {avails.shape}\"\n    # assert all data are valid\n    assert torch.isfinite(pred).all(), \"invalid value found in pred\"\n    assert torch.isfinite(gt).all(), \"invalid value found in gt\"\n    assert torch.isfinite(confidences).all(), \"invalid value found in confidences\"\n    assert torch.isfinite(avails).all(), \"invalid value found in avails\"\n\n    # convert to (batch_size, num_modes, future_len, num_coords)\n    gt = torch.unsqueeze(gt, 1)  # add modes\n    avails = avails[:, None, :, None]  # add modes and cords\n\n    # error (batch_size, num_modes, future_len)\n    error = torch.sum(((gt - pred) * avails) ** 2, dim=-1)  # reduce coords and use availability\n\n    with np.errstate(divide=\"ignore\"):  # when confidence is 0 log goes to -inf, but we're fine with it\n        # error (batch_size, num_modes)\n        error = torch.log(confidences) - 0.5 * torch.sum(error, dim=-1)  # reduce time\n\n    # use max aggregator on modes for numerical stability\n    # error (batch_size, num_modes)\n    max_value, _ = error.max(dim=1, keepdim=True)  # error are negative at this point, so max() gives the minimum one\n    error = -torch.log(torch.sum(torch.exp(error - max_value), dim=-1, keepdim=True)) - max_value  # reduce modes\n    # print(\"error\", error)\n    return torch.mean(error)\n\n\ndef pytorch_neg_multi_log_likelihood_single(\n    gt: Tensor, pred: Tensor, avails: Tensor\n) -> Tensor:\n    \"\"\"\n\n    Args:\n        gt (Tensor): array of shape (bs)x(time)x(2D coords)\n        pred (Tensor): array of shape (bs)x(time)x(2D coords)\n        avails (Tensor): array of shape (bs)x(time) with the availability for each gt timestep\n    Returns:\n        Tensor: negative log-likelihood for this example, a single float number\n    \"\"\"\n    # pred (bs)x(time)x(2D coords) --> (bs)x(mode=1)x(time)x(2D coords)\n    # create confidence (bs)x(mode=1)\n    batch_size, future_len, num_coords = pred.shape\n    confidences = pred.new_ones((batch_size, 1))\n    return pytorch_neg_multi_log_likelihood_batch(gt, pred.unsqueeze(1), confidences, avails)","execution_count":null,"outputs":[]},{"metadata":{"hidden":true,"trusted":false},"cell_type":"code","source":"# Louis improved\ndef pytorch_neg_multi_log_likelihood_batch_imp(\n    true: Tensor, pred: Tensor, confidences: Tensor, avails: Tensor, test_run=test_run,\n) -> Tensor:\n    \"\"\"\n    Args:\n        true (Tensor): array of shape (bs)x(time)x(2D coords)\n        pred (Tensor): array of shape (bs)x(modes)x(time)x(2D coords)\n        confidences (Tensor): array of shape (bs)x(modes) with a confidence for each mode in each sample\n        avails (Tensor): array of shape (bs)x(time) with the availability for each true timestep\n    Returns:\n        Tensor: negative log-likelihood for this example, a single float number\n    \"\"\"\n    if test_run:\n        assert len(pred.shape) == 4, f\"expected 3D (MxTxC) array for pred, got {pred.shape}\"\n        batch_size, num_modes, future_len, num_coords = pred.shape\n\n        assert true.shape == (batch_size, future_len, num_coords), f\"expected 2D (Time x Coords) array for true, got {true.shape}\"\n        assert confidences.shape == (batch_size, num_modes), f\"expected 1D (Modes) array for true, got {confidences.shape}\"\n        assert torch.allclose(torch.sum(confidences, dim=1), confidences.new_ones((batch_size,))), \"confidences should sum to 1\"\n        assert avails.shape == (batch_size, future_len), f\"expected 1D (Time) array for true, got {avails.shape}\"\n        # assert all data are valid\n        assert torch.isfinite(pred).all(), \"invalid value found in pred\"\n        assert torch.isfinite(true).all(), \"invalid value found in true\"\n        assert torch.isfinite(confidences).all(), \"invalid value found in confidences\"\n        assert torch.isfinite(avails).all(), \"invalid value found in avails\"\n\n    # convert to (batch_size, num_modes, future_len, num_coords)\n    true = torch.unsqueeze(true, 1)  # add modes\n    avails = avails[:, None, :, None]  # add modes and cords\n\n    # error (batch_size, num_modes, future_len)\n    error = torch.sum(((pred - true) * avails) ** 2, dim=-1)  # reduce coords and use availability\n    error = torch.sum(error, dim=-1)  # reduce time\n    error = torch.log(confidences + 1e-8) - error / 2\n    error = -torch.logsumexp(error, dim=-1)\n    return torch.mean(error)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.046377,"end_time":"2020-11-05T04:10:02.576670","exception":false,"start_time":"2020-11-05T04:10:02.530293","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Model\nNext we define the baseline model. Note that this model will return three possible trajectories together with confidence score for each trajectory."},{"metadata":{"trusted":false},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"code_folding":[0],"execution":{"iopub.execute_input":"2020-11-05T04:10:02.688826Z","iopub.status.busy":"2020-11-05T04:10:02.683676Z","iopub.status.idle":"2020-11-05T04:10:02.698037Z","shell.execute_reply":"2020-11-05T04:10:02.697436Z"},"papermill":{"duration":0.074399,"end_time":"2020-11-05T04:10:02.698168","exception":false,"start_time":"2020-11-05T04:10:02.623769","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"# class OriginalLyftMultiModel(nn.Module):\n#     def __init__(self, cfg: Dict):\n#         super().__init__()\n\n#         architecture = cfg['model_params']['model_architecture']\n#         backbone = eval(architecture)(pretrained=True, progress=True)\n#         self.backbone = backbone\n\n#         num_history_channels = (cfg['model_params']['history_num_frames'] + 1) * 2\n#         num_in_channels = 3 + num_history_channels\n\n#         self.backbone.conv1 = nn.Conv2d(\n#             num_in_channels,\n#             self.backbone.conv1.out_channels,\n#             kernel_size=self.backbone.conv1.kernel_size,\n#             stride=self.backbone.conv1.stride,\n#             padding=self.backbone.conv1.padding,\n#             bias=False,\n#         )\n\n#         # This is 512 for resnet18 and resnet34\n#         # And it is 2048 for the other resnets        \n#         if architecture == 'resnet50':\n#             backbone_out_features = 2048\n#         else:\n#             backbone_out_features = 512\n\n#         # X, Y coords for the future positions (output shape: batch_sizex50x2)\n#         self.future_len = cfg['model_params']['future_num_frames']\n#         num_targets = 2 * self.future_len\n#         num_modes = cfg['model_params']['num_modes']\n\n#         # You can add more layers here.\n#         self.head = nn.Sequential(\n#             # nn.Dropout(0.2),\n#             nn.Linear(in_features=backbone_out_features, out_features=4096),\n#         )\n\n#         self.num_preds = num_targets * num_modes\n#         self.num_modes = num_modes\n\n#         self.logit = nn.Linear(4096, out_features=self.num_preds + num_modes)\n\n#     def forward(self, x):\n#         x = self.backbone.conv1(x)\n#         x = self.backbone.bn1(x)\n#         x = self.backbone.relu(x)\n#         x = self.backbone.maxpool(x)\n\n#         x = self.backbone.layer1(x)\n#         x = self.backbone.layer2(x)\n#         x = self.backbone.layer3(x)\n#         x = self.backbone.layer4(x)\n\n#         x = self.backbone.avgpool(x)\n#         x = torch.flatten(x, 1)\n\n#         x = self.head(x)\n#         x = self.logit(x)\n\n#         # pred (batch_size)x(modes)x(time)x(2D coords)\n#         # confidences (batch_size)x(modes)\n#         bs, _ = x.shape\n#         pred, confidences = torch.split(x, self.num_preds, dim=1)\n#         pred = pred.view(bs, self.num_modes, self.future_len, 2)\n#         assert confidences.shape == (bs, self.num_modes)\n#         confidences = torch.softmax(confidences, dim=1)\n#         return pred, confidences","execution_count":null,"outputs":[]},{"metadata":{"code_folding":[0],"trusted":false},"cell_type":"code","source":"# not used\ndef extrapolate_positions(history_positions, history_avail):\n    velocity_avail = history_avail[:, 1:]\n    avg_velocities = -10 * history_positions[:, 1:] / torch.arange(1, 11, device=device, requires_grad=False)[None, :, None]  # assume 0.1 s/frame \n    # weighted_avg_velocities\n    weighted_avg_velocities = (\n        (avg_velocities * velocity_avail[:, :, None]).sum(axis=1)\n        / (velocity_avail.sum(axis=1)[:, None] + 1e-8)  # 1e-8 to avoid devide by zero error\n    )\n    # speed threshold\n    speed_threshold = (weighted_avg_velocities**2).sum(axis=1) > 1.0  # filter out speed^2 < (1.0m/s)^2 cases\n    weighted_avg_velocities_th = weighted_avg_velocities * speed_threshold[:, None]\n    # extrapolation_positions_th\n    # assume 0.1 s/frame\n    return weighted_avg_velocities_th[:, None, :] * torch.arange(1, 51, device=device, requires_grad=False)[None, :, None] * 0.1    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def get_weighted_avg_velocities_and_rotation(history_positions, history_avail):\n    velocity_avail = history_avail[:, 1:]\n    # assume 0.1 s/frame \n    avg_velocities = (\n        -10 * history_positions[:, 1:] / torch.arange(1, 11, device=device, requires_grad=False)[None, :, None]\n    )\n    # weighted_avg_velocities\n    weighted_avg_velocities = (\n        (avg_velocities * velocity_avail[:, :, None]).sum(axis=1)\n        / (velocity_avail.sum(axis=1)[:, None] + 1e-8)  # 1e-8 to avoid devide by zero error\n    )\n    # speed threshold\n    speed_square = (weighted_avg_velocities**2).sum(axis=1)\n    speed_threshold = speed_square > 1.0  # filter out speed^2 < (1.0m/s)^2 cases\n    speed_th = torch.sqrt(speed_square * speed_threshold)\n    weighted_avg_velocities_th = weighted_avg_velocities * speed_threshold[:, None]\n    # Construct rotation matrix from velocity\n    cs = weighted_avg_velocities_th / (speed_th + 1e-8)[:, None]  # (cos(theta), sin(theta))\n    c = cs[:, 0]\n    s = cs[:, 1]\n    rotation_matrix = torch.stack((\n        torch.stack((c, -s), dim=1),\n        torch.stack((s, c), dim=1),\n    ), dim=1)\n    # for speed=0 case, we use identity matrix (meaning no rotation)\n    identity_matrix = (\n        torch.eye(2, device=device, requires_grad=False)[None, :, :] * (~speed_threshold)[:, None, None]\n    )\n    rotation_matrix = rotation_matrix + identity_matrix\n    return speed_th, rotation_matrix\ndef extrapolate_position_x(speed):\n    # assume 0.1 s/frame\n    velocity = torch.stack((speed, torch.zeros_like(speed)), dim=1)\n    return velocity[:, None, :] * torch.arange(1, 51, device=device, requires_grad=False)[None, :, None] * 0.1    ","execution_count":null,"outputs":[]},{"metadata":{"code_folding":[],"execution":{"iopub.execute_input":"2020-11-05T04:10:02.810729Z","iopub.status.busy":"2020-11-05T04:10:02.805611Z","iopub.status.idle":"2020-11-05T04:10:02.820400Z","shell.execute_reply":"2020-11-05T04:10:02.819742Z"},"papermill":{"duration":0.074991,"end_time":"2020-11-05T04:10:02.820510","exception":false,"start_time":"2020-11-05T04:10:02.745519","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"class LyftMultiModel(nn.Module):\n    def __init__(self, cfg: Dict):\n        super().__init__()\n\n        architecture = cfg['model_params']['model_architecture']\n        backbone = eval(architecture)(pretrained=True, progress=True)\n        self.backbone = backbone\n\n        num_history_channels = (cfg['model_params']['history_num_frames'] + 1) * 2\n        num_in_channels = 3 + num_history_channels\n\n        self.backbone.conv1 = nn.Conv2d(\n            num_in_channels,\n            self.backbone.conv1.out_channels,\n            kernel_size=self.backbone.conv1.kernel_size,\n            stride=self.backbone.conv1.stride,\n            padding=self.backbone.conv1.padding,\n            bias=False,\n        )\n\n        # This is 512 for resnet18 and resnet34\n        # And it is 2048 for the other resnets        \n        if architecture == 'resnet50':\n            backbone_out_features = 2048\n        else:\n            backbone_out_features = 512\n\n        # X, Y coords for the future positions (output shape: batch_sizex50x2)\n        self.future_len = cfg['model_params']['future_num_frames']\n        num_targets = 2 * self.future_len\n        num_modes = cfg['model_params']['num_modes']\n\n        # You can add more layers here.\n        self.head = nn.Sequential(\n            # nn.Dropout(0.2),\n            nn.Linear(in_features=backbone_out_features, out_features=4096),\n        )\n\n        self.num_preds = num_targets * num_modes\n        self.num_modes = num_modes\n\n        self.logit = nn.Linear(4096, out_features=self.num_preds + num_modes)\n\n#     def forward(self, x, history_positions, history_avail):\n    def forward(self, x):\n        x = self.backbone.conv1(x)\n        x = self.backbone.bn1(x)\n        x = self.backbone.relu(x)\n        x = self.backbone.maxpool(x)\n\n        x = self.backbone.layer1(x)\n        x = self.backbone.layer2(x)\n        x = self.backbone.layer3(x)\n        x = self.backbone.layer4(x)\n\n        x = self.backbone.avgpool(x)\n        x = torch.flatten(x, 1)\n\n        x = self.head(x)\n        x = self.logit(x)\n        \n#         # Compute weighted mean velocity and rotation matrix\n#         # speed (batch_size)\n#         # rotation_matrix (batch_size) x (2D coords target) x (2D coords inner)\n#         speed, rotation_matrix = get_weighted_avg_velocities_and_rotation(history_positions, history_avail)\n#         # extrapolate using the speed in x direction\n#         extrapolation_positions_th = extrapolate_position_x(speed)\n        \n#         # Extrapolate historical positions using weighted mean velocity\n#         extrapolation_positions_th = extrapolate_positions(history_positions, history_avail)\n\n        # pred (batch_size)x(modes)x(time)x(2D coords)\n        # confidences (batch_size)x(modes)\n        bs, _ = x.shape\n        pred, confidences = torch.split(x, self.num_preds, dim=1)\n        pred = pred.view(bs, self.num_modes, self.future_len, 2)\n#         # pred = pred + extrapolation_positions_th[:, None, :, :]\n#         pred0, pred12 = torch.split(pred, [1, 2], dim=1)\n#         pred0 = pred0 + extrapolation_positions_th[:, None, :, :]  # only add to the first mode\n#         pred = torch.cat((pred0, pred12), dim=1)\n#         # rotate from inner space to target space\n#         pred = torch.sum(pred[:, :, :, None, :] * rotation_matrix[:, None, None, :, :], dim=-1)\n        # assert confidences.shape == (bs, self.num_modes)\n        confidences = torch.softmax(confidences, dim=1)\n        return pred, confidences","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:10:02.923927Z","iopub.status.busy":"2020-11-05T04:10:02.922794Z","iopub.status.idle":"2020-11-05T04:10:02.926205Z","shell.execute_reply":"2020-11-05T04:10:02.925603Z"},"papermill":{"duration":0.059541,"end_time":"2020-11-05T04:10:02.926329","exception":false,"start_time":"2020-11-05T04:10:02.866788","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"def forward(data, model, device, criterion=pytorch_neg_multi_log_likelihood_batch_imp, compute_loss=True):\n    image = data['image'].to(device)\n#     history_positions = data['history_positions'].to(device)\n#     history_avail = data['history_availabilities'].to(device)\n    targets = data['target_positions'].to(device)\n    target_avail = data['target_availabilities'].to(device)\n    # Forward pass\n#     preds, confidences = model(image, history_positions, history_avail)\n    preds, confidences = model(image)\n    loss = criterion(targets, preds, confidences, target_avail) if compute_loss else -1\n    return loss, preds, confidences","execution_count":null,"outputs":[]},{"metadata":{"code_folding":[0],"execution":{"iopub.execute_input":"2020-11-05T04:10:02.923927Z","iopub.status.busy":"2020-11-05T04:10:02.922794Z","iopub.status.idle":"2020-11-05T04:10:02.926205Z","shell.execute_reply":"2020-11-05T04:10:02.925603Z"},"papermill":{"duration":0.059541,"end_time":"2020-11-05T04:10:02.926329","exception":false,"start_time":"2020-11-05T04:10:02.866788","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"# def forward(data, model, device, criterion=pytorch_neg_multi_log_likelihood_batch, compute_loss=True):\n#     image = data['image'].to(device)\n#     target_availabilities = data['target_availabilities'].to(device)\n#     targets = data['target_positions'].to(device)\n#     history_availabilities = data['history_availabilities'].to(device)\n#     history_positions = data['history_positions'].to(device)\n#     history_yaws = data['history_yaws'].to(device)\n#     # Forward pass\n#     preds, confidences = model(image, history_positions, history_availabilities, history_yaws)\n#     loss = criterion(targets, preds, confidences, target_availabilities) if compute_loss else -1\n#     return loss, preds, confidences","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:10:03.030967Z","iopub.status.busy":"2020-11-05T04:10:03.029961Z","iopub.status.idle":"2020-11-05T04:10:03.033590Z","shell.execute_reply":"2020-11-05T04:10:03.033024Z"},"papermill":{"duration":0.06074,"end_time":"2020-11-05T04:10:03.033714","exception":false,"start_time":"2020-11-05T04:10:02.972974","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"def set_train_for_resnet(model, is_train):\n    for child in model.children():\n        if isinstance(child, torchvision.models.resnet.ResNet):\n            for param in child.parameters():\n                param.requires_grad = is_train\n\ndef check_resnet_train(model):\n    is_train = []\n    for child in model.children():\n        if isinstance(child, torchvision.models.resnet.ResNet):\n            for param in child.parameters():\n                is_train.append(param.requires_grad)\n    return is_train","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:10:03.540808Z","iopub.status.busy":"2020-11-05T04:10:03.539873Z","iopub.status.idle":"2020-11-05T04:10:13.202708Z","shell.execute_reply":"2020-11-05T04:10:13.203335Z"},"papermill":{"duration":10.123004,"end_time":"2020-11-05T04:10:13.203513","exception":false,"start_time":"2020-11-05T04:10:03.080509","status":"completed"},"scrolled":true,"tags":[],"trusted":false},"cell_type":"code","source":"%%time\n# ==== INIT MODEL=================\nmodel = LyftMultiModel(cfg)\n\n#load weight if there is a pretrained model\nif cfg['model_params']['load_pretrain_weight']:\n    print('load pretrained model..')\n    model.load_state_dict(torch.load(cfg[\"model_params\"][\"weight_path\"]))\n\n# turn off training for resnet (don't work since we have another layer)\n# set_train_for_resnet(model, False)\n    \nmodel.to(device)\nprint(f'device {device}')\n# optimizer = optim.Adam(model.parameters(), lr=cfg['model_params']['lr'])\noptimizer = optim.SGD(model.parameters(), lr=cfg['model_params']['lr'], momentum=0.9)\nif cfg['model_params']['load_pretrain_optimizer']:\n    print('load pretrained optimizer..')\n    optimizer.load_state_dict(torch.load(cfg['model_params']['optimizer_path']))\n    # overwrite learning rate\n    optimizer.param_groups[0]['lr'] = cfg['model_params']['lr']","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:10:13.309122Z","iopub.status.busy":"2020-11-05T04:10:13.307975Z","iopub.status.idle":"2020-11-05T04:10:13.320196Z","shell.execute_reply":"2020-11-05T04:10:13.321017Z"},"papermill":{"duration":0.06811,"end_time":"2020-11-05T04:10:13.321232","exception":false,"start_time":"2020-11-05T04:10:13.253122","status":"completed"},"scrolled":true,"tags":[],"trusted":false},"cell_type":"code","source":"print(model)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:10:13.437423Z","iopub.status.busy":"2020-11-05T04:10:13.436649Z","iopub.status.idle":"2020-11-05T04:10:13.443823Z","shell.execute_reply":"2020-11-05T04:10:13.443059Z"},"papermill":{"duration":0.062299,"end_time":"2020-11-05T04:10:13.443999","exception":false,"start_time":"2020-11-05T04:10:13.381700","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"print(check_resnet_train(model))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"for param_group in optimizer.param_groups:\n    print(param_group['lr'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"if cfg['model_params']['lr_scheduler']:\n    scheduler = OneCycleLR(optimizer, max_lr=cfg[\"model_params\"]['max_lr'], \n                           total_steps=cfg[\"train_params\"][\"steps\"]*cfg['model_params']['lr_scheduler_expect_rounds'])\n    if cfg['model_params']['load_pretrain_scheduler']:\n        print('load pretrained scheduler..')\n        scheduler.load_state_dict(torch.load(cfg['model_params']['scheduler_path']))\n    print(scheduler.state_dict())","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.050454,"end_time":"2020-11-05T04:10:13.544517","exception":false,"start_time":"2020-11-05T04:10:13.494063","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Training loop\nNext let us implement the training loop, when the **train** parameter is set to True. "},{"metadata":{"trusted":false},"cell_type":"code","source":"train_dataset_total_batches = int(np.ceil(len(train_dataset) / cfg['train_data_loader']['batch_size']))\nprint('Number of batches in train:', train_dataset_total_batches)\nprint('We will only train:', cfg[\"train_params\"][\"steps\"], 'batches (%.4f%%)'%(cfg[\"train_params\"][\"steps\"] * 100 / train_dataset_total_batches))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"len(train_dataset)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"train\"]:\n    tr_it = iter(train_dataloader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# New in v11\n# from collections import deque\nclass ReplayMemory(object):\n    ''' storage class for sample reuse '''\n    def __init__(self, capacity):\n        self.capacity = capacity\n        self.memory = []\n\n    def push(self, item):\n        \"\"\"Saves a transition.\"\"\"\n        if len(self.memory) >= self.capacity:\n            del self.memory[0]\n        self.memory.append(item)\n\n    def sample(self, last_item=False):\n        if last_item:\n            item = self.memory[-1]\n        else:\n            item = self.memory[np.random.randint(len(self.memory))]\n        return item\n\n    def __len__(self):\n        return len(self.memory)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"lr_reduce = cfg['model_params']['lr_reduce']\nlr_reduce_steps = cfg['model_params']['lr_reduce_steps']\nlr_reduce, lr_reduce_steps","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def reduce_learning_rate(optimizer, reduce_factor, verbose=True):\n    for i, param_group in enumerate(optimizer.param_groups):\n        old_lr = float(param_group['lr'])\n        new_lr = old_lr * reduce_factor\n        param_group['lr'] = new_lr\n        if verbose:\n            print('Reduce learning rate of group {} from {:.4e} to {:.4e}.'.format(i, old_lr, new_lr))","execution_count":null,"outputs":[]},{"metadata":{"code_folding":[],"execution":{"iopub.execute_input":"2020-11-05T04:10:13.664410Z","iopub.status.busy":"2020-11-05T04:10:13.662335Z","iopub.status.idle":"2020-11-05T04:10:13.669191Z","shell.execute_reply":"2020-11-05T04:10:13.666069Z"},"papermill":{"duration":0.074048,"end_time":"2020-11-05T04:10:13.669353","exception":false,"start_time":"2020-11-05T04:10:13.595305","status":"completed"},"scrolled":true,"tags":[],"trusted":false},"cell_type":"code","source":"%%time\nif cfg[\"model_params\"][\"train\"]:\n    replay_memory = ReplayMemory(cfg['train_params']['replay_cache'])\n    n_replay_steps = cfg['train_params']['replay_steps']\n    n_steps = cfg[\"train_params\"][\"steps\"]\n    progress_bar = tqdm_notebook(range(1, 1 + n_steps), mininterval=5.)\n    losses_all = []\n    iterations = []\n    all_metrics = []\n    metrics = []\n    memorys = []\n    times = []\n    lr_history = []\n    update_steps = cfg['train_params']['update_steps']\n    checkpoint_steps = cfg['train_params']['checkpoint_steps']\n    t_start = time.time()\n    i_epochs = 1\n    torch.set_grad_enabled(True)\n    \n    for i in progress_bar:\n        try:\n            data = next(tr_it)\n        except StopIteration:\n            tr_it = iter(train_dataloader)\n            data = next(tr_it)\n            i_epochs += 1\n        if n_replay_steps > 0:\n            # Replay\n            replay_memory.push(data)\n            for r in range(n_replay_steps):\n                data = replay_memory.sample(r == 0)\n                model.train()\n                loss, _, _ = forward(data, model, device)\n\n                # (change after the run v11)\n                if r == 0:\n                    loss_v = loss.item()\n\n                # Backward pass\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n        else:\n            model.train()\n            loss, _, _ = forward(data, model, device)\n\n            loss_v = loss.item()\n\n            # Backward pass\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n        \n        losses_all.append(loss_v)\n        \n        if i % update_steps == 0:\n            mean_losses_all = np.mean(losses_all)\n            mean_losses_update = np.mean(losses_all[-update_steps:])\n            timespent = (time.time() - t_start) / 60\n            current_lr = optimizer.param_groups[0]['lr']\n            print('i: %5d'%i, 'epochs:', i_epochs,\n                  'loss: %.5f'%loss_v, 'loss(avg): %.5f'%mean_losses_update, \n                  'loss(all): %.5f'%mean_losses_all, '%.2fmins'%timespent, 'lr=%.2e'%current_lr, end=' | ')\n            mem = memory()\n            if i % checkpoint_steps == 0:\n                torch.save(model.state_dict(), f'{model_name}-{train_round}-{i}.pth')\n                torch.save(optimizer.state_dict(), f'{model_name}_optimizer-{train_round}-{i}.pth')\n            iterations.append(i)\n            all_metrics.append(mean_losses_all)\n            metrics.append(mean_losses_update)  # use update_steps mean\n            lr_history.append(current_lr)\n            memorys.append(mem)\n            times.append(timespent)\n\n        if cfg['model_params']['lr_scheduler']:\n            scheduler.step()\n        if i % lr_reduce_steps == 0:\n            reduce_learning_rate(optimizer, lr_reduce)        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"del loss","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"train\"]:\n    torch.save(model.state_dict(), f'{model_name}-{train_round}-final.pth')\n    torch.save(optimizer.state_dict(), f'{model_name}_optimizer-{train_round}-final.pth')\n    if cfg['model_params']['lr_scheduler']:\n        torch.save(scheduler.state_dict(), f'{model_name}_scheduler-{train_round}-final.pth')\n    results = pd.DataFrame({\n        'iterations': iterations, \n        'metrics (all avg)': all_metrics, \n        'metrics (update avg)': metrics,\n        'elapsed_time (mins)': times,\n        'memory (GB)': memorys,\n        'learning rate': lr_history,\n    })\n    results.to_csv(f\"train_metrics_{model_name}-{train_round}-{n_steps}.csv\", index=False)\n    print(f\"Total training time is {(time.time() - t_start) / 60} mins\")\n    print('training speed:', iterations[-1] / (60*times[-1]), 'it/s')\n    memory()\n    display(results)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- === resnet50 ===\n- 15.3s         - 8 workers 4 prefetch\n- 14.6s (28s)   - 6 workers 2 prefetch\n- 16.7s         - 4 workers 2 prefetch\n- === resnet18 ===\n- 9.22s - 8 workers 8 prefetch\n- 13.3  - 8 workers 4 prefetch 2 replay 10 replay_cache\n- 13.3  - 8 workers 6 prefetch 2 replay 10 replay_cache\n- 13.4  - 8 workers 8 prefetch 2 replay 10 replay_cache\n- 17.6  - 8 workers 4 prefetch 3 replay 10 replay_cache\n- 9.76  - 6 workers 32 prefetch 2 replay 50 replay_cache \n- 9.76  - 6 workers 32 prefetch 2 replay 50 replay_cache, drop columns "},{"metadata":{"papermill":{"duration":0.050472,"end_time":"2020-11-05T04:10:13.771539","exception":false,"start_time":"2020-11-05T04:10:13.721067","status":"completed"},"tags":[]},"cell_type":"markdown","source":"622: skip join before relu  \n605: skip join after relu  \n709: skip join after relu additional bn for other  \n743: split into 2  "},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:10:14.018729Z","iopub.status.busy":"2020-11-05T04:10:14.016275Z","iopub.status.idle":"2020-11-05T04:10:14.022309Z","shell.execute_reply":"2020-11-05T04:10:14.021640Z"},"papermill":{"duration":0.199628,"end_time":"2020-11-05T04:10:14.022437","exception":false,"start_time":"2020-11-05T04:10:13.822809","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"gc_memory()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"train\"]:\n    print('training speed:', iterations[-1] / (60*times[-1]), 'it/s')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:10:14.134476Z","iopub.status.busy":"2020-11-05T04:10:14.132786Z","iopub.status.idle":"2020-11-05T04:10:14.135241Z","shell.execute_reply":"2020-11-05T04:10:14.135796Z"},"papermill":{"duration":0.062114,"end_time":"2020-11-05T04:10:14.135942","exception":false,"start_time":"2020-11-05T04:10:14.073828","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"train\"]:\n    plt.figure(figsize=(12, 4))\n    plt.plot(iterations, all_metrics, label='all avg')\n    plt.plot(iterations, metrics, label=f'{update_steps}-batch avg', alpha=0.7)\n    plt.xlabel('steps'); plt.ylabel('metrics (avg)')\n    plt.legend()\n    plt.grid(); plt.show()","execution_count":null,"outputs":[]},{"metadata":{"code_folding":[],"trusted":false},"cell_type":"code","source":"# expected learning rate (not necessarily the original one)\n# lr_sim = cfg['model_params']['lr']\n# lr_history = []\n# for i in iterations:\n#     lr_history.append(lr_sim)\n#     if i % lr_reduce_steps == 0:\n#         lr_sim = lr_sim * lr_reduce\nif cfg[\"model_params\"][\"train\"]:\n    plt.figure(figsize=(12, 3))\n    plt.plot(iterations, lr_history, label='all avg')\n    plt.xlabel('steps'); plt.ylabel('learning rate')\n    plt.legend()\n    plt.grid(); plt.show()","execution_count":null,"outputs":[]},{"metadata":{"code_folding":[0],"execution":{"iopub.execute_input":"2020-11-05T04:10:14.249806Z","iopub.status.busy":"2020-11-05T04:10:14.248346Z","iopub.status.idle":"2020-11-05T04:10:14.250588Z","shell.execute_reply":"2020-11-05T04:10:14.251222Z"},"papermill":{"duration":0.062975,"end_time":"2020-11-05T04:10:14.251380","exception":false,"start_time":"2020-11-05T04:10:14.188405","status":"completed"},"scrolled":true,"tags":[],"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"train\"]:\n    plt.figure(figsize=(12, 4))\n    plt.plot(iterations, memorys)\n    plt.xlabel('steps'); plt.ylabel('memory (GB)')\n    plt.grid(); plt.show()","execution_count":null,"outputs":[]},{"metadata":{"code_folding":[0],"execution":{"iopub.execute_input":"2020-11-05T04:10:14.362533Z","iopub.status.busy":"2020-11-05T04:10:14.361649Z","iopub.status.idle":"2020-11-05T04:10:14.365614Z","shell.execute_reply":"2020-11-05T04:10:14.366324Z"},"papermill":{"duration":0.062871,"end_time":"2020-11-05T04:10:14.366480","exception":false,"start_time":"2020-11-05T04:10:14.303609","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"train\"]:\n    plt.figure(figsize=(12, 4))\n    plt.plot(iterations, times, label='measured data')\n    plt.plot([iterations[0], iterations[-1]], [times[0], times[-1]], alpha=0.5, label='linear ref')\n    plt.xlabel('steps'); plt.ylabel('elapsed_time (mins)')\n    plt.legend(); plt.grid(); plt.show()\n    # if we see the measured data below the linear, means the training getting slower when the steps increase","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"train\"]:\n    tr_it._shutdown_workers()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"del tr_it","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"del replay_memory","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"del train_dataloader","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"gc_memory()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Examine last layer"},{"metadata":{"trusted":false},"cell_type":"code","source":"w = model.logit.weight\nb = model.logit.bias","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"b.shape","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"b_pred, b_confidences = torch.split(b, model.num_preds, dim=0)\nb_pred, b_confidences","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"b_pred_v = b_pred.view(3, model.future_len, 2)\nb_pred_v","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"b_pred_np = b_pred_v.detach().cpu().numpy().copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# bias on the last layer\nfor i in range(3):\n    plt.plot(*b_pred_np[i].T, marker='.', label=f'mode = {i}', alpha=0.8)\nplt.grid(); plt.legend(); plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# bias overall\nfor i in range(3):\n    plt.plot(*b_pred_np[i].T, marker='.', label=f'mode = {i}', alpha=0.8, )\n# plt.plot(*target_positions_mean.cpu().numpy().copy().T, linestyle='--', alpha=0.5, label='target mean')\nplt.grid(); plt.legend(); plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# confidence bias\ntorch.softmax(b_confidences.detach(), dim=0).cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{"heading_collapsed":true},"cell_type":"markdown","source":"# Score and Plot functions"},{"metadata":{"code_folding":[],"hidden":true,"trusted":false},"cell_type":"code","source":"# Plot and score functions\nfrom collections import OrderedDict, defaultdict\nfrom l5kit.evaluation import read_gt_csv, read_pred_csv\nfrom l5kit.evaluation.extract_metrics import validate_dicts\n\ndef compute_csv_nnl_by_row(truth_path: str, pred_path: str) -> tuple:\n    truth = OrderedDict()\n    preds = OrderedDict()\n    scores = OrderedDict()\n\n    for el in read_gt_csv(truth_path):\n        truth[el[\"track_id\"], el[\"timestamp\"]] = el\n    for el in read_pred_csv(pred_path):\n        preds[el[\"track_id\"], el[\"timestamp\"]] = el\n\n    if not validate_dicts(truth, preds):\n        raise ValueError(\"Error validating csv, see above for details.\")\n\n    for key, truth_value in truth.items():\n        true_coord = truth_value[\"coord\"]\n        avail = truth_value[\"avail\"]\n\n        pred_coords = preds[key][\"coords\"]\n        conf = preds[key][\"conf\"]\n        scores[key] = neg_multi_log_likelihood(true_coord, pred_coords, conf, avail)\n        \n    # compute average of each metric\n    return truth, preds, scores\n\n# plot functions\nimport matplotlib.patches as mpatches\n\ndef row_to_confs(row):\n    return [row[f'conf_{i}'] for i in range(3)]\ndef row_to_coords(row):\n    return row[3:].values.reshape(3, 50, 2)\n\ndef row_truth_to_avail(row):\n    return row[:50].values\ndef row_truth_to_coords(row):\n    return row[50:].values.reshape(50, 2)\n\n# here I use matplotlib default colors\ncmap = plt.get_cmap(\"tab10\")\nmatplotlib_colors_in_rgb_int = [\n    [int(255 * x) for x in cmap(i)[:3]] for i in range(10)\n]\n\ndef generate_image_predicted_trajectory(dataset, df_sub, index):\n    data = dataset[index]\n    im = data['image'].transpose(1, 2, 0)\n    im = dataset.rasterizer.to_rgb(im)\n    row = df_sub.loc[(data['timestamp'], data['track_id'])]\n    # note submission coordinate system = world - centroid\n    predicted_target_positions_in_sub = row_to_coords(row)\n    predicted_target_positions_in_world = predicted_target_positions_in_sub + data['centroid']\n    for i, coords in enumerate(predicted_target_positions_in_world):\n        target_positions_pixels = transform_points(coords, data['raster_from_world'])\n        draw_trajectory(im, target_positions_pixels, rgb_color=matplotlib_colors_in_rgb_int[i])\n    return im, row_to_confs(row)\n\ndef plot_predicted_trajectory(dataset, df_sub, indices, width=15, height=5, n_cols=3, title=''):\n    if not isinstance(indices, (list, np.ndarray)):\n        indices = [indices]\n    n_rows = len(indices) // n_cols + len(indices) % n_cols\n    plt.figure(figsize=(width, height*n_rows))\n    for k, index in enumerate(indices):\n        plt.subplot(n_rows, n_cols, 1+k).set_title(str(index))\n        im, confs = generate_image_predicted_trajectory(dataset, df_sub, index)\n        patches = [mpatches.Patch(color=cmap(m), label='%.3f'%conf) for m, conf in enumerate(confs)]\n        plt.imshow(im, origin='lower')\n        plt.legend(handles=patches)\n    if title:\n        plt.suptitle(title)\n    plt.show()\n\ndef generate_image_from_submission(data, rasterizer, pred_row=None, truth_row=None, truth_from_data=False):\n    im = data['image'].transpose(1, 2, 0)\n    im = rasterizer.to_rgb(im)\n    target_avail = np.ones(50, dtype='bool')\n    if truth_from_data:\n        target_avail = data['target_availabilities'] > 0.5\n        target_positions_pixels = transform_points(\n            data[\"target_positions\"][target_avail], data[\"raster_from_agent\"])\n        yaws = data[\"target_yaws\"][target_avail]\n        draw_trajectory(im, target_positions_pixels, TARGET_POINTS_COLOR, radius=1, yaws=yaws)\n    if truth_row is not None:\n        target_avail = row_truth_to_avail(truth_row) > 0.5\n        target_positions_in_sub = row_truth_to_coords(truth_row)[target_avail]\n        target_positions_in_world = target_positions_in_sub + data['centroid']\n        target_positions_pixels = transform_points(\n            target_positions_in_world, data['raster_from_world'])\n        draw_trajectory(im, target_positions_pixels, TARGET_POINTS_COLOR, radius=2)\n    if pred_row is not None:\n        # note submission coordinate system = world - centroid\n        predicted_target_positions_in_sub = row_to_coords(pred_row)\n        predicted_target_positions_in_world = predicted_target_positions_in_sub + data['centroid']\n        for i, coords in enumerate(predicted_target_positions_in_world):\n            pred_positions_pixels = transform_points(coords[target_avail], data['raster_from_world'])\n            draw_trajectory(im, pred_positions_pixels, rgb_color=matplotlib_colors_in_rgb_int[i])\n    return im\n\ndef plot_validated_trajectory(dataset, df_sub, df_val_truth, indices, width=14, height=7, n_cols=2, suptitle='',\n                              scores_val=None):\n    if not isinstance(indices, (list, np.ndarray)):\n        indices = [indices]\n    n_rows = len(indices) // n_cols + len(indices) % n_cols\n    plt.figure(figsize=(width, height*n_rows))\n    for k, index in enumerate(indices):\n        data = dataset[index]\n        timestamp = data['timestamp']\n        track_id = data['track_id']\n        title = f'{index} - {track_id}, {timestamp}'\n        if scores_val is not None:\n            score = scores_val[f'{track_id}', f'{timestamp}']\n            title = title + f' {score}'\n        print(title)\n        pred_row = df_sub.loc[(timestamp, track_id)]\n        truth_row = df_val_truth.loc[(timestamp, track_id)]\n        plt.subplot(n_rows, n_cols, 1+k).set_title(title)\n        im = generate_image_from_submission(data, dataset.rasterizer, \n                                            pred_row=pred_row, truth_row=truth_row,\n                                            truth_from_data=False)\n        confs = row_to_confs(pred_row)\n        patches = [mpatches.Patch(color=cmap(m), label='%.3f'%conf) for m, conf in enumerate(confs)]\n        plt.imshow(im, origin='lower')\n        plt.legend(handles=patches)\n    if title:\n        plt.suptitle(suptitle)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validation 100"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:09:53.495318Z","iopub.status.busy":"2020-11-05T04:09:53.494266Z","iopub.status.idle":"2020-11-05T04:09:54.093058Z","shell.execute_reply":"2020-11-05T04:09:54.092465Z"},"papermill":{"duration":0.645852,"end_time":"2020-11-05T04:09:54.093196","exception":false,"start_time":"2020-11-05T04:09:53.447344","status":"completed"},"scrolled":true,"tags":[],"trusted":false},"cell_type":"code","source":"%%time\n# Validate dataset\nif cfg['model_params']['validate']:\n    from parallelized_lyft_dataset6 import LyftDataset, lyft_dataset_worker_init_func\n#     from parallelized_lyft_dataset6 import lyft_dataset_worker_init_func\n#     # Build rasterizer\n#     if not cfg[\"model_params\"]['predict']:\n#         rasterizer = build_rasterizer(cfg, dm)\n\n    val_cfg = cfg['val_100_data_loader']\n#     val_zarr = ChunkedDataset(dm.require(val_cfg[\"key\"])).open(cached=False)  # try to turn off cache\n#     val_mask = np.load(val_cfg['mask_path'])[\"arr_0\"]\n#     val_dataset = AgentDataset(cfg, val_zarr, rasterizer, agents_mask=val_mask) \n    val_dataset = LyftDataset(cfg, dm, 'val_100_data_loader', agents_mask_path=val_cfg['mask_path']) \n    set_seed(3322)\n#     val_dataloader = DataLoader(val_dataset, shuffle=val_cfg[\"shuffle\"],\n#                                 batch_size=val_cfg[\"batch_size\"], num_workers=val_cfg[\"num_workers\"])\n    val_dataloader = DataLoader(\n        val_dataset, shuffle=val_cfg['shuffle'], batch_size=val_cfg['batch_size'],\n        num_workers=val_cfg['num_workers'], persistent_workers=False,\n        worker_init_fn=lyft_dataset_worker_init_func, pin_memory=False,  # update in v11\n        prefetch_factor=val_cfg['prefetch_factor'],  # update in v11\n    )\n#     print(val_dataset)\n    print('val set size:', len(val_dataset))\n    \n    total_val_batches = int(np.ceil(len(val_dataset) / cfg['val_100_data_loader']['batch_size']))\n    print('Number of batches for validation:', total_val_batches)\n    # print('Evaluating on:', cfg['val_data_loader']['num_batches'], 'batches (%.4f%%)'%(cfg['val_data_loader']['num_batches'] *100/ total_val_batches))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:10:14.704221Z","iopub.status.busy":"2020-11-05T04:10:14.703020Z","iopub.status.idle":"2020-11-05T06:24:21.509427Z","shell.execute_reply":"2020-11-05T06:24:21.505071Z"},"papermill":{"duration":8046.870251,"end_time":"2020-11-05T06:24:21.509642","exception":false,"start_time":"2020-11-05T04:10:14.639391","status":"completed"},"scrolled":true,"tags":[],"trusted":false},"cell_type":"code","source":"%%time\nif cfg[\"model_params\"][\"validate\"]:\n    \n    model.eval()\n    torch.set_grad_enabled(False)\n\n    # store information for evaluation\n    val_loss = []\n    future_coords_offsets_pd = []\n    timestamps = []\n    confidences_list = []\n    agent_ids = []\n    memorys_val = []\n    t0 = time.time()\n    times_val = []\n    iterations_val = []\n    i_update = 5 if test_run else 50\n    \n#     val_it = iter(val_dataloader)\n\n#     for i, data in enumerate(tqdm_notebook(range(cfg['val_data_loader']['num_batches']), mininterval=5.)):\n    for i, data in enumerate(tqdm_notebook(val_dataloader, mininterval=5.)):\n#         data = next(val_it)\n        loss, preds, confidences = forward(data, model, device, compute_loss=True)\n    \n        preds = torch.einsum('bmti,bji->bmtj', \n                             preds.double(), \n                             data[\"world_from_agent\"].to(device)[:, :2, :2]).cpu().numpy()\n        \n        val_loss.append(loss.item())\n        timestamps.append(data[\"timestamp\"].numpy().copy())\n        agent_ids.append(data[\"track_id\"].numpy().copy()) \n        future_coords_offsets_pd.append(preds.copy())\n        confidences_list.append(confidences.cpu().numpy().copy())\n        \n        if i % i_update == 0:\n            t = ((time.time() - t0) / 60)\n            print('%4d'%i, '% 7.3f'%np.mean(val_loss), '%6.2fmins'%t, end=' | ')\n            mem = memory()\n            iterations_val.append(i)\n            memorys_val.append(mem)\n            times_val.append(t)\n            if test_run and i >= 50:\n                break\n    print('Validation loss:', np.mean(val_loss))\n    print('Total timespent: %6.2fmins'%((time.time() - t0) / 60))\n    memory()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# train_round = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"validate\"]:\n    val_path = f'{model_name}-val_100-submission-{train_round}.csv'\n    write_pred_csv(\n        val_path,\n        timestamps=np.concatenate(timestamps),\n        track_ids=np.concatenate(agent_ids),\n        coords=np.concatenate(future_coords_offsets_pd),\n        confs=np.concatenate(confidences_list),\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"validate\"] and not test_run:\n    compute_metrics_results = compute_metrics_csv(\n        cfg['val_100_data_loader']['truth_path'], val_path, \n        [neg_multi_log_likelihood, time_displace],\n    )\n    for metric_name, metric_mean in compute_metrics_results.items():\n        print(metric_name, metric_mean)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Examing validation result"},{"metadata":{"trusted":false},"cell_type":"code","source":"%%time\ntruth_val, preds_val, scores_val = compute_csv_nnl_by_row(cfg['val_100_data_loader']['truth_path'], val_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"scores_val_list = sorted(list(scores_val.items()), key=lambda x: x[1], reverse=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"np.mean([x[1] for x in scores_val_list])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# worse case\nscores_val_list[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# good case\nscores_val_list[-5:]","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"n_plot = 10\nfigsize = (8, 8)\n# plt.figure(figsize)\nfor row in scores_val_list[:n_plot]:\n    s = row[1]\n    key = row[0]\n    t = truth_val[key]\n    avail = t[\"avail\"].astype('bool')\n    true_coord = t[\"coord\"][avail]\n\n    pred_coords = preds_val[key][\"coords\"]\n    confs = preds_val[key][\"conf\"]\n    plt.figure(figsize=figsize)\n    plt.plot(*true_coord.T, label='true', linestyle='--', marker='.', alpha=0.8)\n    for m, (coord, conf) in enumerate(zip(pred_coords, confs)):\n        plt.plot(*coord[avail].T, label=f'pred_{m}: {conf}', alpha=0.6, marker='+')\n    plt.legend()\n    plt.title(f'{key}, score {s}')\n    plt.grid(); plt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"n_plot = 10\nfigsize = (8, 8)\n# plt.figure(figsize)\nfor row in scores_val_list[-n_plot:]:\n    s = row[1]\n    key = row[0]\n    t = truth_val[key]\n    avail = t[\"avail\"].astype('bool')\n    true_coord = t[\"coord\"][avail]\n\n    pred_coords = preds_val[key][\"coords\"]\n    confs = preds_val[key][\"conf\"]\n    plt.figure(figsize=figsize)\n    plt.plot(*true_coord.T, label='true', linestyle='--', marker='.', alpha=0.8)\n    for m, (coord, conf) in enumerate(zip(pred_coords, confs)):\n        plt.plot(*coord[avail].T, label=f'pred_{m}: {conf}', alpha=0.6, marker='+')\n    plt.legend()\n    plt.title(f'{key}, score {s}')\n    plt.grid(); plt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false},"cell_type":"code","source":"df_val = pd.read_csv(val_path)\ndf_val = df_val.set_index(['timestamp', 'track_id'])\ndisplay(df_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"val_truth_path = cfg['val_100_data_loader']['truth_path']\ndf_val_truth = pd.read_csv(val_truth_path)\ndf_val_truth = df_val_truth.set_index(['timestamp', 'track_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"validate\"]:\n    rasterizer = build_rasterizer(cfg, dm)\n    val_cfg = cfg[\"val_100_data_loader\"]\n    val_zarr = ChunkedDataset(dm.require(val_cfg[\"key\"])).open(cached=False)  # try to turn off cache\n    val_mask = np.load(val_cfg['mask_path'])[\"arr_0\"]\n    val_dataset = AgentDataset(cfg, val_zarr, rasterizer, agents_mask=val_mask)\n    print(val_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"validate\"]:\n    plot_validated_trajectory(val_dataset, df_val, df_val_truth, [18431], width=6, height=6, n_cols=1, \n                              scores_val=scores_val)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"validate\"]:\n    i_plots = np.random.randint(len(val_dataset), size=9)\n    plot_validated_trajectory(val_dataset, df_val, df_val_truth, i_plots, scores_val=scores_val)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.051964,"end_time":"2020-11-05T04:10:14.470762","exception":false,"start_time":"2020-11-05T04:10:14.418798","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Prediction\nFinally we implement the inference to submit to Kaggle when **predict** param is set to True."},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:09:53.495318Z","iopub.status.busy":"2020-11-05T04:09:53.494266Z","iopub.status.idle":"2020-11-05T04:09:54.093058Z","shell.execute_reply":"2020-11-05T04:09:54.092465Z"},"papermill":{"duration":0.645852,"end_time":"2020-11-05T04:09:54.093196","exception":false,"start_time":"2020-11-05T04:09:53.447344","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"%%time\n# Test dataset\nif cfg[\"model_params\"]['predict']:\n    from parallelized_lyft_dataset6 import LyftDataset, lyft_dataset_worker_init_func\n#     # Build rasterizer\n#     rasterizer = build_rasterizer(cfg, dm)\n\n    test_cfg = cfg[\"test_data_loader\"]\n    test_dataset = LyftDataset(cfg, dm, 'test_data_loader', agents_mask_path=test_cfg['mask_path']) \n    test_dataloader = DataLoader(\n        test_dataset, shuffle=test_cfg['shuffle'], batch_size=test_cfg['batch_size'],\n        num_workers=test_cfg['num_workers'], persistent_workers=False,\n        worker_init_fn=lyft_dataset_worker_init_func, pin_memory=False,\n        prefetch_factor=test_cfg['prefetch_factor'],  # update in v11\n    )\n    \n#     test_zarr = ChunkedDataset(dm.require(test_cfg[\"key\"])).open(cached=False)  # try to turn off cache\n#     test_mask = np.load(f\"{DIR_INPUT}/scenes/mask.npz\")[\"arr_0\"]\n#     test_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)\n#     test_dataloader = DataLoader(test_dataset, shuffle=test_cfg[\"shuffle\"],\n#                                  batch_size=test_cfg[\"batch_size\"], num_workers=test_cfg[\"num_workers\"])\n#     print(test_dataset)\n    print('test set size:', len(test_dataset))\n    \n    print('Number of batches for predictoin:', int(np.ceil(len(test_dataset) / cfg['test_data_loader']['batch_size'])))    ","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:10:14.704221Z","iopub.status.busy":"2020-11-05T04:10:14.703020Z","iopub.status.idle":"2020-11-05T06:24:21.509427Z","shell.execute_reply":"2020-11-05T06:24:21.505071Z"},"papermill":{"duration":8046.870251,"end_time":"2020-11-05T06:24:21.509642","exception":false,"start_time":"2020-11-05T04:10:14.639391","status":"completed"},"scrolled":true,"tags":[],"trusted":false},"cell_type":"code","source":"%%time\nif cfg[\"model_params\"][\"predict\"]:\n    \n    model.eval()\n    torch.set_grad_enabled(False)\n\n    # store information for evaluation\n    future_coords_offsets_pd = []\n    timestamps = []\n    confidences_list = []\n    agent_ids = []\n    memorys_pred = []\n    t0 = time.time()\n    times_pred = []\n    iterations_pred = []\n    i_update = 10 if test_run else 50\n    \n    n_test_batches = len(test_dataloader)\n    # test_iter = iter(test_dataloader)\n\n    for i, data in enumerate(tqdm_notebook(test_dataloader, total=n_test_batches, mininterval=5.)):\n        \n        _, preds, confidences = forward(data, model, device, compute_loss=False)\n    \n        preds = torch.einsum('bmti,bji->bmtj', \n                             preds.double(), \n                             data[\"world_from_agent\"].to(device)[:, :2, :2]).cpu().numpy()\n\n        future_coords_offsets_pd.append(preds.copy())\n        confidences_list.append(confidences.cpu().numpy().copy())\n        timestamps.append(data[\"timestamp\"].numpy().copy())\n        agent_ids.append(data[\"track_id\"].numpy().copy()) \n        \n        if i % i_update == 0:\n            t = ((time.time() - t0) / 60)\n            print('%4d'%i, '%6.2fmins'%t, end=' | ')\n            mem = memory()\n            iterations_pred.append(i)\n            memorys_pred.append(mem)\n            times_pred.append(t)\n            if test_run and i >= 50:\n                break\n    print('Total timespent: %6.2fmins'%((time.time() - t0) / 60))\n    memory()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"preds = None\nconfidences = None\ndata = None\ntest_dataloader = None\n\ndel preds\ndel confidences\ndel data\ndel test_dataloader\n# del optimizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"gc_memory()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T06:24:21.637849Z","iopub.status.busy":"2020-11-05T06:24:21.636684Z","iopub.status.idle":"2020-11-05T06:25:04.742733Z","shell.execute_reply":"2020-11-05T06:25:04.743792Z"},"papermill":{"duration":43.175457,"end_time":"2020-11-05T06:25:04.744055","exception":false,"start_time":"2020-11-05T06:24:21.568598","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"%%time\n# create submission to submit to Kaggle\nif cfg[\"model_params\"][\"predict\"]:\n    pred_path = 'submission.csv' if on_kaggle else f'{model_name}-submission-{train_round}.csv'\n    write_pred_csv(\n        pred_path,\n        timestamps=np.concatenate(timestamps),\n        track_ids=np.concatenate(agent_ids),\n        coords=np.concatenate(future_coords_offsets_pd),\n        confs=np.concatenate(confidences_list),\n    )","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T06:25:04.882846Z","iopub.status.busy":"2020-11-05T06:25:04.872939Z","iopub.status.idle":"2020-11-05T06:25:05.077079Z","shell.execute_reply":"2020-11-05T06:25:05.076342Z"},"papermill":{"duration":0.271638,"end_time":"2020-11-05T06:25:05.077259","exception":false,"start_time":"2020-11-05T06:25:04.805621","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"predict\"]:\n    plt.figure(figsize=(12, 4))\n    plt.plot(iterations_pred, memorys_pred)\n    plt.xlabel('steps'); plt.ylabel('memory (GB)')\n    plt.grid(); plt.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T06:25:05.220608Z","iopub.status.busy":"2020-11-05T06:25:05.212969Z","iopub.status.idle":"2020-11-05T06:25:05.440451Z","shell.execute_reply":"2020-11-05T06:25:05.441089Z"},"papermill":{"duration":0.297674,"end_time":"2020-11-05T06:25:05.441322","exception":false,"start_time":"2020-11-05T06:25:05.143648","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"predict\"]:\n    plt.figure(figsize=(12, 4))\n    plt.plot(iterations_pred, times_pred, label='measured')\n    plt.plot([iterations_pred[0], iterations_pred[-1]], [times_pred[0], times_pred[-1]], label='linear ref', alpha=0.5)\n    plt.xlabel('steps'); plt.ylabel('elapsed_time (mins)')\n    plt.legend(); plt.grid(); plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# All training history"},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"local_files = [entry.name for entry in os.scandir('.') if entry.is_file()]\ntrain_records = [f for f in local_files \n                 if f.startswith(f'train_metrics_{model_name}-') and f.endswith('.csv')]\noriginal_train_records = [f for f in local_files \n                          if f.startswith(f'train_metrics_{original_model_name}-') and f.endswith('.csv')]\ntrain_records, original_train_records","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# For this case\ntrain_records_used = original_train_records[:train_round-1] + train_records\ntrain_records_used","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_records_used[0].replace('_', '-').replace('-', '.').split('.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def extract_info_from_name(filename):\n    filename_s = filename.replace('_', '-').replace('-', '.').split('.')\n    return {'round': int(filename_s[-3]), \n            'n_steps': int(filename_s[-2]),\n            'filename': filename}","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_records_explain = [extract_info_from_name(r) for r in train_records_used]\ntrain_records_explain","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_history_dfs = {record['round']: pd.read_csv(record['filename']) for record in train_records_explain}\nn_steps_history = {record['round']: record['n_steps'] for record in train_records_explain}\nn_steps_history","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_hs = []\ncumulative_steps = 0\ncumulative_time = 0\nfor r in range(1, train_round + 1):\n    df_h = train_history_dfs[r].copy()\n    round_timespent = df_h['elapsed_time (mins)'].values[-1]\n    df_h['iterations'] = df_h['iterations'] + cumulative_steps\n    df_h['elapsed_time (mins)'] = df_h['elapsed_time (mins)'] + cumulative_time\n    df_h['round'] = r\n    df_hs.append(df_h)\n    cumulative_steps += n_steps_history[r]\n    cumulative_time += round_timespent","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"df_history = pd.concat(df_hs, ignore_index=True)\ndisplay(df_history)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T04:10:14.134476Z","iopub.status.busy":"2020-11-05T04:10:14.132786Z","iopub.status.idle":"2020-11-05T04:10:14.135241Z","shell.execute_reply":"2020-11-05T04:10:14.135796Z"},"papermill":{"duration":0.062114,"end_time":"2020-11-05T04:10:14.135942","exception":false,"start_time":"2020-11-05T04:10:14.073828","status":"completed"},"scrolled":false,"tags":[],"trusted":false},"cell_type":"code","source":"# if cfg[\"model_params\"][\"train\"]:\nplt.figure(figsize=(12, 4))\nplt.plot(df_history['iterations'], df_history['metrics (all avg)'], label='all avg')\nplt.plot(df_history['iterations'], df_history['metrics (update avg)'], label=f'{update_steps}-batch avg')\nplt.xlabel('steps'); plt.ylabel('metrics (avg)')\nplt.legend()\nplt.grid(); plt.show()\n\nplt.figure(figsize=(12, 4))\nplt.plot(df_history['iterations'], df_history['metrics (all avg)'], label='all avg')\nplt.plot(df_history['iterations'], df_history['metrics (update avg)'], label=f'{update_steps}-batch avg')\nplt.xlabel('steps'); plt.ylabel('metrics (avg)')\nplt.ylim(-1, 100)\nplt.legend()\nplt.grid(); plt.show()\n\nplt.figure(figsize=(12, 3))\nplt.plot(df_history['iterations'], df_history['memory (GB)'])\nplt.xlabel('steps'); plt.ylabel('memory (GB)')\nplt.grid(); plt.show()\n\nplt.figure(figsize=(12, 3))\nplt.plot(df_history['iterations'], df_history['elapsed_time (mins)'], label='measured data')\nplt.plot(\n    [df_history['iterations'].values[0], df_history['iterations'].values[-1]], \n    [df_history['elapsed_time (mins)'].values[0], df_history['elapsed_time (mins)'].values[-1]],\n    alpha=0.5, label='linear ref')\nplt.xlabel('steps'); plt.ylabel('elapsed_time (mins)')\nplt.legend(); plt.grid(); plt.show()\n# if we see the measured data below the linear, means the training getting slower when the steps increase","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.064474,"end_time":"2020-11-05T06:25:05.571112","exception":false,"start_time":"2020-11-05T06:25:05.506638","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Examine submission"},{"metadata":{"execution":{"iopub.execute_input":"2020-11-05T06:25:10.526171Z","iopub.status.busy":"2020-11-05T06:25:10.524716Z","iopub.status.idle":"2020-11-05T06:25:10.576376Z","shell.execute_reply":"2020-11-05T06:25:10.577079Z"},"papermill":{"duration":0.13618,"end_time":"2020-11-05T06:25:10.577258","exception":false,"start_time":"2020-11-05T06:25:10.441078","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"predict\"]:\n    df_sub = pd.read_csv(pred_path)\n    df_sub = df_sub.set_index(['timestamp', 'track_id'])\n    display(df_sub)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"predict\"]:\n    # Build rasterizer\n    if not cfg[\"model_params\"][\"validate\"]:\n        rasterizer = build_rasterizer(cfg, dm)\n\n    test_cfg = cfg[\"test_data_loader\"]\n    test_zarr = ChunkedDataset(dm.require(test_cfg[\"key\"])).open(cached=False)  # try to turn off cache\n    test_mask = np.load(test_cfg['mask_path'])[\"arr_0\"]\n    test_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)\n    print(test_dataset)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"predict\"]:\n    plot_predicted_trajectory(test_dataset, df_sub, [18431], width=6, height=6, n_cols=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"if cfg[\"model_params\"][\"predict\"]:\n    i_plots = np.random.randint(len(test_dataset), size=9)\n    plot_predicted_trajectory(test_dataset, df_sub, i_plots)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}