{"cells":[{"metadata":{"papermill":{"duration":0.026659,"end_time":"2020-09-15T12:45:19.00577","exception":false,"start_time":"2020-09-15T12:45:18.979111","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Motion prediction applied to self-driving vehicles"},{"metadata":{"papermill":{"duration":0.024211,"end_time":"2020-09-15T12:45:19.054508","exception":false,"start_time":"2020-09-15T12:45:19.030297","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## 1. Competition goal and introduction"},{"metadata":{"papermill":{"duration":0.026133,"end_time":"2020-09-15T12:45:19.105735","exception":false,"start_time":"2020-09-15T12:45:19.079602","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Self-driving cars: where are we in 2020? A small video intro:"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-15T12:45:19.164826Z","iopub.status.busy":"2020-09-15T12:45:19.163942Z","iopub.status.idle":"2020-09-15T12:45:19.167808Z","shell.execute_reply":"2020-09-15T12:45:19.168306Z"},"papermill":{"duration":0.036908,"end_time":"2020-09-15T12:45:19.168465","exception":false,"start_time":"2020-09-15T12:45:19.131557","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# a brief explanation is found in this video\nfrom IPython.display import HTML\n\n\nHTML('<center><iframe  width=\"850\" height=\"450\" src=\"https://www.youtube.com/watch?v=EzylsrXtkxI\" frameborder=\"0\" allow=\"accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture\" allowfullscreen></iframe></center>')\n","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.025783,"end_time":"2020-09-15T12:45:19.220822","exception":false,"start_time":"2020-09-15T12:45:19.195039","status":"completed"},"tags":[]},"cell_type":"markdown","source":"The goal for this competition is then to predict the motion of extraneous vehicles, people, animals (called \"agents\").\nFull details about the dateset can be found on this academic paper https://arxiv.org/pdf/2006.14480.pdf\nIn section 2 the most important concept will be introduced. \nIn section 3 a predicting model will be proposed."},{"metadata":{"papermill":{"duration":0.024672,"end_time":"2020-09-15T12:45:19.270211","exception":false,"start_time":"2020-09-15T12:45:19.245539","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## 2. Import libraries and  explore data"},{"metadata":{"papermill":{"duration":0.024985,"end_time":"2020-09-15T12:45:19.320132","exception":false,"start_time":"2020-09-15T12:45:19.295147","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Before importing/installing libraries I added the following datasets: kaggle-l5kit, lyft-config-files, lyft-resnet18-baseline"},{"metadata":{"papermill":{"duration":0.024539,"end_time":"2020-09-15T12:45:19.370026","exception":false,"start_time":"2020-09-15T12:45:19.345487","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### 2.1 basic libraries and l5kit"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-09-15T12:45:19.426876Z","iopub.status.busy":"2020-09-15T12:45:19.426119Z","iopub.status.idle":"2020-09-15T12:45:19.50034Z","shell.execute_reply":"2020-09-15T12:45:19.499231Z"},"papermill":{"duration":0.106,"end_time":"2020-09-15T12:45:19.500491","exception":false,"start_time":"2020-09-15T12:45:19.394491","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport scipy as sp\n\nimport gc\nimport os\nfrom pathlib import Path\nimport random\nimport sys\nimport yaml\nfrom tqdm.notebook import tqdm\nimport time\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!jupyter nbconvert --version\n!papermill --version\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-15T12:45:19.560811Z","iopub.status.busy":"2020-09-15T12:45:19.559975Z","iopub.status.idle":"2020-09-15T12:45:58.092379Z","shell.execute_reply":"2020-09-15T12:45:58.092917Z"},"papermill":{"duration":38.565689,"end_time":"2020-09-15T12:45:58.093076","exception":false,"start_time":"2020-09-15T12:45:19.527387","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"#!pip install --no-index -f ../input/kaggle-l5kit pip==20.2.2 >/dev/nul\n#!pip install --no-index -f ../input/kaggle-l5kit -U l5kit > /dev/nul","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TORCH XLA\nimport torch_xla\nimport torch_xla.debug.metrics as met\nimport torch_xla.distributed.data_parallel as dp\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.utils.utils as xu\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla.test.test_utils as test_utils","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## this script transports l5kit and dependencies\nos.system('pip uninstall typing -y')\nos.system('pip install --ignore-installed --target=/kaggle/working l5kit')\n\n# Hold back nbconvert to avoid https://github.com/jupyter/nbconvert/issues/1384\nos.system('pip install --upgrade --ignore-installed --target=/kaggle/working \"nbconvert==5.6.1\"')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-15T12:45:58.155819Z","iopub.status.busy":"2020-09-15T12:45:58.154857Z","iopub.status.idle":"2020-09-15T12:45:58.162778Z","shell.execute_reply":"2020-09-15T12:45:58.163535Z"},"papermill":{"duration":0.044134,"end_time":"2020-09-15T12:45:58.163721","exception":false,"start_time":"2020-09-15T12:45:58.119587","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"\nimport l5kit\nassert l5kit.__version__ == \"1.1.0\"\n\nprint ('l5kit imported')\nprint(\"l5kit version:\", l5kit.__version__)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-15T12:45:58.222984Z","iopub.status.busy":"2020-09-15T12:45:58.222084Z","iopub.status.idle":"2020-09-15T12:46:07.865842Z","shell.execute_reply":"2020-09-15T12:46:07.869517Z"},"papermill":{"duration":9.678664,"end_time":"2020-09-15T12:46:07.869699","exception":false,"start_time":"2020-09-15T12:45:58.191035","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"!pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2020-09-15T12:46:07.993792Z","iopub.status.busy":"2020-09-15T12:46:07.991185Z","iopub.status.idle":"2020-09-15T12:46:11.135471Z","shell.execute_reply":"2020-09-15T12:46:11.1343Z"},"papermill":{"duration":3.21674,"end_time":"2020-09-15T12:46:11.135607","exception":false,"start_time":"2020-09-15T12:46:07.918867","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from l5kit.configs import load_config_data\nfrom l5kit.data import LocalDataManager, ChunkedDataset\nfrom l5kit.data import PERCEPTION_LABELS\nfrom l5kit.dataset import AgentDataset, EgoDataset\nfrom l5kit.rasterization import build_rasterizer\nfrom itertools import islice\nfrom typing import Dict\n\n\nimport torch\nfrom torch import nn, optim\n\n\nfrom efficientnet_pytorch import model as enet\n\nfrom l5kit.evaluation.csv_utils import write_pred_csv\nfrom l5kit.evaluation import compute_metrics_csv, read_gt_csv, create_chopped_dataset\nfrom l5kit.evaluation.chop_dataset import MIN_FUTURE_STEPS\nfrom l5kit.evaluation.metrics import neg_multi_log_likelihood, time_displace\nfrom l5kit.geometry import transform_points\nfrom l5kit.visualization import PREDICTED_POINTS_COLOR, TARGET_POINTS_COLOR, draw_trajectory\n\n\nfrom l5kit.configs import load_config_data\n\n\nfrom tqdm import tqdm\nfrom collections import Counter\n\nfrom prettytable import PrettyTable\n\n\nfrom IPython.display import display, clear_output\nfrom IPython.display import HTML\n\nimport PIL\nimport matplotlib.pyplot as plt\nfrom matplotlib import animation, rc\nrc('animation', html='jshtml')\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-15T12:46:11.230365Z","iopub.status.busy":"2020-09-15T12:46:11.229348Z","iopub.status.idle":"2020-09-15T12:46:14.116024Z","shell.execute_reply":"2020-09-15T12:46:14.115253Z"},"papermill":{"duration":2.925938,"end_time":"2020-09-15T12:46:14.116141","exception":false,"start_time":"2020-09-15T12:46:11.190203","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"\n\n# --- plotly ---\nfrom plotly import tools, subplots\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.express as px\nimport plotly.figure_factory as ff\nimport plotly.io as pio\npio.templates.default = \"plotly_dark\"\n\n# --- models ---\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import KFold\n\nfrom torch import nn\nfrom typing import Dict\nfrom pathlib import Path\n\n\n\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data.dataset import Subset\n\nfrom transformers import AdamW, get_linear_schedule_with_warmup, get_constant_schedule\n","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.030327,"end_time":"2020-09-15T12:46:14.178202","exception":false,"start_time":"2020-09-15T12:46:14.147875","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### 2.2 Data import"},{"metadata":{"papermill":{"duration":0.030287,"end_time":"2020-09-15T12:46:14.239845","exception":false,"start_time":"2020-09-15T12:46:14.209558","status":"completed"},"tags":[]},"cell_type":"markdown","source":"The dataset consists of 170,000 scenes capturing the environment around the autonomous vehicle. Each scene encodes the state of the vehicle’s surroundings at a given point in time. "},{"metadata":{"execution":{"iopub.execute_input":"2020-09-15T12:46:14.310935Z","iopub.status.busy":"2020-09-15T12:46:14.310081Z","iopub.status.idle":"2020-09-15T12:46:14.3141Z","shell.execute_reply":"2020-09-15T12:46:14.313631Z"},"papermill":{"duration":0.044439,"end_time":"2020-09-15T12:46:14.314193","exception":false,"start_time":"2020-09-15T12:46:14.269754","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"l5kit_data_folder = \"../input/lyft-motion-prediction-autonomous-vehicles\"\nos.environ[\"L5KIT_DATA_FOLDER\"] = l5kit_data_folder\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-15T12:46:14.389686Z","iopub.status.busy":"2020-09-15T12:46:14.387862Z","iopub.status.idle":"2020-09-15T12:46:14.392746Z","shell.execute_reply":"2020-09-15T12:46:14.392154Z"},"papermill":{"duration":0.047629,"end_time":"2020-09-15T12:46:14.392845","exception":false,"start_time":"2020-09-15T12:46:14.345216","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"IMG_SIZE = 224\n# --- Lyft configs ---\ncfg = {\n          'model_params': {'model_architecture': 'efficientnet-b0',\n          'history_num_frames': 0,\n          'history_step_size': 1,\n          'history_delta_time': 0.1,\n          'future_num_frames': 50,\n          'future_step_size': 1,\n          'future_delta_time': 0.1},\n\n        'raster_params': {'raster_size': [IMG_SIZE, IMG_SIZE],\n          'pixel_size': [0.5, 0.5],\n          'ego_center': [0.25, 0.5],\n          'map_type': 'py_semantic',\n          'satellite_map_key': 'aerial_map/aerial_map.png',\n          'semantic_map_key': 'semantic_map/semantic_map.pb',\n          'dataset_meta_key': 'meta.json',\n          'filter_agents_threshold': 0.5},\n\n        'train_data_loader': {'key': 'scenes/train.zarr',\n          'batch_size': 4,\n          'shuffle': True,\n          'num_workers': 0},\n\n        \"valid_data_loader\":{\"key\": \"scenes/validation.zarr\",\n                            \"batch_size\": 4,\n                            \"shuffle\": False,\n                            \"num_workers\": 0},\n    \n        \"sample_data_loader\": {\n        'key': 'scenes/sample.zarr',\n        'batch_size': 8,\n        'shuffle': False,\n        'num_workers': 0},\n         \n        \"test_data_loader\":{\n        'key': \"scenes/test.zarr\",\n        'batch_size': 8,\n        'shuffle': False,\n        'num_workers': 0},\n\n    \n        'train_params': {'checkpoint_every_n_steps': 1000,\n          'max_num_steps':3000,\n          'eval_every_n_steps': 1000}\n        }\nprint(cfg)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.030921,"end_time":"2020-09-15T12:46:14.45462","exception":false,"start_time":"2020-09-15T12:46:14.423699","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### 2.3 Data exploration"},{"metadata":{"papermill":{"duration":0.030491,"end_time":"2020-09-15T12:46:14.516611","exception":false,"start_time":"2020-09-15T12:46:14.48612","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Credits to https://www.kaggle.com/nxrprime/lyft-understanding-the-data-baseline-on-tpus\n\nInput data files are available in the read-only \"../input/\" directory.\nThe dataset is structured as follows:\n- aerial_map\n- scenes\n- semantic_map\n\nAerial map subfolder contains images. Scenes subfolder contains information about foreing street objects classified in zarr files as following\n\n- sample.zarr\n- test.zarr\n- train.zarr\n- validate.zarr\n\nFor who never heard about \"zarr\" (for Italians, nothing to do with \"zarro\"):\n\n\"Zarr\" provides classes and functions for working with N-dimensional arrays that behave like NumPy arrays but whose data is divided into chunks and each chunk is compressed.\nUseful for handling big data.\nMore info at http://alimanfoo.github.io/2016/04/14/to-hdf5-and-beyond.html\n\nThe dataset consists of frames and agent states. A frame is a snapshot in time which consists of ego pose, time, and multiple agent states. Each agent state describes the position, orientation, bounds, and type."},{"metadata":{"papermill":{"duration":0.030589,"end_time":"2020-09-15T12:46:14.578726","exception":false,"start_time":"2020-09-15T12:46:14.548137","status":"completed"},"tags":[]},"cell_type":"markdown","source":"sample.zarr data is used for visualization, train.zarr / validate.zarr / test.zarr are those needed for actual model training/validation/prediction.\nWe're building a LocalDataManager object. This will resolve relative paths from the config using the L5KIT_DATA_FOLDER env variable that was set above."},{"metadata":{"execution":{"iopub.execute_input":"2020-09-15T12:46:14.647173Z","iopub.status.busy":"2020-09-15T12:46:14.646386Z","iopub.status.idle":"2020-09-15T12:46:14.697343Z","shell.execute_reply":"2020-09-15T12:46:14.697843Z"},"papermill":{"duration":0.087328,"end_time":"2020-09-15T12:46:14.697962","exception":false,"start_time":"2020-09-15T12:46:14.610634","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"dm = LocalDataManager()\ndataset_path = dm.require(cfg[\"sample_data_loader\"][\"key\"]) # for the EDA we use samples dataset. Smaller and RAM-ready\nzarr_dataset = ChunkedDataset(dataset_path)\nzarr_dataset.open()\nprint(zarr_dataset)\n\n","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.031919,"end_time":"2020-09-15T12:46:14.762563","exception":false,"start_time":"2020-09-15T12:46:14.730644","status":"completed"},"tags":[]},"cell_type":"markdown","source":"#### Instead of working with raw data, L5Kit provides PyTorch ready datasets. It's much easier to use this wrapped dataset class to access data."},{"metadata":{"papermill":{"duration":0.03133,"end_time":"2020-09-15T12:46:14.82581","exception":false,"start_time":"2020-09-15T12:46:14.79448","status":"completed"},"tags":[]},"cell_type":"markdown","source":"2 dataset class is implemented.\n\n    EgoDataset: this dataset iterates over the AV (Autonomous Vehicle) annotations\n    AgentDataset: this dataset iterates over other agents annotations\n\nLet's see each class in detail. What kind of attributes/methods they have? What kind of data structure for each attributes?\n\nAs written in Class diagram, both classes are instantiated by:\n\n    cfg: configuration file\n    ChunkedDataset: Internal data class which holds 4 raw data scenes, frames, agents and tl_faces (described later).\n    rasterizer: Rasterizer converts raw data into image.\n"},{"metadata":{"papermill":{"duration":0.031185,"end_time":"2020-09-15T12:46:14.888656","exception":false,"start_time":"2020-09-15T12:46:14.857471","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### 2.4 Data examples\n\n#### Objects in our maps, defined as \"agents\""},{"metadata":{"execution":{"iopub.execute_input":"2020-09-15T12:46:14.963124Z","iopub.status.busy":"2020-09-15T12:46:14.962296Z","iopub.status.idle":"2020-09-15T12:46:16.458682Z","shell.execute_reply":"2020-09-15T12:46:16.459185Z"},"papermill":{"duration":1.537016,"end_time":"2020-09-15T12:46:16.45932","exception":false,"start_time":"2020-09-15T12:46:14.922304","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"agents = zarr_dataset.agents\nprobabilities = agents[\"label_probabilities\"]\nlabels_indexes = np.argmax(probabilities, axis=1)\ncounts = []\nfor idx_label, label in enumerate(PERCEPTION_LABELS):\n    counts.append(np.sum(labels_indexes == idx_label))\n    \ntable = PrettyTable(field_names=[\"label\", \"counts\"])\nfor count, label in zip(counts, PERCEPTION_LABELS):\n    table.add_row([label, count])\nprint(table)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":1.101229,"end_time":"2020-09-15T12:47:33.631718","exception":false,"start_time":"2020-09-15T12:47:32.530489","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## 3. Predicting model"},{"metadata":{"papermill":{"duration":1.174202,"end_time":"2020-09-15T12:47:35.912615","exception":false,"start_time":"2020-09-15T12:47:34.738413","status":"completed"},"tags":[]},"cell_type":"markdown","source":"MODEL_NAME: It can be one of-\n\n    \"efficientnet-b0\"\n    \"efficientnet-b1\"\n    \"efficientnet-b2\"\n    \"efficientnet-b3\"\n    \"efficientnet-b4\"\n    \"efficientnet-b5\"\n    \"efficientnet-b6\"\n    \"efficientnet-b7\"\n\nIMG_SIZE: I use 224 for efficientnet-b0\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    WEIGHT_FILE = \"/kaggle/input/lyftpretrained-resnet101/lyft_efficientnetb0.pth\" # Model state_dict path of previously trained model\n    \n    MODEL_NAME = \"efficientnet-b0\" # b0-b7 could be the different choices.\n    \n    IMG_SIZE = IMG_SIZE # stated above, together with cfg\n    \n    PIXEL_SIZE = 0.4\n        \n    EPOCHS = 2 # Epochs to train the model for.\n    VALIDATION = True # A hyperparameter you could use to toggle for validating the model\n    l_rate = 1e-4 # Learning rate\n\n    scheduler_params = dict(\n        mode='max',\n        factor=0.7,\n        patience=0,\n        verbose=False, \n        threshold=0.0001,\n        threshold_mode='abs',\n        cooldown=0, \n        min_lr=1e-5,\n        eps=1e-08\n    )\n    \n    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau # Scheduler for learning rate.\n    \n    criterion = torch.nn.MSELoss(reduction=\"none\") # Loss function.\n     \n    verbose_steps = 500 # Steps to print model's training status after.\n    \nconfig = Config()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataloader(config, zarr_data, map_type=\"py_semantic\"):\n    \"\"\"Creates DataLoader instance for the given dataset.\"\"\"\n    cfg[\"raster_params\"][\"map_type\"] = map_type\n    rasterizer = build_rasterizer(cfg, dm)\n    chunk_data = ChunkedDataset(zarr_data).open()\n    agent_data = AgentDataset(cfg, chunk_data, rasterizer)\n    dataloader = DataLoader(agent_data, \n                            batch_size=config[\"batch_size\"],\n                            num_workers=config[\"num_workers\"],\n                            shuffle=config[\"shuffle\"]\n                           )\n    return dataloader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TPUFitter:\n    def __init__(self, model, device):\n        self.model = model\n        self.device = device\n\n        # Following some lines are for setting up the AdamW optimizer.\n        # See below explanation for details.\n        param_optimizer = list(self.model.named_parameters())\n        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n        optimizer_grouped_parameters = [\n            {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n             'weight_decay': 0.001},\n            \n            {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n             'weight_decay': 0.0}\n        ]\n\n        self.optimizer = AdamW(optimizer_grouped_parameters, lr=config.l_rate*xm.xrt_world_size())\n        self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)\n        \n        # Following function is used for printing to output efficiently. \n        xm.master_print(f'Model Loaded on {self.device}')\n\n    def fit(self, train_loader, validation_loader):\n        \"\"\"Function to fit the model.\"\"\"\n        losses = []\n        losses_mean = []\n        \n        progress = tqdm(range(cfg[\"train_params\"][\"max_num_steps\"]))\n        \n        for e in range(config.EPOCHS):\n            \n            t = time.time() # Get a measurement of time.\n            para_loader = pl.ParallelLoader(train_loader, [self.device]) # Distributed loading of the model.\n            loss = self.forward(para_loader.per_device_loader(self.device))\n            xm.master_print(\n                            f'[RESULT]: Train. iter: {e+1}, ' + \\\n                            f'train_loss: {loss:.5f}, '+ \\\n                            f' time: {(time.time() - t)/60:.3f}'\n                           )\n            xm.master_print(\"\\n\")\n            losses.append(loss.item())\n            losses_mean.append(np.mean(losses))\n           \n\n    def validation(self, val_loader):\n        \"\"\"Function to validate the model's predictions.\"\"\"\n        val_losses = []\n        val_losses_mean = []\n        # Setting model to evaluation mode.\n        self.model.eval()\n        t = time.time()\n        \n        for step, data in enumerate(val_loader):\n            with torch.no_grad():\n                inputs = data[\"image\"].to(self.device)\n                targets = data[\"target_positions\"].to(self.device)\n                target_availabilities = data[\"target_availabilities\"].unsqueeze(-1).to(self.device)\n                \n                outputs = self.model(inputs)\n                outputs = outputs.reshape(targets.shape)\n                val_loss = config.criterion(outputs, targets)\n                val_loss = val_loss * target_availabilities\n                val_loss = val_loss.mean()\n                val_losses.append(val_loss.item())\n                val_losses_mean.append(np.mean(val_losses))\n                if step % config.verbose_steps == 0:\n                    xm.master_print(\n                        f'Valid Step {step}, val_loss: ' + \\\n                        f'{loss:.4f}' + \\\n                        f' time: {(time.time() - t)/60:.3f}'\n                    )                \n        return val_losses_mean\n         \n    def forward(self, train_loader):\n        \"\"\"Function to perform custom forward propagation.\"\"\"\n        \n        # Setting model to training mode.\n        self.model.train()\n        \n        t = time.time()\n        for step, data in enumerate(train_loader):\n            inputs = data[\"image\"].to(self.device)\n            target_availabilities = data[\"target_availabilities\"].unsqueeze(-1).to(self.device)\n            targets = data[\"target_positions\"].to(self.device)\n    \n            outputs = self.model(inputs)\n            outputs = outputs.reshape(targets.shape)\n            loss = config.criterion(outputs, targets)\n    \n            loss = loss * target_availabilities\n            loss = loss.mean()\n\n            if step % config.verbose_steps == 0:\n                xm.master_print(\n                    f'Train Step {step}, loss: ' + \\\n                    f'{loss:.4f}' + \\\n                    f'time: {(time.time() - t)/60:.3f}'\n                )\n            self.optimizer.zero_grad()\n        \n            loss.backward()\n            xm.optimizer_step(self.optimizer)\n        \n        self.model.eval()\n        self.save('last-checkpoint.bin')\n        return loss\n\n    def save(self, path):\n        \"\"\"Function to save the model's current state.\"\"\"\n        xm.save(self.model.state_dict(), path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Implementation of class to load the particular EfficientNet model.\nclass LyftModel(torch.nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.backbone = enet.EfficientNet.from_name(config.MODEL_NAME)\n        \n        num_history_channels = (cfg[\"model_params\"][\"history_num_frames\"] + 1) * 2\n        num_in_channels = 3 + num_history_channels\n        num_targets = 2*cfg[\"model_params\"][\"future_num_frames\"]\n    \n        self.backbone._conv_stem = nn.Conv2d(\n            num_in_channels,\n            self.backbone._conv_stem.out_channels,\n            kernel_size=self.backbone._conv_stem.kernel_size,\n            stride=self.backbone._conv_stem.stride,\n            padding=self.backbone._conv_stem.padding,\n            bias=False\n        )\n    \n        self.backbone._fc = nn.Linear(in_features=self.backbone._fc.in_features, out_features=num_targets)\n    \n    def forward(self, x):\n        \"\"\"Function to perform forward propagation.\"\"\"\n        x = self.backbone(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataloader(config, zarr_data, subset_len, map_type=\"py_semantic\"):\n    \"\"\"Creates DataLoader instance for the given dataset.\"\"\"\n    \n    cfg[\"raster_params\"][\"map_type\"] = map_type\n    rasterizer = build_rasterizer(cfg, dm)\n    chunk_data = ChunkedDataset(zarr_data).open()\n    agent_data = AgentDataset(cfg, chunk_data, rasterizer)\n    \n    # Getting Subset of the dataset.\n    subset_data = torch.utils.data.Subset(agent_data, range(0, subset_len))\n    \n    dataloader = DataLoader(subset_data, \n                            batch_size=config[\"batch_size\"],\n                            num_workers=config[\"num_workers\"],\n                            shuffle=config[\"shuffle\"]\n                           )\n    return dataloader\n\ndef train():\n    device = xm.xla_device()\n    model = LyftModel(cfg).to(device)\n    fitter = TPUFitter(model, device)\n    \n    xm.master_print(\"Preparing the dataloader..\")\n    train_dataloader = get_dataloader(cfg[\"train_data_loader\"], dm.require(\"scenes/train.zarr\"), 40000)\n    val_dataloader   = get_dataloader(cfg[\"valid_data_loader\"], dm.require(\"scenes/validate.zarr\"), 3000)\n    \n    xm.master_print(\"Training the model..\")\n    fitter.fit(train_dataloader, val_dataloader)\n    fitter.save(\"lyft_model.pth\")\n    return fitter","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-15T12:47:43.321922Z","iopub.status.busy":"2020-09-15T12:47:43.320775Z","iopub.status.idle":"2020-09-15T13:03:59.418102Z","shell.execute_reply":"2020-09-15T13:03:59.417328Z"},"papermill":{"duration":977.20042,"end_time":"2020-09-15T13:03:59.418277","exception":false,"start_time":"2020-09-15T12:47:42.217857","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"\nmodel = train()\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-15T13:04:02.436097Z","iopub.status.busy":"2020-09-15T13:04:02.435176Z","iopub.status.idle":"2020-09-15T13:04:02.686702Z","shell.execute_reply":"2020-09-15T13:04:02.685816Z"},"papermill":{"duration":1.774717,"end_time":"2020-09-15T13:04:02.686835","exception":false,"start_time":"2020-09-15T13:04:00.912118","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"#print(\"Saving the model...\")\n#torch.save(model.state_dict(), \"lyft_model.pth\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-15T13:04:09.326043Z","iopub.status.busy":"2020-09-15T13:04:09.32532Z","iopub.status.idle":"2020-09-15T13:04:22.781148Z","shell.execute_reply":"2020-09-15T13:04:22.782079Z"},"papermill":{"duration":14.97988,"end_time":"2020-09-15T13:04:22.782277","exception":false,"start_time":"2020-09-15T13:04:07.802397","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"PATH_TO_DATA = '/kaggle/input/lyft-motion-prediction-autonomous-vehicles/'\n\ntest_cfg = cfg[\"test_data_loader\"]\n\n# Rasterizer\nrasterizer = build_rasterizer(cfg, dm)\n\n# Test dataset/dataloader\ntest_zarr = ChunkedDataset(dm.require(test_cfg[\"key\"])).open()\ntest_mask = np.load(f\"{PATH_TO_DATA}/scenes/mask.npz\")[\"arr_0\"]\ntest_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)\ntest_dataloader = DataLoader(test_dataset,\n                             shuffle=test_cfg[\"shuffle\"],\n                             batch_size=test_cfg[\"batch_size\"],\n                             num_workers=test_cfg[\"num_workers\"])\n\n\nprint(test_dataset)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _test():\n    \n\n    future_coords_offsets_pd = []\n    timestamps = []\n    agent_ids = []\n    device = 'xla:0'\n    print(f\"device: {device} ready!\")\n    model = LyftModel(cfg)\n    ckpt = torch.load('../input/lyftmodelall/effnet0l2binay_368.bin')\n    model.load_state_dict(ckpt)\n    model = model.to(device)\n    model.eval()\n    with torch.no_grad():\n        dataiter = tqdm(test_dataloader)\n\n        for data in dataiter:\n            inputs = data[\"image\"].to(device)\n            target_availabilities = data[\"target_availabilities\"].unsqueeze(-1).to(device)\n            targets = data[\"target_positions\"].to(device)\n\n            outputs = model(inputs).reshape(targets.shape)\n\n            future_coords_offsets_pd.append(outputs.cpu().numpy().copy())\n            timestamps.append(data[\"timestamp\"].numpy().copy())\n            agent_ids.append(data[\"track_id\"].numpy().copy())\n    write_pred_csv('submission.csv',\n               timestamps=np.concatenate(timestamps),\n               track_ids=np.concatenate(agent_ids),\n               coords=np.concatenate(future_coords_offsets_pd))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def forward(data, model, device):\n    inputs = data[\"image\"].to(device)\n    target_availabilities = data[\"target_availabilities\"].unsqueeze(-1).to(device)\n    targets = data[\"target_positions\"].to(device)\n    # Forward pass\n    outputs = model(inputs).reshape(targets.shape)\n    return outputs\n\ndef _test():\n    future_coords_offsets_pd = []\n    timestamps = []\n    agent_ids = []\n    device = 'xla:0'\n    print(f\"device: {device} ready!\")\n    model = LyftModel(cfg)\n    ckpt = torch.load('./lyft_model.pth')\n    model.load_state_dict(ckpt)\n    model = model.to(device)\n    model.eval()\n    with torch.no_grad():\n        \n    # store information for evaluation\n        future_coords_offsets_pd = []\n        timestamps = []\n        agent_ids = []\n\n        progress_bar = tqdm(test_dataloader)\n        for data in progress_bar:\n\n        # convert agent coordinates into world offsets\n            agents_coords = forward(data, model, device).cpu().numpy().copy()\n            world_from_agents = data[\"world_from_agent\"].numpy()\n            centroids = data[\"centroid\"].numpy()\n            coords_offset = []\n\n        for agent_coords, world_from_agent, centroid in zip(agents_coords, world_from_agents, centroids):\n            coords_offset.append(transform_points(agent_coords, world_from_agent) - centroid[:2])\n            future_coords_offsets_pd.append(np.stack(coords_offset))\n            timestamps.append(data[\"timestamp\"].numpy().copy())\n            agent_ids.append(data[\"track_id\"].numpy().copy())\n\n\n    write_pred_csv(\n        \"submission.csv\",\n        timestamps=np.concatenate(timestamps),\n        track_ids=np.concatenate(agent_ids),\n        coords=np.concatenate(future_coords_offsets_pd),\n    )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _mp_fn(rank, flags):\n#     torch.set_default_tensor_type('torch.FloatTensor')\n    _test()\n\nFLAGS={}\n\nxmp.spawn(_mp_fn, args=(FLAGS,), nprocs=1, start_method='fork')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The trained efficientnet is finally run on the test dataset and the submission.csv file has been created, ready to be submitted"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}