{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install transforms3d\n!pip install zarr\n!pip install pymap3d","metadata":{"execution":{"iopub.status.busy":"2021-05-31T21:07:29.477976Z","iopub.execute_input":"2021-05-31T21:07:29.478364Z","iopub.status.idle":"2021-05-31T21:07:55.435488Z","shell.execute_reply.started":"2021-05-31T21:07:29.478284Z","shell.execute_reply":"2021-05-31T21:07:55.434559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.insert(0, '/kaggle/input/nfl5kit-dev/l5kit')","metadata":{"execution":{"iopub.status.busy":"2021-05-31T21:07:55.437116Z","iopub.execute_input":"2021-05-31T21:07:55.437441Z","iopub.status.idle":"2021-05-31T21:07:55.445245Z","shell.execute_reply.started":"2021-05-31T21:07:55.437413Z","shell.execute_reply":"2021-05-31T21:07:55.444217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from l5kit.configs import load_config_data\nfrom l5kit.data import ChunkedDataset, LocalDataManager\nfrom l5kit.dataset import EgoDataset, AgentDataset\nfrom l5kit.rasterization import build_rasterizer","metadata":{"execution":{"iopub.status.busy":"2021-05-31T21:07:55.447779Z","iopub.execute_input":"2021-05-31T21:07:55.448221Z","iopub.status.idle":"2021-05-31T21:07:57.452785Z","shell.execute_reply.started":"2021-05-31T21:07:55.448182Z","shell.execute_reply":"2021-05-31T21:07:57.451937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2021-05-31T21:07:57.454321Z","iopub.execute_input":"2021-05-31T21:07:57.454666Z","iopub.status.idle":"2021-05-31T21:07:57.460446Z","shell.execute_reply.started":"2021-05-31T21:07:57.454627Z","shell.execute_reply":"2021-05-31T21:07:57.459514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set env variable for data\nos.environ[\"L5KIT_DATA_FOLDER\"] = \"../input/lyft-motion-prediction-autonomous-vehicles/\"\ndm = LocalDataManager(None)\n\n# get config\ncfg = load_config_data(\"../input/lyftconfigfiles/agent_motion_prediction/agent_motion_config.yaml\")","metadata":{"execution":{"iopub.status.busy":"2021-05-31T21:07:57.461870Z","iopub.execute_input":"2021-05-31T21:07:57.462423Z","iopub.status.idle":"2021-05-31T21:07:57.483221Z","shell.execute_reply.started":"2021-05-31T21:07:57.462381Z","shell.execute_reply":"2021-05-31T21:07:57.482406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg['model_params']['history_num_frames'] = 10\ncfg['train_params']['max_num_steps'] = 25000\n# cfg['train_params']['max_num_steps'] = 5000\ncfg['train_params']['checkpoint_every_n_steps'] = 10000\n# cfg['train_params']['checkpoint_every_n_steps'] = 1000\ncfg['train_data_loader']['batch_size'] = 12\ncfg['train_data_loader']['num_workers'] = 4\ncfg['train_data_loader']['key'] = 'scenes/train.zarr'\n# cfg['train_data_loader']['key'] = 'scenes/sample.zarr'\n\ncfg['model_params']['render_ego_history'] = True\ncfg['model_params']['history_num_frames'] = 10","metadata":{"execution":{"iopub.status.busy":"2021-05-31T21:07:57.484537Z","iopub.execute_input":"2021-05-31T21:07:57.484899Z","iopub.status.idle":"2021-05-31T21:07:57.490431Z","shell.execute_reply.started":"2021-05-31T21:07:57.484862Z","shell.execute_reply":"2021-05-31T21:07:57.489509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader\nfrom torchvision.models.mobilenet import mobilenet_v2","metadata":{"execution":{"iopub.status.busy":"2021-05-31T21:07:57.491874Z","iopub.execute_input":"2021-05-31T21:07:57.492245Z","iopub.status.idle":"2021-05-31T21:07:57.642185Z","shell.execute_reply.started":"2021-05-31T21:07:57.492206Z","shell.execute_reply":"2021-05-31T21:07:57.641317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Identity(nn.Module):\n    def __init__(self):\n        super(Identity, self).__init__()\n\n    def forward(self, x):\n        return x\n    \nclass MobilenetV2LSTM(nn.Module):\n    def __init__(self, config):\n        super(MobilenetV2LSTM, self).__init__()\n        self.cfg = config\n        self.batch_size = self.cfg['train_data_loader']['batch_size']\n        self.hist_frames = self.cfg['model_params']['history_num_frames']\n#         self.fc_infeatures = 1280 + (2 * (self.hist_frames + 1)) + (2 * self.hist_frames) + (self.hist_frames + 1)\n        self.fc_infeatures = 1311\n        self.num_targets = 2 * self.cfg[\"model_params\"][\"future_num_frames\"]\n        self.cnn = self.build_basecnn()\n        self.fc1 = nn.Sequential(\n            nn.Dropout(p=0.2, inplace=False),\n            nn.Linear(in_features=self.fc_infeatures, out_features=4096),\n            nn.ReLU(inplace=True),\n            nn.Dropout(p=0.2, inplace=False),\n#             nn.Linear(in_features=4096, out_features=self.num_targets)\n            nn.Linear(in_features=4096, out_features=128)\n        )\n        # LSTM DECODER\n#         self.lstm = nn.LSTM(input_size=128, hidden_size=128)\n#         self.fc2 = nn.Linear(in_features=128, out_features=self.num_targets)\n        \n        self.lstm = nn.LSTM(input_size=1, hidden_size=128, batch_first=True)\n        self.fc2 = nn.Linear(in_features=128, out_features=self.num_targets)\n        \n        \n#     def forward(self, x, vel, accel, yaw):\n#         x = self.cnn(x)\n#         vel = vel.reshape(self.batch_size, -1)\n#         accel = accel.reshape(self.batch_size, -1)\n#         yaw = yaw.reshape(self.batch_size, -1)\n#         x = torch.cat([x, vel, accel, yaw], dim=1)\n#         x = self.fc1(x)\n\n#         return x\n    \n#     def forward(self, x, vel, yaw):\n#         x = self.cnn(x)\n#         vel = vel.reshape(self.batch_size, -1)\n#         yaw = yaw.reshape(self.batch_size, -1)\n#         x = torch.cat([x, vel, yaw], dim=1)\n#         x = self.fc1(x)\n\n#         return x\n    \n    # WITH LSTM DECODER\n    def forward(self, x, vel, yaw):\n        x = self.cnn(x)\n        vel = vel.reshape(self.batch_size, -1)\n        yaw = yaw.reshape(self.batch_size, -1)\n        x = torch.cat([x, vel, yaw], dim=1)\n        x = self.fc1(x)\n#         x = x.view(self.batch_size, 1, 128) #??\n#         x = x.view(1, self.batch_size, 128) #??\n        x = x.view(self.batch_size, 128, 1) # IF BATCH_FIRST=TRUE\n        lstm_out, lstm_hidden = self.lstm(x)\n        lstm_out = lstm_hidden[0].view(self.batch_size, 128) # USING HIDDEN INSTEAD OF LSTM_OUT\n        x = self.fc2(lstm_out)\n\n        return x\n\n    def build_basecnn(self):\n        # change input channels number to match the rasterizer's output\n        mnet = mobilenet_v2(pretrained=True)\n        num_history_channels = (self.cfg[\"model_params\"][\"history_num_frames\"] + 1) * 2\n        num_in_channels = 3 + num_history_channels\n        mnet.features[0][0] = nn.Conv2d(\n            num_in_channels,\n            mnet.features[0][0].out_channels,\n            kernel_size=mnet.features[0][0].kernel_size,\n            stride=mnet.features[0][0].stride,\n            padding=mnet.features[0][0].padding,\n            bias=False,\n        )\n\n        mnet.classifier = Identity()\n        \n        return mnet","metadata":{"execution":{"iopub.status.busy":"2021-05-31T21:07:57.643665Z","iopub.execute_input":"2021-05-31T21:07:57.644005Z","iopub.status.idle":"2021-05-31T21:07:57.659902Z","shell.execute_reply.started":"2021-05-31T21:07:57.643970Z","shell.execute_reply":"2021-05-31T21:07:57.659103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def forward(data, model, device, criterion):\n#     im_inputs = data[\"image\"].to(device)\n#     vel_inputs = data[\"history_velocities\"].to(device)\n#     accel_inputs = data[\"history_accels\"][:, :-1, :].to(device)  # removing last history frame since we don't have accel for it\n#     yaw_inputs = data[\"history_yaws\"].to(device)\n\n#     target_availabilities = data[\"target_availabilities\"].unsqueeze(-1).to(device)\n#     targets = data[\"target_positions\"].to(device)\n\n#     # Forward pass\n#     outputs = model(im_inputs, vel_inputs, accel_inputs, yaw_inputs).reshape(targets.shape)\n#     loss = criterion(outputs, targets)\n#     # not all the output steps are valid, but we can filter them out from the loss using availabilities\n#     loss = loss * target_availabilities\n#     loss = loss.mean()\n#     return loss, outputs\n\n\ndef forward(data, model, device, criterion):\n    im_inputs = data[\"image\"].to(device)\n    vel_inputs = data[\"history_velocities\"].to(device)\n    yaw_inputs = data[\"history_yaws\"].to(device)\n\n    target_availabilities = data[\"target_availabilities\"].unsqueeze(-1).to(device)\n    targets = data[\"target_positions\"].to(device)\n\n    # Forward pass\n    outputs = model(im_inputs, vel_inputs, yaw_inputs).reshape(targets.shape)\n    loss = criterion(outputs, targets)\n    # not all the output steps are valid, but we can filter them out from the loss using availabilities\n    loss = loss * target_availabilities\n    loss = loss.mean()\n    return loss, outputs","metadata":{"execution":{"iopub.status.busy":"2021-05-31T21:07:57.663626Z","iopub.execute_input":"2021-05-31T21:07:57.663951Z","iopub.status.idle":"2021-05-31T21:07:57.673967Z","shell.execute_reply.started":"2021-05-31T21:07:57.663923Z","shell.execute_reply":"2021-05-31T21:07:57.673091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rasterizer = build_rasterizer(cfg, dm)\ntrain_cfg = cfg[\"train_data_loader\"]\ntrain_zarr = ChunkedDataset(dm.require(train_cfg[\"key\"])).open()\ntrain_dataset = AgentDataset(cfg, train_zarr, rasterizer)\n\n# create training splits for transfer learning --- wait for pytorch update on kaggle kernels\n# train1_size = 25000\n# trainrem_size = len(train_dataset) - train1_size\n# train_dataset, _ = random_split(train_dataset, [train1_size, trainrem_size], generator=torch.Generator().manual_seed(42))\n\ntrain_dataloader = DataLoader(train_dataset, shuffle=train_cfg[\"shuffle\"], batch_size=train_cfg[\"batch_size\"], \n                             num_workers=train_cfg[\"num_workers\"])\nprint(train_dataset)\nprint(len(train_dataset))\nprint(len(train_dataloader))","metadata":{"execution":{"iopub.status.busy":"2021-05-31T21:07:57.675688Z","iopub.execute_input":"2021-05-31T21:07:57.676065Z","iopub.status.idle":"2021-05-31T21:10:25.772541Z","shell.execute_reply.started":"2021-05-31T21:07:57.676028Z","shell.execute_reply":"2021-05-31T21:10:25.771682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ==== INIT MODEL\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel = MobilenetV2LSTM(cfg).to(device)\n# optimizer = optim.Adam(model.parameters(), lr=1e-3)\noptimizer = optim.AdamW(model.parameters())\ncriterion = nn.MSELoss(reduction=\"none\")","metadata":{"execution":{"iopub.status.busy":"2021-05-31T21:10:25.773848Z","iopub.execute_input":"2021-05-31T21:10:25.774207Z","iopub.status.idle":"2021-05-31T21:10:31.553386Z","shell.execute_reply.started":"2021-05-31T21:10:25.774170Z","shell.execute_reply":"2021-05-31T21:10:31.552585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-05-31T21:10:31.554713Z","iopub.execute_input":"2021-05-31T21:10:31.555053Z","iopub.status.idle":"2021-05-31T21:10:31.560702Z","shell.execute_reply.started":"2021-05-31T21:10:31.555003Z","shell.execute_reply":"2021-05-31T21:10:31.559167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, device, train_loader, criterion, optimizer, max_steps, checkpoint_steps):\n    # Switch model to training mode. This is necessary for layers like dropout, batchnorm etc \n    # which behave differently in training and evaluation mode\n    model.train()\n    losses_train = []\n    step = 1\n    \n    # We loop over the data iterator, and feed the inputs to the network and adjust the weights.\n    train_pbar = tqdm(enumerate(train_loader), desc=\"Training steps\", leave=True, total=max_steps)\n    for batch_idx, data in train_pbar:\n        # Forward pass to calculate loss\n        loss, _ = forward(data, model, device, criterion)\n\n        # Reset the gradients to 0 for all learnable weight parameters\n        optimizer.zero_grad()\n\n        # Backward pass: compute the gradients of the loss w.r.t. the model's parameters\n        loss.backward()\n\n        # Update the model weights\n        optimizer.step()\n\n        # Get average loss of iterations so far\n        losses_train.append(loss.item())\n        avg_train_loss = np.mean(losses_train)\n\n        train_pbar.set_description(f\" Avg train loss: {avg_train_loss}\")\n        \n        if step % checkpoint_steps == 0 and step != max_steps:\n            print('Training Loss: {l}, Avg Training Loss: {a}'.format(l=loss.item(), a=avg_train_loss))\n        \n        if step >= max_steps:\n            return avg_train_loss\n        step += 1\n    \n    return avg_train_loss","metadata":{"execution":{"iopub.status.busy":"2021-05-31T21:10:31.562322Z","iopub.execute_input":"2021-05-31T21:10:31.562715Z","iopub.status.idle":"2021-05-31T21:10:31.582281Z","shell.execute_reply.started":"2021-05-31T21:10:31.562676Z","shell.execute_reply":"2021-05-31T21:10:31.581308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(cfg['train_params']['max_num_steps'])\ncheckpoint_steps =  cfg['train_params']['checkpoint_every_n_steps']   \ntrain_loss = train(model, device, train_dataloader, criterion, optimizer, cfg['train_params']['max_num_steps'], checkpoint_steps)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T21:10:31.583846Z","iopub.execute_input":"2021-05-31T21:10:31.584299Z","iopub.status.idle":"2021-06-01T02:42:28.028604Z","shell.execute_reply.started":"2021-05-31T21:10:31.584256Z","shell.execute_reply":"2021-06-01T02:42:28.023489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model.state_dict(), 'testrun_mobilenetv2_withlstm_full_train_25k.pth')","metadata":{"execution":{"iopub.status.busy":"2021-06-01T02:42:28.039577Z","iopub.execute_input":"2021-06-01T02:42:28.040108Z","iopub.status.idle":"2021-06-01T02:42:28.533677Z","shell.execute_reply.started":"2021-06-01T02:42:28.040047Z","shell.execute_reply":"2021-06-01T02:42:28.532695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}