{"cells":[{"metadata":{},"cell_type":"markdown","source":"# L5kit dataloader memory usage"},{"metadata":{"trusted":true},"cell_type":"code","source":"import psutil\ndef memory(verbose=True):\n    mem = psutil.virtual_memory()\n    gb = 1024*1024*1024\n    if verbose:\n        print('Physical memory:',\n              '%.2f GB (used),'%((mem.total - mem.available) / gb),\n              '%.2f GB (available)'%((mem.available) / gb), '/',\n              '%.2f GB'%(mem.total / gb))\n    return (mem.total - mem.available) / gb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"memory()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"# Environment setup\n\n - Please add [pestipeti/lyft-l5kit-unofficial-fix](https://www.kaggle.com/pestipeti/lyft-l5kit-unofficial-fix) as utility script.\n    - Official utility script \"[philculliton/kaggle-l5kit](https://www.kaggle.com/mathurinache/kaggle-l5kit)\" does not work with pytorch GPU.\n\nClick \"File\" botton on top-left, and choose \"Add utility script\". For the pop-up search window, you need to remove \"Your Work\" filter, and search [pestipeti/lyft-l5kit-unofficial-fix](https://www.kaggle.com/pestipeti/lyft-l5kit-unofficial-fix) on top-right of the search window. Then you can add the kaggle-l5kit utility script. It is much faster to do this rather than !pip install l5kit every time you run the notebook. \n\nIf successful, you can see \"usr/lib/lyft-l5kit-unofficial-fix\" is added to the \"Data\" section of this kernel page on right side of the kernel.\n\n- Also please add [pretrained baseline model](https://www.kaggle.com/huanvo/lyft-pretrained-model-hv)\n\nClick on the button \"Add data\" in the \"Data\" section and search for lyft-pretrained-model-hv. If you find the model useful, please upvote it as well.  "},{"metadata":{"trusted":true},"cell_type":"code","source":"from typing import Dict\n\nfrom tempfile import gettempdir\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader\nfrom torchvision.models.resnet import resnet50, resnet18, resnet34, resnet101\nfrom tqdm import tqdm\n\nimport l5kit\nfrom l5kit.configs import load_config_data\nfrom l5kit.data import LocalDataManager, ChunkedDataset\nfrom l5kit.dataset import AgentDataset, EgoDataset\nfrom l5kit.rasterization import build_rasterizer\nfrom l5kit.evaluation import write_pred_csv, compute_metrics_csv, read_gt_csv, create_chopped_dataset\nfrom l5kit.evaluation.chop_dataset import MIN_FUTURE_STEPS\nfrom l5kit.evaluation.metrics import neg_multi_log_likelihood, time_displace\nfrom l5kit.geometry import transform_points\nfrom l5kit.visualization import PREDICTED_POINTS_COLOR, TARGET_POINTS_COLOR, draw_trajectory\nfrom prettytable import PrettyTable\nfrom pathlib import Path\n\nimport matplotlib.pyplot as plt\n\nimport os\nimport random\nimport time\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"l5kit.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\nset_seed(42)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configs"},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- Lyft configs ---\ncfg = {\n    'format_version': 4,\n    'data_path': '/kaggle/input/lyft-motion-prediction-autonomous-vehicles',\n    'model_params': {\n        'model_architecture': 'resnet34',\n        'history_num_frames': 10,\n        'history_step_size': 1,\n        'history_delta_time': 0.1,\n        'future_num_frames': 50,\n        'future_step_size': 1,\n        'future_delta_time': 0.1,\n        'model_name': \"model_resnet34_output\",\n        'lr': 1e-6,\n        'weight_path': '/kaggle/input/lyft-pretrained-model-hv/model_multi_update_lyft_public.pth',\n        'train': True,\n        'predict': True,\n    },\n    'raster_params': {\n        'raster_size': [224, 224],\n        'pixel_size': [0.5, 0.5],\n        'ego_center': [0.25, 0.5],\n        'map_type': 'py_semantic',\n        'satellite_map_key': 'aerial_map/aerial_map.png',\n        'semantic_map_key': 'semantic_map/semantic_map.pb',\n        'dataset_meta_key': 'meta.json',\n        'filter_agents_threshold': 0.5,\n    },\n    'train_data_loader': {\n        'key': 'scenes/train.zarr',\n        'batch_size': 16,\n        'shuffle': True,\n        'num_workers': 0,\n    },    \n    'test_data_loader': {\n        'key': 'scenes/test.zarr',\n        'batch_size': 32,\n        'shuffle': False,\n        'num_workers': 0,\n    },\n    'train_params': {\n        'max_num_steps': 4001,\n        'checkpoint_every_n_steps': 40,\n    }\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load the train and test datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# set env variable for data\nDIR_INPUT = cfg[\"data_path\"]\nos.environ[\"L5KIT_DATA_FOLDER\"] = DIR_INPUT\ndm = LocalDataManager()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# Build rasterizer\nrasterizer = build_rasterizer(cfg, dm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# Train dataset\ntrain_cfg = cfg[\"train_data_loader\"]\ntrain_zarr = ChunkedDataset(dm.require(train_cfg[\"key\"])).open(cached=False)\ntrain_dataset = AgentDataset(cfg, train_zarr, rasterizer)\ntrain_dataloader = DataLoader(train_dataset, shuffle=train_cfg[\"shuffle\"], \n                              batch_size=train_cfg[\"batch_size\"], num_workers=train_cfg[\"num_workers\"])\nprint(train_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# Test dataset\ntest_cfg = cfg[\"test_data_loader\"]\ntest_zarr = ChunkedDataset(dm.require(test_cfg[\"key\"])).open(cached=False)\ntest_mask = np.load(f\"{DIR_INPUT}/scenes/mask.npz\")[\"arr_0\"]\ntest_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)\ntest_dataloader = DataLoader(test_dataset, shuffle=test_cfg[\"shuffle\"],\n                             batch_size=test_cfg[\"batch_size\"], num_workers=test_cfg[\"num_workers\"])\nprint(test_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Training data\nNext let us implement the training loop, when the **train** parameter is set to True. "},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import display","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm_notebook","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nif cfg[\"model_params\"][\"train\"]:\n    tr_it = iter(train_dataloader)\n    progress_bar = tqdm_notebook(range(1, 1 + cfg[\"train_params\"][\"max_num_steps\"]), mininterval=5)\n    num_iter = cfg[\"train_params\"][\"max_num_steps\"]\n    losses_train = []\n    iterations = []\n    metrics = []\n    times = []\n    model_name = cfg[\"model_params\"][\"model_name\"]\n    t_start = time.time()\n    t_prev = t_start\n    memory_used_prev = memory()\n    print('== Start loading train data ==')\n    for i in progress_bar:\n        try:\n            data = next(tr_it)\n        except StopIteration:\n            tr_it = iter(train_dataloader)\n            data = next(tr_it)\n        \n        if i % cfg['train_params']['checkpoint_every_n_steps'] == 0:\n            memory_used = memory(False)\n            t = time.time()\n            iterations.append(i)\n            metrics.append(memory_used)\n            times.append((t - t_start) / 60)\n            print('%5d -'%i, \n                  'dt = %5.3fmins,'%((t - t_prev) / 60), \n                  'dm = %5.3fGB,'%(memory_used - memory_used_prev), \n                  'elapsed_time = %6.2fmins,'%((t - t_start) / 60),\n                  'memory_used = %6.3fGB'%memory_used,\n                 )\n            t_prev = t\n            memory_used_prev = memory_used\n\n    results = pd.DataFrame({'iterations': iterations, 'memory (GB)': metrics, 'elapsed_time (mins)': times})\n    results.to_csv(f\"train_loading_time.csv\", index = False)\n    print(f\"Total training time is {(time.time() - t_start) / 60} mins\")\n    memory()\n    display(results)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(results['elapsed_time (mins)'], results['memory (GB)'])\nplt.xlabel('elapsed_time (mins)')\nplt.ylabel('memory usage (GB)')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(results['iterations'], results['memory (GB)'])\nplt.xlabel('iterations')\nplt.ylabel('memory usage (GB)')\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}