{"cells":[{"metadata":{},"cell_type":"markdown","source":"Memory seems to accumulate while iterating through dataloader\n\nversion 4 with caching disabled still appears to be accumulating memory"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\n# torch.multiprocessing.set_sharing_strategy('file_system')\nfrom torch.utils.data import DataLoader\nfrom tqdm.notebook import tqdm\n\nfrom l5kit.data import LocalDataManager, ChunkedDataset\nfrom l5kit.rasterization import build_rasterizer\nfrom l5kit.dataset import AgentDataset\n\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DIR_INPUT = \"../input/lyft-motion-prediction-autonomous-vehicles\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg = {\n    'format_version': 4,\n    'model_params': {\n        'history_num_frames': 10,\n        'history_step_size': 1,\n        'history_delta_time': 0.1,\n        'future_num_frames': 50,\n        'future_step_size': 1,\n        'future_delta_time': 0.1\n    },\n    \n    'raster_params': {\n        'raster_size': [300, 300],\n        'pixel_size': [0.5, 0.5],\n        'ego_center': [0.25, 0.5],\n        'map_type': 'py_semantic',\n        'satellite_map_key': 'aerial_map/aerial_map.png',\n        'semantic_map_key': 'semantic_map/semantic_map.pb',\n        'dataset_meta_key': 'meta.json',\n        'filter_agents_threshold': 0.5\n    },\n    \n    'test_data_loader': {\n        'key': 'scenes/test.zarr',\n        'batch_size': 8,\n        'shuffle': False,\n        'num_workers': 4\n    },\n    \n    'train_data_loader': {\n        'key': 'scenes/train.zarr',\n        'batch_size': 32,\n        'shuffle': True,\n        'num_workers': 4\n    },\n    \n    'val_data_loader': {\n        'key': 'scenes/validate.zarr',\n        'batch_size': 8,\n        'shuffle': True,\n        'num_workers': 4\n    },\n    'train_params':{\n      'checkpoint_every_n_steps': 10000,\n      'max_num_steps': 2000,\n      'eval_every_n_steps': 10000\n                    }\n\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.environ[\"L5KIT_DATA_FOLDER\"] = DIR_INPUT\ndm = LocalDataManager(None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rasterizer = build_rasterizer(cfg, dm)\ntrain_zarr = ChunkedDataset(dm.require(cfg['train_data_loader'][\"key\"])).open(cached=False)\ntrain_dataset = AgentDataset(cfg, train_zarr, rasterizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 4\ntrain_dataloader = DataLoader(train_dataset,\n                 shuffle=True,\n                 batch_size=cfg['train_data_loader']['batch_size'],\n                 num_workers=cfg['train_data_loader']['num_workers'])\nfor i in range(epochs):\n    tr_it = iter(train_dataloader)\n    progress_bar = tqdm(range(cfg[\"train_params\"][\"max_num_steps\"]))\n    for _ in progress_bar:\n        try:\n            data = next(tr_it)\n        except StopIteration:\n            tr_it = iter(train_dataloader)\n            data = next(tr_it)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}