{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nfrom pathlib import Path\nimport psutil\nimport gc\nimport cv2\n\nprocess = psutil.Process(os.getpid())\n\nimg_path = '../input/herbarium-2020-fgvc7/nybg2020/train/images'\nimages = (Path(path)/fn\n          for path, subdirs, fns in os.walk(img_path) if not subdirs\n          for fn in fns)\n\n# Session Info: RAM 348.9MB, Max 16GB\n# Python kernel: 0.15 GB\nprint(process.memory_info().rss / 1024 / 1024 / 1024, 'GB')\n\nfor i, image in enumerate(images):\n    foo = cv2.imread(str(image))\n    if i > 50000: break\n\n# Session Info: RAM 3 GB\n# Python kernel: 0.17 GB\nprint(process.memory_info().rss / 1024 / 1024 / 1024, 'GB')\n\ngc.collect()\n\n# Python kernel: 0.17 GB\nprint(process.memory_info().rss / 1024 / 1024 / 1024, 'GB') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# As the loop is executed, RAM usage grows continuously, reaching 3 GB after 50 k images read.\n# After about 500 k images, the kernel dies as it runs out of memory.\n# If the kernel also writes something to disk it may die before with \n#     OSError: [Errno 28] No space left on device\n# instead, which is not caused by a full disk but (presumably) some file cashing can't allocate mem.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Not even ps shows the cause of memory drain:\n!ps -Al | sort -rnk 10 | head","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}