{"cells":[{"metadata":{"_uuid":"c7b9bd0bec9adbc9067f4c96bbbe7293a5938a1c"},"cell_type":"markdown","source":"Loading images is pretty slow, especially when you are reading 4 images per example. Here I attempt to create a HDF5 datastore for faster loading of data."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport h5py\nfrom tqdm import tqdm\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\nchannels = ['red', 'green', 'blue', 'yellow']\nhdf_path = f'./train.hdf5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b32eae5b6232846317526c5e881b2d1129489745"},"cell_type":"code","source":"def load_image(id):\n    img = np.zeros((4, 512, 512), dtype=np.uint8)\n    for c, ch in enumerate(channels):\n        img[c, ...] = cv2.imread('../input/train/{}_{}.png'.format(id, ch), cv2.IMREAD_GRAYSCALE)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4153b0b4101478493c835dca2bee97ec8d7efe90"},"cell_type":"code","source":"with h5py.File(hdf_path, mode='w') as train_hdf5:\n    train_hdf5.create_dataset(\"train\", (len(train_df), 4, 512, 512), np.uint8)\n    for i, id in tqdm(enumerate(train_df['Id'][:100])):    #Remove the [:100] for full dataset\n        img = load_image(id)\n        train_hdf5['train'][i, ...] = img","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb0f081a156451fe4092cc897be7666014f4e343"},"cell_type":"markdown","source":"**Rough Benchmark**"},{"metadata":{"trusted":true,"_uuid":"17ce16472799b8734506f1cac7cf18232e881c89"},"cell_type":"code","source":"randind = np.random.randint(0, len(train_df), 8)\nrandind = np.sort(randind)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d51eeab96d9254157c2fa56c31e97dd5857f56a4"},"cell_type":"code","source":"train_hdf5 = h5py.File(hdf_path, \"r\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"557ee308807abde5ade4bbdc2bbc510bb866585b"},"cell_type":"markdown","source":"Loading from the HDF5 Datastore"},{"metadata":{"trusted":true,"_uuid":"311ddb99b5d85375aac4cea40ebde57e1d8ff4ab"},"cell_type":"code","source":"%%timeit\n# with h5py.File(hdf_path, \"r\") as train_hdf5:       # Causes 20% slowdown :(\nbatch = train_hdf5['train'][randind, ...]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf2e9e07482d9088d96e87d14e238fe7fa3f7901"},"cell_type":"code","source":"train_hdf5.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2223db8a6389a7d9fcbbc9a0e8eb1816e56469cf"},"cell_type":"code","source":"%%timeit\nbatch = np.zeros((8, 4, 512, 512), dtype=np.uint8)\nfor i, ind in enumerate(randind):\n    batch[i, ...] = load_image(train_df['Id'][ind])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e3ff7d339de7c2e085f5292a4e7f03a1b7ee94b4"},"cell_type":"markdown","source":"This is my first kernel, and the first time I'm experimenting with HDF5, so suggestions and feedback are welcome.\n\nCan someone tell me what happens if I don't close an open datastore? Opening and closing per batch is slow, and I want to know if I will corrupt the data if I interrupt training without closing."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}