{"cells":[{"metadata":{},"cell_type":"markdown","source":"To speed up the loading of training samples, we can pre-scale the images to a smaller size (here we use 512) and use a format that loads faster, such as jpg."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.notebook import tqdm\n\nimport cv2\nfrom PIL import Image\n\nfrom matplotlib import pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Experiment\nWe compare the loading speed of different loading strategies\n+ do nothing: load 4 full size png images \n+ load 1 single scaled-down png image with 4 channels\n+ load 4 scaled-down jpg image\n+ load a jpg image with rgb channel and a jpg image with yellow channel"},{"metadata":{"trusted":true},"cell_type":"code","source":"im = cv2.imread('../input/hpa-single-cell-image-classification/train/000a6c98-bb9b-11e8-b2b9-ac1f6b6435d0_blue.png', cv2.IMREAD_UNCHANGED)\nplt.imshow(im)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(cv2.resize(im, (512, 512), interpolation=cv2.INTER_LINEAR))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prepare images"},{"metadata":{"trusted":true},"cell_type":"code","source":"imb = cv2.imread('../input/hpa-single-cell-image-classification/train/000a6c98-bb9b-11e8-b2b9-ac1f6b6435d0_blue.png', cv2.IMREAD_UNCHANGED)\nimg = cv2.imread('../input/hpa-single-cell-image-classification/train/000a6c98-bb9b-11e8-b2b9-ac1f6b6435d0_green.png', cv2.IMREAD_UNCHANGED)\nimr = cv2.imread('../input/hpa-single-cell-image-classification/train/000a6c98-bb9b-11e8-b2b9-ac1f6b6435d0_red.png', cv2.IMREAD_UNCHANGED)\nimy = cv2.imread('../input/hpa-single-cell-image-classification/train/000a6c98-bb9b-11e8-b2b9-ac1f6b6435d0_yellow.png', cv2.IMREAD_UNCHANGED)\n\nimb = cv2.resize(imb, (512, 512), interpolation=cv2.INTER_LINEAR)\nimg = cv2.resize(img, (512, 512), interpolation=cv2.INTER_LINEAR)\nimr = cv2.resize(imr, (512, 512), interpolation=cv2.INTER_LINEAR)\nimy = cv2.resize(imy, (512, 512), interpolation=cv2.INTER_LINEAR)\n\nim = np.stack((imb, img, imr, imy)).transpose((1, 2, 0))\ncv2.imwrite('im.png', im)\n\ncv2.imwrite('imb.jpg', imb)\ncv2.imwrite('img.jpg', img)\ncv2.imwrite('imr.jpg', imr)\ncv2.imwrite('imy.jpg', imy)\n\ncv2.imwrite('imgrgb.jpg', np.stack((imb, img, imr)).transpose((1, 2, 0)))  # opencv uses BGR format","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -lh","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"do nothing: load 4 full size png images"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%timeit\nimb = cv2.imread('../input/hpa-single-cell-image-classification/train/000a6c98-bb9b-11e8-b2b9-ac1f6b6435d0_blue.png', cv2.IMREAD_UNCHANGED)\nimg = cv2.imread('../input/hpa-single-cell-image-classification/train/000a6c98-bb9b-11e8-b2b9-ac1f6b6435d0_green.png', cv2.IMREAD_UNCHANGED)\nimr = cv2.imread('../input/hpa-single-cell-image-classification/train/000a6c98-bb9b-11e8-b2b9-ac1f6b6435d0_red.png', cv2.IMREAD_UNCHANGED)\nimy = cv2.imread('../input/hpa-single-cell-image-classification/train/000a6c98-bb9b-11e8-b2b9-ac1f6b6435d0_yellow.png', cv2.IMREAD_UNCHANGED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"load 1 single scaled-down png image with 4 channels"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%timeit\ncv2.imread('im.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"load 4 scaled-down jpg image"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%timeit\ncv2.imread('imb.jpg')\ncv2.imread('img.jpg')\ncv2.imread('imr.jpg')\ncv2.imread('imy.jpg')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"load a jpg image with rgb channel and a jpg image with yellow channel, loading time reduced from 164 ms to 2.87 ms"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%timeit\ncv2.imread('imrgb.jpg')\ncv2.imread('imy.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imrgb = cv2.imread('imgrgb.jpg', cv2.IMREAD_UNCHANGED)\nimrgb = cv2.cvtColor(imrgb, cv2.COLOR_BGR2RGB)\nplt.imshow(imrgb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imy = cv2.imread('imy.jpg', cv2.IMREAD_UNCHANGED)\nplt.imshow(imy)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Process the whole dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf *\n!mkdir train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/hpa-single-cell-image-classification/train.csv')\ndf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from joblib import Parallel, delayed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '../input/hpa-single-cell-image-classification/train'\n\ndef save(ID):\n    imb = cv2.imread(f'{path}/{ID}_blue.png', cv2.IMREAD_UNCHANGED)\n    img = cv2.imread(f'{path}/{ID}_green.png', cv2.IMREAD_UNCHANGED)\n    imr = cv2.imread(f'{path}/{ID}_red.png', cv2.IMREAD_UNCHANGED)\n    imy = cv2.imread(f'{path}/{ID}_yellow.png', cv2.IMREAD_UNCHANGED)\n    \n    imb = cv2.resize(imb, (512, 512), interpolation=cv2.INTER_LINEAR)\n    img = cv2.resize(img, (512, 512), interpolation=cv2.INTER_LINEAR)\n    imr = cv2.resize(imr, (512, 512), interpolation=cv2.INTER_LINEAR)\n    imy = cv2.resize(imy, (512, 512), interpolation=cv2.INTER_LINEAR)\n    \n    cv2.imwrite(f'train/{ID}_rgb.jpg', np.stack((imb, img, imr)).transpose((1, 2, 0)))\n    cv2.imwrite(f'train/{ID}_yellow.jpg', imy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Parallel(n_jobs=4)(delayed(save)(row.ID) for row in tqdm(df.itertuples(), total=len(df)))\n'' # suppress output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!du -sh train\n!ls -f train | wc -l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp ../input/hpa-single-cell-image-classification/train.csv .","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}