{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"If file size is not an issue, reading a pickle or npy file would be the fastest option.\n\nGrateful to: https://www.kaggle.com/code/anyai28/load-image-benchmark","metadata":{}},{"cell_type":"code","source":"import os\nimport subprocess\nimport pickle\nfrom pathlib import Path\n\nfrom tqdm import tqdm\nimport numpy as np\nimport cv2\nfrom PIL import Image","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:15:27.128116Z","iopub.execute_input":"2023-03-18T00:15:27.128724Z","iopub.status.idle":"2023-03-18T00:15:27.135385Z","shell.execute_reply.started":"2023-03-18T00:15:27.128681Z","shell.execute_reply":"2023-03-18T00:15:27.134464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jpg_dir = Path('./jpg')\nnpy_dir = Path('./npy')\npkl_dir = Path('./pkl')\n\njpg_dir.mkdir(exist_ok=True)\nnpy_dir.mkdir(exist_ok=True)\npkl_dir.mkdir(exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:15:27.137718Z","iopub.execute_input":"2023-03-18T00:15:27.138425Z","iopub.status.idle":"2023-03-18T00:15:27.152203Z","shell.execute_reply.started":"2023-03-18T00:15:27.138384Z","shell.execute_reply":"2023-03-18T00:15:27.151119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mp4 -> jpg\nvideo = '58168_003392_Endzone.mp4'\nsubprocess.run(f'ffmpeg -i /kaggle/input/nfl-player-contact-detection/test/{video} -q:v 2 -f image2 /kaggle/working/jpg/{video}_%04d.jpg -hide_banner -loglevel error', shell=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:15:27.153970Z","iopub.execute_input":"2023-03-18T00:15:27.154681Z","iopub.status.idle":"2023-03-18T00:15:33.215001Z","shell.execute_reply.started":"2023-03-18T00:15:27.154641Z","shell.execute_reply":"2023-03-18T00:15:33.213456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# directory size\n!du -sh {str(jpg_dir)}","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:15:33.219194Z","iopub.execute_input":"2023-03-18T00:15:33.219641Z","iopub.status.idle":"2023-03-18T00:15:34.316403Z","shell.execute_reply.started":"2023-03-18T00:15:33.219595Z","shell.execute_reply":"2023-03-18T00:15:34.314713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_name = jpg_dir / f'{video}_0001.jpg'\nprint(str(img_name))","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:15:34.318260Z","iopub.execute_input":"2023-03-18T00:15:34.318654Z","iopub.status.idle":"2023-03-18T00:15:34.326342Z","shell.execute_reply.started":"2023-03-18T00:15:34.318617Z","shell.execute_reply":"2023-03-18T00:15:34.324844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# jpg -> npy\njpg_files = list(jpg_dir.glob(\"*.jpg\"))\n\nfor jpg_file in tqdm(jpg_files):\n    img = cv2.imread(jpg_file.as_posix())\n    np.save(npy_dir / f\"{jpg_file.stem}.npy\", img.astype(np.uint8))","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:15:34.327856Z","iopub.execute_input":"2023-03-18T00:15:34.328192Z","iopub.status.idle":"2023-03-18T00:15:47.262009Z","shell.execute_reply.started":"2023-03-18T00:15:34.328161Z","shell.execute_reply":"2023-03-18T00:15:47.260542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# directory size\n!du -sh {str(npy_dir)}","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:15:47.263852Z","iopub.execute_input":"2023-03-18T00:15:47.264269Z","iopub.status.idle":"2023-03-18T00:15:48.356913Z","shell.execute_reply.started":"2023-03-18T00:15:47.264223Z","shell.execute_reply":"2023-03-18T00:15:48.355385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_name_npy = npy_dir / f'{video}_0001.npy'\nprint(str(img_name_npy))","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:15:48.358752Z","iopub.execute_input":"2023-03-18T00:15:48.359130Z","iopub.status.idle":"2023-03-18T00:15:48.366205Z","shell.execute_reply.started":"2023-03-18T00:15:48.359093Z","shell.execute_reply":"2023-03-18T00:15:48.364837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# jpg -> pkl\njpg_files = list(jpg_dir.glob(\"*.jpg\"))\n\nfor jpg_file in tqdm(jpg_files):\n    img = cv2.imread(jpg_file.as_posix())\n    with open(pkl_dir / f\"{jpg_file.stem}.pkl\", mode=\"wb\") as f:\n        pickle.dump(img, f, protocol=4)","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:15:48.369942Z","iopub.execute_input":"2023-03-18T00:15:48.370322Z","iopub.status.idle":"2023-03-18T00:16:02.173894Z","shell.execute_reply.started":"2023-03-18T00:15:48.370286Z","shell.execute_reply":"2023-03-18T00:16:02.172788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# directory size\n!du -sh {str(pkl_dir)}","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:16:02.174982Z","iopub.execute_input":"2023-03-18T00:16:02.175345Z","iopub.status.idle":"2023-03-18T00:16:03.287415Z","shell.execute_reply.started":"2023-03-18T00:16:02.175312Z","shell.execute_reply":"2023-03-18T00:16:03.286157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_name_pkl = pkl_dir / f'{video}_0001.pkl'\nprint(str(img_name_pkl))","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:16:03.288925Z","iopub.execute_input":"2023-03-18T00:16:03.289766Z","iopub.status.idle":"2023-03-18T00:16:03.297544Z","shell.execute_reply.started":"2023-03-18T00:16:03.289723Z","shell.execute_reply":"2023-03-18T00:16:03.296112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_img1():\n    img = Image.open(img_name)\n    np.array(img)\n    \n%timeit load_img1()","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:16:03.298929Z","iopub.execute_input":"2023-03-18T00:16:03.299374Z","iopub.status.idle":"2023-03-18T00:16:04.988033Z","shell.execute_reply.started":"2023-03-18T00:16:03.299339Z","shell.execute_reply":"2023-03-18T00:16:04.986764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"left = 100\ntop = 100\nright = 200\nbottom = 200\n\ndef load_img2():\n    for _ in range(1):\n        img = Image.open(img_name)\n        np.array(img.crop((left, top, right, bottom)))\n    \n%timeit load_img2()","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:16:04.989448Z","iopub.execute_input":"2023-03-18T00:16:04.989785Z","iopub.status.idle":"2023-03-18T00:16:17.537794Z","shell.execute_reply.started":"2023-03-18T00:16:04.989755Z","shell.execute_reply":"2023-03-18T00:16:17.536321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_img_opencv():\n    img = cv2.imread(str(img_name))\n    np.array(img)\n    \n%timeit load_img_opencv()","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:16:17.539641Z","iopub.execute_input":"2023-03-18T00:16:17.540603Z","iopub.status.idle":"2023-03-18T00:16:28.155754Z","shell.execute_reply.started":"2023-03-18T00:16:17.540508Z","shell.execute_reply":"2023-03-18T00:16:28.154471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_img_npy():\n    img = np.load(img_name_npy, allow_pickle=True)\n\n%timeit load_img_npy()","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:16:28.157477Z","iopub.execute_input":"2023-03-18T00:16:28.157979Z","iopub.status.idle":"2023-03-18T00:16:33.743112Z","shell.execute_reply.started":"2023-03-18T00:16:28.157929Z","shell.execute_reply":"2023-03-18T00:16:33.741757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_img_pkl():\n    with open(img_name_pkl, mode=\"rb\") as f:\n        # numpy.ndarray\n        img = pickle.load(f)\n\n%timeit load_img_pkl()","metadata":{"execution":{"iopub.status.busy":"2023-03-18T00:16:33.744773Z","iopub.execute_input":"2023-03-18T00:16:33.745279Z","iopub.status.idle":"2023-03-18T00:16:38.363127Z","shell.execute_reply.started":"2023-03-18T00:16:33.745227Z","shell.execute_reply":"2023-03-18T00:16:38.361658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}