{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Here's a faster way to download images using the principles of concurrency and parallelism. In other words, asynchronous tasking and multiprocessing. It works best on machines with 8+ cores and when downloading large files like videos.\n\nPlease let me know if you have other good methods for downloading large numbers of files!","metadata":{}},{"cell_type":"code","source":"# Libraries for concurrency and parallelism\n! pip install asks trio","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-15T15:34:06.509564Z","iopub.execute_input":"2021-09-15T15:34:06.510398Z","iopub.status.idle":"2021-09-15T15:34:19.261892Z","shell.execute_reply.started":"2021-09-15T15:34:06.510363Z","shell.execute_reply":"2021-09-15T15:34:19.260829Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport requests\nfrom os import cpu_count\n\nimport datatable as dt\nimport asks\nimport trio","metadata":{"execution":{"iopub.status.busy":"2021-09-15T16:05:41.173492Z","iopub.execute_input":"2021-09-15T16:05:41.173784Z","iopub.status.idle":"2021-09-15T16:05:41.178722Z","shell.execute_reply.started":"2021-09-15T16:05:41.173748Z","shell.execute_reply":"2021-09-15T16:05:41.177806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Path('./pics').mkdir(exist_ok=True)\n\nimg_urls = dt.fread(\"../input/wikipedia-image-caption/test.tsv\", sep='\\t', columns={'image_url'})\nlinks = img_urls.to_list()[0][:1000]","metadata":{"execution":{"iopub.status.busy":"2021-09-15T16:05:41.929122Z","iopub.execute_input":"2021-09-15T16:05:41.929444Z","iopub.status.idle":"2021-09-15T16:05:41.996233Z","shell.execute_reply.started":"2021-09-15T16:05:41.929412Z","shell.execute_reply":"2021-09-15T16:05:41.995141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# fast way\n\nasync def fetch_pic(s, url):\n    r = await s.get(url)\n    return r.content\n\n\nasync def save_pic(s, url):\n    content = await fetch_pic(s, url)\n    filename = f\"pics/{url.split('/')[-1][:100]}\"\n    with open(filename,'wb') as f:\n        f.write(content)\n\n        \nasync def main(links):\n    dname = 'https://upload.wikimedia.org'\n    s = asks.sessions.Session(dname, connections=cpu_count()*2)\n    async with trio.open_nursery() as n:\n        for url in links:\n            n.start_soon(save_pic, s, url)\n\n            \ntrio.run(main, links)","metadata":{"execution":{"iopub.status.busy":"2021-09-15T16:05:42.096208Z","iopub.execute_input":"2021-09-15T16:05:42.097324Z","iopub.status.idle":"2021-09-15T16:06:33.851861Z","shell.execute_reply.started":"2021-09-15T16:05:42.097273Z","shell.execute_reply":"2021-09-15T16:06:33.850518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\nls pics | wc -l\nls pics -U | head -6","metadata":{"execution":{"iopub.status.busy":"2021-09-15T15:46:42.541695Z","iopub.execute_input":"2021-09-15T15:46:42.542722Z","iopub.status.idle":"2021-09-15T15:46:42.585259Z","shell.execute_reply.started":"2021-09-15T15:46:42.542681Z","shell.execute_reply":"2021-09-15T15:46:42.584228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# regular way\n\nfor url in links:\n    r = requests.get(url, stream=True)\n    content=r.content\n    filename = f\"pics/{url.split('/')[-1][:100]}\"\n    with open(filename,'wb') as f:\n        f.write(content)\n    ","metadata":{"execution":{"iopub.status.busy":"2021-09-15T16:07:47.373853Z","iopub.execute_input":"2021-09-15T16:07:47.374528Z","iopub.status.idle":"2021-09-15T16:09:13.364970Z","shell.execute_reply.started":"2021-09-15T16:07:47.374477Z","shell.execute_reply":"2021-09-15T16:09:13.364119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}