{"cells":[{"metadata":{"_uuid":"bc192d84c117c07c7e6035b4ddf5547adae1f5c3","_cell_guid":"8b374130-47eb-4281-ab77-f97304fae911","collapsed":true,"trusted":false},"cell_type":"code","source":"import os\nfrom zipfile import ZipFile\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom dask import bag, threaded\nfrom dask.diagnostics import ProgressBar\nimport matplotlib.pyplot as plt\nimport imagehash","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false,"collapsed":true},"cell_type":"code","source":"# get filenames\nzipped = ZipFile('../input/test_jpg.zip')\nfilenames = zipped.namelist()[1:] # exclude the initial directory listing\nprint(len(filenames))\n#filenames = filenames[0:10]","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"60288fa1-8d73-42af-a89d-cbe1fa132388","_uuid":"0da84ba8eba13f850d4260b5ac6cc5a4e085f082","trusted":false},"cell_type":"code","source":"# define function\ndef get_hashes2(file):\n    exfile = zipped.read(file)\n    arr = np.frombuffer(exfile, np.uint8)\n    if arr.size > 0:   # exclude dirs and blanks\n        imz = cv2.imdecode(arr, flags=cv2.IMREAD_GRAYSCALE)\n        shp = imz.shape\n    else: \n        shp = [0,0,0] \n    return (file, shp)\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"d3ec43b6-434f-44ac-bb7e-9f653ef8cbaf","_uuid":"1d01e1ead21af3e698987602d493dc5381450091","trusted":false,"collapsed":true},"cell_type":"code","source":"exfile = zipped.read(filenames[1])\narr = np.frombuffer(exfile, np.uint8)\nimz = cv2.imdecode(arr, flags=cv2.IMREAD_GRAYSCALE)\nshp = imz.shape\nshp\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"222a5d9d-486f-4100-ad68-cb14e672e25b","_uuid":"016d97c5a8764ec0bd0b1fb806cb5f218bdb66ee","trusted":false,"collapsed":true},"cell_type":"code","source":"file = [zinfo.filename for zinfo in  zipped.filelist[1:]]\nfile[-5:]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b83315b5-0ee4-4e00-bfff-3eb100fad277","_uuid":"8ba7644034e3d4c3e5600ca2a5630045544f6ad2","trusted":false,"collapsed":true},"cell_type":"code","source":"size_train = [zinfo.file_size//1000 for zinfo in  zipped.filelist[1:]]\nsize_train[-5:]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"fea1755d-9613-405f-841d-d2c4dbf5fb94","_uuid":"7581ec995aa153a91a5823444c98bf9a94cee042","trusted":false,"collapsed":true},"cell_type":"code","source":"compress_size_train =[zinfo.compress_size/1000 for zinfo in  zipped.filelist[1:]]\ncompress_size_train[-5:]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a2e4ef36-39cf-41fc-be54-21e928540153","_uuid":"4db521b155e671dc3e28485ac8dcf4491bc2ba15","trusted":false,"collapsed":true},"cell_type":"code","source":"out = pd.DataFrame({'image':file, 'size':size_train,  'csize':compress_size_train})\nout['image'] = out.image.str.replace(\"data/competition_files/test_jpg/\", \"\")\nout['image'] = out.image.str.replace(\".jpg\", \"\")\nout.sort_values('image').head()\nout.to_csv('test_img_files_info.csv')\nout.head()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"644fd01b-e8db-4117-832d-cb162933da42","_uuid":"58b96efd3655bab6269337446ddd5894d89744e4","trusted":false},"cell_type":"code","source":"# define function\ndef get_hashes3(file):\n    exfile = zipped.read(file)\n    arr = np.frombuffer(exfile, np.uint8)\n    if arr.size > 0:   # exclude dirs and blanks\n        img = cv2.imdecode(arr, flags=cv2.IMREAD_UNCHANGED)\n        average_color = [img[:, :, i].mean() for i in range(img.shape[-1])]\n        average_color\n    else: \n        average_color = [0,0,0] \n    return (file, average_color)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6d5eafcf-d254-4cc4-8dd8-8e2fb943e6e8","_uuid":"30c914ae6583618f59d42cf452846aa4d572a359","trusted":false,"collapsed":true},"cell_type":"code","source":"exfile = zipped.read(filenames[1])\narr = np.frombuffer(exfile, np.uint8)\nimg = cv2.imdecode(arr, flags=cv2.IMREAD_UNCHANGED)\naverage_color = [img[:, :, i].mean() for i in range(img.shape[-1])]\naverage_color\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"10197deb-56c6-463c-944e-0469316bc006","_uuid":"211398eb0c75bbd3838b1ec5e54373e2f03ae2ca"},"cell_type":"markdown","source":"Here is the parallel processing. I use Dask (my new favorite) to distribute the workload and pull hash pairs back into a list. It seems really fast considering it's cpu-driven. Also, the simplicity of the progress bar is excellent compared to directly using tqdm with multiprocessing and multiple arguments!"},{"metadata":{"_cell_guid":"2f93cfc8-d187-426e-a605-e4be761a943c","_uuid":"62ffa93d85e6610664beaa1b7d16449997b0a69c","trusted":false,"collapsed":true},"cell_type":"code","source":"b = bag.from_sequence(filenames).map(get_hashes2)\nwith ProgressBar():\n    h_list_trn = b.compute(get=threaded.get)\nprint(h_list_trn[0:10])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"53e31bb7-458c-4691-9621-069701674eac","_uuid":"692909874ebb590dc1c3e43d6c4d91f7d795774e","trusted":false,"collapsed":true},"cell_type":"code","source":"b = bag.from_sequence(filenames).map(get_hashes3)\nwith ProgressBar():\n    c_list_trn = b.compute(get=threaded.get)\nprint(c_list_trn[0:10])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f5d89b0e-d647-464c-8c12-33d80728dfa1","_uuid":"86f6ec58952df2dc615f807f9ce4e50986061906","scrolled":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"pd.options.display.max_colwidth = 100   # show the annoyingly long strings\nnames_ = [h[0] for h in h_list_trn]\ndim_ = [h[1] for h in h_list_trn]\ncol_ = [h[1] for h in c_list_trn]\n\nhash_df_trn = pd.DataFrame({'image':names_, 'dim':dim_,  'colors':col_})\nhash_df_trn['image'] = hash_df_trn.image.str.replace(\"data/competition_files/test_jpg/\", \"\")\nhash_df_trn['image'] = hash_df_trn.image.str.replace(\".jpg\", \"\")\nhash_df_trn.sort_values('image').head()\nhash_df_trn.to_csv('train_img_feat.csv')\nhash_df_trn.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}