{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nfrom pathlib import Path\n\nimport imagehash\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\n\ntqdm.pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Paths\nBASE_DIR = Path('/kaggle/input/cassava-leaf-disease-classification/')\nTRAIN_DIR = BASE_DIR / 'train_images'\nLBL_FP = BASE_DIR / 'train.csv'\nLBL_MAP_FP = BASE_DIR / 'label_num_to_disease_map.json'\nTEST_DIR = BASE_DIR / 'test_images'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(LBL_FP)\nprint(f'df shape: {df.shape}')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# hash images\ndef get_image_hash(fn, hash_size=16):\n    return str(imagehash.average_hash(Image.open(TRAIN_DIR / fn), hash_size=hash_size))\n\ndf['image_hash'] = df['image_id'].progress_apply(lambda x: get_image_hash(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"duplicates = df[df['image_hash'].duplicated(keep=False)]\nprint(f'# of possible duplicates: {duplicates.shape[0]}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['image_hash'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('So, There are possibly no duplicated images based on average_hashing method.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}