{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Hello!\n\nDuplicates are always harmful for training process: differently labeled duplicates produce noise in the dataset, while equally labeled duplicates lead to data leakage. \n\nIn this short notebook I am looking through image hash of **Plant Pathology 2021** competition dataset with `image_hash` library, finding more than 50 duplicates.\n\n### Update\nDue to recent changes in the `train.csv` file mentioned in **[this discussion](https://www.kaggle.com/c/plant-pathology-2021-fgvc8/discussion/228465)**, we have no more the `cider_apple_rust` class. This version (6) is made after the changes.","metadata":{}},{"cell_type":"markdown","source":"### Imports","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n# Preliminaries\nimport os\nfrom pathlib import Path\nimport glob\nfrom tqdm import tqdm\ntqdm.pandas()\nimport json\nimport pandas as pd\nimport numpy as np\n\n## Image hash\nimport imagehash\n# Visuals and CV2\nimport seaborn as sn\nimport matplotlib.pyplot as plt\nimport cv2\n#from PIL import Image\n\n\n# albumentations for augs\nimport albumentations\nfrom albumentations.pytorch.transforms import ToTensorV2\n\n# clustering and dimension reduction\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score\n\n# # Keras and TensorFlow\nfrom keras.preprocessing.image import load_img\n# from keras.preprocessing.image import img_to_array \nfrom keras.applications.resnet50 import preprocess_input \n# # from keras.applications.resnet18 import preprocess_input \n\n\n# models \nfrom keras.applications.resnet50 import ResNet50\n# from keras.applications.resnet18 import ResNet18\nfrom keras.models import Model\n\n#torch\nimport torch\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset,DataLoader","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from tqdm.notebook import tqdm\n# import matplotlib.pyplot as plt\n# import tensorflow as tf\n# import pandas as pd\n# import numpy as np\n# import imagehash\nimport PIL\n# import os","metadata":{"_uuid":"4a78d0be9bc398c6f69dffdb9709c7e32361f47d","scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = '../input/plant-pathology-2021-fgvc8'\ntrain = pd.read_csv(os.path.join(BASE_DIR, 'train.csv'))\n\n\nlabels_list = list(set(train.labels))\nlabels_list.sort()\nmapping = {label:i for i, label in enumerate(labels_list)}\n# print(labels_list, '\\n',mapping)\n\ntrain['labels_id'] = train['labels'].map(mapping)\ntrain","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Learn the distribution.","metadata":{}},{"cell_type":"code","source":"num_labels = []\nfor label in labels_list:\n    num_labels.append(train[train['labels']==label].count().labels)\nfor i, label in enumerate(labels_list):\n    print(f'{mapping[label]} {label} : {num_labels[i]}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_cts=train.labels.value_counts()\nfig = plt.figure(figsize=(12,6))\nsn.barplot(y=target_cts.sort_values(ascending=False).index, x=target_cts.sort_values(ascending=False).values, palette='winter')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_images(class_id, label, images_number, verbose=0, square_flag = False):\n   \n    plot_list = train[train[\"labels_id\"] == class_id].sample(images_number)['image'].tolist()\n    \n    if verbose:\n        print(plot_list)\n        \n    labels = [label for i in range(len(plot_list))]\n    size = np.sqrt(images_number)\n    if int(size)*int(size) < images_number:\n        size = int(size) + 1\n        \n    plt.figure(figsize=(20, 20))\n    \n    for ind, (image_id, label) in enumerate(zip(plot_list, labels)):\n        if square_flag:\n            plt.subplot(size, size, ind + 1)\n        else:\n            plt.subplot(1, images_number, ind + 1)\n        image = cv2.imread(os.path.join(BASE_DIR, 'train_images', image_id))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        plt.imshow(image)\n        plt.title(label, fontsize=12)\n        plt.axis(\"off\")\n    \n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(12):\n    plot_images(class_id=i,label=labels_list[i],images_number=4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG():\n    \n    threshold = .9\n    img_size = 512\n    seed = 42","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1. Saving downscaled images to boost performance\nComputing hash over original images of very high quality would take nearly 5 hours, thus we downscaling first.","metadata":{}},{"cell_type":"code","source":"root = '/kaggle/input/plant-pathology-2021-fgvc8/train_images'\n\npaths = os.listdir(root)\n\ndf = pd.read_csv('/kaggle/input/plant-pathology-2021-fgvc8/train.csv', index_col='image')\n\n# for path in tqdm(paths, total=len(paths)):\n#     image = tf.io.read_file(os.path.join(root, path))\n#     image = tf.image.decode_jpeg(image, channels=3)\n#     image = tf.image.resize(image, [CFG.img_size, CFG.img_size])\n#     image = tf.cast(image, tf.uint8).numpy()\n#     plt.imsave(path, image)","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. Hash computation","metadata":{}},{"cell_type":"code","source":"hash_functions = [\n    imagehash.average_hash,\n    imagehash.phash,\n    imagehash.dhash,\n    imagehash.whash]\n\nimage_ids = []\nhashes = []\n\npaths = tf.io.gfile.glob('./*.jpg')\n\nfor path in tqdm(paths, total=len(paths)):\n\n    image = PIL.Image.open(path)\n\n    hashes.append(np.array([x(image).hash for x in hash_functions]).reshape(-1,))\n    image_ids.append(path.split('/')[-1])\n    \nhashes = np.array(hashes)\nimage_ids = np.array(image_ids)","metadata":{"_uuid":"3a94ec9c45f58e7bc62bfeee6c2cdf06d7d92d92","scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3. Run search across hashed images\nWe firstly compare each image hash with all the hashes and then leave only unique pairs of matches","metadata":{}},{"cell_type":"code","source":"duplicate_ids = []\n\nfor i in tqdm(range(len(hashes)), total=len(hashes)):\n    similarity = (hashes[i] == hashes).mean(axis=1)\n    duplicate_ids.append(list(image_ids[similarity > CFG.threshold]))\n    \nduplicates = [frozenset([x] + y) for x, y in zip(image_ids, duplicate_ids)]\nduplicates = set([x for x in duplicates if len(x) > 1])","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4. Let's see what is found","metadata":{}},{"cell_type":"code","source":"print(f'Found {len(duplicates)} duplicate pairs:')\nfor row in duplicates:\n    print(', '.join(row))","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Writing duplicates to \"duplicates.csv\".')\nwith open('duplicates.csv', 'w') as file:\n    for row in duplicates:\n        file.write(','.join(row) + '\\n')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for row in duplicates:\n    \n    figure, axes = plt.subplots(1, len(row), figsize=[5 * len(row), 5])\n\n    for i, image_id in enumerate(row):\n        image = plt.imread(image_id)\n        axes[i].imshow(image)\n\n        axes[i].set_title(df.loc[image_id, 'labels'])\n        axes[i].axis('off')\n\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Clear working folder to avoid output pollution","metadata":{}},{"cell_type":"code","source":"for file in tf.io.gfile.glob('./*.jpg'):\n    os.remove(file)","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Acknowledgements\n\nThis work is Copy&Edit form @appian **[notebook](https://www.kaggle.com/appian/let-s-find-out-duplicate-images-with-imagehash)** with a lot of changes, but still highly inspired. If you find this notebook useful, please, upvote his work too.","metadata":{}}]}