{"cells":[{"metadata":{},"cell_type":"markdown","source":"This kernel is a little bit modified version of [this kernel](https://www.kaggle.com/manojprabhaakr/similar-duplicate-images-in-aptos-data).   \nCredit to [@ManojPrabhakar](https://www.kaggle.com/manojprabhaakr).  \n<br>\nYou can download the duplicated list from **output**.\n\n---\n# <font color=dimgray> Import Package </font>"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd \nimport os\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport matplotlib.pyplot as plt\nimport imagehash\nimport psutil\n\nfrom PIL import Image\nfrom joblib import Parallel, delayed\n\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom IPython import display\nimport time\n\n%matplotlib inline\n\nplt.style.use('ggplot')\npd.set_option('display.max_columns', 300)\npd.set_option('display.max_rows', 100)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# <font color=dimgray> Load CSV </font>"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# <font color=dimgray> Get Info of train images </font>\n\nGetting the path of the Image"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = train[['diagnosis']]\ndf['path'] = glob.glob('../input/train_images/*.png')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Calculating the Hash, Shape, Mode, Length and Ratio of each image"},{"metadata":{"trusted":true},"cell_type":"code","source":"def getImageMetaData(file_path):\n    with Image.open(file_path) as img:\n        img_hash = imagehash.phash(img)\n        return img.size, img.mode, str(img_hash), file_path\n\n    \nimg_meta_l = Parallel(n_jobs=psutil.cpu_count(), verbose=1)(\n    (delayed(getImageMetaData)(fp) for fp in glob.glob('../input/train_images/*.png'))\n)\nimg_meta_df = pd.DataFrame(np.array(img_meta_l))\nimg_meta_df.columns = ['Size', 'Mode', 'Hash', 'path']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df.merge(img_meta_df, on='path', how='left')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_csv('./image_info.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_gb = df.groupby('Hash').count().reset_index()\ndf_gb.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_gb_dup = df_gb.query('path > 1')\ndf_gb_dup","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_gb_dup['path'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dup_hash_l = df_gb_dup['Hash'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_dup = df.loc[df['Hash'].isin(dup_hash_l)].sort_values('Hash')\ndf_dup.to_csv('./duplicated_info.csv')\ndf_dup.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"samp_hash = df_dup['Hash'].sample(1).values[0]\nprint(samp_hash)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dups = df_dup.query('Hash == @samp_hash')['path'].values\n\nfig, ax = plt.subplots(len(dups), 1, figsize=(7, 5 * len(dups)))\nfor i, d in enumerate(dups):\n    ax[i].imshow(mpimg.imread(d))\n    ax[i].grid(alpha=0.1)\nfig.tight_layout();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dups = df_dup.query('Hash == \"969a6b60246f3967\"')['path'].values\n\nfig, ax = plt.subplots(len(dups), 1, figsize=(7, 5 * len(dups)))\nfor i, d in enumerate(dups):\n    ax[i].imshow(mpimg.imread(d))\n    ax[i].grid(alpha=0.1)\nfig.tight_layout();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"---\n# <font color=dimgray> EOF </font>"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}