{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Find Duplicates\nModified from https://www.kaggle.com/appian/let-s-find-out-duplicate-images-with-imagehash","metadata":{}},{"cell_type":"code","source":"# !conda install -c conda-forge imagehash --yes","metadata":{"ExecuteTime":{"end_time":"2021-05-19T06:38:26.540851Z","start_time":"2021-05-19T06:38:26.538834Z"},"execution":{"iopub.status.busy":"2021-05-28T04:58:16.289137Z","iopub.execute_input":"2021-05-28T04:58:16.289429Z","iopub.status.idle":"2021-05-28T04:58:16.293080Z","shell.execute_reply.started":"2021-05-28T04:58:16.289367Z","shell.execute_reply":"2021-05-28T04:58:16.292232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext autoreload\n%autoreload 2\n%matplotlib inline\n\nimport glob\nimport itertools\nimport collections\n\nfrom PIL import Image\nimport cv2\nfrom tqdm import tqdm_notebook as tqdm\nimport pandas as pd\nimport numpy as np\nimport torch\nimport imagehash\n\nimport matplotlib.pyplot as plt\n\n","metadata":{"ExecuteTime":{"end_time":"2021-05-19T06:38:27.402761Z","start_time":"2021-05-19T06:38:26.541836Z"},"execution":{"iopub.status.busy":"2021-05-28T04:58:16.313363Z","iopub.execute_input":"2021-05-28T04:58:16.313602Z","iopub.status.idle":"2021-05-28T04:58:17.581282Z","shell.execute_reply.started":"2021-05-28T04:58:16.313579Z","shell.execute_reply":"2021-05-28T04:58:17.580466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nCalc similalities between all image pairs\n\nI use imagehash library to calculate hash value of image. https://github.com/JohannesBuchner/imagehash\n\nThere are several hash functions provided and I used 4 of them and combined the calculated hash values.\n\n    average hashing (aHash)\n    perception hashing (pHash)\n    difference hashing (dHash)\n    wavelet hashing (wHash)\n\nI used profile image(1st image) of pet images to calculate hash values.\n","metadata":{}},{"cell_type":"code","source":"def run():\n\n    funcs = [\n        imagehash.average_hash,\n        imagehash.phash,\n        imagehash.dhash,\n        imagehash.whash,\n        #lambda x: imagehash.whash(x, mode='db4'),\n    ]\n\n    SOPInstanceUIDs = []\n    hashes = []\n    for path in tqdm(glob.glob('../input/siim-covid19-resized-to-512px-png/*/*.png')):\n\n        image = Image.open(path)\n        imageid = path.split('/')[-1].split('.')[0]\n\n        SOPInstanceUIDs.append(imageid)\n        hashes.append(np.array([f(image).hash for f in funcs]).reshape(256))\n\n    return SOPInstanceUIDs, np.array(hashes)\n\n%time SOPInstanceUIDs, hashes_all = run()\n\n","metadata":{"ExecuteTime":{"start_time":"2021-05-19T06:42:45.455Z"},"scrolled":true,"execution":{"iopub.status.busy":"2021-05-28T04:58:17.584426Z","iopub.execute_input":"2021-05-28T04:58:17.584708Z","iopub.status.idle":"2021-05-28T05:02:58.683352Z","shell.execute_reply.started":"2021-05-28T04:58:17.584681Z","shell.execute_reply":"2021-05-28T05:02:58.682485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hashes_all = torch.Tensor(hashes_all.astype(int)).cuda()","metadata":{"ExecuteTime":{"start_time":"2021-05-19T06:42:49.3Z"},"execution":{"iopub.status.busy":"2021-05-28T05:02:58.685928Z","iopub.execute_input":"2021-05-28T05:02:58.686256Z","iopub.status.idle":"2021-05-28T05:03:02.789975Z","shell.execute_reply.started":"2021-05-28T05:02:58.686222Z","shell.execute_reply":"2021-05-28T05:03:02.789096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\nCalculate similarities among all image pairs. Divide the value by 256 to normalize (0-1).\n","metadata":{}},{"cell_type":"code","source":"%time sims = np.array([(hashes_all[i] == hashes_all).sum(dim=1).cpu().numpy()/256 for i in range(hashes_all.shape[0])])\n","metadata":{"ExecuteTime":{"start_time":"2021-05-19T06:42:50.625Z"},"execution":{"iopub.status.busy":"2021-05-28T05:03:02.791490Z","iopub.execute_input":"2021-05-28T05:03:02.791857Z","iopub.status.idle":"2021-05-28T05:03:04.849874Z","shell.execute_reply.started":"2021-05-28T05:03:02.791820Z","shell.execute_reply":"2021-05-28T05:03:04.848872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices1 = np.where(sims > 0.99)\nindices2 = np.where(indices1[0] != indices1[1])\nSOPInstanceUID1 = [SOPInstanceUIDs[i] for i in indices1[0][indices2]]\nSOPInstanceUID2 = [SOPInstanceUIDs[i] for i in indices1[1][indices2]]\ndups = {tuple(sorted([SOPInstanceUID1,SOPInstanceUID2])):True for SOPInstanceUID1, SOPInstanceUID2 in zip(SOPInstanceUID1, SOPInstanceUID2)}\nprint('found %d duplicates' % len(dups))","metadata":{"ExecuteTime":{"start_time":"2021-05-19T06:42:51.186Z"},"execution":{"iopub.status.busy":"2021-05-28T05:03:04.851156Z","iopub.execute_input":"2021-05-28T05:03:04.851677Z","iopub.status.idle":"2021-05-28T05:03:05.063485Z","shell.execute_reply.started":"2021-05-28T05:03:04.851635Z","shell.execute_reply":"2021-05-28T05:03:05.062645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dups","metadata":{"ExecuteTime":{"start_time":"2021-05-19T06:42:51.625Z"},"execution":{"iopub.status.busy":"2021-05-28T05:03:05.064839Z","iopub.execute_input":"2021-05-28T05:03:05.065193Z","iopub.status.idle":"2021-05-28T05:03:05.108135Z","shell.execute_reply.started":"2021-05-28T05:03:05.065153Z","shell.execute_reply":"2021-05-28T05:03:05.107241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/read-dicom-metadate/alldicomtrain.csv')\ntest = pd.read_csv('../input/read-dicom-metadate/alldicomtest.csv')\ntrainbox = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\ntrainbox['SOPInstanceUID']=trainbox.id.str[:-6]\ntrain.loc[:,'Category'] = 'train'\ntrainmerge=pd.merge(train, trainbox,on='SOPInstanceUID')\n\ntest.loc[:,'Category'] = 'test'\ntest.loc[:,'boxes'] = np.nan\n\ndf1 = pd.concat([trainmerge, test], sort=False)\n","metadata":{"ExecuteTime":{"start_time":"2021-05-19T06:42:52.546Z"},"execution":{"iopub.status.busy":"2021-05-28T05:03:05.109422Z","iopub.execute_input":"2021-05-28T05:03:05.110021Z","iopub.status.idle":"2021-05-28T05:03:05.448269Z","shell.execute_reply.started":"2021-05-28T05:03:05.109960Z","shell.execute_reply":"2021-05-28T05:03:05.447401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=df1[['SOPInstanceUID','PatientName','Category', 'PatientID', 'PatientSex','ImageType', 'boxes','SeriesInstanceUID', 'StudyID', 'SeriesNumber', 'InstanceNumber']]","metadata":{"ExecuteTime":{"start_time":"2021-05-19T06:42:53.153Z"},"execution":{"iopub.status.busy":"2021-05-28T05:03:05.451614Z","iopub.execute_input":"2021-05-28T05:03:05.451932Z","iopub.status.idle":"2021-05-28T05:03:05.487447Z","shell.execute_reply.started":"2021-05-28T05:03:05.451904Z","shell.execute_reply":"2021-05-28T05:03:05.486540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df1[['SOPInstanceUID','PatientName', 'PatientID', 'PatientSex','ImageType','fname', 'boxes','SeriesInstanceUID', 'StudyID', 'SeriesNumber', 'InstanceNumber']]","metadata":{"ExecuteTime":{"start_time":"2021-05-19T06:42:53.899Z"},"execution":{"iopub.status.busy":"2021-05-28T05:03:05.489142Z","iopub.execute_input":"2021-05-28T05:03:05.489732Z","iopub.status.idle":"2021-05-28T05:03:05.518834Z","shell.execute_reply.started":"2021-05-28T05:03:05.489693Z","shell.execute_reply":"2021-05-28T05:03:05.518033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testbox['SOPInstanceUID']=testbox.id.str[:-7]","metadata":{"ExecuteTime":{"start_time":"2021-05-19T06:42:54.402Z"},"scrolled":true,"execution":{"iopub.status.busy":"2021-05-28T05:03:05.521463Z","iopub.execute_input":"2021-05-28T05:03:05.521814Z","iopub.status.idle":"2021-05-28T05:03:05.550465Z","shell.execute_reply.started":"2021-05-28T05:03:05.521789Z","shell.execute_reply":"2021-05-28T05:03:05.549667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns","metadata":{"ExecuteTime":{"start_time":"2021-05-19T06:42:54.881Z"},"execution":{"iopub.status.busy":"2021-05-28T05:03:05.553143Z","iopub.execute_input":"2021-05-28T05:03:05.553387Z","iopub.status.idle":"2021-05-28T05:03:05.583834Z","shell.execute_reply.started":"2021-05-28T05:03:05.553363Z","shell.execute_reply":"2021-05-28T05:03:05.583093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"detail = {SOPInstance:df[df.SOPInstanceUID == SOPInstance] for SOPInstance in itertools.chain.from_iterable(list(dups))}","metadata":{"ExecuteTime":{"start_time":"2021-05-19T06:42:55.393Z"},"execution":{"iopub.status.busy":"2021-05-28T05:03:05.585015Z","iopub.execute_input":"2021-05-28T05:03:05.585339Z","iopub.status.idle":"2021-05-28T05:03:06.069921Z","shell.execute_reply.started":"2021-05-28T05:03:05.585303Z","shell.execute_reply":"2021-05-28T05:03:06.069106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ntype(detail)","metadata":{"ExecuteTime":{"start_time":"2021-05-19T06:42:56.201Z"},"execution":{"iopub.status.busy":"2021-05-28T05:03:06.071222Z","iopub.execute_input":"2021-05-28T05:03:06.071581Z","iopub.status.idle":"2021-05-28T05:03:06.103205Z","shell.execute_reply.started":"2021-05-28T05:03:06.071543Z","shell.execute_reply":"2021-05-28T05:03:06.102242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show(row1, row2):\n    try:\n        print('Image: %s / %s' % (row1.SOPInstanceUID.iloc[-1], row2.SOPInstanceUID.iloc[-1]))\n        print('boxes: %s / %s' % (row1.boxes.iloc[-1],row2.boxes.iloc[-1]))\n        print('Category: %s / %s' % (row1.Category.iloc[-1], row2.Category.iloc[-1]))\n#         print('Box: %s / %s' % (row1.boxes.iloc[-1], row2.boxes.iloc[-1]))\n        #     print('Breed1: %d / %d' % (row1.Breed1, row2.Breed1))\n        #     print('Age: %d / %d' % (row1.Age, row2.Age))\n        #     print('RescuerID:\\n%s\\n%s' % (row1.RescuerID, row2.RescuerID))\n    \n\n    \n        image1 = cv2.imread(f'../input/siim-covid19-resized-to-512px-png/{row1.Category.iloc[-1]}/{row1.SOPInstanceUID.iloc[-1]}.png' )\n        image2 = cv2.imread(f'../input/siim-covid19-resized-to-512px-png/{row2.Category.iloc[-1]}/{row2.SOPInstanceUID.iloc[-1]}.png' )\n        image1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY)\n        image2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)\n    \n    \n        fig = plt.figure(figsize=(10, 20))\n        fig.add_subplot(1,2,1)\n        plt.imshow(image1,cmap='gray')\n        fig.add_subplot(1,2, 2)\n        plt.imshow(image2,cmap='gray')\n        plt.show()\n    except:\n        print('SOPInstanceUID.iloc[-1]')","metadata":{"ExecuteTime":{"start_time":"2021-05-19T06:42:57.064Z"},"execution":{"iopub.status.busy":"2021-05-28T05:03:06.104631Z","iopub.execute_input":"2021-05-28T05:03:06.104977Z","iopub.status.idle":"2021-05-28T05:03:06.138410Z","shell.execute_reply.started":"2021-05-28T05:03:06.104938Z","shell.execute_reply":"2021-05-28T05:03:06.137628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for SOPInstanceUID1, SOPInstanceUID2 in sorted(list(dups)):\n    \n        row1 = detail[SOPInstanceUID1]\n#         \n        row2 = detail[SOPInstanceUID2]\n#         try:\n#             if row1.Category.iloc[-1] != row2.Category.iloc[-1]:\n        show(row1, row2)\n#         except:\n#             print(f'error {SOPInstanceUID1}-{SOPInstanceUID2}')","metadata":{"ExecuteTime":{"start_time":"2021-05-19T06:42:57.873Z"},"execution":{"iopub.status.busy":"2021-05-28T05:03:06.139511Z","iopub.execute_input":"2021-05-28T05:03:06.139867Z","iopub.status.idle":"2021-05-28T05:03:59.050225Z","shell.execute_reply.started":"2021-05-28T05:03:06.139828Z","shell.execute_reply":"2021-05-28T05:03:59.049332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}