{"cells":[{"metadata":{"_uuid":"4b036627ed8dfa93c70d71902c40b29d6b3d3854"},"cell_type":"markdown","source":"### Overview\nTraining data for this competition is generated by cropping a big image into 768x768 patches with 256 step. Therefore, it creates numerous overlaps in the train dataset and does not allow to use a random subset of images for validation. To exclude images that overlap with validation set from train set, I created this kernel. It checks 'train_v2' dataset and creates a file with a list of overlapping images, which can be used during train/val split."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import cv2\nimport os\nimport numpy as np\nimport pandas as pd\nfrom collections import defaultdict\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a1706ac5d868feb48bb2d8aa1ca0cee3cb91498"},"cell_type":"code","source":"TRAIN = '../input/train_v2/'\nnames = os.listdir(TRAIN)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"724624577dc347f060c8013020c4e410263f0ec0"},"cell_type":"code","source":"def get_hash(img):\n    result = []\n    sz = 256 #images are composed of 256x256 overlapping patches\n    for ix in range(0,768,sz):\n        for jx in range(0,768,sz):\n            result.append(hash(img[ix:ix+sz,jx:jx+sz,:].tobytes()))\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35779b800b5856bf0ad59703d49b43ff0bfd6562"},"cell_type":"code","source":"hash_dict = defaultdict(list)\nfor name in tqdm(names):\n    img = cv2.imread(os.path.join(TRAIN,name))\n    hashes = get_hash(img)\n    for h in hashes:\n        hash_dict[h].append(name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47bca6fb73ab8e41745451160f42b7f227cb2406"},"cell_type":"code","source":"img_dict = {name:list() for name in names}\nfor key,val in hash_dict.items():\n    items = set(val)\n    if(len(items) > 1):\n        for item_i in items:\n            for item_j in items:\n                if item_i == item_j: continue\n                img_dict[item_i].append(item_j)\n                \nduplicate_dict = dict()\nfor key,val in img_dict.items():\n    duplicates = set(val)\n    if len(duplicates) == 0: duplicate_dict[key] = np.nan\n    else: duplicate_dict[key] = ' '.join(duplicates)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27024a21af94ac20f4fb9fe0e2f1e12fd71b7642"},"cell_type":"code","source":"df = pd.DataFrame(pd.Series(duplicate_dict), columns=['duplicates'])\ndf['ImageId'] = df.index\ndf.to_csv('duplicates.csv',index=False)\ndf.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}