{"cells":[{"metadata":{},"cell_type":"markdown","source":"part 2 -> [notebook](https://www.kaggle.com/keremt/cassava-eda-part2-cnn-dedup-with-rapids/)"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -qqU fastai==2.1.7","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import fastai; print(\"fastai:\", fastai.__version__)\nimport torch; print(\"torch:\", torch.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.vision.all import *\nimport torchvision ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"new_data_path = Path(\"../input/cassava-leaf-disease-classification//\")\nold_data_path = Path(\"../input/cassavaold/\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data"},{"metadata":{},"cell_type":"markdown","source":"### a) New Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"new_data_path.ls().map(lambda o: o.name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images = get_image_files(new_data_path/'train_images')\ntest_images = get_image_files(new_data_path/'test_images')\ntrain_df = pd.read_csv(new_data_path/'train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_images), len(test_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_images = train_images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labeldict = json.loads((new_data_path/'label_num_to_disease_map.json').open().read())\nlabeldict = {int(k):v for k,v in labeldict.items()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['label'].map(labeldict).value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%timeit\nimg1 = PILImage.create(train_images[0]) \nimg1 = ToTensor()(img1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%timeit\nimg2 = torchvision.io.read_image(train_images[0].as_posix())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### b) Old Data\n\nPlease upvote if you use: https://www.kaggle.com/keremt/cassavaold"},{"metadata":{"trusted":true},"cell_type":"code","source":"old_train_images = get_image_files(old_data_path/'train')\nold_test_images = get_image_files(old_data_path/'test')\nold_unsup_images = get_image_files(old_data_path/'extraimages')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"old_images = old_train_images + old_test_images + old_unsup_images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(old_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## 1) Image Hash based Dedup\n\nLet's see if we can use old competition data or not...\n\nNote that below we are doing exact match of hash codes, so we are not looking at soft similarity scores, for that skip to cnn based dedup."},{"metadata":{"trusted":true},"cell_type":"code","source":"import imagehash, PIL","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_imagehash(path, hashfunc=imagehash.average_hash, hash_size=8):\n    img = PIL.Image.open(path)\n    return str(hashfunc(img, hash_size=hash_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_image_hashes = parallel(partial(get_imagehash, hashfunc=imagehash.phash, hash_size=8), train_images)\nold_image_hashes = parallel(partial(get_imagehash, hashfunc=imagehash.phash, hash_size=8), old_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(new_image_hashes), len(old_image_hashes)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dups between old and new datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"common_hashes = list(set(new_image_hashes).intersection(set(old_image_hashes))); len(common_hashes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Total of {len(common_hashes)}/{len(old_image_hashes)} pairs might be same images\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"common_hashes[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hash2new_images = defaultdict(list)\nhash2old_images = defaultdict(list)\n\nfor h, im in zip(new_image_hashes, train_images):\n    hash2new_images[h].append(im)\n\nfor h, im in zip(old_image_hashes, old_images):\n    hash2old_images[h].append(im)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hash2new_images[common_hashes[0]], hash2old_images[common_hashes[0]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imgs = []\nfor h in common_hashes:\n    newimgs = [PILImage.create(o) for o in hash2new_images[h]]\n    oldimgs = [PILImage.create(o) for o in hash2old_images[h]]\n    imgs += newimgs\n    imgs += oldimgs\n    \n    if len(imgs) > 64: break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's plot first 64 sample due to notebook limit. We can see that these are indeed same images. But 4902 is not too bad if that's all the duplicates. I am not very experienced with image hash methods, so using a different hash function with different parameters might also change the result. For that reason I will let you be the judge on whether to use old competition data or not. Let me know what you think down in the comments, as I am also very interested to hear about it!\n\nYou may see new images are all horizontal images and old ones are resized in new dataset to make them horizontal too. It's probably due to how photo was originally take with the phone."},{"metadata":{"trusted":true},"cell_type":"code","source":"show_images(imgs[:64], nrows=8, ncols=8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"You may check how average hash performs in this link: https://johannesbuchner.github.io/imagehash/art2.html. It looks like it has a high false positive rate and brings non-duplicates as similar when they are different. This is pretty good for us and gives us confidence that there is probably not any other duplicates."},{"metadata":{"trusted":true},"cell_type":"code","source":"duplicate_old_images = [hash2old_images[o] for o in common_hashes]\nduplicate_new_images = [hash2new_images[o] for o in common_hashes]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(duplicate_old_images), len(duplicate_new_images)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We might prefer to use either the old data or the new one for dups. It looks like old data is resized."},{"metadata":{"trusted":true},"cell_type":"code","source":"old_new_duplicate_pairs = list(zip(duplicate_old_images, duplicate_new_images))[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.to_pickle(old_new_duplicate_pairs, \"old_new_duplicate_pairs.pkl\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dups within new dataset or old dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"new_image_dups = [v for k,v in hash2new_images.items() if len(v) > 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_image_dups","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"old_image_dups = [v for k,v in hash2old_images.items() if len(v) > 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(old_image_dups), len(np.concatenate(old_image_dups))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dups = [(Path(o.parent.name)/o.name, PILImage.create(o)) for o in np.random.choice(old_image_dups)]\ntitles, imgs = zip(*dups)\nshow_images(imgs, titles=titles)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.to_pickle(old_image_dups, \"old_image_dups.pkl\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## 2) CNN Based Dedup"},{"metadata":{},"cell_type":"markdown","source":"### Normalize labels "},{"metadata":{"trusted":true},"cell_type":"code","source":"len(old_images), len(new_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oldlabeldict = {'cbsd': 'Cassava Brown Streak Disease (CBSD)',\n                 'healthy': 'Healthy',\n                 'cmd': 'Cassava Mosaic Disease (CMD)',\n                 'cgm': 'Cassava Green Mottle (CGM)',\n                 'cbb': 'Cassava Bacterial Blight (CBB)',\n                 '0': 'Unsup', # test\n                 'extraimages': 'Unsup'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labeldict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"old_images2labels = dict(zip(old_images, [oldlabeldict[o] for o in old_images.map(lambda o: o.parent.name)]))\n\nnew_images2labels = dict(zip(train_df['image_id'], train_df['label']))\nnew_images2labels = {k:labeldict[v] for k,v in new_images2labels.items()}\nnew_images2labels = {o:new_images2labels[o.name] for o in new_images}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Counter(old_images2labels.values())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Counter(new_images2labels.values())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_images2label = {**old_images2labels, **new_images2labels}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Counter(all_images2label.values())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(all_images2label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_vocab = {'Cassava Bacterial Blight (CBB)':0,\n             'Cassava Brown Streak Disease (CBSD)':1,\n             'Cassava Green Mottle (CGM)':2,\n             'Cassava Mosaic Disease (CMD)':3,\n             'Healthy':4, \n             'Unsup':5}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2) Get Embeddings "},{"metadata":{"trusted":true},"cell_type":"code","source":"all_images = old_images + new_images; len(all_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Torchvision\nsize = (224,224)\nbs = 64\ndef open_image(fn):    return TensorImage(torchvision.io.read_image(str(fn)))\n\ntfms = [[open_image, torchvision.transforms.Resize(size, )], \n        [lambda o: all_images2label[o], Categorize(label_vocab)]]\nbatch_tfms = [IntToFloatTensor, Normalize.from_stats(*imagenet_stats)]\n\ndsets = Datasets(all_images, tfms=tfms, splits=None)\ndls = dsets.dataloaders(bs=bs, after_batch=batch_tfms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_image(dsets[0][0]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndls.show_batch(max_n=25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Fastai\n# size = (224,224)\n# bs = 64\n\n# tfms = [[PILImage.create, ToTensor, Resize(size, method='squish')], \n#         [lambda o: all_images2label[o], Categorize(label_vocab)]]\n\n# dsets = Datasets(all_images, tfms=tfms, splits=None)\n\n# batch_tfms = [IntToFloatTensor, Normalize.from_stats(*imagenet_stats)]\n# dls = dsets.dataloaders(bs=bs, after_batch=batch_tfms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show_image(dsets[0][0]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# dls.show_batch(max_n=25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = create_cnn_model(resnet34, 1, pretrained=True)\nmodel = nn.Sequential(model[0], model[1][:2])\nlearner = Learner(dls, model, loss_func=CrossEntropyLossFlat)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# generate embeddings\nembedding_dl = dls.test_dl(all_images)\nembeddings, _ = learner.get_preds(dl=embedding_dl, act=noop)\ntorch.save(embeddings, \"embeddings.pth\")\npd.to_pickle(all_images, \"all_images_filenames.pkl\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load\nembeddings = torch.load(\"embeddings.pth\")\nembeddings.shape, len(all_images)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"CPU RAM on this GPU kernel is not enough, so we need to chunk embeddings in to rows and columns. Also it's pretty slow, so let me know down in the comments if you have a better solution for this!\n\nFor demonstration I will look at first 1000 images and to see if there are any dups for them."},{"metadata":{"trusted":true},"cell_type":"code","source":"gpu_kernel = torch.cuda.is_available()\nchunk_idxs = list(chunked(range(len(embeddings)), chunk_sz=1000))\n\nsims = []\nfor i, row_idxs in enumerate(progress_bar(chunk_idxs)):\n    row_sims = []\n    for col_idxs in progress_bar(chunk_idxs):\n        sim = F.cosine_similarity(embeddings[row_idxs].unsqueeze(0), embeddings[col_idxs].unsqueeze(1), dim=-1)    \n        row_sims.append(sim)\n    \n    if gpu_kernel: \n        if i == 2: break\n    \n    row_sims = torch.cat(row_sims, dim=0).T\n    sims.append(row_sims)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sims = torch.cat(sims)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sims.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sims = torch.triu(sims, diagonal=1)\nthresh = 0.95\nsimilar_idxs = [(i,j) for i,j in list(zip(torch.where(sims > thresh)[0].numpy(), torch.where(sims > thresh)[1].numpy())) if i != j]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"We found {len(similar_idxs)}/{len(sims)} similar pairs\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's plot a few random similar pairs"},{"metadata":{"trusted":true},"cell_type":"code","source":"similar_files = []\n\nfor i,j in np.random.permutation(similar_idxs)[:20]:\n    fn1, fn2 = all_images[i], all_images[j]\n    similar_files.append((fn1, fn2))\n    \n    imgs = [open_image(fn1), open_image(fn2)]\n    titles = [all_images2label[fn1], all_images2label[fn2]]\n    titles = [all_images[i], all_images[j]]\n    show_images(imgs, titles=titles, imsize=10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Up next: Rapids CuML clustering!"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}