{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"'''\n    refer to https://www.kaggle.com/isaienkov/cassava-leaf-disease-classification-data-analysis , https://www.kaggle.com/tanulsingh077/how-to-become-leaf-doctor-with-deep-learning, https://www.kaggle.com/jirkaborovec/plant-pathology-data-exploration and https://www.kaggle.com/nickuzmenkov/pp2021-duplicates-revealing\n'''","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n# Preliminaries\nimport os\nfrom pathlib import Path\nimport glob\nfrom tqdm import tqdm\ntqdm.pandas()\nimport json\nimport pandas as pd\nimport numpy as np\n\n## Image hash\nimport imagehash\n\n# Visuals and CV2\nimport seaborn as sn\nimport matplotlib.pyplot as plt\nimport cv2\nfrom PIL import Image\n\n\n# albumentations for augs\nimport albumentations\nfrom albumentations.pytorch.transforms import ToTensorV2\n\n# clustering and dimension reduction\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score\n\n# # Keras and TensorFlow\nfrom keras.preprocessing.image import load_img\n# from keras.preprocessing.image import img_to_array \nfrom keras.applications.resnet50 import preprocess_input \n# # from keras.applications.resnet18 import preprocess_input \n\n\n# models \nfrom keras.applications.resnet50 import ResNet50\n# from keras.applications.resnet18 import ResNet18\nfrom keras.models import Model\n\n#torch\nimport torch\nimport timm\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset,DataLoader\n\nfrom pprint import pprint","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = '../input/plant-pathology-2021-fgvc8'\ntrain = pd.read_csv(os.path.join(BASE_DIR, 'train.csv'))\n\n\nlabels_list = list(set(train.labels))\nlabels_list.sort()\nmapping = {label:i for i, label in enumerate(labels_list)}\n# print(labels_list, '\\n',mapping)\n\ntrain['labels_id'] = train['labels'].map(mapping)\ntrain","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Learn the distribution.","metadata":{}},{"cell_type":"code","source":"train['nb_classes'] = [len(lbs.split(\" \")) for lbs in train['labels']]\nlb_hist = dict(zip(range(10), np.bincount(train['nb_classes'])))\npprint(lb_hist)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import itertools\nimport seaborn as sns\n\n# import pdb;pdb.set_trace()\nlabels_all = list(itertools.chain(*[lbs.split(\" \") for lbs in train['labels']]))\n\nax = sns.countplot(y=sorted(labels_all), orient='v')\nax.grid()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['labels_sorted'] = [\" \".join(sorted(lbs.split(\" \"))) for lbs in train['labels']]\nnb_samples = 6\nn, m = len(np.unique(train['labels_sorted'])), nb_samples,\nfig, axarr = plt.subplots(nrows=n, ncols=m, figsize=(m * 2, n * 2))\nfor ilb, (lb, df_) in enumerate(train.groupby('labels_sorted')):\n    img_names = list(df_['image'])\n    for i in range(m):\n        img_name = img_names[i]\n        img = plt.imread(os.path.join(BASE_DIR, f\"train_images/{img_name}\"))\n        axarr[ilb, i].imshow(img)\n        if i == 1:\n            axarr[ilb, i].set_title(f\"{lb} #{len(df_)}\")\n        axarr[ilb, i].set_xticks([])\n        axarr[ilb, i].set_yticks([])\nplt.axis('off')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_labels = []\nfor label in labels_list:\n    num_labels.append(train[train['labels']==label].count().labels)\nfor i, label in enumerate(labels_list):\n    print(f'{mapping[label]} {label} : {num_labels[i]}')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Calculate the distribution of the image size.It may take more than half an hour.**","metadata":{}},{"cell_type":"code","source":"check_dict = dict()\n\nfor filename in tqdm(os.listdir('/kaggle/input/plant-pathology-2021-fgvc8/train_images/')):\n#     import pdb;pdb.set_trace()\n    img = cv2.imread('/kaggle/input/plant-pathology-2021-fgvc8/train_images/' + filename)\n    try:\n        check_dict[img.shape] += 1\n    except:\n        check_dict[img.shape] = 1\ncheck_dict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"for multi-class","metadata":{}},{"cell_type":"code","source":"# train['labels_id'].hist(grid=False, bins=2*len(labels_list), alpha=0.5);\n# cal_train = pd.Series(num_labels,index=labels_list)\n# cal_train.plot(grid=True, kind='barh',alpha=0.5)\n# # cal_train.plot(grid=True, kind='pie')\n# plt.show()\n\n# train.labels.value_counts().plot(kind='bar', figsize=(16,6))\n\n# sn.distplot(train['labels_id'], kde=False)\ntarget_cts=train.labels.value_counts()\nfig = plt.figure(figsize=(12,6))\nsn.barplot(y=target_cts.sort_values(ascending=False).index, x=target_cts.sort_values(ascending=False).values, palette='winter')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_images(class_id, label, images_number, verbose=0, square_flag = False):\n   \n    plot_list = train[train[\"labels_id\"] == class_id].sample(images_number)['image'].tolist()\n    \n    if verbose:\n        print(plot_list)\n        \n    labels = [label for i in range(len(plot_list))]\n    size = np.sqrt(images_number)\n    if int(size)*int(size) < images_number:\n        size = int(size) + 1\n        \n    plt.figure(figsize=(20, 20))\n    \n    for ind, (image_id, label) in enumerate(zip(plot_list, labels)):\n        if square_flag:\n            plt.subplot(size, size, ind + 1)\n        else:\n            plt.subplot(1, images_number, ind + 1)\n        image = cv2.imread(os.path.join(BASE_DIR, 'train_images', image_id))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        plt.imshow(image)\n        plt.title(label, fontsize=12)\n        plt.axis(\"off\")\n    \n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's view some samples of every class.","metadata":{}},{"cell_type":"code","source":"for i in range(6):\n    plot_images(class_id=i,label=labels_list[i],images_number=4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(6,12):\n    plot_images(class_id=i,label=labels_list[i],images_number=4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***To Tell if there is any noise within every class.***","metadata":{}},{"cell_type":"code","source":"def extract_features(image_id, model):\n    file = os.path.join(BASE_DIR, 'train_images', image_id)\n    # load the image as a 224x224 array\n    img = load_img(file, target_size=(224,224))\n    # convert from 'PIL.Image.Image' to numpy array\n    img = np.array(img) \n    # reshape the data for the model reshape(num_of_samples, dim 1, dim 2, channels)\n    reshaped_img = img.reshape(1,224,224,3) \n    # prepare image for model\n    imgx = preprocess_input(reshaped_img)\n    # get the feature vector\n    features = model.predict(imgx, use_multiprocessing=True)\n    \n    return features","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ResNet50()\nmodel = Model(inputs=model.inputs, outputs=model.layers[-2].output)\nrust_complex = train[train['labels_id']==7]\nrust_complex['features'] = rust_complex['image'].progress_apply(lambda x:extract_features(x,model))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = np.array(rust_complex['features'].values.tolist()).reshape(-1,2048)\nimage_ids = np.array(rust_complex['image'].values.tolist())\n\n# Clustering\nkmeans = KMeans(n_clusters=2,n_jobs=-1, random_state=22)\nkmeans.fit(features)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"groups = {}\nfor file, cluster in zip(image_ids,kmeans.labels_):\n    if cluster not in groups.keys():\n        groups[cluster] = []\n        groups[cluster].append(file)\n    else:\n        groups[cluster].append(file)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def view_cluster(cluster):\n    plt.figure(figsize = (25,25));\n    # gets the list of filenames for a cluster\n    files = groups[cluster]\n    # only allow up to 30 images to be shown at a time\n    if len(files) > 30:\n        print(f\"Clipping cluster size from {len(files)} to 25\")\n        start = np.random.randint(0,len(files))\n        files = files[start:start+25]\n    # plot each image in the cluster\n    for index, file in enumerate(files):\n        plt.subplot(5,5,index+1);\n        img = load_img(os.path.join(BASE_DIR, 'train_images', file))\n        img = np.array(img)\n        plt.imshow(img)\n        plt.title(file)\n        plt.axis('off')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"view_cluster(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"view_cluster(0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* We can find that there exists difference within the same class, although they are not noisy images maybe.\n* Next we will explore the possibility of duplicate images in the dataset.","metadata":{}},{"cell_type":"markdown","source":"Resize the images to speed the latter computation.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nroot = '/kaggle/input/plant-pathology-2021-fgvc8/train_images'\npaths = os.listdir(root)\n\ndf = pd.read_csv('/kaggle/input/plant-pathology-2021-fgvc8/train.csv', index_col='image')\n\nfor path in tqdm(paths, total=len(paths)):\n    image = tf.io.read_file(os.path.join(root, path))\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.image.resize(image, [512, 512])\n    image = tf.cast(image, tf.uint8).numpy()\n    plt.imsave(path, image)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"funcs = [\n        imagehash.average_hash,\n        imagehash.phash,\n        imagehash.dhash,\n        imagehash.whash,\n    ]\n\nimage_ids = []\nhashes = []\n\nfor path in tqdm(glob.glob('./*.jpg' )):\n    image = Image.open(path)\n    image_id = os.path.basename(path)\n    image_ids.append(image_id)\n    hashes.append(np.array([f(image).hash for f in funcs]).reshape(256))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hashes_all = np.array(hashes)\nhashes_all = torch.Tensor(hashes_all.astype(int)).cuda()\n\n%time sims = np.array([(hashes_all[i] == hashes_all).sum(dim=1).cpu().numpy()/256 for i in range(hashes_all.shape[0])])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices1 = np.where(sims > 0.9)\nindices2 = np.where(indices1[0] != indices1[1])\nimage_ids1 = [image_ids[i] for i in indices1[0][indices2]]\nimage_ids2 = [image_ids[i] for i in indices1[1][indices2]]\n\ndups = {tuple(sorted([image_ids1,image_ids2])):True for image_ids1, image_ids2 in zip(image_ids1, image_ids2)}\nprint('found %d duplicates' % len(dups))\nfor row in dups:\n    print(','.join(row))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicate_image_ids = sorted(list(dups))\n\nfig, axs = plt.subplots(2, 2, figsize=(15,15))\n\nfor row in range(2):\n        for col in range(2):\n            img_id = duplicate_image_ids[row][col]\n            img = Image.open(os.path.join(BASE_DIR,'train_images',img_id))\n            label =str(train.loc[train['image'] == img_id].labels.values[0])\n            axs[row, col].imshow(img)\n            axs[row, col].set_title(\"image_id : \"+ img_id + \"  label : \" + label)\n            axs[row, col].axis('off')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for row in dups:\n    \n    figure, axes = plt.subplots(1, len(row), figsize=[5 * len(row), 5])\n\n    for i, image_id in enumerate(row):\n        image = plt.imread(image_id)\n        axes[i].imshow(image)\n\n        axes[i].set_title(df.loc[image_id, 'labels'])\n        axes[i].axis('off')\n\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}