{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\n#sys function provides the name of the existing python modules which have been imported.\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n#The OS module in Python provides functions for creating and removing a directory (folder), fetching its contents, changing and identifying the current directory, etc.\nfrom pathlib import Path\n#Pathlib module in Python provides various classes representing file system paths with semantics appropriate for different operating systems. \nimport glob\n#the glob module is used to retrieve files/pathnames matching a specified pattern.\nfrom tqdm import tqdm\n#tqdm is used to display bar library with good support for nested loops and Jupyter/IPython notebooks.\ntqdm.pandas()\n#when tqdm.pandas() is used then you can simply replace all your .apply() functions with .progress_apply() it’s really that simple!\nimport json\n#for reading json file and for mapping operation.\nimport pandas as pd\n#for reading the reading the data of csv file and converting into pandas dataframe\nimport numpy as np\n#numpy array used for scientific computational purpose.\n\nimport imagehash\n#using different algorithm of imagehash to detect two particular images are similar or not.\n\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n#using seaborn and matplotlib for visulization and different plotting purpose.\nimport cv2\n#Cv2 is computervison library for reading the images.\nfrom PIL import Image\n#PIL is python imaging library for reading the images.\n\n\nimport albumentations\n#albumentations for augmentation purpose\nfrom albumentations.pytorch.transforms import ToTensorV2\n#ToTensorV2 convert image and mask to torch.Tensor\n\nimport torch\nimport timm\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset,DataLoader\n\nfrom keras.preprocessing.image import load_img\n#load the image in PIL format.\nfrom keras.preprocessing.image import img_to_array \n#Converts a PIL Image instance to a Numpy array.\nfrom keras.applications.resnet50 import preprocess_input \n#Preprocesses a tensor or Numpy array encoding a batch of images.(RGB---->BGR && floating point--->>float32)\n\n\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.models import Model\n\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score\n\n\n\n\n\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_dir=Path(\"../input/cassava-leaf-disease-classification\")\nBase_dir=\"../input/cassava-leaf-disease-classification\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df=pd.read_csv(base_dir/\"train.csv\")\ndf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(base_dir/\"label_num_to_disease_map.json\") as f:\n    mapping=json.loads(f.read())\n    mapping={int(k):v for k,v in mapping.items()}\nmapping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"diseases_label\"]=df[\"label\"].map(mapping)\ndf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_images(label,diseases_label,count,mode=0):\n    plot_list = df[df[\"label\"] == label].sample(count)['image_id'].tolist()\n    \n    # Printing list of images\n    if mode:\n        print(plot_list)\n        \n    labels = [diseases_label for i in range(len(plot_list))]\n    size = np.sqrt(count)\n    if int(size)*int(size) < count:\n        size = int(size) + 1\n        \n    plt.figure(figsize=(20, 20))\n    \n    for idx, (image_id, label) in enumerate(zip(plot_list, labels)):\n        plt.subplot(size, size, idx + 1)\n        image = cv2.imread(str(base_dir/\"train_images\"/image_id))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        plt.imshow(image)\n        plt.title(label, fontsize=12)\n        plt.axis(\"off\")\n    \n    plt.show()\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_images(0,\"Cassava Bacterial Blight (CBB)\",9,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_images(1,\"Cassava Brown Streak Disease (CBSD)\",9,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_images(2,\"Cassava Green Mottle (CGM)\",9,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_images(3,\"Cassava Mosaic Disease (CMD)\",9,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"NUMBER OF CMD IMAGES IN THE TRAINING DATASET:\")\ndf[df[\"label\"]==3][\"image_id\"].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_images(4,\"Healthy\",9,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"NUMBER OF HEAlTHY IMAGES IN THE TRAINING DATASET:\")\ndf[df[\"label\"]==4][\"image_id\"].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,10))\nsns.countplot(y='diseases_label',data = df)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_features(image_id, model):\n    file = str(base_dir/'train_images'/image_id)\n    # load the image as an instance of PIL\n    img = load_img(file, target_size=(224,224))\n    # convert from 'PIL.Image.Image' to numpy array\n    img = np.array(img) \n    # reshape the data for the model reshape(num_of_samples, dim 1, dim 2, channels)\n    reshaped_img = img.reshape(1,224,224,3) \n    # prepare image for model\n    imgx = preprocess_input(reshaped_img)\n    # get the feature vector\n    features = model.predict(imgx, use_multiprocessing=True)\n    \n    return features\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_model = ResNet50()\n# create the base pre-trained model\n\nmodel = Model(inputs = base_model.inputs, outputs = base_model.layers[-2].output)\n# this is the model we will train\n\n\nhealthy = df[df['label']==4]\nhealthy['features'] = healthy['image_id'].progress_apply(lambda x:extract_features(x,model))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.array(healthy['features'].values.tolist()).reshape(-1,2048)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.array(healthy[\"image_id\"].values.tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features=np.array(healthy['features'].values.tolist()).reshape(-1,2048)\nimage_ids=np.array(healthy[\"image_id\"].values.tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kmeans = KMeans(n_clusters=5,n_jobs=-1, random_state=22)\nkmeans.fit(features)\nkmeans.labels_\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group1 = {}\n\nfor file, cluster in zip(image_ids,kmeans.labels_):\n    if cluster not in group1.keys():\n        group1[cluster] = []\n        group1[cluster].append(file)\n    else:\n        group1[cluster].append(file)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group1\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def view_cluster(cluster):\n    plt.figure(figsize=(25,25))\n    files=group1[cluster]\n    if len(files)>25:\n        print(f\"The number of sample images is reduced from {len(files)} to 25\")\n        rand=np.random.randint(0,len(files))\n        files=files[rand:rand+25]\n        for index,file in enumerate(files):\n            plt.subplot(5,5,index+1)\n            image=cv2.imread(Base_dir+\"/train_images/\"+file)\n            image=cv2.cvtColor(image,cv2.COLOR_BGR2RGB)\n            plt.imshow(image)\n            plt.axis(\"off\")\n            plt.title(file)\n        plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_cluster(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_cluster(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_cluster(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_cluster(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_cluster(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cbsd= df[df['label']==1]\ncbsd['features'] = cbsd['image_id'].progress_apply(lambda x:extract_features(x,model))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cbsd_features=np.array(cbsd['features'].values.tolist()).reshape(-1,2048)\ncbsd_image_ids=np.array(cbsd[\"image_id\"].values.tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cbsd_kmeans = KMeans(n_clusters=5,n_jobs=-1, random_state=22)\ncbsd_kmeans.fit(cbsd_features)\ncbsd_kmeans.labels_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group2 = {}\n\nfor file, cluster in zip(cbsd_image_ids,cbsd_kmeans.labels_):\n    if cluster not in group2.keys():\n        group2[cluster] = []\n        group2[cluster].append(file)\n    else:\n        group2[cluster].append(file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def view_cluster(cluster):\n    plt.figure(figsize=(25,25))\n    files=group2[cluster]\n    if len(files)>25:\n        print(f\"The number of sample images is reduced from {len(files)} to 25\")\n        rand=np.random.randint(0,len(files))\n        files=files[rand:rand+25]\n        for index,file in enumerate(files):\n            plt.subplot(5,5,index+1)\n            image=cv2.imread(Base_dir+\"/train_images/\"+file)\n            image=cv2.cvtColor(image,cv2.COLOR_BGR2RGB)\n            plt.imshow(image)\n            plt.axis(\"off\")\n            plt.title(file)\n        plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_cluster(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_cluster(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_cluster(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_cluster(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"view_cluster(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_image_path=[os.path.join(Base_dir+\"/train_images/\",k)for k in df.image_id.values]\ntrain_targets=df.label.values\n\nprint(len(train_image_path))\nprint(len(train_targets))\ntrain_image_path\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision.transforms import transforms\nclass CassavaDataset(Dataset):\n    def __init__(self,image_ids,labels,dimension=None,augmentations=None):\n        super().__init__()\n        self.image_ids=image_ids\n        self.labels=labels\n        self.dim=dimension\n        self.augmentations=augmentations\n        \n    def __len__(self):\n        return len(self.image_ids)\n    \n    def __getitem__(self,idx):\n        imge=cv2.imread(self.image_ids[idx])\n        imge=cv2.cvtColor(imge,cv2.COLOR_BGR2RGB)\n        \n        if self.dim:\n            imge=cv2.resize(imge,self.dim)\n            \n        if self.augmentations:\n            aug_img=self.augmentations(image=imge)\n            imge=aug_img[\"image\"]\n            \n        return {\"image\":transforms.ToTensor()(imge),\n                \"label\":torch.tensor(self.labels[idx])}    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##---------","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}