{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Initial Exploratory Data Analysis for Plant Pathology 2021"},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading packages"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport PIL\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining environment parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_ROOT = os.path.join('..', 'input'); DATA_ROOT","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_COMPT = os.path.join(DATA_ROOT, 'plant-pathology-2021-fgvc8'); DATA_COMPT","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_TRAIN_IMAGES = os.path.join(DATA_COMPT, 'train_images'); DATA_TRAIN_IMAGES","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_TEST_IMAGES = os.path.join(DATA_COMPT, 'test_images'); DATA_TEST_IMAGES","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_IMG_STATS = os.path.join(DATA_ROOT, 'plant-pathology-2021-metadata-with-image-stats'); DATA_IMG_STATS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_OUTPUT = './'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Exploring data consistency"},{"metadata":{},"cell_type":"markdown","source":"## 1) Metadata"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(DATA_COMPT, 'train.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sample_submission = pd.read_csv(os.path.join(DATA_COMPT, 'sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sample_submission.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Checking for duplicate records in train metadata. If return > 1 there are duplicates."},{"metadata":{"trusted":true},"cell_type":"code","source":"len(df_train)/len(set(df_train['image']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Checking for data leak between train and sample_submission subsets with respect to 'image'"},{"metadata":{"trusted":true},"cell_type":"code","source":"set(df_train['image']).intersection(set(df_sample_submission['image']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Looking up labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['labels'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2) Metadata & image files consistency"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_image_files = [f for f in os.listdir(DATA_TRAIN_IMAGES) \\\n if os.path.isfile(os.path.join(DATA_TRAIN_IMAGES, f))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_image_files[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image_files = [f for f in os.listdir(DATA_TEST_IMAGES) \\\n if os.path.isfile(os.path.join(DATA_TEST_IMAGES, f))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_image_files)==len(set(train_image_files))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_image_files)==len(df_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"set(train_image_files) - set(df_train['image'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(test_image_files)==len(df_sample_submission)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"set(test_image_files) - set(df_sample_submission['image'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"sample_submission.csv file contains metadata for the subset of test images which host made avalibale"},{"metadata":{},"cell_type":"markdown","source":"## 3) Image data"},{"metadata":{"trusted":true},"cell_type":"code","source":"image = PIL.Image.open(os.path.join(DATA_TRAIN_IMAGES, f\"{df_train.sample()['image'].values[0]}\")); image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.array(image).shape # (height, width, channels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.array(image).min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.array(image).max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_image_stats(df:pd.DataFrame, path:str, image_id_col, suffix:str='')->pd.DataFrame:\n    \n    image_stats = {image_id_col:[], 'height':[], 'width':[], 'channels':[], 'pixl_mean':[], 'pixl_std':[]}\n    \n    for image_id in tqdm(df[image_id_col]):\n        if suffix=='': image = PIL.Image.open( os.path.join(path, f'{image_id}') )\n        else: image = PIL.Image.open( os.path.join(path, f'{image_id}.{suffix}') )\n        \n        image = np.array(image)\n        image_shape = image.shape\n        image_stats[image_id_col].append(image_id)\n        \n        if len(image_shape)==3: image_stats['channels'].append(image_shape[2])\n        else: image_stats['channels'].append(1)\n        \n        image_stats['height'].append(image_shape[0])\n        image_stats['width'].append(image_shape[1])\n        image_stats['pixl_mean'].append(image.mean())\n        image_stats['pixl_std'].append(image.std())\n    \n    return df.merge(right=pd.DataFrame(data=image_stats), how='inner', on=image_id_col)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    df_train = pd.read_csv(os.path.join(DATA_IMG_STATS, 'train_stats.csv'))\nexcept:\n    df_train = get_image_stats(df=df_train, path=DATA_TRAIN_IMAGES, image_id_col='image')\n    df_train.to_csv(os.path.join(DATA_COMPT, 'train_stats.csv'), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    df_sample_submission = pd.read_csv(os.path.join(DATA_IMG_STATS, 'sample_submission_stats.csv'))\nexcept:\n    df_sample_submission = get_image_stats(df=df_sample_submission, path=DATA_TEST_IMAGES, image_id_col='image')\n    df_sample_submission.to_csv(os.path.join(DATA_OUTPUT, 'sample_submission_stats.csv'), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sample_submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['channels'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['height'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['width'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['height2width'] = df_train['height']/df_train['width']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['height2width'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['height2width'].max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train[df_train['height2width']>1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.loc[df_train['height2width']<=1, 'height2width'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.loc[df_train['height2width']<=1, 'height2width'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"for imgFileName in df_train.loc[df_train['height2width']>1, 'image']:\n    im = PIL.Image.open(os.path.join(DATA_TRAIN_IMAGES, f\"{imgFileName}\"))\n    \n    fig, ax = plt.subplots(figsize=(10,10))\n    ax.imshow(im)\n    ax.title.set_text(imgFileName)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Nothing wrong or special about these images, they were simpy rotated, so that height and width switched around. It is good to keep it mind when building pre-processing steps in pipeline as similarly rotated images can be in the test set. One might consider adding a step into pipeline which checks for rotated images and deals with them accordingly."},{"metadata":{},"cell_type":"markdown","source":"### Let's look for duplicate images via image statistics"},{"metadata":{},"cell_type":"markdown","source":"**Method Description:**\n\nIn case of exact and trivial duplicate images (pixel to pixel same images) it, in principal, should be possible to find duplicates via image statistics.\n\n\nFirst, calculate for each image mean and std over its pixel values.\n\n\nSecond, detect possible suspects for dublicate images via selecting groups of images with exactly the same pairs of values of mean and std within each group.\n\n\nThird, visually inspect detected suspects on whether they are duplicates."},{"metadata":{"trusted":true},"cell_type":"code","source":"subset = ['pixl_mean','pixl_std']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_mask_duplicates = df_train.duplicated(subset=subset, keep=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_mask_duplicates.sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_mask_duplicates = df_sample_submission.duplicated(subset=subset, keep=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_mask_duplicates.sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_test_outer_mask_duplicates =\\\npd.concat([df_train[~train_mask_duplicates],df_sample_submission[~test_mask_duplicates]]).reset_index(drop=True).duplicated(subset=subset, keep=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_test_outer_mask_duplicates.sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Looks like there are duplicate suspects in train data (54 images) but there is no duplicate suspect images between train and test sets nor within the small sample of the test data alone."},{"metadata":{},"cell_type":"markdown","source":"Let's visually inspect suspects for duplicate images:"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_dupl_train =\\\ndf_train.loc[train_mask_duplicates, ['image']+subset].groupby(by=subset)['image'].apply(list).reset_index()\n\ndf_dupl_train['num_dupls'] = df_dupl_train['image'].apply(lambda x: len(x)).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_dupl_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = 'labels'\nimages_target_inconsistent = []\nfor i in range(len(df_dupl_train)):\n    image_id1 = df_dupl_train.loc[i, 'image'][0]\n    image_id2 = df_dupl_train.loc[i, 'image'][1]\n    \n    image1 = PIL.Image.open( os.path.join(DATA_TRAIN_IMAGES, f'{image_id1}') )\n    image1 = np.array(image1)\n    image2 = PIL.Image.open( os.path.join(DATA_TRAIN_IMAGES, f'{image_id2}') )\n    image2 = np.array(image2)\n    \n    fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(15,15))\n    ax1.imshow(image1)\n    ax2.imshow(image2)\n    \n    l1 = df_train.loc[df_train['image']==image_id1, target].tolist()[0].split()\n    l2 = df_train.loc[df_train['image']==image_id2, target].tolist()[0].split()\n    \n    ax1.title.set_text(f'Image:{image_id1}\\nLabel:{l1}')\n    ax2.title.set_text(f'Image:{image_id2}\\nLabel:{l2}')\n    plt.show()\n    \n    if set(l1)!=set(l2): images_target_inconsistent.extend([image_id1,image_id2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(images_target_inconsistent)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Visual inspection confirmed that the suspcts were indeed duplicate images. There are total of 54 duplicates with groups of 2 of the same images thus, resulting in 27 unique pairs of duplicates.\n\nThe other issue with these duplicates is that their respected labels are all different for each pair, so there is target inconsistency among duplicate images. This issue might not be only present for this subset of images but also for the rest of the train and test data sets, meaning that we might have noisy label in this data!"},{"metadata":{},"cell_type":"markdown","source":"I will drop all dublicates because there is issue with their labels."},{"metadata":{"trusted":true},"cell_type":"code","source":"len(df_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df_train[~df_train['image'].isin(images_target_inconsistent)].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(df_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Exploring image stats more in depth"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_dist_stats(df, col, **kwargs):\n    mn  = round(df[col].min(), 2)\n    mx  = round(df[col].max(), 2)\n    avg = round(df[col].mean(), 2)\n    std = round(df[col].std(), 2)\n\n    df[col].hist(label=f'min, max = ({mn}, {mx})\\navg, std = ({avg}, {std})', **kwargs)\n    plt.legend()\n    plt.title(col)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_dist_stats(df=df_train, col='pixl_mean')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_dist_stats(df=df_train, col='pixl_std')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# least bright image:\nPIL.Image.open( os.path.join(DATA_TRAIN_IMAGES, f\"{df_train.loc[df_train['pixl_mean']==df_train['pixl_mean'].min(), 'image'].tolist()[0]}\") )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# most bright image:\nPIL.Image.open( os.path.join(DATA_TRAIN_IMAGES, f\"{df_train.loc[df_train['pixl_mean']==df_train['pixl_mean'].max(), 'image'].tolist()[0]}\") )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# least contrast image:\nPIL.Image.open( os.path.join(DATA_TRAIN_IMAGES, f\"{df_train.loc[df_train['pixl_std']==df_train['pixl_std'].min(), 'image'].tolist()[0]}\") )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# most contrast image:\nPIL.Image.open( os.path.join(DATA_TRAIN_IMAGES, f\"{df_train.loc[df_train['pixl_std']==df_train['pixl_std'].max(), 'image'].tolist()[0]}\") )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Exploring label"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_dist_bar(df, col, **kwargs):\n    ds = df[col].value_counts()\n    \n    height = ds.values\n    xticks = list(ds.index)\n    x = np.arange(len(xticks))\n    \n    plt.bar(x=x, height=height)\n    plt.xticks(x, xticks, **kwargs)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['labels'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_dist_bar(df=df_train, col='labels', rotation=90)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It is a multi-label classification problem i.e., single image can have a label comprised of a few classes at the same time."},{"metadata":{},"cell_type":"markdown","source":"### Unique classes"},{"metadata":{},"cell_type":"markdown","source":"Are **'cider_apple_rust'** and **'rust'** just two labels for the same class? Depending on the answer there are 7 or 6 unique classes."},{"metadata":{"trusted":true},"cell_type":"code","source":"clas_counts = df_train['labels'].value_counts().to_dict()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"uclasses =\\\npd.DataFrame([(clas,clas_counts[label]) for label in clas_counts \n              for clas in label.split()]).groupby(by=0).sum().reset_index().rename(columns={0:'class',\n                                                                                            1:'counts'}).sort_values(by='counts',\n                                                                                                                     ascending=False).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"uclasses","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"uclasses['class_single_rust'] = uclasses['class']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"uclasses.loc[uclasses['class']=='cider_apple_rust', 'class_single_rust'] = 'rust'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"uclasses_single_rust =\\\nuclasses.groupby(by='class_single_rust').sum().sort_values(by='counts', ascending=False).reset_index().copy()\nuclasses = uclasses.drop(labels=['class_single_rust'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"uclasses_single_rust","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rot=60\n\nheight1 = uclasses['counts']\nxticks1 = uclasses['class'].tolist()\nx1 = np.arange(len(xticks1))\n\nheight2 = uclasses_single_rust['counts']\nxticks2 = uclasses_single_rust['class_single_rust'].tolist()\nx2 = np.arange(len(xticks2))\n\nfig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(15,5))\n\nax1.bar(x=x1, height=height1)\nax1.set_xticks(x1)\nax1.set_xticklabels(xticks1, rotation=rot)\nax1.title.set_text(f'Unique class counts')\n\nax2.bar(x=x2, height=height2)\nax2.set_xticks(x2)\nax2.set_xticklabels(xticks2, rotation=rot)\nax2.title.set_text(f'Unique class single rust counts')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}