{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#  Loading libraries","metadata":{}},{"cell_type":"code","source":"import json\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom tabulate import tabulate","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading and cleaning data","metadata":{}},{"cell_type":"code","source":"#loading json file for train data\ndata_path_train = \"../input/herbarium-2021-fgvc8/train/metadata.json\"\nwith open(data_path_train) as json_file:\n    meta_train = json.load(json_file)\n    \n#loading json file for test data\ndata_path_test = \"../input/herbarium-2021-fgvc8/test/metadata.json\"\nwith open(data_path_test) as json_file:\n    meta_test = json.load(json_file)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Finding keys in dictionary\nmeta_train.keys()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating seperate dataframes from metadata\nannotations_train =  pd.json_normalize(meta_train ['annotations'])\ncategories_train =  pd.json_normalize(meta_train ['categories'])\nimages_train =  pd.json_normalize(meta_train ['images'])\ninfo_train =  pd.json_normalize(meta_train ['info'])\nlicenses_train =  pd.json_normalize(meta_train ['licenses'])\ninstitutions_train =  pd.json_normalize(meta_train ['institutions'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now lets check how each dataframe looks like","metadata":{}},{"cell_type":"code","source":"annotations_train.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categories_train.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_train.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"info_train.head(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"licenses_train.head(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"institutions_train.head(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we saw there is not much usefull information in info, licences and institutions","metadata":{}},{"cell_type":"code","source":"#Removing unused dataframe\ndel info_train\ndel licenses_train\ndel institutions_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Finding keys in dictionary\nmeta_test.keys()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating seperate dataframes from metadata\nimages_test=  pd.json_normalize(meta_test ['images'])\ninfo_test =  pd.json_normalize(meta_test ['info'])\nlicenses_test=  pd.json_normalize(meta_test ['licenses'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking what each dataframe looks like\nimages_test.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"info_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"licenses_test.head(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Removing unwanted dataframes\ndel info_test\ndel licenses_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now manipulate dataframes to make a dataset in required format","metadata":{}},{"cell_type":"code","source":"#creating test data\ndf_test = images_test.drop(['height','license','width'], axis=1)\n#renaming id to image_id \ndf_test = df_test.rename(columns={\"id\": \"image_id\"})\n\n# changing order of column\ndf_test = df_test[['image_id','file_name']]\ndf_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Before creating train dataset lets have some checks","metadata":{}},{"cell_type":"code","source":"#checking whether id, image_id in annotaion_train and images_train are same.\nsame = pd.DataFrame(np.where(annotations_train['id'] == annotations_train['image_id'], 'True', 'False'))\nsame.columns = ['an_id_vs_an_im_id']                   \nsame['im_id_vs_an_id'] = np.where(annotations_train['id'] == images_train['id'], 'True', 'False')\n\n# find true or false. true means id and image_id are same\nprint(same.head()) \n\n# find if both true and false is present\nprint('number of unique items in first column',len(same['an_id_vs_an_im_id'].unique()))\nprint('number of unique items in second column',len(same['im_id_vs_an_id'].unique()))\n\ndel same","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After checking whether id and image_id in annotation_train are same, we found that indeed the are same. Only one unique item in both columns and the item is True so id in annotations_train, image_id in annotations_train and id in images_train are same and in same order. Now lets start merging dataframes","metadata":{}},{"cell_type":"code","source":"#merging annotations and images df\ndf_merge = pd.merge(annotations_train[['category_id', 'image_id','id']],images_train[['file_name','id','height','width']] , on='id')\n#removing unwanted columns\ndf_merge=df_merge.drop(['id'], axis=1)\ndf_merge.head()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Before merging df_merge with categories_train, lets have some checks","metadata":{}},{"cell_type":"code","source":"#checking if category_id in df_merge is having same limits as id in categories_train\n\nif df_merge['category_id'].min() == categories_train['id'].min():\n    print('true')\nelse:\n    print('false')\n\nif df_merge['category_id'].max() == categories_train['id'].max():\n    print('true')\nelse:\n    print('false')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the results we see that category_id in df_merge is same as id in categories_train. so we merge those two dataframes","metadata":{}},{"cell_type":"code","source":"#renaming id to category_id \ncategories_train = categories_train.rename(columns={\"id\": \"category_id\"})\n\n#merging label with data. creating train data\ndf_train = pd.merge(df_merge[['image_id','file_name','category_id','height','width']],categories_train[['category_id','name','family','order']] , on='category_id')\n\n#Add a colum containing file fath pointing towards location of images\ndf_train['file_path']=\"../input/herbarium-2021-fgvc8/train/\"+df_train['file_name']\n\ndf_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#removing null values\ndf_train = df_train.dropna(how = 'all')\n#cheking for missing data \ndf_train.isnull().sum()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking for duplicates\ndf_train['file_name'].duplicated().any()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We found that some data in order cloumn is 'unknown' after checking unique values in order column","metadata":{}},{"cell_type":"code","source":"#finding missing data\nlen(df_train.loc[df_train['order'] == 'Unknown'])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dropping missing data\ndf_train = df_train.drop(df_train[df_train.order == 'Unknown'].index)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets create a seperate dataset for image dimention data and then remove them from training dataset","metadata":{}},{"cell_type":"code","source":"dimentions = pd.DataFrame(df_train[[ 'image_id','category_id','height','width']])\ndimentions","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#removing unwanted columns\ndf_train = df_train.drop(['height','width'], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sorting test df on image_id\ndf_test.sort_values(by=['image_id'])\n#sorting train df on image_id\ndf_train.sort_values(by=['image_id'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Removing unused dataframes\ndel annotations_train\ndel categories_train\ndel images_train\ndel images_test\ndel df_merge","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"markdown","source":"Lets check the number of items by each category","metadata":{}},{"cell_type":"code","source":"len(df_train['image_id'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_test['image_id'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_train['category_id'].unique())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_train['family'].unique())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_train['order'].unique())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that there are 2257710 images in the dataset which can be divided into 64488 specices.\nThere are 243020 images in test dataset\nThe name column gives the name of the plant as species+genus.\nThese specices can be further divided into 450 familes and further into 80 orders. \nLets create a table for this.\n","metadata":{}},{"cell_type":"code","source":"#finding name of orders\nn = df_train['order'].unique().tolist()\n\n\n#finding number of families in each order\no_f_n=[]\nfor i in range(len(n)):    \n    o_f_n.append(len(df_train.loc[df_train['order']==n[i],'family' ].unique()))\n\n#finding number of species in each order\no_s_n=[]\nfor i in range(len(n)):    \n    o_s_n.append(len(df_train.loc[df_train['order']==n[i],'category_id' ].unique()))  \n    \n#finding number of images in each order\no_i_n=[]\nfor i in range(len(n)):\n    o_i_n.append(len(df_train.loc[df_train['order']==n[i]]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table_order =pd.DataFrame(df_train['order'].unique(),columns =['order'])\ntable_order['Number_of_families_in_order'] = o_f_n\ntable_order['Number_of_species_in_order'] = o_s_n\ntable_order['Number_of_images_in_order'] = o_i_n\n\ntable_order = table_order.sort_values(by=['Number_of_images_in_order'], ascending=False)\n\nprint(table_order.to_markdown())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating table to classify by sample size\nList = [10,100,150,200,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000]\nsample_size = pd.DataFrame({'Sample_size':['more than 10','more than 100','more than 150','more than 200',\n                                           'more than 250','more than 300','more than 350','more than 400',\n                                           'more than 450','more than 500','more than 550','more than 600',\n                                           'more than 650','more than 700','more than 750','more than 800',\n                                           'more than 850','more than 900','more than 950','more than 1000']})\n\n#finding number of species having particular sample size\ns_n=[]\nfor i in List:\n    more= df_train['category_id'].value_counts() > i\n    s_n.append(len(more.index[more==True]))  \n\n#finding number of families having particular sample size\nf_n=[]\nfor i in List:\n    more= df_train['family'].value_counts() > i\n    f_n.append(len(more.index[more==True]))\n    \n#finding number of orders having particular sample size\no_n=[]\nfor i in List:\n    more= df_train['order'].value_counts() > i\n    o_n.append(len(more.index[more==True]))\n\n    \n\nsample_size['Number_of_species'] = s_n\nsample_size['Number_of_families'] = f_n\nsample_size['Number_of_orders'] = o_n\n\nprint(sample_size.to_markdown())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see from the table, number of images if classified by order is heavily  imbalenced. Only 27 species have sample sizes more than 1000.","metadata":{}},{"cell_type":"markdown","source":"# Visualize","metadata":{}},{"cell_type":"code","source":"#Number of samoles in each order\nplt.figure(figsize=(15, 10))\ndf_train['order'].value_counts().plot.bar()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plotting image by image id for single image\ndef  visualize(image_id):\n    \n    path = df_train.loc[df_train['image_id'] == image_id, 'file_path'].iloc[0]\n    family = df_train.loc[df_train['image_id'] == image_id, 'family'].iloc[0]\n    order = df_train.loc[df_train['image_id'] == image_id, 'order'].iloc[0]\n    name = df_train.loc[df_train['image_id'] == image_id, 'name'].iloc[0]\n    plt.figure(figsize=(10, 10))\n    image = cv2.imread(path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    plt.imshow(image)\n    plt.title(f\"ORDER: {order}FAMILY: {family} \\n NAME:{name}\\n Image_id:{image_id}\", fontsize=10,)\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize(23445)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}