{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":33679,"databundleVersionId":3212216,"sourceType":"competition"}],"dockerImageVersionId":30162,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport glob\nimport json\nimport os\nimport seaborn as sns\nimport cv2\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-23T07:22:55.784021Z","iopub.execute_input":"2022-02-23T07:22:55.784349Z","iopub.status.idle":"2022-02-23T07:22:56.130006Z","shell.execute_reply.started":"2022-02-23T07:22:55.784313Z","shell.execute_reply":"2022-02-23T07:22:56.129301Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<!-- The training set images are organized in subfolders **h22-train/images/subfolder1/subfolder2/image_id.jpg**, where subfolder1 and subfolder2 comes from the **first three and the last two digits of the image_id**. **Image_id is a result of combination between category_id and unique numbers that differentiates images within plant taxa.** -->","metadata":{}},{"cell_type":"code","source":"INPUT_BASE_FILES = glob.glob('../input/herbarium-2022-fgvc9/*')\n\ntrain_metadata_json = INPUT_BASE_FILES[0]\nsample_submission_csv = INPUT_BASE_FILES[1]\ntest_metadata_json = INPUT_BASE_FILES[2]\ntrain_images_folder = INPUT_BASE_FILES[3]\ntest_images_folder = INPUT_BASE_FILES[4]\n","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:22:12.396537Z","iopub.execute_input":"2022-02-23T07:22:12.396741Z","iopub.status.idle":"2022-02-23T07:22:12.402393Z","shell.execute_reply.started":"2022-02-23T07:22:12.396716Z","shell.execute_reply":"2022-02-23T07:22:12.401457Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open(train_metadata_json) as json_file:\n    train_metadata = json.load(json_file)\n    \nwith open(test_metadata_json) as json_file:\n    test_metadata = json.load(json_file)","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:22:12.403725Z","iopub.execute_input":"2022-02-23T07:22:12.404199Z","iopub.status.idle":"2022-02-23T07:22:38.758123Z","shell.execute_reply.started":"2022-02-23T07:22:12.404161Z","shell.execute_reply":"2022-02-23T07:22:38.756454Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_metadata.keys()) # A dictionary","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:22:38.760842Z","iopub.execute_input":"2022-02-23T07:22:38.761797Z","iopub.status.idle":"2022-02-23T07:22:38.769503Z","shell.execute_reply.started":"2022-02-23T07:22:38.76176Z","shell.execute_reply":"2022-02-23T07:22:38.768522Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(test_metadata[:2]) # A list","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:22:38.771967Z","iopub.execute_input":"2022-02-23T07:22:38.772489Z","iopub.status.idle":"2022-02-23T07:22:38.780228Z","shell.execute_reply.started":"2022-02-23T07:22:38.772442Z","shell.execute_reply":"2022-02-23T07:22:38.779647Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for k,v in train_metadata.items():\n    print(f'| Key : {k}   >>  Total values  : {len(v)} ')","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:22:38.781568Z","iopub.execute_input":"2022-02-23T07:22:38.781911Z","iopub.status.idle":"2022-02-23T07:22:38.795249Z","shell.execute_reply.started":"2022-02-23T07:22:38.781882Z","shell.execute_reply":"2022-02-23T07:22:38.794172Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gen = train_metadata.get('genera')\ngenera_dict = {}\nfor i in gen:\n    genera_dict[i.get('genus_id')] = i.get('genus')","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:34:14.243809Z","iopub.execute_input":"2022-02-23T07:34:14.24426Z","iopub.status.idle":"2022-02-23T07:34:14.250401Z","shell.execute_reply.started":"2022-02-23T07:34:14.244223Z","shell.execute_reply":"2022-02-23T07:34:14.249516Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Sample Values of each keys .. \\n')\n\nprint('[+] Images ---\\n')\nprint(train_metadata.get('images')[0])\nprint('\\n')\nprint('[+] Annotations ---\\n')\nprint(train_metadata.get('annotations')[0])\nprint('\\n')\nprint('[+] Categories ---\\n')\nprint(train_metadata.get('categories')[0])\nprint('\\n')\nprint('[+] Genera --- \\n ')\nprint(train_metadata.get('genera')[0])\nprint('\\n')\nprint('[+] Distances --- \\n ')\nprint(train_metadata.get('distances')[0])\nprint('\\n')\nprint('[+] Institutions --- \\n ')\nprint(train_metadata.get('institutions')[0])\nprint('\\n')\nprint('[+] License --- \\n ')\nprint(train_metadata.get('license')[0])","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:22:38.796907Z","iopub.execute_input":"2022-02-23T07:22:38.797625Z","iopub.status.idle":"2022-02-23T07:22:38.813786Z","shell.execute_reply.started":"2022-02-23T07:22:38.797578Z","shell.execute_reply":"2022-02-23T07:22:38.812994Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Image information\nfile_names = []\nimage_ids = []\ngenus_ids = []\ngenus_names = []\ncategory_ids = []\ninstitution_ids = []\nimage_paths = []\n\nfor i,j in zip(train_metadata.get('images'),train_metadata.get('annotations')):\n    \n    image_id_im = i.get('image_id')\n    image_id_anno = j.get('image_id')\n    \n    if image_id_im == image_id_anno:\n        file_name = i.get('file_name')\n        genus_id = j.get('genus_id')\n        category_id = j.get('category_id')\n        institution_id = j.get('institution_id')\n        \n        \n        file_names.append(file_name)\n        image_ids.append(image_id_anno)\n        genus_ids.append(genus_id)\n        genus_names.append(genera_dict.get(genus_id))\n        category_ids.append(category_id)\n        institution_ids.append(institution_id)\n        image_paths.append(os.path.join(train_images_folder,file_name))","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:36:10.750358Z","iopub.execute_input":"2022-02-23T07:36:10.750697Z","iopub.status.idle":"2022-02-23T07:36:14.150307Z","shell.execute_reply.started":"2022-02-23T07:36:10.750663Z","shell.execute_reply":"2022-02-23T07:36:14.149409Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"training_images_df = pd.DataFrame.from_dict({'FileNames' : file_names, 'ImageID' : image_ids, 'GenusID' : genus_ids,'GenusNames':genus_names,\n                                             'CategoryID' : category_ids,'InstitutionID' : institution_ids,'ImagePath':image_paths})","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:36:51.027128Z","iopub.execute_input":"2022-02-23T07:36:51.027411Z","iopub.status.idle":"2022-02-23T07:36:52.469521Z","shell.execute_reply.started":"2022-02-23T07:36:51.027378Z","shell.execute_reply":"2022-02-23T07:36:52.468537Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"training_images_df.sample(5)","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:36:53.557049Z","iopub.execute_input":"2022-02-23T07:36:53.557445Z","iopub.status.idle":"2022-02-23T07:36:53.595615Z","shell.execute_reply.started":"2022-02-23T07:36:53.557414Z","shell.execute_reply":"2022-02-23T07:36:53.594789Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Briet data information","metadata":{}},{"cell_type":"code","source":"print('Genus ID information')\nid,count = np.unique(genus_ids,return_counts=True)\ngenus_count_df = pd.DataFrame.from_dict({'Genus ID' : id,'Count' : count}).sort_values(by=['Count'],ascending=False)\ngenus_count_df['Count'].hist(bins=100, figsize=(18, 6), grid=True)\nplt.title('Histogram of Genus ID counts')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:22:43.545958Z","iopub.execute_input":"2022-02-23T07:22:43.546174Z","iopub.status.idle":"2022-02-23T07:22:44.20859Z","shell.execute_reply.started":"2022-02-23T07:22:43.546148Z","shell.execute_reply":"2022-02-23T07:22:44.207663Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Category ID Information') \nid,count = np.unique(category_ids,return_counts=True)\ncategory_id_df = pd.DataFrame.from_dict({'Category ID' : id,'Count' : count}).sort_values(by=['Count'],ascending=False)\ncategory_id_df['Count'].hist(bins=100, figsize=(18, 6), grid=True)\nplt.title('Histogram of Category ID counts')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:22:44.209648Z","iopub.execute_input":"2022-02-23T07:22:44.20988Z","iopub.status.idle":"2022-02-23T07:22:44.69568Z","shell.execute_reply.started":"2022-02-23T07:22:44.209845Z","shell.execute_reply":"2022-02-23T07:22:44.694631Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Institution ID information')\nid,count = np.unique(institution_ids,return_counts=True)\ninstitution_id_df = pd.DataFrame.from_dict({'Institution ID' : id,'Count' : count}).sort_values(by=['Count'],ascending=False)\ninstitution_id_df['Count'].hist(bins=100, figsize=(18, 6), grid=True)\nplt.title('Histogram of Institution ID counts')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:22:44.697089Z","iopub.execute_input":"2022-02-23T07:22:44.697753Z","iopub.status.idle":"2022-02-23T07:22:45.165445Z","shell.execute_reply.started":"2022-02-23T07:22:44.697703Z","shell.execute_reply":"2022-02-23T07:22:45.164635Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Image Visualization","metadata":{}},{"cell_type":"code","source":"def visualize_data(df,show_by='Random',genus_name = None):\n    \n    if show_by == 'Genus':\n        df = df[df['GenusNames']==genus_name]\n            \n    data = df.sample(10)\n    \n    image_paths = data['ImagePath'].to_list()\n    genus_ids = data['GenusNames'].to_list()\n    category_ids = data['CategoryID'].to_list()\n    institution_ids = data['InstitutionID'].to_list()\n    \n    plt.figure(figsize=(13,13))\n    \n    for indx,im in enumerate(image_paths):\n        plt.subplot(2,5,indx+1)\n        image = cv2.imread(im)\n        plt.imshow(image[:,:,::-1])\n        plt.title(f'GeniusNames :{genus_ids[indx]},\\nCategoryID : {category_ids[indx]},\\nInstitutionID : {institution_ids[indx]}')\n        plt.axis('off')\n    plt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:48:58.685074Z","iopub.execute_input":"2022-02-23T07:48:58.685381Z","iopub.status.idle":"2022-02-23T07:48:58.694801Z","shell.execute_reply.started":"2022-02-23T07:48:58.685346Z","shell.execute_reply":"2022-02-23T07:48:58.694169Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize random 10 image\nvisualize_data(training_images_df,show_by='Random')","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:49:17.604445Z","iopub.execute_input":"2022-02-23T07:49:17.605173Z","iopub.status.idle":"2022-02-23T07:49:20.36238Z","shell.execute_reply.started":"2022-02-23T07:49:17.605134Z","shell.execute_reply":"2022-02-23T07:49:20.361598Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize random 10 image for a particular genus\nvisualize_data(training_images_df,show_by='Genus',genus_name='Asimina')","metadata":{"execution":{"iopub.status.busy":"2022-02-23T07:50:19.727922Z","iopub.execute_input":"2022-02-23T07:50:19.728502Z","iopub.status.idle":"2022-02-23T07:50:21.984335Z","shell.execute_reply.started":"2022-02-23T07:50:19.728453Z","shell.execute_reply":"2022-02-23T07:50:21.983644Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}