{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Herbarium2022\nThe Herbarium 2022: Flora of North America is a part of a project of the New York Botanical Garden funded by the National Science Foundation to build tools to identify novel plant species around the world. The dataset strives to represent all known vascular plant taxa in North America, using images gathered from 60 different botanical institutions around the world. \nFor each image Id, you should predict the corresponding image label (category_id) in the Predicted column. \n\n<img align='center' src='https://www.floridamuseum.ufl.edu/wp-content/uploads/sites/23/2016/12/herbarium-specimen-sheets-montage-header-600x326.jpg'>\n\nI'm trying overview and generate the csv file train and test data\n\nDay1: Baseline\n\n[https://www.kaggle.com/jagannathrk/herbarium-2020/data](http://)","metadata":{}},{"cell_type":"markdown","source":"# Import necessary library","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\nimport matplotlib.pyplot as plt\nimport PIL\nimport pathlib\nimport cv2\nimport json,codecs","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:51:18.367161Z","iopub.execute_input":"2022-02-15T15:51:18.367625Z","iopub.status.idle":"2022-02-15T15:51:18.644542Z","shell.execute_reply.started":"2022-02-15T15:51:18.367532Z","shell.execute_reply":"2022-02-15T15:51:18.643757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Path**","metadata":{}},{"cell_type":"code","source":"!ls ../input/herbarium-2022-fgvc9","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:51:18.645706Z","iopub.execute_input":"2022-02-15T15:51:18.646618Z","iopub.status.idle":"2022-02-15T15:51:19.328824Z","shell.execute_reply.started":"2022-02-15T15:51:18.646577Z","shell.execute_reply":"2022-02-15T15:51:19.328027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Image counts**","metadata":{}},{"cell_type":"code","source":"#path\ntrain_dir = '../input/herbarium-2022-fgvc9/train_images'\ntest_dir = '../input/herbarium-2022-fgvc9/test_images'\n\ntrain_dir = pathlib.Path(train_dir)\ntest_dir = pathlib.Path(test_dir)\n\n#count the images train and test\nimage_train_count = len(list(train_dir.glob('*/*/*.jpg')))\nimage_test_count = len(list(test_dir.glob('*/*.jpg')))\nprint(f'Image_train_count: {image_train_count},Image_test_count: {image_test_count}')\n\n","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:51:19.330188Z","iopub.execute_input":"2022-02-15T15:51:19.330542Z","iopub.status.idle":"2022-02-15T15:55:37.517767Z","shell.execute_reply.started":"2022-02-15T15:51:19.330508Z","shell.execute_reply":"2022-02-15T15:55:37.51695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#counts the subfolder images\n\nfolder000 = len(list(train_dir.glob('000/*/*.jpg')))\nfolder000_00 = len(list(train_dir.glob('000/00/*.jpg')))\nprint(f'Train_subfolder_000:{folder000},Train_000_One_of_Subfolder:{folder000_00}')\n\nfolder001 = len(list(train_dir.glob('001/*/*.jpg')))\nfolder001_00 = len(list(train_dir.glob('001/00/*.jpg')))\nprint(f'Train_subfolder_001:{folder001},Train_001_One_of_Subfolder:{folder001_00}')\n","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:55:37.520125Z","iopub.execute_input":"2022-02-15T15:55:37.520554Z","iopub.status.idle":"2022-02-15T15:55:37.670692Z","shell.execute_reply.started":"2022-02-15T15:55:37.520513Z","shell.execute_reply":"2022-02-15T15:55:37.6699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Visualize the data**","metadata":{}},{"cell_type":"code","source":"#Plot the images\nPlan_000_00 = list(train_dir.glob('000/00/*'))\nPIL.Image.open(str(Plan_000_00[1]))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:55:37.672167Z","iopub.execute_input":"2022-02-15T15:55:37.67274Z","iopub.status.idle":"2022-02-15T15:55:37.972915Z","shell.execute_reply.started":"2022-02-15T15:55:37.672697Z","shell.execute_reply":"2022-02-15T15:55:37.972023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **load metadata**","metadata":{}},{"cell_type":"code","source":"#Focus on train metadataset\n\nwith codecs.open(\"../input/herbarium-2022-fgvc9/train_metadata.json\", 'r',\n                 encoding='utf-8', errors='ignore') as f:\n    train_data = json.load(f)\n    \nwith codecs.open(\"../input/herbarium-2022-fgvc9/train_metadata.json\", 'r',\n                 encoding='utf-8', errors='ignore') as f:\n    test_data = json.load(f)\n    \n#sample\nsample = pd.read_csv('../input/herbarium-2022-fgvc9/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:55:37.974066Z","iopub.execute_input":"2022-02-15T15:55:37.974325Z","iopub.status.idle":"2022-02-15T15:55:57.904872Z","shell.execute_reply.started":"2022-02-15T15:55:37.974291Z","shell.execute_reply":"2022-02-15T15:55:57.904058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(test_data.keys())","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:55:57.906202Z","iopub.execute_input":"2022-02-15T15:55:57.906482Z","iopub.status.idle":"2022-02-15T15:55:57.913755Z","shell.execute_reply.started":"2022-02-15T15:55:57.906444Z","shell.execute_reply":"2022-02-15T15:55:57.913072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_data.keys())","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:55:57.914837Z","iopub.execute_input":"2022-02-15T15:55:57.915595Z","iopub.status.idle":"2022-02-15T15:55:57.923997Z","shell.execute_reply.started":"2022-02-15T15:55:57.915554Z","shell.execute_reply":"2022-02-15T15:55:57.922718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **convert to build dataframe**","metadata":{}},{"cell_type":"code","source":"#TrainDataset\ntrain_data_df = pd.DataFrame(train_data['annotations'])\ntrain_data_img = pd.DataFrame(train_data[\"images\"])\ntrain_data_cat = pd.DataFrame(train_data[\"categories\"])\ntrain_data_gen = pd.DataFrame(train_data[\"genera\"])\ntrain_data_ins = pd.DataFrame(train_data[\"institutions\"])\ntrain_data_dis = pd.DataFrame(train_data[\"distances\"])\ntrain_data_lic = pd.DataFrame(train_data[\"license\"])","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:55:57.926025Z","iopub.execute_input":"2022-02-15T15:55:57.927216Z","iopub.status.idle":"2022-02-15T15:56:04.881264Z","shell.execute_reply.started":"2022-02-15T15:55:57.927176Z","shell.execute_reply":"2022-02-15T15:56:04.880457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_dis.sample(2)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:56:04.882719Z","iopub.execute_input":"2022-02-15T15:56:04.882972Z","iopub.status.idle":"2022-02-15T15:56:04.988475Z","shell.execute_reply.started":"2022-02-15T15:56:04.882938Z","shell.execute_reply":"2022-02-15T15:56:04.987611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Merge important data**","metadata":{}},{"cell_type":"code","source":"train_data_df = train_data_df.merge(train_data_cat,on='category_id',how='outer')\ntrain_data_df = train_data_df.merge(train_data_img,on='image_id',how='outer')\ntrain_data_df = train_data_df.merge(train_data_gen,on='genus_id',how='outer')\n","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:56:04.990026Z","iopub.execute_input":"2022-02-15T15:56:04.990338Z","iopub.status.idle":"2022-02-15T15:56:06.161013Z","shell.execute_reply.started":"2022-02-15T15:56:04.990301Z","shell.execute_reply":"2022-02-15T15:56:06.160244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:56:39.819652Z","iopub.execute_input":"2022-02-15T15:56:39.819942Z","iopub.status.idle":"2022-02-15T15:56:40.458727Z","shell.execute_reply.started":"2022-02-15T15:56:39.819909Z","shell.execute_reply":"2022-02-15T15:56:40.457814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df.sample(2)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:56:44.922225Z","iopub.execute_input":"2022-02-15T15:56:44.922759Z","iopub.status.idle":"2022-02-15T15:56:45.269265Z","shell.execute_reply.started":"2022-02-15T15:56:44.922718Z","shell.execute_reply":"2022-02-15T15:56:45.268424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#handle to remove nan values\ntrain_data_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:57:16.225435Z","iopub.execute_input":"2022-02-15T15:57:16.226213Z","iopub.status.idle":"2022-02-15T15:57:16.864303Z","shell.execute_reply.started":"2022-02-15T15:57:16.226169Z","shell.execute_reply":"2022-02-15T15:57:16.863617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Neglected null data**","metadata":{}},{"cell_type":"code","source":"#remove nan\nnull_identify = train_data_df.file_name.isna()\ndata_keep= [x for x in range(train_data_df.shape[0]) if not null_identify[x]]\ntrain_df = train_data_df.iloc[data_keep]","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:59:36.412652Z","iopub.execute_input":"2022-02-15T15:59:36.412926Z","iopub.status.idle":"2022-02-15T15:59:40.900541Z","shell.execute_reply.started":"2022-02-15T15:59:36.412895Z","shell.execute_reply":"2022-02-15T15:59:40.899239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Test_data**","metadata":{}},{"cell_type":"code","source":"#test data\ntest_df = pd.DataFrame(test_data['images'])\nprint(test_df.info())\ndisplay(test_df.sample(2))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T16:04:58.226503Z","iopub.execute_input":"2022-02-15T16:04:58.227293Z","iopub.status.idle":"2022-02-15T16:04:59.263125Z","shell.execute_reply.started":"2022-02-15T16:04:58.227228Z","shell.execute_reply":"2022-02-15T16:04:59.262328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Save important datas---> csv file**","metadata":{}},{"cell_type":"code","source":"train_data_df.to_csv('full_train_data.csv', index=False)\ntest_df.to_csv('full_test_data.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-02-15T16:05:34.06551Z","iopub.execute_input":"2022-02-15T16:05:34.066328Z","iopub.status.idle":"2022-02-15T16:05:41.855109Z","shell.execute_reply.started":"2022-02-15T16:05:34.066252Z","shell.execute_reply":"2022-02-15T16:05:41.854031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T16:18:21.379599Z","iopub.execute_input":"2022-02-15T16:18:21.380631Z","iopub.status.idle":"2022-02-15T16:18:21.390751Z","shell.execute_reply.started":"2022-02-15T16:18:21.380584Z","shell.execute_reply":"2022-02-15T16:18:21.389952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Next_Process_Coming soon.....................................**","metadata":{}},{"cell_type":"markdown","source":"## **Thankyou for visiting guys**","metadata":{}}]}