{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Some genera were not included during the data curation, but their genus_id still exists in the `distances` metadata. Thanks @kalelpark for reporting this.\n\nThis notebook is just to double-check if genus_ids presented in the data are consistent throughout all metadata, except the `distances` metadata.","metadata":{}},{"cell_type":"code","source":"import json\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nos.listdir('/kaggle/input/herbarium-2022-fgvc9')","metadata":{"execution":{"iopub.status.busy":"2022-02-16T07:14:24.383328Z","iopub.execute_input":"2022-02-16T07:14:24.384321Z","iopub.status.idle":"2022-02-16T07:14:24.391488Z","shell.execute_reply.started":"2022-02-16T07:14:24.384266Z","shell.execute_reply":"2022-02-16T07:14:24.39051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/input/herbarium-2022-fgvc9/train_metadata.json', 'r') as f:\n    train_metadata= json.load(f)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T07:20:12.565919Z","iopub.execute_input":"2022-02-16T07:20:12.567179Z","iopub.status.idle":"2022-02-16T07:20:27.470006Z","shell.execute_reply.started":"2022-02-16T07:20:12.567107Z","shell.execute_reply":"2022-02-16T07:20:27.469372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata.keys()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T07:20:29.704874Z","iopub.execute_input":"2022-02-16T07:20:29.705917Z","iopub.status.idle":"2022-02-16T07:20:29.712776Z","shell.execute_reply.started":"2022-02-16T07:20:29.705881Z","shell.execute_reply":"2022-02-16T07:20:29.711871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata['genera'][0], train_metadata['genera'][-1], len(train_metadata['genera'])","metadata":{"execution":{"iopub.status.busy":"2022-02-16T07:46:48.967317Z","iopub.execute_input":"2022-02-16T07:46:48.967629Z","iopub.status.idle":"2022-02-16T07:46:48.976105Z","shell.execute_reply.started":"2022-02-16T07:46:48.967599Z","shell.execute_reply":"2022-02-16T07:46:48.974644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that the ID number for genus starts from 1 and ends at 2584, however the length is 2564, because some genera were removed during the data curation. \nFirst thing to do is to check if the genus_ids in the `genera` metadata match exactly with the genus_ids in the `annotations` metadata:","metadata":{}},{"cell_type":"code","source":"#Get unique genus id from key==genera\nS_genus_id=set(pd.DataFrame(train_metadata['genera']).genus_id)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T07:33:42.272434Z","iopub.execute_input":"2022-02-16T07:33:42.272742Z","iopub.status.idle":"2022-02-16T07:33:42.281868Z","shell.execute_reply.started":"2022-02-16T07:33:42.272712Z","shell.execute_reply":"2022-02-16T07:33:42.281089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Get unique genus id from image annotations data\nS_genus_id_from_annotations=set(pd.DataFrame(train_metadata['annotations']).genus_id)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T07:33:43.587659Z","iopub.execute_input":"2022-02-16T07:33:43.587965Z","iopub.status.idle":"2022-02-16T07:33:45.315203Z","shell.execute_reply.started":"2022-02-16T07:33:43.587935Z","shell.execute_reply":"2022-02-16T07:33:45.314522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make sure two sets match: if true, it means genera metadata has all record matching the image metadata \nS_genus_id==S_genus_id_from_annotations","metadata":{"execution":{"iopub.status.busy":"2022-02-16T07:34:05.406544Z","iopub.execute_input":"2022-02-16T07:34:05.407388Z","iopub.status.idle":"2022-02-16T07:34:05.413346Z","shell.execute_reply.started":"2022-02-16T07:34:05.407347Z","shell.execute_reply":"2022-02-16T07:34:05.41261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The genus_ids match, OK.\n\nNow we check that if the `distances` metadata has all genus_id from 1 to 2584","metadata":{}},{"cell_type":"code","source":"#Check length of genera of distance metadata:\nS_genus_id_x=set(pd.DataFrame(train_metadata['distances']).genus_id_x)\nS_genus_id_y=set(pd.DataFrame(train_metadata['distances']).genus_id_y)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T07:48:41.058711Z","iopub.execute_input":"2022-02-16T07:48:41.059056Z","iopub.status.idle":"2022-02-16T07:48:52.616584Z","shell.execute_reply.started":"2022-02-16T07:48:41.059025Z","shell.execute_reply":"2022-02-16T07:48:52.615554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[id_num for id_num  in range(1,2584) if id_num not in S_genus_id_x],[id_num for id_num  in range(1,2584) if id_num not in S_genus_id_y]","metadata":{"execution":{"iopub.status.busy":"2022-02-16T07:49:18.717766Z","iopub.execute_input":"2022-02-16T07:49:18.718087Z","iopub.status.idle":"2022-02-16T07:49:18.726751Z","shell.execute_reply.started":"2022-02-16T07:49:18.718054Z","shell.execute_reply":"2022-02-16T07:49:18.725669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, we get genus_id that are not used in the herbarium 2022 dataset:","metadata":{}},{"cell_type":"code","source":"# Check the genera id that does not present in the current image dataset\n[id_num for id_num  in range(1,2584) if id_num not in S_genus_id ]","metadata":{"execution":{"iopub.status.busy":"2022-02-16T07:45:27.618431Z","iopub.execute_input":"2022-02-16T07:45:27.619302Z","iopub.status.idle":"2022-02-16T07:45:27.626088Z","shell.execute_reply.started":"2022-02-16T07:45:27.619265Z","shell.execute_reply":"2022-02-16T07:45:27.625406Z"},"trusted":true},"execution_count":null,"outputs":[]}]}