{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport PIL\nimport tensorflow as tf\nimport pathlib\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-14T23:00:28.00552Z","iopub.execute_input":"2022-02-14T23:00:28.006122Z","iopub.status.idle":"2022-02-14T23:00:34.782729Z","shell.execute_reply.started":"2022-02-14T23:00:28.006082Z","shell.execute_reply":"2022-02-14T23:00:34.781644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = '../input/herbarium-2022-fgvc9/train_images/'\ntest_dir = '../input/herbarium-2022-fgvc9/test_images/'\ntrain_dir = pathlib.Path(train_dir)\ntest_dir = pathlib.Path(test_dir)\ntrain_dir","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:30:58.505939Z","iopub.execute_input":"2022-02-14T23:30:58.506208Z","iopub.status.idle":"2022-02-14T23:30:58.512931Z","shell.execute_reply.started":"2022-02-14T23:30:58.50618Z","shell.execute_reply":"2022-02-14T23:30:58.512141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_train_count = len(list(train_dir.glob('*/*/*.jpg')))\nprint(image_train_count)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:26:59.0869Z","iopub.execute_input":"2022-02-14T23:26:59.087753Z","iopub.status.idle":"2022-02-14T23:29:02.329642Z","shell.execute_reply.started":"2022-02-14T23:26:59.08771Z","shell.execute_reply":"2022-02-14T23:29:02.328737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_test_count = len(list(test_dir.glob('*/*.jpg')))\nprint(image_test_count)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:31:00.926938Z","iopub.execute_input":"2022-02-14T23:31:00.927609Z","iopub.status.idle":"2022-02-14T23:32:04.480951Z","shell.execute_reply.started":"2022-02-14T23:31:00.927559Z","shell.execute_reply":"2022-02-14T23:32:04.48007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*  We have 839772 images in the train dataset\n*  while we have 210407 in test dataset","metadata":{}},{"cell_type":"code","source":"# our train data set follow the following structure:\n\"\"\"\nTrain dataset/\n  000/\n    00/\n    01/\n    02/\n    ...\n  001/\n    00/\n    01/\n    02/\n    ...\n  002/\n  003/\n  004/\n  ...\n    \"\"\"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folder000 = len(list(data_dir.glob('000/*/*.jpg')))\nprint(folder000)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:06:24.246935Z","iopub.execute_input":"2022-02-14T23:06:24.247257Z","iopub.status.idle":"2022-02-14T23:06:24.408448Z","shell.execute_reply.started":"2022-02-14T23:06:24.247226Z","shell.execute_reply":"2022-02-14T23:06:24.407673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* The first folder \"000\" contains 5832 images.","metadata":{}},{"cell_type":"code","source":"folder000_00 = len(list(data_dir.glob('000/00/*.jpg')))\nprint(folder000_00)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:07:28.186801Z","iopub.execute_input":"2022-02-14T23:07:28.187461Z","iopub.status.idle":"2022-02-14T23:07:28.196584Z","shell.execute_reply.started":"2022-02-14T23:07:28.187423Z","shell.execute_reply":"2022-02-14T23:07:28.195731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* the subfolder \"00\" of the folder \"000\" contains 57 images.","metadata":{}},{"cell_type":"code","source":"folder001 = len(list(data_dir.glob('001/*/*.jpg')))\nprint(folder001)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:06:53.906418Z","iopub.execute_input":"2022-02-14T23:06:53.907626Z","iopub.status.idle":"2022-02-14T23:06:54.66998Z","shell.execute_reply.started":"2022-02-14T23:06:53.907582Z","shell.execute_reply":"2022-02-14T23:06:54.667635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* The second folder \"001\" contains 5616 images.","metadata":{}},{"cell_type":"markdown","source":"# let's plot some plants","metadata":{}},{"cell_type":"markdown","source":"## 1. Train images","metadata":{}},{"cell_type":"code","source":"plant_000_00 = list(train_dir.glob('000/00/*'))\nPIL.Image.open(str(plant_000_00[0]))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:52:17.086188Z","iopub.execute_input":"2022-02-14T23:52:17.086561Z","iopub.status.idle":"2022-02-14T23:52:17.432841Z","shell.execute_reply.started":"2022-02-14T23:52:17.086526Z","shell.execute_reply":"2022-02-14T23:52:17.431731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plant_000_00 = list(train_dir.glob('000/00/*'))\nPIL.Image.open(str(plant_000_00[1]))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:52:29.505321Z","iopub.execute_input":"2022-02-14T23:52:29.505678Z","iopub.status.idle":"2022-02-14T23:52:29.846079Z","shell.execute_reply.started":"2022-02-14T23:52:29.505641Z","shell.execute_reply":"2022-02-14T23:52:29.845156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plant_000_00 = list(train_dir.glob('000/00/*'))\nPIL.Image.open(str(plant_000_00[2]))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:52:54.865336Z","iopub.execute_input":"2022-02-14T23:52:54.866002Z","iopub.status.idle":"2022-02-14T23:52:55.244272Z","shell.execute_reply.started":"2022-02-14T23:52:54.865963Z","shell.execute_reply":"2022-02-14T23:52:55.24096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plant_001_00 = list(train_dir.glob('001/00/*'))\nPIL.Image.open(str(plant_001_00[0]))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:53:09.14574Z","iopub.execute_input":"2022-02-14T23:53:09.146051Z","iopub.status.idle":"2022-02-14T23:53:09.454085Z","shell.execute_reply.started":"2022-02-14T23:53:09.146019Z","shell.execute_reply":"2022-02-14T23:53:09.452982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plant_002_00 = list(train_dir.glob('002/00/*'))\nPIL.Image.open(str(plant_002_00[0]))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:53:21.154277Z","iopub.execute_input":"2022-02-14T23:53:21.155501Z","iopub.status.idle":"2022-02-14T23:53:21.28469Z","shell.execute_reply.started":"2022-02-14T23:53:21.155447Z","shell.execute_reply":"2022-02-14T23:53:21.281822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Test images","metadata":{}},{"cell_type":"code","source":"plant_000 = list(test_dir.glob('000/*'))\nPIL.Image.open(str(plant_000[0]))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:54:36.833577Z","iopub.execute_input":"2022-02-14T23:54:36.833901Z","iopub.status.idle":"2022-02-14T23:54:36.937212Z","shell.execute_reply.started":"2022-02-14T23:54:36.833868Z","shell.execute_reply":"2022-02-14T23:54:36.936336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plant_000 = list(test_dir.glob('000/*'))\nPIL.Image.open(str(plant_000[1]))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:54:56.432996Z","iopub.execute_input":"2022-02-14T23:54:56.43331Z","iopub.status.idle":"2022-02-14T23:54:56.793323Z","shell.execute_reply.started":"2022-02-14T23:54:56.433279Z","shell.execute_reply":"2022-02-14T23:54:56.791961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plant_000 = list(test_dir.glob('000/*'))\nPIL.Image.open(str(plant_000[2]))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:55:05.525059Z","iopub.execute_input":"2022-02-14T23:55:05.525399Z","iopub.status.idle":"2022-02-14T23:55:05.863749Z","shell.execute_reply.started":"2022-02-14T23:55:05.525341Z","shell.execute_reply":"2022-02-14T23:55:05.862923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plant_001 = list(test_dir.glob('001/*'))\nPIL.Image.open(str(plant_001[0]))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:55:36.465349Z","iopub.execute_input":"2022-02-14T23:55:36.465794Z","iopub.status.idle":"2022-02-14T23:55:36.55794Z","shell.execute_reply.started":"2022-02-14T23:55:36.465728Z","shell.execute_reply":"2022-02-14T23:55:36.556831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plant_002 = list(test_dir.glob('002/*'))\nPIL.Image.open(str(plant_002[0]))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:55:57.105653Z","iopub.execute_input":"2022-02-14T23:55:57.10656Z","iopub.status.idle":"2022-02-14T23:55:57.349352Z","shell.execute_reply.started":"2022-02-14T23:55:57.106481Z","shell.execute_reply":"2022-02-14T23:55:57.348734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plant_003 = list(test_dir.glob('003/*'))\nPIL.Image.open(str(plant_003[0]))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T23:56:14.365894Z","iopub.execute_input":"2022-02-14T23:56:14.366231Z","iopub.status.idle":"2022-02-14T23:56:14.750608Z","shell.execute_reply.started":"2022-02-14T23:56:14.366165Z","shell.execute_reply":"2022-02-14T23:56:14.749148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train & Test metadata","metadata":{}},{"cell_type":"code","source":"import json\ntrain = open('../input/herbarium-2022-fgvc9/train_metadata.json')\ntest = open('../input/herbarium-2022-fgvc9/test_metadata.json')\ndata_train = json.load(train)\ndata_test = json.load(test)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:02:02.046789Z","iopub.execute_input":"2022-02-15T00:02:02.047146Z","iopub.status.idle":"2022-02-15T00:02:16.309299Z","shell.execute_reply.started":"2022-02-15T00:02:02.0471Z","shell.execute_reply":"2022-02-15T00:02:16.308245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train.keys()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:24:05.265075Z","iopub.execute_input":"2022-02-15T00:24:05.265613Z","iopub.status.idle":"2022-02-15T00:24:05.27095Z","shell.execute_reply.started":"2022-02-15T00:24:05.265578Z","shell.execute_reply":"2022-02-15T00:24:05.270176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Test Dataset","metadata":{}},{"cell_type":"code","source":"df_test = pd.DataFrame(data_test)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:25:23.083916Z","iopub.execute_input":"2022-02-15T00:25:23.084267Z","iopub.status.idle":"2022-02-15T00:25:24.990527Z","shell.execute_reply.started":"2022-02-15T00:25:23.084224Z","shell.execute_reply":"2022-02-15T00:25:24.989325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:25:31.365676Z","iopub.execute_input":"2022-02-15T00:25:31.366011Z","iopub.status.idle":"2022-02-15T00:25:31.38164Z","shell.execute_reply.started":"2022-02-15T00:25:31.365978Z","shell.execute_reply":"2022-02-15T00:25:31.380912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Train Dataset","metadata":{}},{"cell_type":"code","source":"df_train = pd.DataFrame(data_train['annotations'])\ndf_train","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:24:51.344938Z","iopub.execute_input":"2022-02-15T00:24:51.345241Z","iopub.status.idle":"2022-02-15T00:24:51.361932Z","shell.execute_reply.started":"2022-02-15T00:24:51.345211Z","shell.execute_reply":"2022-02-15T00:24:51.360841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.category_id.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:40:27.666042Z","iopub.execute_input":"2022-02-15T00:40:27.66639Z","iopub.status.idle":"2022-02-15T00:40:27.69061Z","shell.execute_reply.started":"2022-02-15T00:40:27.666344Z","shell.execute_reply":"2022-02-15T00:40:27.689307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Images data","metadata":{}},{"cell_type":"code","source":"images = pd.DataFrame(data_train[\"images\"])\nimages","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:34:11.866101Z","iopub.execute_input":"2022-02-15T00:34:11.866416Z","iopub.status.idle":"2022-02-15T00:34:12.929888Z","shell.execute_reply.started":"2022-02-15T00:34:11.866359Z","shell.execute_reply":"2022-02-15T00:34:12.929007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Categories data","metadata":{}},{"cell_type":"code","source":"categories = pd.DataFrame(data_train[\"categories\"])\ncategories","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:26:51.166284Z","iopub.execute_input":"2022-02-15T00:26:51.167152Z","iopub.status.idle":"2022-02-15T00:26:51.210065Z","shell.execute_reply.started":"2022-02-15T00:26:51.167108Z","shell.execute_reply":"2022-02-15T00:26:51.20936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Genera data","metadata":{}},{"cell_type":"code","source":"genera = pd.DataFrame(data_train[\"genera\"])\ngenera","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:35:32.806294Z","iopub.execute_input":"2022-02-15T00:35:32.80664Z","iopub.status.idle":"2022-02-15T00:35:32.826051Z","shell.execute_reply.started":"2022-02-15T00:35:32.806607Z","shell.execute_reply":"2022-02-15T00:35:32.824881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Institutions data ","metadata":{}},{"cell_type":"code","source":"institutions = pd.DataFrame(data_train[\"institutions\"])\ninstitutions","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:27:42.346121Z","iopub.execute_input":"2022-02-15T00:27:42.346424Z","iopub.status.idle":"2022-02-15T00:27:42.361104Z","shell.execute_reply.started":"2022-02-15T00:27:42.346393Z","shell.execute_reply":"2022-02-15T00:27:42.360336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7. Distances data ","metadata":{}},{"cell_type":"code","source":"distances = pd.DataFrame(data_train[\"distances\"])\ndistances","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:37:45.205917Z","iopub.execute_input":"2022-02-15T00:37:45.206225Z","iopub.status.idle":"2022-02-15T00:37:50.530816Z","shell.execute_reply.started":"2022-02-15T00:37:45.206194Z","shell.execute_reply":"2022-02-15T00:37:50.529959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8. License data ","metadata":{}},{"cell_type":"code","source":"license = pd.DataFrame(data_train[\"license\"])\nlicense","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:29:51.906272Z","iopub.execute_input":"2022-02-15T00:29:51.907233Z","iopub.status.idle":"2022-02-15T00:29:51.91911Z","shell.execute_reply.started":"2022-02-15T00:29:51.907186Z","shell.execute_reply":"2022-02-15T00:29:51.918199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv('../input/herbarium-2022-fgvc9/sample_submission.csv')\nsub","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:42:02.606365Z","iopub.execute_input":"2022-02-15T00:42:02.606911Z","iopub.status.idle":"2022-02-15T00:42:02.710406Z","shell.execute_reply.started":"2022-02-15T00:42:02.606878Z","shell.execute_reply":"2022-02-15T00:42:02.709679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['Predicted'] = 2774\nsub.to_csv('submission.csv', index=False)\nsub","metadata":{"execution":{"iopub.status.busy":"2022-02-15T00:42:27.515228Z","iopub.execute_input":"2022-02-15T00:42:27.51559Z","iopub.status.idle":"2022-02-15T00:42:27.928249Z","shell.execute_reply.started":"2022-02-15T00:42:27.515553Z","shell.execute_reply":"2022-02-15T00:42:27.927175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}