{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport json\nimport os\nimport random\nimport matplotlib.pyplot as plt\n#from tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport tensorflow as tf\nfrom tensorflow.keras import layers \nfrom sklearn.preprocessing import LabelEncoder","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-25T17:10:22.219359Z","iopub.execute_input":"2022-07-25T17:10:22.220811Z","iopub.status.idle":"2022-07-25T17:10:22.228650Z","shell.execute_reply.started":"2022-07-25T17:10:22.220742Z","shell.execute_reply":"2022-07-25T17:10:22.227451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DIR = \"../input/herbarium-2022-fgvc9/train_images/\"\nTEST_DIR = \"../input/herbarium-2022-fgvc9/test_images/\"\n\nwith open(\"../input/herbarium-2022-fgvc9/train_metadata.json\") as json_file:\n    train_meta = json.load(json_file)\nwith open(\"../input/herbarium-2022-fgvc9/test_metadata.json\") as json_file:\n    test_meta = json.load(json_file)\n#Create a meta-data df that can be used to call in images\nids = []\ncategories = []\npaths = []\n\nfor annotation, image in zip(train_meta['annotations'], train_meta['images']):\n    ids.append(image[\"image_id\"])\n    categories.append(annotation['category_id'])\n    paths.append(image[\"file_name\"])\n\ndf_meta = pd.DataFrame({\"id\":ids, \"category\":categories, \"path\":paths})\ndf_meta.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T17:10:22.886462Z","iopub.execute_input":"2022-07-25T17:10:22.887530Z","iopub.status.idle":"2022-07-25T17:10:36.961195Z","shell.execute_reply.started":"2022-07-25T17:10:22.887485Z","shell.execute_reply":"2022-07-25T17:10:36.959867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sci_name = {cat[\"category_id\"]:cat[\"scientificName\"] for cat in train_meta['categories']}\nfamily = {cat[\"category_id\"]:cat[\"family\"] for cat in train_meta['categories']}\ngenus = {cat[\"category_id\"]:cat[\"genus\"] for cat in train_meta['categories']}\nspecies = {cat[\"category_id\"]:cat[\"species\"] for cat in train_meta['categories']}\n\ndf_meta[\"scientific_name\"] = df_meta[\"category\"].map(sci_name)\ndf_meta[\"family\"] = df_meta[\"category\"].map(family)\ndf_meta[\"genus\"] = df_meta[\"category\"].map(genus)\ndf_meta[\"species\"] = df_meta[\"category\"].map(species)\ndf_meta.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T17:10:36.963699Z","iopub.execute_input":"2022-07-25T17:10:36.964087Z","iopub.status.idle":"2022-07-25T17:10:37.161075Z","shell.execute_reply.started":"2022-07-25T17:10:36.964052Z","shell.execute_reply":"2022-07-25T17:10:37.159891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_image(index):\n    path=os.path.join(\"../input/herbarium-2022-fgvc9/train_images\",df_meta[\"path\"][index])\n    print(df_meta[\"scientific_name\"][index])\n    plt.imshow(plt.imread(path)/255)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T17:10:37.162601Z","iopub.execute_input":"2022-07-25T17:10:37.163213Z","iopub.status.idle":"2022-07-25T17:10:37.170944Z","shell.execute_reply.started":"2022-07-25T17:10:37.163174Z","shell.execute_reply":"2022-07-25T17:10:37.169152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_image(20)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T17:10:37.174442Z","iopub.execute_input":"2022-07-25T17:10:37.175494Z","iopub.status.idle":"2022-07-25T17:10:37.568074Z","shell.execute_reply.started":"2022-07-25T17:10:37.175441Z","shell.execute_reply":"2022-07-25T17:10:37.566587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LABEL=len(train_meta['categories'])\nLABEL","metadata":{"execution":{"iopub.status.busy":"2022-07-25T17:10:37.570015Z","iopub.execute_input":"2022-07-25T17:10:37.571015Z","iopub.status.idle":"2022-07-25T17:10:37.580159Z","shell.execute_reply.started":"2022-07-25T17:10:37.570964Z","shell.execute_reply":"2022-07-25T17:10:37.578663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta['categories'][1]","metadata":{"execution":{"iopub.status.busy":"2022-07-25T17:10:37.581961Z","iopub.execute_input":"2022-07-25T17:10:37.582533Z","iopub.status.idle":"2022-07-25T17:10:37.592297Z","shell.execute_reply.started":"2022-07-25T17:10:37.582497Z","shell.execute_reply":"2022-07-25T17:10:37.591075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"family_id=LabelEncoder()\ngenus_id=LabelEncoder()\nfamily_id.fit([x [\"family\"] for x in train_meta[\"categories\"]])\ngenus_id.fit([x [\"genus\"] for x in train_meta[\"categories\"]])","metadata":{"execution":{"iopub.status.busy":"2022-07-25T17:10:37.594295Z","iopub.execute_input":"2022-07-25T17:10:37.594971Z","iopub.status.idle":"2022-07-25T17:10:37.629228Z","shell.execute_reply.started":"2022-07-25T17:10:37.594934Z","shell.execute_reply":"2022-07-25T17:10:37.627732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#adding sublabels to make this job easier\ndata_df=df_meta.drop(columns=[\"species\",\"scientific_name\"])\ndata_df[\"family\"]=family_id.transform(data_df[\"family\"])\ndata_df[\"genus\"]=genus_id.transform(data_df[\"genus\"])\ndata_df","metadata":{"execution":{"iopub.status.busy":"2022-07-25T17:10:37.630659Z","iopub.execute_input":"2022-07-25T17:10:37.631064Z","iopub.status.idle":"2022-07-25T17:10:38.389008Z","shell.execute_reply.started":"2022-07-25T17:10:37.631024Z","shell.execute_reply":"2022-07-25T17:10:38.387703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"family_id.inverse_transform([195]),genus_id.inverse_transform([0])","metadata":{"execution":{"iopub.status.busy":"2022-07-25T17:10:38.390677Z","iopub.execute_input":"2022-07-25T17:10:38.391060Z","iopub.status.idle":"2022-07-25T17:10:38.401321Z","shell.execute_reply.started":"2022-07-25T17:10:38.391026Z","shell.execute_reply":"2022-07-25T17:10:38.399879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_numbers(index,data=data_df,prefix=\"../input/herbarium-2022-fgvc9/train_images\"):\n    path=os.path.join(prefix,data[\"path\"][index])\n    cat=data[\"category\"][index]\n    family=data[\"family\"][index]\n    genus=data[\"genus\"][index]\n    x=tf.keras.preprocessing.image.load_img(path,target_size=(256,256))\n    x=tf.keras.utils.img_to_array(x)\n    \n    return({\"full_name\":tf.constant(cat,tf.int64),\n            \"family\":tf.constant(family,tf.int64),\n            \"genus\":tf.constant(genus,tf.int64),\n            \"x\":tf.constant(x)})\nb=get_numbers(3)\nb.items()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T17:38:33.552969Z","iopub.execute_input":"2022-07-25T17:38:33.553591Z","iopub.status.idle":"2022-07-25T17:38:33.588634Z","shell.execute_reply.started":"2022-07-25T17:38:33.553539Z","shell.execute_reply":"2022-07-25T17:38:33.587720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"specs={k:tf.TensorSpec.from_tensor(v) for k,v in b.items()}\nspecs","metadata":{"execution":{"iopub.status.busy":"2022-07-25T17:38:38.574593Z","iopub.execute_input":"2022-07-25T17:38:38.575435Z","iopub.status.idle":"2022-07-25T17:38:38.583861Z","shell.execute_reply.started":"2022-07-25T17:38:38.575382Z","shell.execute_reply":"2022-07-25T17:38:38.582731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class generator():\n    def __init__(self,data):\n        self.cap=len(data)\n        self.data=data \n    def generate(self):\n        for index in range(self.cap):\n            yield get_numbers(index,self.data)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T17:40:39.028131Z","iopub.execute_input":"2022-07-25T17:40:39.028812Z","iopub.status.idle":"2022-07-25T17:40:39.038823Z","shell.execute_reply.started":"2022-07-25T17:40:39.028754Z","shell.execute_reply":"2022-07-25T17:40:39.037554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a=generator(data_df) \nb=next(a.generate())\nb","metadata":{"execution":{"iopub.status.busy":"2022-07-25T17:40:40.359268Z","iopub.execute_input":"2022-07-25T17:40:40.359767Z","iopub.status.idle":"2022-07-25T17:40:40.389862Z","shell.execute_reply.started":"2022-07-25T17:40:40.359728Z","shell.execute_reply":"2022-07-25T17:40:40.388883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset=tf.data.Dataset.from_generator(generator(data_df).generate,output_signature=specs)\nnext(dataset.as_numpy_iterator())","metadata":{"execution":{"iopub.status.busy":"2022-07-25T17:41:55.761647Z","iopub.execute_input":"2022-07-25T17:41:55.762080Z","iopub.status.idle":"2022-07-25T17:41:55.870713Z","shell.execute_reply.started":"2022-07-25T17:41:55.762039Z","shell.execute_reply":"2022-07-25T17:41:55.868927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}