{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-26T04:49:33.177406Z","iopub.execute_input":"2022-03-26T04:49:33.177805Z","iopub.status.idle":"2022-03-26T04:49:33.207027Z","shell.execute_reply.started":"2022-03-26T04:49:33.177707Z","shell.execute_reply":"2022-03-26T04:49:33.206174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip uninstall -y tensorflow_datasets\n! pip install tensorflow_datasets==4.4.0","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-26T04:49:33.208403Z","iopub.execute_input":"2022-03-26T04:49:33.208868Z","iopub.status.idle":"2022-03-26T04:49:49.303054Z","shell.execute_reply.started":"2022-03-26T04:49:33.208722Z","shell.execute_reply":"2022-03-26T04:49:49.302209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nfrom sklearn.preprocessing import  LabelEncoder\nfrom tqdm.auto import tqdm\nimport tensorflow as tf\nimport dill\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nimport tensorflow_datasets as tfds\nimport glob","metadata":{"execution":{"iopub.status.busy":"2022-03-26T04:49:49.304392Z","iopub.execute_input":"2022-03-26T04:49:49.304687Z","iopub.status.idle":"2022-03-26T04:49:55.712846Z","shell.execute_reply.started":"2022-03-26T04:49:49.304617Z","shell.execute_reply":"2022-03-26T04:49:55.712116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# This notebook shows the easiest way to convert any dataset to TFRecords","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/sorghum-id-fgvc-9/train_cultivar_mapping.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-03-26T04:49:55.715119Z","iopub.execute_input":"2022-03-26T04:49:55.715580Z","iopub.status.idle":"2022-03-26T04:49:55.766662Z","shell.execute_reply.started":"2022-03-26T04:49:55.715538Z","shell.execute_reply":"2022-03-26T04:49:55.765843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.DataFrame({\n    'filename':[os.path.basename(name) for name in glob.glob(\"../input/sorghum-id-fgvc-9/test/*.png\")]\n})\ntest_df['cultivar'] = ''\ntest_df.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T04:49:55.768366Z","iopub.execute_input":"2022-03-26T04:49:55.769007Z","iopub.status.idle":"2022-03-26T04:49:56.507187Z","shell.execute_reply.started":"2022-03-26T04:49:55.768963Z","shell.execute_reply":"2022-03-26T04:49:56.505975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T04:49:56.508502Z","iopub.execute_input":"2022-03-26T04:49:56.508893Z","iopub.status.idle":"2022-03-26T04:49:56.524223Z","shell.execute_reply.started":"2022-03-26T04:49:56.508861Z","shell.execute_reply":"2022-03-26T04:49:56.523687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['jpeg_image'] = [path[:-3]+\"jpg\" for path in train_df.image]\ntest_df['jpeg_image'] = [path[:-3]+\"jpg\" for path in test_df.filename]","metadata":{"execution":{"iopub.status.busy":"2022-03-26T04:49:56.525461Z","iopub.execute_input":"2022-03-26T04:49:56.525907Z","iopub.status.idle":"2022-03-26T04:49:56.549965Z","shell.execute_reply.started":"2022-03-26T04:49:56.525875Z","shell.execute_reply":"2022-03-26T04:49:56.549100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Filter entries with no image associated","metadata":{}},{"cell_type":"code","source":"def image_exists(path):\n    return os.path.exists(path)\ntrain_df = train_df[[image_exists(f\"../input/sorghum-id-fgvc-9/train_images/{img}\") for img in train_df.image]]","metadata":{"execution":{"iopub.status.busy":"2022-03-26T04:49:56.551260Z","iopub.execute_input":"2022-03-26T04:49:56.551579Z","iopub.status.idle":"2022-03-26T04:50:23.308254Z","shell.execute_reply.started":"2022-03-26T04:49:56.551552Z","shell.execute_reply":"2022-03-26T04:50:23.306806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape,test_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-26T04:50:23.311305Z","iopub.execute_input":"2022-03-26T04:50:23.311547Z","iopub.status.idle":"2022-03-26T04:50:23.317882Z","shell.execute_reply.started":"2022-03-26T04:50:23.311520Z","shell.execute_reply":"2022-03-26T04:50:23.317101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2022-03-26T04:50:23.319092Z","iopub.execute_input":"2022-03-26T04:50:23.319530Z","iopub.status.idle":"2022-03-26T04:50:23.408865Z","shell.execute_reply.started":"2022-03-26T04:50:23.319491Z","shell.execute_reply":"2022-03-26T04:50:23.407467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Encode cultivar","metadata":{}},{"cell_type":"code","source":"le = LabelEncoder()\nle = le.fit(train_df.cultivar)\ndill.dump(le,open(\"le.dill\",'wb'))\ntrain_df[\"target\"] = le.transform(train_df.cultivar)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T04:50:23.410581Z","iopub.execute_input":"2022-03-26T04:50:23.411772Z","iopub.status.idle":"2022-03-26T04:50:23.430081Z","shell.execute_reply.started":"2022-03-26T04:50:23.411701Z","shell.execute_reply":"2022-03-26T04:50:23.428857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-03-26T04:50:23.431995Z","iopub.execute_input":"2022-03-26T04:50:23.432489Z","iopub.status.idle":"2022-03-26T04:50:23.450853Z","shell.execute_reply.started":"2022-03-26T04:50:23.432373Z","shell.execute_reply":"2022-03-26T04:50:23.449678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define the dataset class\n\nWe need to save the class in a .py file and import it","metadata":{}},{"cell_type":"code","source":"%%writefile fgvc_dataset.py -a\n\nimport tensorflow_datasets as tfds\nimport tensorflow as tf\n\nclass FGVCDataset(tfds.core.GeneratorBasedBuilder):\n    VERSION = tfds.core.Version('0.1.0')\n    \n    def _split_generators(self, dl_manager):\n        arr = [\n            tfds.core.SplitGenerator(name=f'train',gen_kwargs={\"split\":\"train\"}),\n            tfds.core.SplitGenerator(name=f'test',gen_kwargs={\"split\":\"test\"})\n        ]\n        return arr\n    \n    def _info(self):\n        return tfds.core.DatasetInfo(\n            builder=self,\n            description=(\"\"),\n            #disable_shuffling=True,\n            features=tfds.features.FeaturesDict({\n                \"img\": tfds.features.Image(encoding_format='jpeg'),#dtype=tf.uint8,shape=(self.WIDTH,self.HEIGHT,3),\n                \"name\": tfds.features.Tensor(dtype=tf.string,shape=()),\n                \"cultivar\": tfds.features.Tensor(dtype=tf.string,shape=()),\n                \"target\": tfds.features.Tensor(dtype=tf.int32,shape=()),\n            }),\n        )\n    \n    def _generate_examples(self,**args):\n        print(args)\n        split = args[\"split\"]\n        \n        if split == 'train':\n            for i in range(len(self.train_df)):\n                row = self.train_df.iloc[i]\n                img = f\"../input/sorghum100cultivarjpgimages512x512/train_images/{row.jpeg_image}\"\n                yield i, {\n                    'img':img,\n                    'cultivar':row.cultivar,\n                    'name':row.image,\n                    'target':row.target,\n                }\n                \n        if split == 'test':\n            for i in range(len(self.test_df)):\n                row = self.test_df.iloc[i]\n                img = f\"../input/sorghum100cultivarjpgimages512x512/test/{row.jpeg_image}\"\n                yield i, {\n                    'img':img,\n                    'name':row.filename,\n                    'cultivar':'',\n                    'target':-1,\n                }","metadata":{"execution":{"iopub.status.busy":"2022-03-26T04:50:23.452604Z","iopub.execute_input":"2022-03-26T04:50:23.453438Z","iopub.status.idle":"2022-03-26T04:50:23.463447Z","shell.execute_reply.started":"2022-03-26T04:50:23.453383Z","shell.execute_reply":"2022-03-26T04:50:23.462722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Images to TFRecords","metadata":{}},{"cell_type":"code","source":"from fgvc_dataset import FGVCDataset","metadata":{"execution":{"iopub.status.busy":"2022-03-26T04:50:23.465084Z","iopub.execute_input":"2022-03-26T04:50:23.466083Z","iopub.status.idle":"2022-03-26T04:50:23.480773Z","shell.execute_reply.started":"2022-03-26T04:50:23.466052Z","shell.execute_reply":"2022-03-26T04:50:23.479686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_dir='fgvc_dataset'\nFGVCDataset.train_df = train_df#.iloc[:100]\nFGVCDataset.test_df = test_df#.iloc[:100]\n\nbuilder = FGVCDataset(data_dir=data_dir)\nbuilder.download_and_prepare() ","metadata":{"execution":{"iopub.status.busy":"2022-03-26T04:50:23.482212Z","iopub.execute_input":"2022-03-26T04:50:23.482715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test the dataset","metadata":{}},{"cell_type":"code","source":"builder = FGVCDataset(data_dir=data_dir)\nbuilder.download_and_prepare() ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"builder.as_dataset()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = builder.as_dataset()['train']\ntest_ds = builder.as_dataset()['test']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x in train_ds.take(1):\n    img = x['img']\n    cultivar = x['cultivar']\n    target = x['target']\n    print(img.shape,cultivar,target)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(img.numpy())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check Training Notebook at  https://www.kaggle.com/tchaye59/efficientnetb0-tensorflow-baseline-tpu","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}