{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n    #for filename in filenames:\n        #print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom keras.preprocessing.sequence import pad_sequences\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os, cv2\nimport dill\nimport tensorflow_datasets.public_api as tfds\nimport albumentations as A\nimport tensorflow as tf\nimport json\nfrom mt_utils import *\nimport dill","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the API key for the dataset\n! mkdir -p /root/.kaggle/\n! cp ../input/api-token/kaggle.json /root/.kaggle/kaggle.json\n! mkdir -p /kaggle/tmp/mt_train\n! kaggle datasets init -p /kaggle/tmp/mt_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\necho \"{\n  \\\"title\\\": \\\"MTCustomVocabImg\\\",\n  \\\"id\\\": \\\"tchaye59/MTCUSTOMVOCABIMG\\\",\n  \\\"licenses\\\": [\n    {\n      \\\"name\\\": \\\"CC0-1.0\\\"\n    }\n  ]\n}\" > /kaggle/tmp/mt_train/dataset-metadata.json","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/bms-molecular-translation/train_labels.csv')\nsub_df = pd.read_csv('../input/bms-molecular-translation/sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get the tokenizer from https://www.kaggle.com/tchaye59/mt-utils","metadata":{}},{"cell_type":"code","source":"tokenizer = CstTokenizer()\nN_VOCAB = len(tokenizer.word_index)+1\ntokenizer.word_index","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nstart = '<start>'\nend = '<end>'\n# Load tokenized labels\nlabels = dill.load(open('/kaggle/usr/lib/mt_utils/labels.dill','rb'))\ncount_elements = dill.load(open('/kaggle/usr/lib/mt_utils/count_elements.dill','rb'))\n\niids = df.image_id.values\nmax_seq = max([len(l) for l in labels])\nmax_seq,N_VOCAB","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class Dataset:\n    \n    def __init__(self, iids,targets=None,counts=None,max_seq=max_seq):\n        \n        self.iids,self.targets,self.counts = iids,targets,counts\n        self.max_seq = max_seq\n\n    def __len__(self):\n        return len(self.iids)\n\n    def __getitem__(self, index):\n        iid = self.iids[index]\n        if self.targets is None:\n            return self.get_image(iid),iid\n        label = self.targets[index]\n        label = pad_sequences([label], maxlen=self.max_seq, padding='post')[0].astype(np.int8)\n        return self.get_image(iid),label,self.counts[index]\n\n    def get_image(self, iid):\n        name = 'train' if self.targets is not None else 'test'\n        path = \"../input/bms-molecular-translation/\"+name+\"/{}/{}/{}/{}.png\"\n        path = path.format(iid[0], iid[1], iid[2], iid)\n        return cv2.imread(path, cv2.IMREAD_UNCHANGED)[:,:,np.newaxis]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = Dataset(iids,labels,count_elements)\ntest_dataset = Dataset(sub_df.image_id.values)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(dataset[100][0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(test_dataset[3][0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset to tf records ","metadata":{}},{"cell_type":"code","source":"class TrainDataset(tfds.core.GeneratorBasedBuilder):\n    VERSION = tfds.core.Version('0.1.0')\n    \n    def _split_generators(self, dl_manager):\n        return [\n            tfds.core.SplitGenerator(\n                    name=f'train',\n                    gen_kwargs={\n                    },\n            )\n        ]\n    \n    def _info(self):\n        return tfds.core.DatasetInfo(\n            builder=self,\n            description=(\"\"),\n            features=tfds.features.FeaturesDict({\n                \"image\": tfds.features.Image(shape=(None,None,1)),\n                \"target\": tfds.features.Tensor(shape=(max_seq,),dtype=tf.int8),\n                \"count\": tfds.features.Tensor(dtype=tf.int32,shape=()),\n            }),\n        )\n    \n    def _generate_examples(self,**args):\n        print(f\"Data size: {len(dataset)}\")\n        for i in range(len(dataset)):\n            image,target,count = dataset[i]\n            yield i, {\n                'image':image,\n                'target':target,\n                'count':count,\n            }","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataset(tfds.core.GeneratorBasedBuilder):\n    VERSION = tfds.core.Version('0.1.0')\n    \n    def _split_generators(self, dl_manager):\n        return [\n            tfds.core.SplitGenerator(\n                    name=f'test',\n                    gen_kwargs={\n                    },\n            )\n        ]\n    \n    def _info(self):\n        return tfds.core.DatasetInfo(\n            builder=self,\n            description=(\"\"),\n            features=tfds.features.FeaturesDict({\n                \"image\": tfds.features.Image(shape=(None,None,1),),\n                \"image_id\": tfds.features.Text(),\n            }),\n        )\n    \n    def _generate_examples(self,**args):\n        print(f\"Data size: {len(test_dataset)}\")\n        for i in range(len(test_dataset)):\n            image,image_id = test_dataset[i]\n            yield i, {\n                'image':image,\n                'image_id':image_id,\n            }","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training dataset","metadata":{}},{"cell_type":"code","source":"#! cp -rv ../input/mtcustomvocabimg/* /kaggle/tmp/mt_train","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_dir='/kaggle/tmp/mt_train' \nbuilder = TrainDataset(data_dir=data_dir)\n# The following line creates the train dataset folder containing the tf records files in /kaggle/input\nbuilder.download_and_prepare() ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test data","metadata":{}},{"cell_type":"code","source":"! cp -rv ../input/mtcustomvocabimg/test_dataset/ /kaggle/tmp/mt_train","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_dir='/kaggle/tmp/mt_train' \nbuilder = TestDataset(data_dir=data_dir)\n# The following line creates the test dataset folder containing the tf records files in /kaggle/input\nbuilder.download_and_prepare() ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Upload or Update dataset","metadata":{}},{"cell_type":"code","source":"! kaggle datasets version -p /kaggle/tmp/mt_train -m \"update\"  --dir-mode tar\n#! kaggle datasets create -p /kaggle/tmp/mt_train/ -u --dir-mode tar","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Delete Kaggle API key\n! rm -rf /root/.kaggle/kaggle.json","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Dataset link : https://www.kaggle.com/tchaye59/mtcustomvocabimg\n#### Pretraining : https://www.kaggle.com/tchaye59/mt-pretraining\n#### Training: https://www.kaggle.com/tchaye59/mt-fast-distributed-training-tpu","metadata":{}}]}