{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport os\nimport re\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\ntqdm.pandas()\nimport torch\n\n# ====================================================\n# Data Loading\n# ====================================================\ntrain = pd.read_csv('../input/bms-molecular-translation/train_labels.csv')\nprint(f'train.shape: {train.shape}')\n\n# ====================================================\n# Preprocess functions\n# ====================================================\ndef split_form(form):\n    string = ''\n    for i in re.findall(r\"[A-Z][^A-Z]*\", form):\n        elem = re.match(r\"\\D+\", i).group()\n        num = i.replace(elem, \"\")\n        if num == \"\":\n            string += f\"{elem} \"\n        else:\n            string += f\"{elem} {str(num)} \"\n    return string.rstrip(' ')\n\ndef split_form2(form):\n    string = ''\n    for i in re.findall(r\"[a-z][^a-z]*\", form):\n        elem = i[0]\n        num = i.replace(elem, \"\").replace('/', \"\")\n        num_string = ''\n        for j in re.findall(r\"[0-9]+[^0-9]*\", num):\n            num_list = list(re.findall(r'\\d+', j))\n            assert len(num_list) == 1, f\"len(num_list) != 1\"\n            _num = num_list[0]\n            if j == _num:\n                num_string += f\"{_num} \"\n            else:\n                extra = j.replace(_num, \"\")\n                num_string += f\"{_num} {' '.join(list(extra))} \"\n        string += f\"/{elem} {num_string}\"\n    return string.rstrip(' ')\n\n# ====================================================\n# Tokenizer\n# ====================================================\nclass Tokenizer(object):\n    \n    def __init__(self):\n        self.stoi = {}\n        self.itos = {}\n\n    def __len__(self):\n        return len(self.stoi)\n    \n    def fit_on_texts(self, texts):\n        vocab = set()\n        for text in texts:\n            vocab.update(text.split(' '))\n        vocab = sorted(vocab)\n        vocab.append('<sos>')\n        vocab.append('<eos>')\n        vocab.append('<pad>')\n        for i, s in enumerate(vocab):\n            self.stoi[s] = i\n        self.itos = {item[1]: item[0] for item in self.stoi.items()}\n        \n    def text_to_sequence(self, text):\n        sequence = []\n        sequence.append(self.stoi['<sos>'])\n        for s in text.split(' '):\n            sequence.append(self.stoi[s])\n        sequence.append(self.stoi['<eos>'])\n        return sequence\n    \n    def texts_to_sequences(self, texts):\n        sequences = []\n        for text in texts:\n            sequence = self.text_to_sequence(text)\n            sequences.append(sequence)\n        return sequences\n\n    def sequence_to_text(self, sequence):\n        return ''.join(list(map(lambda i: self.itos[i], sequence)))\n    \n    def sequences_to_texts(self, sequences):\n        texts = []\n        for sequence in sequences:\n            text = self.sequence_to_text(sequence)\n            texts.append(text)\n        return texts\n    \n    def predict_caption(self, sequence):\n        caption = ''\n        for i in sequence:\n            if i == self.stoi['<eos>'] or i == self.stoi['<pad>']:\n                break\n            caption += self.itos[i]\n        return caption\n    \n    def predict_captions(self, sequences):\n        captions = []\n        for sequence in sequences:\n            caption = self.predict_caption(sequence)\n            captions.append(caption)\n        return captions","metadata":{"_uuid":"01b3b45d-ded2-4582-ad2c-43cb0395fe2f","_cell_guid":"61ee2a6e-1535-43b6-ae7f-6202c7ddda7a","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# main\n# ====================================================\ndef main():\n    # ====================================================\n    # preprocess train.csv\n    # ====================================================\n    train['InChI_1'] = train['InChI'].progress_apply(lambda x: x.split('/')[1])\n    train['InChI_text'] = train['InChI_1'].progress_apply(split_form) + ' ' + \\\n                            train['InChI'].apply(lambda x: '/'.join(x.split('/')[2:])).progress_apply(split_form2).values\n    # ====================================================\n    # create tokenizer\n    # ====================================================\n    tokenizer = Tokenizer()\n    tokenizer.fit_on_texts(train['InChI_text'].values)\n    torch.save(tokenizer, 'tokenizer2.pth')\n    print('Saved tokenizer')\n    # ====================================================\n    # preprocess train.csv\n    # ====================================================\n    lengths = []\n    tk0 = tqdm(train['InChI_text'].values, total=len(train))\n    for text in tk0:\n        seq = tokenizer.text_to_sequence(text)\n        length = len(seq) - 2\n        lengths.append(length)\n    train['InChI_length'] = lengths\n    train.to_pickle('train2.pkl')\n    print('Saved preprocessed train.pkl')","metadata":{"_uuid":"120e6571-1b41-4163-b38d-0d0080fe3228","_cell_guid":"e6f3e263-4c4b-44d6-87c5-840dbb5e7fe3","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    main()","metadata":{"_uuid":"d18d7d26-e1ce-4274-9613-351ce3f5f425","_cell_guid":"975662bf-4106-4d4a-aa7a-66168c01b279","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}