{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport os\nimport re\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\ntqdm.pandas()\nimport torch\n\n# ====================================================\n# Preprocess functions\n# ====================================================\ndef split_form(form):\n    string = ''\n    for i in re.findall(r\"[A-Z][^A-Z]*\", form):\n        elem = re.match(r\"\\D+\", i).group()\n        num = i.replace(elem, \"\")\n        if num == \"\":\n            string += f\"{elem} \"\n        else:\n            string += f\"{elem} {str(num)} \"\n    return string.rstrip(' ')\n\ndef split_form2(form):\n    string = ''\n    for i in re.findall(r\"[a-z][^a-z]*\", form):\n        elem = i[0]\n        num = i.replace(elem, \"\").replace('/', \"\")\n        num_string = ''\n        for j in re.findall(r\"[0-9]+[^0-9]*\", num):\n            num_list = list(re.findall(r'\\d+', j))\n            assert len(num_list) == 1, f\"len(num_list) != 1\"\n            _num = num_list[0]\n            if j == _num:\n                num_string += f\"{_num} \"\n            else:\n                extra = j.replace(_num, \"\")\n                num_string += f\"{_num} {' '.join(list(extra))} \"\n        string += f\"/{elem} {num_string}\"\n    return string.rstrip(' ')\n\nimport re\nPATTEN = re.compile('\\d+|[A-Z][a-z]?|[^A-Za-z\\d/]|/[a-z]')\ndef l_split(s):\n    return ' '.join(re.findall(PATTEN,s))\n\n# ====================================================\n# Tokenizer\n# ====================================================\nclass Tokenizer(object):\n    \n    def __init__(self):\n        self.stoi = {}\n        self.itos = {}\n\n    def __len__(self):\n        return len(self.stoi)\n    \n    def fit_on_texts(self): \n        vocab = [str(i) for i in range(180)]\n        vocab.extend(['(', ')', '+', ',', '-', '/b', '/c', '/h', '/i', '/m', '/s', '/t'\n                         , 'B', 'Br', 'C', 'Cl', 'F', 'H', 'I', 'N', 'O', 'P', 'S', 'Si', 'T', 'D'\n                         , '<sos>', '<eos>', '<pad>'])\n#         ['B', 'Br', 'C', 'Cl', 'F', 'H', 'I', 'N', 'O', 'P', 'S', 'Si']\n#         {'b', 'm', 'i', 'c', 't', 's', 'h'}\n        for i, s in enumerate(vocab):\n            self.stoi[s] = i\n        self.itos = {item[1]: item[0] for item in self.stoi.items()}\n\n    \n#     def fit_on_texts(self, texts):\n#         vocab = set()\n#         for text in texts:\n#             vocab.update(text.split(' '))\n#         vocab = sorted(vocab)\n#         vocab.append('<sos>')\n#         vocab.append('<eos>')\n#         vocab.append('<pad>')\n#         for i, s in enumerate(vocab):\n#             self.stoi[s] = i\n#         self.itos = {item[1]: item[0] for item in self.stoi.items()}\n        \n    def text_to_sequence(self, text):\n        sequence = []\n        sequence.append(self.stoi['<sos>'])\n        for s in text.split(' '):\n            sequence.append(self.stoi[s])\n        sequence.append(self.stoi['<eos>'])\n        return sequence\n    \n    def texts_to_sequences(self, texts):\n        sequences = []\n        for text in texts:\n            sequence = self.text_to_sequence(text)\n            sequences.append(sequence)\n        return sequences\n\n    def sequence_to_text(self, sequence):\n        return ''.join(list(map(lambda i: self.itos[i], sequence)))\n    \n    def sequences_to_texts(self, sequences):\n        texts = []\n        for sequence in sequences:\n            text = self.sequence_to_text(sequence)\n            texts.append(text)\n        return texts\n    \n    def predict_caption(self, sequence):\n        caption = ''\n        for i in sequence:\n            if i == self.stoi['<eos>'] or i == self.stoi['<pad>']:\n                break\n            caption += self.itos[i]\n        return caption\n    \n    def predict_captions(self, sequences):\n        captions = []\n        for sequence in sequences:\n            caption = self.predict_caption(sequence)\n            captions.append(caption)\n        return captions","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:19:32.893492Z","iopub.execute_input":"2021-05-24T15:19:32.893813Z","iopub.status.idle":"2021-05-24T15:19:32.920079Z","shell.execute_reply.started":"2021-05-24T15:19:32.893784Z","shell.execute_reply":"2021-05-24T15:19:32.919061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Data Loading\n# ====================================================\ntrain = pd.read_csv('../input/bms-molecular-translation/train_labels.csv')\nprint(f'train.shape: {train.shape}')","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:19:32.922376Z","iopub.execute_input":"2021-05-24T15:19:32.922649Z","iopub.status.idle":"2021-05-24T15:19:40.070345Z","shell.execute_reply.started":"2021-05-24T15:19:32.922622Z","shell.execute_reply":"2021-05-24T15:19:40.069704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# main\n# ====================================================\n# def main():\n# ====================================================\n# preprocess train.csv\n# ====================================================\ntrain['InChI_text'] = train['InChI'].progress_apply(lambda x: l_split(x[9:]))\n# ====================================================\n# create tokenizer\n# ====================================================\ntokenizer = Tokenizer()\ntokenizer.fit_on_texts()\n#     train['InChI_text'].values\nprint(f\"tokenizer.stoi: {tokenizer.stoi.keys()}\")\ntorch.save(tokenizer, 'tokenizer2.pth')\nprint('Saved tokenizer')\n# ====================================================\n# preprocess train.csv\n# ====================================================\nlengths = []\ntk0 = tqdm(train['InChI_text'].values, total=len(train))\nfor text in tk0:\n    seq = tokenizer.text_to_sequence(text)\n    length = len(seq) - 2\n    lengths.append(length)\ntrain['InChI_length'] = lengths\ntrain.to_pickle('train2.pkl')\nprint('Saved preprocessed train.pkl')\nprint(train['InChI_text'].iloc[2])","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:19:40.077422Z","iopub.execute_input":"2021-05-24T15:19:40.077667Z","iopub.status.idle":"2021-05-24T15:19:40.158704Z","shell.execute_reply.started":"2021-05-24T15:19:40.077645Z","shell.execute_reply":"2021-05-24T15:19:40.156759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wrong = []\nfor i in tqdm(range(len(train))):\n    new = l_split(train.InChI[i][9:])\n    if train['InChI_text'][i].replace(\" \", \"\") != train.InChI[i][9:]:\n        wrong.append(i)\nprint(len(wrong))","metadata":{"execution":{"iopub.status.busy":"2021-05-24T15:19:40.159651Z","iopub.execute_input":"2021-05-24T15:19:40.159847Z","iopub.status.idle":"2021-05-24T15:19:40.209866Z","shell.execute_reply.started":"2021-05-24T15:19:40.159826Z","shell.execute_reply":"2021-05-24T15:19:40.208151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}