{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport os\nimport re\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\ntqdm.pandas()\nimport torch\n\n\nimport re\nPATTEN = re.compile('\\d+|[A-Z][a-z]?|[^A-Za-z\\d/]|/[a-z]')\ndef l_split(s):\n    return ' '.join(re.findall(PATTEN,s))\n# ====================================================\n# Tokenizer\n# ====================================================\nclass Tokenizer(object):    \n    def __init__(self):\n        self.stoi = {}\n        self.itos = {}\n        self.col2sos = {\n            'formula': '<sos>', \n            'c': '/c', \n            'h': '/h', \n            'b': '/b', \n            't': '/t', \n            'm': '/m',\n            's': '/s',\n            'i': '/i'}\n# [\"c\", \"h\", \"b\", \"t\", \"m\", \"s\", \"i\"]\n    def __len__(self):\n        return len(self.stoi)\n    \n    def fit_on_texts(self): \n        vocab = [str(i) for i in range(180)]\n        vocab.extend(['(', ')', '+', ',', '-', '/b', '/c', '/h', '/i', '/m', '/s', '/t'\n                         , 'B', 'Br', 'C', 'Cl', 'F', 'H', 'I', 'N', 'O', 'P', 'S', 'Si', 'T', 'D'\n                         , '<sos>', '<eos>', '<pad>', 'X'])\n#         ['B', 'Br', 'C', 'Cl', 'F', 'H', 'I', 'N', 'O', 'P', 'S', 'Si']\n#         {'b', 'm', 'i', 'c', 't', 's', 'h'}, '<empty>'\n        for i, s in enumerate(vocab):\n            self.stoi[s] = i\n        self.itos = {item[1]: item[0] for item in self.stoi.items()}\n        \n    def text_to_sequence(self, text, col):\n        sequence = []\n        sequence.append(self.stoi[self.col2sos[col]])\n#         if col == 'i' and text == 'i':\n#             sequence.append(self.stoi['<empty>'])\n        for s in text.split(' '):\n            sequence.append(self.stoi[s])\n        sequence.append(self.stoi['<eos>'])\n        return sequence\n    \n    def texts_to_sequences(self, texts):\n        sequences = []\n        for text in texts:\n            sequence = self.text_to_sequence(text)\n            sequences.append(sequence)\n        return sequences\n\n    def sequence_to_text(self, sequence):\n        return ''.join(list(map(lambda i: self.itos[i], sequence)))\n    \n    def sequences_to_texts(self, sequences):\n        texts = []\n        for sequence in sequences:\n            text = self.sequence_to_text(sequence)\n            texts.append(text)\n        return texts\n    \n    def predict_caption(self, sequence):\n        caption = ''\n        for i in sequence:\n            if i == self.stoi['<eos>'] or i == self.stoi['<pad>']:\n                break\n            caption += self.itos[i]\n        return caption\n    \n    def predict_captions(self, sequences):\n        captions = []\n        for sequence in sequences:\n            caption = self.predict_caption(sequence)\n            captions.append(caption)\n        return captions\n    \n# assemble: c -- h -- b-- t -- m  -- s  -- i","metadata":{"execution":{"iopub.status.busy":"2021-05-24T23:59:55.672884Z","iopub.execute_input":"2021-05-24T23:59:55.673260Z","iopub.status.idle":"2021-05-24T23:59:56.751694Z","shell.execute_reply.started":"2021-05-24T23:59:55.673182Z","shell.execute_reply":"2021-05-24T23:59:56.750865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Data Loading\n# ====================================================\ntrain = pd.read_csv('../input/arranged-bms-train-labels/arranged_bms_train_labels.csv')\nprint(f'train.shape: {train.shape}')","metadata":{"execution":{"iopub.status.busy":"2021-05-24T23:59:56.755767Z","iopub.execute_input":"2021-05-24T23:59:56.756064Z","iopub.status.idle":"2021-05-25T00:00:25.710873Z","shell.execute_reply.started":"2021-05-24T23:59:56.756038Z","shell.execute_reply":"2021-05-25T00:00:25.709730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = train[:1000]","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:00:25.712535Z","iopub.execute_input":"2021-05-25T00:00:25.712936Z","iopub.status.idle":"2021-05-25T00:00:25.717102Z","shell.execute_reply.started":"2021-05-25T00:00:25.712903Z","shell.execute_reply":"2021-05-25T00:00:25.716085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:00:25.719095Z","iopub.execute_input":"2021-05-25T00:00:25.719545Z","iopub.status.idle":"2021-05-25T00:00:25.731780Z","shell.execute_reply.started":"2021-05-25T00:00:25.719502Z","shell.execute_reply":"2021-05-25T00:00:25.730793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# seq_columns = ['formula', \"c\", \"h\", \"b\", \"t\", \"m\", \"s\", \"i\"]\nseq_columns = ('formula','c', 'h', 'b', 't', 'i')","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:00:25.736049Z","iopub.execute_input":"2021-05-25T00:00:25.736490Z","iopub.status.idle":"2021-05-25T00:00:25.746297Z","shell.execute_reply.started":"2021-05-25T00:00:25.736435Z","shell.execute_reply":"2021-05-25T00:00:25.744957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for c in seq_columns:\n    train[c] = (train[c].progress_apply(l_split))","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:00:25.749947Z","iopub.execute_input":"2021-05-25T00:00:25.750260Z","iopub.status.idle":"2021-05-25T00:01:56.175469Z","shell.execute_reply.started":"2021-05-25T00:00:25.750229Z","shell.execute_reply":"2021-05-25T00:01:56.174205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# c = 'i'\n# train[c] = (train[c].progress_apply(split_form2))","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:01:56.176920Z","iopub.execute_input":"2021-05-25T00:01:56.177306Z","iopub.status.idle":"2021-05-25T00:01:56.181359Z","shell.execute_reply.started":"2021-05-25T00:01:56.177266Z","shell.execute_reply":"2021-05-25T00:01:56.180051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.i[train.i_flg == 1]","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:01:56.182420Z","iopub.execute_input":"2021-05-25T00:01:56.182677Z","iopub.status.idle":"2021-05-25T00:01:56.223473Z","shell.execute_reply.started":"2021-05-25T00:01:56.182651Z","shell.execute_reply":"2021-05-25T00:01:56.222305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.loc[:, seq_columns].head()","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:01:56.224915Z","iopub.execute_input":"2021-05-25T00:01:56.225301Z","iopub.status.idle":"2021-05-25T00:01:56.396330Z","shell.execute_reply.started":"2021-05-25T00:01:56.225269Z","shell.execute_reply":"2021-05-25T00:01:56.395296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = Tokenizer()\ntokenizer.fit_on_texts()\nprint(f\"tokenizer.stoi: {tokenizer.stoi}\")\ntorch.save(tokenizer, 'tokenizer2.pth')\nprint('Saved tokenizer')","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:01:56.397444Z","iopub.execute_input":"2021-05-25T00:01:56.397785Z","iopub.status.idle":"2021-05-25T00:01:56.404471Z","shell.execute_reply.started":"2021-05-25T00:01:56.397757Z","shell.execute_reply":"2021-05-25T00:01:56.403533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_pickle('train2.pkl')\nprint('Saved preprocessed train.pkl')","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:01:56.405728Z","iopub.execute_input":"2021-05-25T00:01:56.406026Z","iopub.status.idle":"2021-05-25T00:02:05.248222Z","shell.execute_reply.started":"2021-05-25T00:01:56.406000Z","shell.execute_reply":"2021-05-25T00:02:05.247408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:02:05.249527Z","iopub.execute_input":"2021-05-25T00:02:05.249926Z","iopub.status.idle":"2021-05-25T00:02:05.272020Z","shell.execute_reply.started":"2021-05-25T00:02:05.249882Z","shell.execute_reply":"2021-05-25T00:02:05.270923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for c in seq_columns[3:]:\n    print(c)\n    for text in tqdm(train[c].values):\n        seq = tokenizer.text_to_sequence(text, c)","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:02:05.273550Z","iopub.execute_input":"2021-05-25T00:02:05.273944Z","iopub.status.idle":"2021-05-25T00:02:14.206387Z","shell.execute_reply.started":"2021-05-25T00:02:05.273904Z","shell.execute_reply":"2021-05-25T00:02:14.205459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=train.replace('X', '')","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:02:14.207621Z","iopub.execute_input":"2021-05-25T00:02:14.207895Z","iopub.status.idle":"2021-05-25T00:02:16.357702Z","shell.execute_reply.started":"2021-05-25T00:02:14.207869Z","shell.execute_reply":"2021-05-25T00:02:16.355430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def assemble(formula, c, h, b, t, m, s, i):    \n    inchis = f\"InChI=1S/{formula}/c{c}\"\n    if h != '':\n        inchis += f'/h{h}'\n    if b != '':\n        inchis += f'/b{b}'\n    if t != '':\n        inchis += f'/t{t}'\n    if m != '':\n        inchis += f'/{m}'\n    if s != '':\n        inchis += f'/{s}'\n    if i != '':\n        inchis += f'/i{i}'\n    return inchis.replace(' ', '')","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:05:01.630448Z","iopub.execute_input":"2021-05-25T00:05:01.630801Z","iopub.status.idle":"2021-05-25T00:05:01.636721Z","shell.execute_reply.started":"2021-05-25T00:05:01.630770Z","shell.execute_reply":"2021-05-25T00:05:01.635658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_columns = ['formula', \"c\", \"h\", \"b\", \"t\", \"m\", \"s\", \"i\"]\ninchi_preds = np.array([assemble(f, c, h, b, t, m, s, i) for f, c, h, b, t, m, s, i in zip(*[train[c1] for c1 in all_columns])])","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:05:02.500441Z","iopub.execute_input":"2021-05-25T00:05:02.500947Z","iopub.status.idle":"2021-05-25T00:05:14.897648Z","shell.execute_reply.started":"2021-05-25T00:05:02.500907Z","shell.execute_reply":"2021-05-25T00:05:14.896704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.m.unique(), train.s.unique()","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:05:21.459444Z","iopub.execute_input":"2021-05-25T00:05:21.460044Z","iopub.status.idle":"2021-05-25T00:05:21.676676Z","shell.execute_reply.started":"2021-05-25T00:05:21.460007Z","shell.execute_reply":"2021-05-25T00:05:21.675942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[train[c1][0] for c1 in seq_columns]","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:05:23.081055Z","iopub.execute_input":"2021-05-25T00:05:23.081653Z","iopub.status.idle":"2021-05-25T00:05:23.087556Z","shell.execute_reply.started":"2021-05-25T00:05:23.081617Z","shell.execute_reply":"2021-05-25T00:05:23.086369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eq = ((inchi_preds) == train.InChI)\neq.all()","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:05:29.572549Z","iopub.execute_input":"2021-05-25T00:05:29.572881Z","iopub.status.idle":"2021-05-25T00:05:31.075410Z","shell.execute_reply.started":"2021-05-25T00:05:29.572829Z","shell.execute_reply":"2021-05-25T00:05:31.074257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neq = np.logical_not(eq)\nneq.sum()","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:05:31.264194Z","iopub.execute_input":"2021-05-25T00:05:31.264508Z","iopub.status.idle":"2021-05-25T00:05:31.273672Z","shell.execute_reply.started":"2021-05-25T00:05:31.264482Z","shell.execute_reply":"2021-05-25T00:05:31.272656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# i=0\n# inchi_preds[neq][0]","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:05:44.368679Z","iopub.execute_input":"2021-05-25T00:05:44.369469Z","iopub.status.idle":"2021-05-25T00:05:44.373463Z","shell.execute_reply.started":"2021-05-25T00:05:44.369426Z","shell.execute_reply":"2021-05-25T00:05:44.372312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(inchi_preds[0])\nprint(train.InChI[0])","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:05:46.133632Z","iopub.execute_input":"2021-05-25T00:05:46.133963Z","iopub.status.idle":"2021-05-25T00:05:46.139417Z","shell.execute_reply.started":"2021-05-25T00:05:46.133931Z","shell.execute_reply":"2021-05-25T00:05:46.138568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# vocabs = ['formula_vocab'] + [f'{c}_vocab' for c in seq_columns]","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:02:34.992816Z","iopub.execute_input":"2021-05-25T00:02:34.993215Z","iopub.status.idle":"2021-05-25T00:02:35.006796Z","shell.execute_reply.started":"2021-05-25T00:02:34.993174Z","shell.execute_reply":"2021-05-25T00:02:35.005975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def length(text):\n    return len(text.split(' '))","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:02:35.008057Z","iopub.execute_input":"2021-05-25T00:02:35.008620Z","iopub.status.idle":"2021-05-25T00:02:35.020708Z","shell.execute_reply.started":"2021-05-25T00:02:35.008568Z","shell.execute_reply":"2021-05-25T00:02:35.019779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_length = {}\nfor v in seq_columns:\n    l = train[v].apply(length)\n    max_length[v] = l.max()\n    print(v, max_length[v])","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:02:35.022395Z","iopub.execute_input":"2021-05-25T00:02:35.022781Z","iopub.status.idle":"2021-05-25T00:02:45.731958Z","shell.execute_reply.started":"2021-05-25T00:02:35.022742Z","shell.execute_reply":"2021-05-25T00:02:45.730797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(max_length, 'max_length.pth')\nprint('Saved max_length')","metadata":{"execution":{"iopub.status.busy":"2021-05-25T00:02:45.733231Z","iopub.execute_input":"2021-05-25T00:02:45.733495Z","iopub.status.idle":"2021-05-25T00:02:45.739226Z","shell.execute_reply.started":"2021-05-25T00:02:45.733466Z","shell.execute_reply":"2021-05-25T00:02:45.738299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}