{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# coding: utf-8\n__author__ = 'ZFTurbo: https://kaggle.com/zfturbo'\n\n\nUSE_PARALLEL = False\nUSE_RDKIT = False\nLIMIT_CHECKS = 500\nPROC_NUM = 16\nDEBUG_LIMIT = 100000000\n\n\nimport sys\nif USE_RDKIT:\n    from rdkit import Chem\n    from rdkit import RDLogger\n\nif __name__ == '__main__':\n    if len(sys.argv) == 2:\n        inchi = sys.argv[1].strip()\n        try:\n            mol = Chem.MolFromInchi(inchi)\n            if mol is None:\n                print(0, end='')\n            else:\n                print(1, end='')\n        except Exception as e:\n            print(0, end='')\n        exit()\n\n\nimport Levenshtein\nimport warnings\nimport subprocess\nimport itertools\nfrom multiprocessing import Process\nimport gc\nimport numpy as np\nimport pandas as pd\nimport os\nimport time\nimport glob\nimport operator\n\n\nwarnings.simplefilter(\"ignore\")\nwarnings.filterwarnings(\"ignore\", message='ERROR:')\n\n\ndef get_avg_strings(s1, s2):\n    dist = Levenshtein.distance(s1, s2)\n    s1_new = s1\n    intermediate_strings = [s1_new]\n    for i in range(dist):\n        e = Levenshtein.editops(s1_new, s2)\n        s1_new = Levenshtein.apply_edit(e[:1], s1_new, s2)\n        intermediate_strings.append(s1_new)\n    check = Levenshtein.distance(s2, s1_new)\n    if check != 0:\n        print('Some error here!')\n        exit()\n    return intermediate_strings[1:-1]\n\n\ndef check_if_valid_exec_v2(inchi: str) -> int:\n    from subprocess import check_output\n    try:\n        script_path = os.path.realpath(__file__)\n        ret = check_output([sys.executable, script_path, inchi], universal_newlines=True)\n        if ret == '1':\n            return 1\n        else:\n            return 0\n    except Exception as e:\n        print('Exception: {}'.format(e))\n        return 0\n\n\ndef normalize_inchi(inchi):\n    try:\n        mol = Chem.MolFromInchi(inchi)\n        return 'bad' if (mol is None) else Chem.MolToInchi(mol)\n    except:\n        return 'bad'\n\n\ndef merge_all_variants(str1):\n    res = dict()\n    # First generate all possible conmbinations\n    data = list(itertools.product(*str1))\n    # Now sum distances for all parts and create new array\n    for i in range(len(data)):\n        dist = 0\n        str_list = []\n        for j in range(len(data[i])):\n            dist += data[i][j][1]\n            str_list.append(data[i][j][0])\n        str_data = '/'.join(str_list)\n        res[str_data] = dist\n\n    res = sort_dict_by_values(res, reverse=False)\n    return res\n\n\ndef sort_dict_by_values(a, reverse=True):\n    sorted_x = sorted(a.items(), key=operator.itemgetter(1), reverse=reverse)\n    return sorted_x\n\n\ndef get_score(y_true, y_pred):\n    scores = np.zeros(len(y_true), dtype=np.float32)\n    for i in range(len(y_true)):\n        score = Levenshtein.distance(y_true[i], y_pred[i])\n        scores[i] = score\n    avg_score = np.mean(scores)\n    return avg_score\n\n\ndef ensemble_v5_partial_with_rdkit(subm_list, use_valid_check, validation, parts_number=16, current_part=None,\n                                   exclude_ids=None):\n    verbose = False\n    error_fixed = 0\n    error_not_fixed = 0\n    no_error = 0\n\n    print('Start part: {} from {}'.format(current_part, parts_number))\n    start_time = time.time()\n    st = []\n    ids_intersection = set()\n    for path, w in subm_list:\n        if DEBUG_LIMIT > 0:\n            s = pd.read_csv(path, nrows=DEBUG_LIMIT)\n        else:\n            s = pd.read_csv(path)\n        if exclude_ids is not None:\n            cond = ~s['image_id'].isin(exclude_ids)\n            s = s[cond]\n        s = s.sort_values('image_id')\n        s.reset_index(inplace=True, drop=True)\n        if current_part is not None:\n            start = current_part * (len(s) // parts_number)\n            if current_part == parts_number - 1:\n                end = len(s)\n            else:\n                end = (current_part + 1) * (len(s) // parts_number)\n            print('Process: {:2d} Start: {:8d} End: {:8d} Subm: {}'.format(current_part, start, end,\n                                                                           os.path.basename(path)))\n            s = s[start:end]\n        if validation is False:\n            s['prediction'] = s['InChI']\n        s.reset_index(inplace=True, drop=True)\n        st.append(s.copy())\n        if len(ids_intersection) == 0:\n            ids_intersection = set(s['image_id'])\n        else:\n            ids_intersection &= set(s['image_id'])\n        s = None\n        gc.collect()\n\n    print('IDs to use: {}'.format(len(ids_intersection)))\n\n    if validation:\n        for i in range(len(st)):\n            score1 = get_score(st[i]['InChI'].values, st[i]['prediction'].values)\n            st[i] = st[i][st[i]['image_id'].isin(ids_intersection)]\n            st[i].reset_index(inplace=True, drop=True)\n            score2 = get_score(st[i]['InChI'].values, st[i]['prediction'].values)\n            print(\"Score full: {:.6f} Score same IDs: {:.6f} File: {}\".format(score1, score2,\n                                                                              os.path.basename(subm_list[i][0])))\n    else:\n        for i in range(len(st)):\n            st[i] = st[i][st[i]['image_id'].isin(ids_intersection)]\n            st[i].reset_index(inplace=True, drop=True)\n\n    if 0:\n        # Subm difference\n        print('Init diff')\n        for i in range(len(st)):\n            for j in range(i + 1, len(st)):\n                diff = get_score(st[i]['prediction'].values, st[j]['prediction'].values)\n                print('Diff between {} and {}: {:.4f}'.format(i, j, diff))\n\n    cond = (st[0]['prediction'] != st[0]['prediction'])\n    for i in range(len(st)):\n        for j in range(len(st)):\n            if i == j:\n                continue\n            cond |= (st[i]['prediction'] != st[j]['prediction'])\n\n    for i in range(len(st)):\n        st[i] = st[i][cond].copy()\n        print(len(st[i]))\n\n    if 0:\n        # Subm difference\n        print('After removing same')\n        for i in range(len(st)):\n            for j in range(i + 1, len(st)):\n                diff = get_score(st[i]['prediction'].values, st[j]['prediction'].values)\n                print('Diff between {} and {}: {:.4f}'.format(i, j, diff))\n\n    s = st[0][['image_id', 'InChI']]\n    feats = []\n    for i in range(len(st)):\n        s[str(i)] = st[i]['prediction']\n        feats.append(str(i))\n\n    good = []\n    match_stat = np.zeros(len(st), dtype=np.int32)\n    ids = s['image_id'].values\n    matrix = s[feats].values\n    invalid_choices = 0\n    valid_choices = 0\n\n    print('After exclude same inchis. IDs to process: {} Size of DF: {}'.format(len(ids), len(s)))\n    for image_number in range(len(ids)):\n        id = ids[image_number]\n        print('ID: {} Number: {} from {} Time: {:.2f} sec'.format(id, image_number, len(ids), time.time() - start_time))\n        p = matrix[image_number]\n\n        parts = []\n        lengths = dict()\n        indexes_order = dict()\n        for i in range(0, len(p)):\n            parts.append(p[i].split('/'))\n            l1 = len(parts[-1])\n            if l1 in lengths:\n                lengths[l1] += 1\n            else:\n                lengths[l1] = 1\n            indexes = []\n            for j in range(2, l1):\n                if len(parts[-1][j]) > 0:\n                    indexes.append(parts[-1][j][0])\n                else:\n                    indexes.append('')\n            indexes = tuple(indexes)\n            if indexes in indexes_order:\n                indexes_order[indexes] += 1\n            else:\n                indexes_order[indexes] = 1\n        lengths = sort_dict_by_values(lengths)\n        indexes_order = sort_dict_by_values(indexes_order)\n\n        for p in parts:\n            print(p)\n        if len(lengths) == 1:\n            print(lengths, indexes_order)\n        else:\n            print(lengths, indexes_order)\n            # exit()\n\n        new_parts = []\n\n        # First part always the same\n        new_parts.append([(parts[0][0], 0)])\n\n        # Append next parts\n        for k in range(1, lengths[0][0]):\n            res = np.zeros(len(parts), dtype=np.int32)\n\n            for i in range(0, len(parts)):\n\n                # We need to skip different length Inchis\n                # if (k > 1) and (len(parts[i]) != lengths[0][0]):\n                if k >= len(parts[i]):\n                    res[i] = 1000000000\n                    continue\n\n                parts1 = parts[i][k]\n                val = 0\n                for j in range(0, len(parts)):\n                    if i == j:\n                        continue\n\n                    # We need to skip different length Inchis\n                    # if (k > 1) and (len(parts[j]) != lengths[0][0]):\n                    if k >= len(parts[j]):\n                        continue\n\n                    parts2 = parts[j][k]\n                    val += Levenshtein.distance(parts1, parts2)\n                res[i] = int(val)\n            sort_index = res.argsort()\n\n            try:\n                partial_res = []\n                for i in range(len(sort_index)):\n                    if k < len(parts[sort_index[i]]):\n                        add_tuple = (parts[sort_index[i]][k], res[sort_index[i]])\n                        if add_tuple not in partial_res:\n                            partial_res.append(add_tuple)\n            except Exception as e:\n                print(len(parts))\n                print(parts)\n                print(res)\n                print(len(res))\n                print(sort_index)\n                exit()\n\n            print('Best match: {} [{}]'.format(partial_res, res))\n            new_parts.append(partial_res)\n\n        new_parts = merge_all_variants(new_parts)\n        print('Length of candidates: {}'.format(len(new_parts)))\n\n        if USE_RDKIT:\n            good_index = -100000\n            for i in range(min(len(new_parts), LIMIT_CHECKS)):\n                if check_if_valid_exec_v2(new_parts[i][0]):\n                    good_index = i\n                    break\n        else:\n            good_index = 0\n\n        if good_index == -100000:\n            print('Error not fixed!')\n            error_not_fixed += 1\n            good_index = 0\n        elif good_index == 0:\n            print('No error!')\n            no_error += 1\n        else:\n            print('Error was fixed!')\n            error_fixed += 1\n\n        good.append(new_parts[good_index][0])\n        print(good[-1])\n\n    if validation is True:\n        s['prediction'] = good\n        if 1:\n            s_final = pd.read_csv(subm_list[0][0])\n            if exclude_ids is not None:\n                cond = ~s_final['image_id'].isin(exclude_ids)\n                s_final = s_final[cond]\n            s_final = s_final.sort_values('image_id')\n            s_final.reset_index(inplace=True, drop=True)\n            if current_part is not None:\n                start = current_part * (len(s_final) // parts_number)\n                if current_part == parts_number - 1:\n                    end = len(s_final)\n                else:\n                    end = (current_part + 1) * (len(s_final) // parts_number)\n                print('Start: {} End: {}'.format(start, end))\n                s_final = s_final[start:end]\n            s_final = s_final[s_final['image_id'].isin(ids_intersection)]\n            s_final.reset_index(inplace=True, drop=True)\n            s_final = s_final[~s_final['image_id'].isin(s['image_id'].values)]\n            print(list(s.columns.values))\n            print(list(s_final.columns.values))\n            s_final = pd.concat((s_final[['image_id', 'InChI', 'prediction']], s[['image_id', 'InChI', 'prediction']]),\n                                axis=0)\n        s_final = s\n        print(len(s_final))\n        s_final[['image_id', 'InChI', 'prediction']].to_csv(\n            subm_list[0][0][:-4] + '_fixed_valid_check_{}_part_{}.csv'.format(use_valid_check, current_part),\n            index=False)\n        score = get_score(s_final['InChI'].values, s_final['prediction'].values)\n        print('Ensemble score: {:.6f}'.format(score))\n    else:\n        s['InChI'] = good\n        if 1:\n            s_final = pd.read_csv(subm_list[0][0])\n            if exclude_ids is not None:\n                cond = ~s_final['image_id'].isin(exclude_ids)\n                s_final = s_final[cond]\n            s_final = s_final.sort_values('image_id')\n            s_final.reset_index(inplace=True, drop=True)\n            if current_part is not None:\n                start = current_part * (len(s_final) // parts_number)\n                if current_part == parts_number - 1:\n                    end = len(s_final)\n                else:\n                    end = (current_part + 1) * (len(s_final) // parts_number)\n                print('Start: {} End: {}'.format(start, end))\n                s_final = s_final[start:end]\n            s_final = s_final[~s_final['image_id'].isin(s['image_id'].values)]\n            s_final = pd.concat((s_final, s[['image_id', 'InChI']]), axis=0)\n        print(len(s_final))\n        out_path = os.path.basename(subm_list[0][0])[:-4] + '_fixed_valid_check_{}_part_{}_from_{}.csv'.format(\n            use_valid_check, current_part, parts_number)\n        s_final[['image_id', 'InChI']].to_csv(out_path, index=False)\n\n    print('Stat for validation check. No error {} Error fixed: {} Error not fixed: {}'.format(no_error, error_fixed,\n                                                                                              error_not_fixed))\n    print('Valid check enabled: {} Time: {:.2f} sec'.format(use_valid_check, time.time() - start_time))\n\n\nif __name__ == '__main__':\n    parts = PROC_NUM\n\n    subm_list = [\n        ('../input/pl-bms-molecular-translation/submission.csv', 1),  # 3.63\n        ('../input/efficientnet-multi-layer-lstm-inference/submission.csv', 1),  # 3.69\n        ('../input/tensorflow-tpu-training-baseline-predictions/submission.csv', 1),  # 3.7\n        ('../input/bms-efficientnetv2-tpu-32-epocs-final-lr-42/submission.csv', 1),  # 4.2\n    ]\n    exclude_ids = np.array([])\n\n    if USE_PARALLEL:\n        plist = []\n        for current_part in range(parts):\n            p = Process(target=ensemble_v5_partial_with_rdkit,\n                        args=(subm_list, False, False, parts, current_part, exclude_ids))\n            plist.append(p)\n\n        for i in range(len(plist)):\n            print('Start {}'.format(i))\n            plist[i].start()\n\n        for i in range(len(plist)):\n            plist[i].join()\n\n        # Merge or parts\n        files = glob.glob(\n            os.path.basename(subm_list[0][0])[:-4] + '_fixed_valid_check_{}_part_*_from_{}.csv'.format(False, parts))\n        all_parts = []\n        for f in files:\n            print('Read: {}'.format(f))\n            s = pd.read_csv(f)\n            all_parts.append(s)\n        s = pd.concat(all_parts, axis=0)\n        s.to_csv(os.path.basename(subm_list[0][0])[:-4] + '_fixed_valid_check_merged.csv', index=False)\n    else:\n        ensemble_v5_partial_with_rdkit(subm_list, False, False, 1, 0, exclude_ids)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-07T06:49:33.797639Z","iopub.execute_input":"2021-06-07T06:49:33.797998Z","iopub.status.idle":"2021-06-07T06:49:43.423563Z","shell.execute_reply.started":"2021-06-07T06:49:33.79797Z","shell.execute_reply":"2021-06-07T06:49:43.421456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}