{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# from sklearnex import patch_sklearn\n# patch_sklearn()\n\nimport numba as nb\nimport numpy as np\nimport pandas as pd \nfrom sklearn.neighbors import BallTree\nfrom tqdm.notebook import tqdm, trange\nfrom collections import deque\nfrom unidecode import unidecode\nfrom Levenshtein import distance, jaro, ratio, setratio, jaro_winkler\n\nfrom catboost import CatBoostClassifier, Pool\nimport gc\n\nfrom sklearn.metrics.pairwise import haversine_distances\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics.pairwise import cosine_similarity\n\nimport time","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-05T13:07:13.811126Z","iopub.execute_input":"2022-07-05T13:07:13.811387Z","iopub.status.idle":"2022-07-05T13:07:13.820618Z","shell.execute_reply.started":"2022-07-05T13:07:13.811358Z","shell.execute_reply":"2022-07-05T13:07:13.819042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n%load_ext Cython","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:07:13.822989Z","iopub.execute_input":"2022-07-05T13:07:13.823539Z","iopub.status.idle":"2022-07-05T13:07:13.834476Z","shell.execute_reply.started":"2022-07-05T13:07:13.82349Z","shell.execute_reply":"2022-07-05T13:07:13.833466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%cython\n# Optimized version\nimport numpy as np\ncimport numpy as np\n\nimport cython\nfrom libc.stdlib cimport malloc, free\n\n\n@cython.boundscheck(False) # turn off bounds-checking for entire function\n@cython.wraparound(False)  # turn off negative index wrapping for entire function\ndef LCS(str S, str T):\n    if len(S) < len(T):\n        S, T = T, S\n\n    cdef int i, j\n    cdef np.uint16_t[:] dp_prev, dp_curr\n    \n    dp_prev = np.zeros(len(T) + 1, dtype=np.uint16)\n    dp_curr = np.zeros(len(T) + 1, dtype=np.uint16)\n\n    for i in range(len(S)):\n        for j in range(len(T)):\n            dp_curr[j + 1]  = max(dp_prev[j] + (1 if S[i] == T[j] else 0), dp_curr[j], dp_prev[j + 1])\n        dp_prev, dp_curr = dp_curr, dp_prev\n    return dp_prev[len(T)]","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:07:13.836495Z","iopub.execute_input":"2022-07-05T13:07:13.837088Z","iopub.status.idle":"2022-07-05T13:07:22.486987Z","shell.execute_reply.started":"2022-07-05T13:07:13.837037Z","shell.execute_reply":"2022-07-05T13:07:22.485259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !rm -r mypip\n# !mkdir mypip\n\n# !tar -czvf mypip/jaconv-0.3.tar.gz -C ../input/pykakasi/jaconv-0.3/jaconv-0.3 .\n# !ls -l mypip","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:07:22.489896Z","iopub.execute_input":"2022-07-05T13:07:22.49021Z","iopub.status.idle":"2022-07-05T13:07:22.497251Z","shell.execute_reply.started":"2022-07-05T13:07:22.490175Z","shell.execute_reply":"2022-07-05T13:07:22.494927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install mypip/jaconv-0.3.tar.gz\n","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:07:22.501179Z","iopub.execute_input":"2022-07-05T13:07:22.501494Z","iopub.status.idle":"2022-07-05T13:07:22.522799Z","shell.execute_reply.started":"2022-07-05T13:07:22.501463Z","shell.execute_reply":"2022-07-05T13:07:22.521622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install ../input/pykakasi/Deprecated-1.2.13-py2.py3-none-any.whl\n# !pip install ../input/pykakasi/wrapt-1.14.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n# !pip install ../input/pykakasi/zipp-3.8.0-py3-none-any.whl\n# !pip install ../input/pykakasi/pykakasi-2.2.1-py3-none-any.whl\n","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:07:22.524617Z","iopub.execute_input":"2022-07-05T13:07:22.525226Z","iopub.status.idle":"2022-07-05T13:07:22.534303Z","shell.execute_reply.started":"2022-07-05T13:07:22.525178Z","shell.execute_reply":"2022-07-05T13:07:22.53365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import pykakasi\n\n# def convert_japanese_alphabet(df: pd.DataFrame):\n#     kakasi = pykakasi.kakasi()\n#     kakasi.setMode('H', 'a')  # Convert Hiragana into alphabet\n#     kakasi.setMode('K', 'a')  # Convert Katakana into alphabet\n#     kakasi.setMode('J', 'a')  # Convert Kanji into alphabet\n#     conversion = kakasi.getConverter()\n\n#     def convert(row):\n#         for column in [\"name\", \"address\", \"city\", \"state\"]:\n#             try:\n#                 row[column] = conversion.do(row[column])\n#             except:\n#                 pass\n#         return row\n\n#     df[df[\"country\"] == \"JP\"] = df[df[\"country\"] == \"JP\"].apply(convert, axis=1)\n#     return df","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:07:22.535923Z","iopub.execute_input":"2022-07-05T13:07:22.536432Z","iopub.status.idle":"2022-07-05T13:07:22.547753Z","shell.execute_reply.started":"2022-07-05T13:07:22.536387Z","shell.execute_reply":"2022-07-05T13:07:22.546273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Parameters","metadata":{}},{"cell_type":"code","source":"N = 60\nsize = 'max'\nN_cat = 1000\n\ndo_only_model_train = True\nis_submission = True\ndefault_nan_string = \"_n_\"","metadata":{"execution":{"iopub.status.busy":"2022-07-05T14:22:15.932999Z","iopub.execute_input":"2022-07-05T14:22:15.933307Z","iopub.status.idle":"2022-07-05T14:22:15.940505Z","shell.execute_reply.started":"2022-07-05T14:22:15.933272Z","shell.execute_reply":"2022-07-05T14:22:15.939016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/foursquare-location-matching/train.csv') #.sort_values('point_of_interest').iloc[:100_000]\nsample_submission = pd.read_csv('../input/foursquare-location-matching/sample_submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:07:22.561296Z","iopub.execute_input":"2022-07-05T13:07:22.561949Z","iopub.status.idle":"2022-07-05T13:07:30.575153Z","shell.execute_reply.started":"2022-07-05T13:07:22.5619Z","shell.execute_reply":"2022-07-05T13:07:30.57418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if size == 'max':\n    size =  len(train) // 2","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:07:30.576588Z","iopub.execute_input":"2022-07-05T13:07:30.576867Z","iopub.status.idle":"2022-07-05T13:07:30.582871Z","shell.execute_reply.started":"2022-07-05T13:07:30.576834Z","shell.execute_reply":"2022-07-05T13:07:30.581793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.country.value_counts().head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:07:30.584293Z","iopub.execute_input":"2022-07-05T13:07:30.584518Z","iopub.status.idle":"2022-07-05T13:07:30.77803Z","shell.execute_reply.started":"2022-07-05T13:07:30.584491Z","shell.execute_reply":"2022-07-05T13:07:30.776839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_cat_stats(df):\n    \n    df['adjusted_cats'] = df.categories.fillna(default_nan_string)\\\n    .apply(unidecode)\\\n    .str.lower()\n\n    cats = []\n    \n    for _, ldf in df[df.adjusted_cats != default_nan_string].groupby('point_of_interest'):\n        \n        cl = \", \".join(list(ldf.adjusted_cats.values))\n        \n        for _ in range(len(ldf)):\n            cats.append(cl)\n        \n    \n    cat_stats = {}\n\n    for c in cats:\n        csplit = c.split(\", \")\n        for c_ in csplit:\n            if c_ not in cat_stats:\n                cat_stats[c_] = [1, {ca: 1 for ca in csplit}]\n            else:\n                prev_cat_stats = cat_stats[c_]\n                prev_cat_stats[0] += 1\n                prev_sims = prev_cat_stats[1]\n\n                for ca in csplit:\n                    prev_sims[ca] = prev_sims.get(ca, 0) + 1\n\n    result = {}\n    \n    for c, val in cat_stats.items():\n\n        overall_count = sum(v for ca, v in val[1].items() if ca != c and v > 1)\n\n        new_probs = {c: 1.0}\n        if overall_count != 0:\n            new_probs.update({ca: round(v / overall_count, 3) for ca, v in val[1].items() if  ca != c and v > 1})\n            result[c] = new_probs\n            \n    return result\n    \ncat_stats = get_cat_stats(train)\n\ndef get_cat_similarity(cat1, cat2):\n    \n    if cat1 == default_nan_string or cat2 == default_nan_string:\n        return np.nan\n    \n    best = 0\n    \n    for c1 in cat1.split(\", \"):\n        for c2 in cat2.split(\", \"):\n            if c1 in cat_stats:\n                best = max(best, cat_stats[c1].get(c2, 0))\n            if c2 in cat_stats:\n                best = max(best, cat_stats[c2].get(c1, 0))\n    \n    return best","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:07:30.780006Z","iopub.execute_input":"2022-07-05T13:07:30.78035Z","iopub.status.idle":"2022-07-05T13:08:57.630713Z","shell.execute_reply.started":"2022-07-05T13:07:30.780306Z","shell.execute_reply":"2022-07-05T13:08:57.629457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_name(col):\n    \n    return col.fillna(default_nan_string)\\\n    .str.lower()\\\n    .apply(unidecode)\\\n    .str.lower()\\\n    .str.replace(\"[^a-zA-Z0-9_ ]\", \"\", regex=True)\\\n    .str.replace(\"the \", \"\", regex=False)\\\n    .str.replace(\" a \", \" \", regex=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:08:57.632793Z","iopub.execute_input":"2022-07-05T13:08:57.633189Z","iopub.status.idle":"2022-07-05T13:08:57.640858Z","shell.execute_reply.started":"2022-07-05T13:08:57.633139Z","shell.execute_reply":"2022-07-05T13:08:57.639537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_name_matches(df):\n    \n    df['clean_name'] = clean_name(df.name)\n    \n    all_names = {k : {} for k in df.clean_name}\n    \n    for _, ldf in df[df.clean_name != default_nan_string].groupby('point_of_interest'):\n        \n        lnames = ldf.clean_name\n        \n        for n in lnames:\n            lall_names = all_names[n]\n            \n            for l in lnames:\n                lall_names[l] = lall_names.get(l, 0) + 1\n                \n    reduced_all_names = {k : {k1 for k1, v1 in v.items() if v1 >= 20 or k == k1} for k, v in all_names.items()}\n    \n    return reduced_all_names\n\n\nname_matches = get_name_matches(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:08:57.642806Z","iopub.execute_input":"2022-07-05T13:08:57.643121Z","iopub.status.idle":"2022-07-05T13:10:35.052712Z","shell.execute_reply.started":"2022-07-05T13:08:57.643075Z","shell.execute_reply":"2022-07-05T13:10:35.051379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_cleaned_name_dict(train):\n    cleaned_names = clean_name(train.name)\n    cleaned_name_dict = {k : i for i, k in enumerate(cleaned_names.value_counts().head(N_cat).index)}\n    return cleaned_name_dict\n\ncleaned_name_dict = create_cleaned_name_dict(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:10:35.054853Z","iopub.execute_input":"2022-07-05T13:10:35.055438Z","iopub.status.idle":"2022-07-05T13:10:46.528375Z","shell.execute_reply.started":"2022-07-05T13:10:35.055401Z","shell.execute_reply":"2022-07-05T13:10:46.527329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_cleaned_cat_dict(train):\n    cleaned_cats = train.categories.fillna(default_nan_string)\\\n    .str.replace(\"'\", \"\", regex=False)\\\n    .apply(unidecode)\\\n    .str.lower()\n    cleaned_cat_dict = {k : i for i, k in enumerate(cleaned_cats.value_counts().head(N_cat).index)}\n    return cleaned_cat_dict\n\ncleaned_cat_dict = create_cleaned_cat_dict(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:10:46.530647Z","iopub.execute_input":"2022-07-05T13:10:46.53102Z","iopub.status.idle":"2022-07-05T13:10:49.048972Z","shell.execute_reply.started":"2022-07-05T13:10:46.530963Z","shell.execute_reply":"2022-07-05T13:10:49.047709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_primary_cats_dict(train):\n    cleaned_cats = train.categories.fillna(default_nan_string)\\\n    .str.replace(\"'\", \"\", regex=False)\\\n    .apply(unidecode)\\\n    .str.lower()\n    all_cats = [a.split(\", \") for a in cleaned_cats]\n    print(f\"{len(all_cats)}\")\n\n    all_cats_set = {}\n    for cat in all_cats:\n        for c in cat:\n            all_cats_set[c] = all_cats_set.get(c, 0) + 1\n\n    print(f\"{len(all_cats_set)}\")\n    \n    sorted_cats = sorted([(k, v) for k, v in all_cats_set.items()], key = lambda x: x[1], reverse = True)\n    \n    return {a[0]: i for i, a in enumerate(sorted_cats[:N_cat])}, all_cats_set\n\nprimary_cats_dict, all_cats_set = create_primary_cats_dict(train)\n\ndef get_primary_cat(col):\n    max_cats = {}\n    \n    for a in col.unique():\n        a_split = a.split(\", \")\n        max_a = default_nan_string\n        max_count_a = 0\n        for a_ in a_split:\n            if all_cats_set.get(a_, 0) > max_count_a:\n                max_count_a = all_cats_set.get(a_, 0)\n                max_a = a_\n                \n        max_cats[a] = max_a\n    \n    return col.map(lambda x: max_cats.get(x, N_cat))\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:10:49.050335Z","iopub.execute_input":"2022-07-05T13:10:49.050806Z","iopub.status.idle":"2022-07-05T13:10:54.817235Z","shell.execute_reply.started":"2022-07-05T13:10:49.050751Z","shell.execute_reply":"2022-07-05T13:10:54.815878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"empty_set = set()\n\ndef get_jaro(col, test_col):\n    \n    jaro_ = np.nan\n\n    if col != default_nan_string and test_col != default_nan_string and len(col) > 0 and len(test_col) > 0:\n        jaro_ = jaro(col, test_col)\n        \n    return jaro_\n        \ndef clean_phone(col):\n    \n    col = col.fillna(default_nan_string)\\\n    .astype(str)\\\n    .apply(unidecode)\\\n    .str.replace(\"[^0-9]\", \"\", regex=True)\\\n    .str.lower()\n        \n    return  col\n \ndef get_x(\n    train,\n    i,\n    j,\n    d,\n    rel_d,\n    tf_score,\n    cat_tf_score,\n    true_matches,\n    all_columns,\n    match_cat,\n    model\n    ):\n    \n    current_true_matches = true_matches[all_columns[\"ids\"][i]] if train else empty_set \n    match = 1.0 if (all_columns[\"ids\"][j] in current_true_matches) else 0.0\n\n    cat, test_cat = all_columns[\"cats\"][i], all_columns[\"cats\"][j]\n    name, test_name = all_columns[\"names\"][i], all_columns[\"names\"][j]\n    name_num, test_name_num = all_columns[\"name_numbers\"][i], all_columns[\"name_numbers\"][j]\n\n\n    name_bucket, test_name_bucket = all_columns[\"name_buckets\"][i], all_columns[\"name_buckets\"][j]\n    cat_bucket, test_cat_bucket = all_columns[\"cat_buckets\"][i], all_columns[\"cat_buckets\"][j]\n    primary_cat_bucket, test_primary_cat_bucket = all_columns[\"primary_cat_buckets\"][i], all_columns[\"primary_cat_buckets\"][j]\n\n    city, test_city = all_columns[\"cities\"][i], all_columns[\"cities\"][j]\n    zip_, test_zip = all_columns[\"zips\"][i], all_columns[\"zips\"][j]\n    url, test_url = all_columns[\"urls\"][i], all_columns[\"urls\"][j]\n    country, test_country = all_columns[\"countries\"][i], all_columns[\"countries\"][j]\n\n    address, test_address = all_columns[\"addresses\"][i], all_columns[\"addresses\"][j]\n    phone, test_phone = all_columns[\"phones\"][i], all_columns[\"phones\"][j]\n    state, test_state = all_columns[\"states\"][i], all_columns[\"states\"][j]\n\n    cat_nan = 1.0 if all_columns[\"cat_nans\"][i] or all_columns[\"cat_nans\"][j] else 0.0\n    cat_comp = 1.0 if (cat_nan < 0.1 and ((test_cat in cat) or (cat in test_cat))) else 0.0\n\n    name_nan = 1.0 if all_columns[\"name_nans\"][i] or all_columns[\"name_nans\"][j] else 0.0\n    name_comp = 1.0 if name_nan < 0.1 and ((test_name in name) or (name in test_name)) else 0.0\n\n    is_common = all_columns[\"common_score\"][i] * all_columns[\"common_score\"][j]\n\n    jaro_cat = np.nan\n    jaro_winkler_cat = np.nan\n    levenshtein_cat = np.nan\n    cat_ratio = np.nan\n    set_ratio_cat = np.nan\n    sim_cat = get_cat_similarity(cat, test_cat)\n\n    if cat_nan < 0.1:\n        jaro_cat = jaro(cat, test_cat)\n        jaro_winkler_cat = jaro_winkler(cat, test_cat)\n        cat_ratio = ratio(cat, test_cat)\n        levenshtein_cat = distance(cat, test_cat) / max(len(cat), len(test_cat))\n\n        set_ratio_cat = setratio(all_columns[\"cat_splits\"][i], all_columns[\"cat_splits\"][j])\n    else:\n        cat_tf_score = np.nan\n\n    jaro_name = np.nan\n    jaro_winkler_name = np.nan\n    levenshtein_name = np.nan\n    name_ratio = np.nan\n    set_ratio_name = np.nan\n    \n    name_num_comp = np.nan\n    name_imb = np.nan\n    name_length = np.nan\n    \n    name_match = np.nan\n    \n    has_bracket = np.nan\n    \n    lcs_name = np.nan\n\n    if len(name) > 0 and len(test_name) > 0 and name_nan < 0.1:\n        \n        lcs_name = LCS(name, test_name)\n\n        jaro_name = jaro(name, test_name)\n        jaro_winkler_name = jaro_winkler(name, test_name)\n\n        levenshtein_name = distance(name, test_name) / max(len(name), len(test_name))\n        name_ratio = ratio(name, test_name)\n        set_ratio_name = setratio(all_columns[\"name_splits\"][i], all_columns[\"name_splits\"][j])\n        \n        name_num_comp = 1.0 if name_num == test_name_num else 0.0\n        \n        name_imb = abs(len(name) - len(test_name)) / (len(name) + len(test_name))\n        name_length = len(name) + len(test_name)\n        \n        if len(name_matches.get(name, empty_set) & name_matches.get(test_name, empty_set)) > 0:\n            name_match = 1.0\n        \n        if all_columns['name_brackets'][i] or all_columns['name_brackets'][j]:\n            has_bracket = 1.0\n        \n    else:\n        tf_score = np.nan\n\n\n    jaro_city = get_jaro(city, test_city)\n    jaro_address = get_jaro(address, test_address)\n    jaro_phone = get_jaro(phone, test_phone)\n    jaro_zip = get_jaro(zip_, test_zip)\n    jaro_url = get_jaro(url, test_url)\n    jaro_state = get_jaro(state, test_state)\n    \n    address_name = 0\n    \n    if address != default_nan_string and len(test_name) > 0:\n        \n        if address in test_name or test_name in address:\n            address_name = max(1.0, address_name)\n        \n        address_name = max(jaro(address, test_name), address_name)\n    \n    if test_address != default_nan_string and len(name) > 0:\n        \n        if test_address in name or name in test_address:\n            address_name = max(1.0, address_name)\n        \n        address_name = max(jaro(test_address, name), address_name)\n        \n    city_name = 0\n    \n    if city != default_nan_string and len(test_name) > 0:\n        \n        if city in test_name or test_name in city:\n            city_name = max(1.0, city_name)\n        \n        city_name = max(jaro(city, test_name), city_name)\n    \n    if test_city != default_nan_string and len(name) > 0:\n        \n        if test_city in name or name in test_city:\n            city_name = max(1.0, city_name)\n        \n        city_name = max(jaro(test_city, name), city_name)\n        \n    cat_name = 0\n    \n    if cat != default_nan_string and len(test_name) > 0:\n        \n        if cat in test_name or test_name in cat:\n            cat_name = max(1.0, cat_name)\n        \n        cat_name = max(jaro(cat, test_name), cat_name)\n    \n    if test_cat != default_nan_string and len(name) > 0:\n        \n        if test_cat in name or name in test_cat:\n            cat_name = max(1.0, cat_name)\n        \n        cat_name = max(jaro(test_cat, name), cat_name)\n\n    \n    full_info = []\n    \n    full_info.extend(all_columns[\"name_splits\"][i])\n    \n    for col in [city, address, state]:\n        if col != default_nan_string:\n            for a in col.split(\",\"):\n                full_info.append(a)\n    \n    full_info_test = []\n    \n    full_info.extend(all_columns[\"name_splits\"][j])\n    \n    for col in [test_city, test_address, test_state]:\n        if col != default_nan_string:\n            for a in col.split(\",\"):\n                full_info_test.append(a)\n                \n    full_info_ratio = setratio(full_info, full_info_test)\n    \n    prim_cat_match = 1.0 if primary_cat_bucket == test_primary_cat_bucket else 0.0\n    rads_ = all_columns[\"rads\"][i]\n        \n    country_match = 0\n    \n    is_jp = 0\n    is_us = 0\n    is_tk = 0\n    is_id = 0\n    \n    if (country != default_nan_string) and (test_country != default_nan_string):\n        if (country == test_country):\n            country_match = 1\n            \n        if country == 'jp' or test_country == 'jp':\n            is_jp = 1\n        elif country == 'tk' or test_country == 'tk':\n            is_tk = 1\n        elif country == 'us' or test_country == 'us':\n            is_us = 1\n        elif country == 'id' or test_country == 'id':\n            is_id = 1\n    \n    exact_location_match = 1.0 if (i != j) and d == 0 else 0.0\n    \n    density = rel_d / (d + 1e-15)\n    \n    \n    x = np.array([\n#         cat_comp, #0\n#         name_comp, \n        d, #2\n        rads_[0], \n        rads_[1], #4\n        is_common,\n        jaro_name, #6\n        jaro_winkler_name,\n        levenshtein_name, #8\n        levenshtein_cat,\n        jaro_cat, #10\n        jaro_winkler_cat,\n        name_ratio, #12\n        cat_ratio,\n        set_ratio_cat, #14\n        set_ratio_name,\n        jaro_city, #16\n        jaro_address,\n        jaro_phone, #18\n        jaro_zip,\n        jaro_url, #20\n        jaro_state,\n        country_match,\n        full_info_ratio,  \n        primary_cat_bucket,\n        test_primary_cat_bucket,\n        sim_cat,\n        tf_score,\n        cat_tf_score,\n        address_name,\n        city_name,\n        cat_name,\n        rel_d,\n        density,\n        name_imb,\n        name_length,\n        name_match,\n        name_num_comp,\n        lcs_name,\n        match,\n        i,\n        j\n    ], dtype=np.float32)\n    \n    return x\n    \ndef get_all_columns(df):\n    \n#     df = convert_japanese_alphabet(df)\n    \n    all_columns = dict()\n    \n    all_columns[\"rads\"] = np.deg2rad(df[['latitude', 'longitude']].values)\n\n    raw_cats = df.categories.fillna(default_nan_string)\\\n    .apply(unidecode)\\\n    .str.lower()\n    \n    col = df.address.fillna(default_nan_string)\\\n    .astype(str)\\\n    .apply(unidecode)\\\n    .str.replace(\"[^a-zA-Z0-9,_ ]\", \"\", regex=True)\\\n    .str.lower()\n    \n    all_columns[\"addresses\"] = col.values\n    \n    col = df.city.fillna(default_nan_string)\\\n    .astype(str)\\\n    .apply(unidecode)\\\n    .str.replace(\"[^a-zA-Z0-9,_ ]\", \"\", regex=True)\\\n    .str.lower()\n    \n    all_columns[\"cities\"] = col.values\n    \n    all_columns[\"urls\"] = df.url.fillna(default_nan_string)\\\n    .apply(unidecode)\\\n    .str.lower()\\\n    .str.replace(\"www.\", \"\", regex=False)\\\n    .str.replace(\"https://\", \"\", regex=False)\\\n    .str.replace(\"http://\", \"\", regex=False)\\\n    .values\n    \n    col = df.zip.fillna(default_nan_string)\\\n    .astype(str)\\\n    .apply(unidecode)\\\n    .str.replace(\"[^a-zA-Z0-9_]\", \"\", regex=True)\\\n    .str.lower()\n    \n    all_columns[\"zips\"] = col.values\n    \n    col = df.country.fillna(default_nan_string)\\\n    .astype(str)\\\n    .apply(unidecode)\\\n    .str.lower()\\\n    .str.replace(\"[^a-z]\", \"\", regex=True)\n    \n    all_columns[\"countries\"] = col.values\n    \n    col = df.state.fillna(default_nan_string)\\\n    .astype(str)\\\n    .apply(unidecode)\\\n    .str.replace(\"[^a-zA-Z0-9_]\", \"\", regex=True)\\\n    .str.lower()\n    \n    all_columns[\"states\"] = col.values    \n    \n    all_columns[\"phones\"] = clean_phone(df.phone).values\n    \n    all_columns[\"primary_cats\"] = get_primary_cat(raw_cats).values\n    all_columns[\"cats\"] = raw_cats.values\n    \n    raw_names = clean_name(df.name)\n    all_columns[\"names\"] = raw_names.values\n    all_columns[\"name_numbers\"] = raw_names.str.replace(\"[^0-9]\", \"\", regex=True).values\n    all_columns[\"name_brackets\"] = raw_names.str.contains('(', regex=False).values\n\n    all_columns[\"ids\"] = df.id.values\n     #np.zeros((len(df) * n, 24))\n        \n    \n#     common_names = {k for k,v in raw_names.value_counts().to_dict().items()}\n#     very_common_names = {k for k,v in raw_names.value_counts().to_dict().items() if v > 20}\n    \n    all_columns[\"name_buckets\"] = raw_names.map(lambda x: cleaned_name_dict.get(x, N_cat)).values\n    all_columns[\"cat_buckets\"] = raw_cats.map(lambda x: cleaned_cat_dict.get(x, N_cat)).values\n\n    all_columns[\"primary_cat_buckets\"] = raw_cats.map(lambda x: primary_cats_dict.get(x, N_cat)).values\n    \n    all_columns[\"name_splits\"] = raw_names.str.split(\" \").values\n    all_columns[\"cat_splits\"] = raw_cats.str.split(\", \").values\n    all_columns[\"name_nans\"] = all_columns[\"names\"] == default_nan_string\n    all_columns[\"cat_nans\"] = all_columns[\"cats\"] == default_nan_string\n    \n    all_columns[\"common_score\"] = raw_names.map(raw_names.value_counts().to_dict()).values / raw_names.nunique()\n    \n    all_columns[\"tf\"] = TfidfVectorizer(\n        token_pattern=r'\\w{2,}',\n        ngram_range=(3, 3), analyzer=\"char_wb\",\n        use_idf = False\n    ).fit_transform(raw_names)\n    all_columns[\"tf_cat\"] = TfidfVectorizer(\n        token_pattern=r'\\w{2,}',\n        use_idf = False\n    ).fit_transform(raw_cats)\n\n#     all_columns[\"cat_tokens\"] = list(map(lambda x: x.vector if x.has_vector else np.zeros(1), nlp.pipe(all_columns[\"cats\"], n_process=-1)))\n#     all_columns[\"name_tokens\"] = list(map(lambda x: x.vector if x.has_vector else np.zeros(1), nlp.pipe(all_columns[\"names\"], n_process=-1)))\n\n    return all_columns\n    \ndef create_train_df(df, true_matches, model, n = 2, train=True):\n    \n    n = min(n, df.shape[0])\n    \n    n_mean = min(n, 10)\n    \n    results = deque()\n    start_time = time.time()\n    all_columns = get_all_columns(df)\n    print(f\"Time to create features: {np.around(time.time() - start_time, 2)}s\")\n\n    rads = all_columns[\"rads\"]\n    tree = BallTree(rads, metric='haversine', leaf_size=5)\n        \n    start_time = time.time()\n    dist, ind = tree.query(rads, k = n, breadth_first=True)\n    print(f\"Time to find nearest neighbors: {np.around(time.time() - start_time, 2)}s\")\n\n    for i in trange(len(df)):\n        \n        dist_filtered = dist[i]\n        ind_filtered = ind[i]\n        \n        tf_sim = cosine_similarity(all_columns['tf'][i], all_columns['tf'][ind_filtered])[0]\n        cat_tf_sim = cosine_similarity(all_columns['tf_cat'][i], all_columns['tf_cat'][ind_filtered])[0]\n\n        k = 0\n        \n        d_mean = np.mean(dist_filtered[:n_mean]) + 1e-15\n        \n        for d, j in zip(dist_filtered, ind_filtered):\n            \n            x = get_x(train, i, j, d, d/d_mean, tf_sim[k], cat_tf_sim[k], true_matches, all_columns, 0, model)\n            k+=1\n            results.append(x)\n            \n    print(f\"Generated {len(results)} samples!\")\n    \n    df.loc[:, \"col_id\"] = list(range(len(df)))\n    \n    chunk_sizes = {\n        \"addresses\": 10_000,\n        \"phones\": 10_000,\n        \"zips\": 5,\n        \"urls\": 10,\n        \"cities\": 50,\n        \"names\": 1_000\n        \n    }\n    \n    for num_, col in enumerate(['addresses', 'phones']):\n        df.loc[:, col] = all_columns[col]\n\n        add = df.loc[df[col] != default_nan_string].sort_values(\"name\")\n        \n        iterations = add[col].nunique()\n        print(f\"Found {iterations} elements for {col}\")\n\n        for a, ldf in tqdm(add.groupby(col, sort = False), total = iterations):\n            if len(ldf) > 1 and len(a) > 0:\n\n                ids_ = ldf.col_id.values\n                \n                chunks = [ids_[x:min(len(ids_), x + chunk_sizes[col])] for x in range(0, len(ids_), chunk_sizes[col])]\n\n                for c in chunks:\n                    for i in c:\n                        \n                        tf_sim = cosine_similarity(all_columns['tf'][i], all_columns['tf'][c])[0]\n                        cat_tf_sim = cosine_similarity(all_columns['tf_cat'][i], all_columns['tf_cat'][c])[0]\n                        \n                        d_mean = np.mean(dist[i,:n_mean]) + 1e-15\n\n\n                        k = 0\n                        for j in c:\n                            if i != j:\n                                d = haversine_distances([rads[i], rads[j]])[0,1]\n                                x = get_x(train, i, j, d, d / d_mean, tf_sim[k], cat_tf_sim[k], true_matches, all_columns, num_ + 1, model)\n                                k+=1\n                                results.append(x)\n\n        print(f\"Generated {len(results)} samples!\")\n\n    return np.array(list(results)).astype(np.float32)\n\ndef get_matches(y_pred, pairs, ids, threshold = 0.5, depth = 5):\n    \n    match_pairs = pairs[y_pred > threshold]\n    \n    predicted_matches = {a : {a} for a in ids}\n    \n    for row in match_pairs:\n        i = ids[int(row[0])]\n        j = ids[int(row[1])]\n        \n        predicted_matches[i].add(j)\n        predicted_matches[j].add(i)\n    \n    predicted_matches = propagate_matches(predicted_matches, depth)\n        \n    return predicted_matches\n\ndef propagate_matches(predicted_matches, depth):\n    \n    for _ in range(depth):\n        final_predicted_matches = {a : {a} for a in predicted_matches.keys()}\n\n        for p, val in predicted_matches.items():\n            new_p = set([p])\n            for p_ in val:\n                new_p.update(predicted_matches[p_])\n\n            final_predicted_matches[p].update(new_p)\n\n        predicted_matches = final_predicted_matches\n        \n    return predicted_matches\n\ndef get_predictions(df, models, threshold, n = 2):\n    \n    n = min(n, df.shape[0])\n    n_mean = min(n, 10)\n\n    all_columns = get_all_columns(df)\n    ids = all_columns[\"ids\"]\n    rads = all_columns[\"rads\"]\n    tree = BallTree(rads, metric='haversine', leaf_size=5)\n        \n    start_time = time.time()\n    dist, ind = tree.query(rads, k = n, breadth_first=True)\n    print(f\"Time to find nearest neighbors: {np.around(time.time() - start_time, 2)}s\")\n        \n    i = 0\n    \n    all_matches = {a : {a} for a in ids}\n    \n    batch_size = 5_000_000 // n\n    \n    while i < len(df):\n        \n        \n        results = deque()\n\n        max_range = min(batch_size, len(df) - i)\n        for _ in trange(min(len(df), max_range)):\n\n            dist_filtered = dist[i]\n            ind_filtered = ind[i]\n\n            tf_sim = cosine_similarity(all_columns['tf'][i], all_columns['tf'][ind_filtered])[0]\n            cat_tf_sim = cosine_similarity(all_columns['tf_cat'][i], all_columns['tf_cat'][ind_filtered])[0]\n            \n            d_mean = np.mean(dist_filtered[:n_mean]) + 1e-15\n\n\n            k = 0\n\n            for d, j in zip(dist_filtered, ind_filtered):\n\n                x = get_x(False, i, j, d, d / d_mean, tf_sim[k], cat_tf_sim[k], None, all_columns, 0, None)\n\n                k+=1\n                results.append(x)\n            \n            i += 1\n\n        sample_arr = np.array(list(results)).astype(np.float32)\n\n        print(f\"sample_arr shape: {sample_arr.shape}\")\n        X = sample_arr[:,:-3]\n\n        pairs = sample_arr[:,-2:]\n        y_pred = np.mean([model.predict_proba(X)[:,1] for model in models], axis = 0)\n        \n        m = get_matches(y_pred, pairs, ids, threshold, depth = 5)\n        \n        for c, v in m.items():\n            all_matches[c].update(v)\n        \n\n          \n    df.loc[:, \"col_id\"] = list(range(len(df)))\n    \n    chunk_sizes = {\n        \"addresses\": 100,\n        \"phones\": 100,\n        \"zips\": 20,\n        \"urls\": 10,\n        \"cities\": 20,\n        \"names\": 5,\n    }\n    \n    for num_, col in enumerate(['names','addresses', 'phones', \"cities\", \"urls\"]):\n        \n        results = deque()\n        \n        df.loc[:, col] = all_columns[col]\n\n        add = df.loc[df[col] != default_nan_string].sort_values(\"names\")\n        \n        print(f\"Found {add[col].nunique()} elements for {col}\")\n\n        for a, ldf in add.groupby(col, sort = False):\n            if len(ldf) > 1 and len(a) > 0:\n\n                ids_ = ldf.col_id.values\n                \n                chunks = [ids_[x:min(len(ids_), x + chunk_sizes[col])] for x in range(0, len(ids_), chunk_sizes[col])]\n\n                for c in chunks:\n                    for i in c:\n                        \n                        tf_sim = cosine_similarity(all_columns['tf'][i], all_columns['tf'][c])[0]\n                        cat_tf_sim = cosine_similarity(all_columns['tf_cat'][i], all_columns['tf_cat'][c])[0]\n                        \n                        d_mean = np.mean(dist[i,:n_mean]) + 1e-15\n\n\n                        k = 0\n                        for j in c:\n                            if i != j:\n                                d = haversine_distances([rads[i], rads[j]])[0,1]\n                                x = get_x(False, i, j, d, d / d_mean, tf_sim[k], cat_tf_sim[k], None, all_columns, num_ + 1, None)\n\n                                k+=1\n                                results.append(x)\n    \n        if len(results) > 0:\n            sample_arr = np.array(list(results)).astype(np.float32)\n            print(f\"sample_arr shape: {sample_arr.shape}\")\n\n            X = sample_arr[:,:-3]\n\n            pairs = sample_arr[:,-2:]\n            y_pred = np.mean([model.predict_proba(X)[:,1] for model in models], axis = 0)\n\n            m = get_matches(y_pred, pairs, ids, threshold, depth = 5)\n\n            for c, v in m.items():\n                all_matches[c].update(v)\n                \n    all_matches = propagate_matches(all_matches, depth = 5)\n\n    return all_matches\n","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:17:11.892754Z","iopub.execute_input":"2022-07-05T13:17:11.893371Z","iopub.status.idle":"2022-07-05T13:17:12.026629Z","shell.execute_reply.started":"2022-07-05T13:17:11.893315Z","shell.execute_reply":"2022-07-05T13:17:12.025806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## True Matches from Training","metadata":{}},{"cell_type":"code","source":"def get_true_matches(df):\n    true_matches = {}\n\n    for i, row in df[['id', 'point_of_interest']].groupby('point_of_interest', sort = False):\n        ids_ = set(row.id.values)\n\n        for id_ in ids_:\n            true_matches[id_] = ids_\n    \n    return true_matches\n","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:17:12.457811Z","iopub.execute_input":"2022-07-05T13:17:12.458382Z","iopub.status.idle":"2022-07-05T13:17:12.465179Z","shell.execute_reply.started":"2022-07-05T13:17:12.458319Z","shell.execute_reply":"2022-07-05T13:17:12.463888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Scoring","metadata":{}},{"cell_type":"code","source":"def get_score(true_matches, predicted_matches):\n#     print(\"scoring...\")\n    score_sum = 0\n    \n    predicted_keys = predicted_matches.keys()\n    \n    assert len(true_matches) == len(predicted_matches), f\"length mismatch: {len(true_matches)} != {len(predicted_matches)}\"\n        \n    for id_ in predicted_keys:\n        a = true_matches.get(id_, set())\n        b = predicted_matches.get(id_, set())\n        score = len(a.intersection(b)) / len(a.union(b))\n        score_sum += score\n        \n#         if score < 1.0:\n#             print(f\"{id_} : {a} != {b}\")\n\n    total_score = round(100.0 * score_sum / len(predicted_keys), 2)\n\n    return total_score","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:17:13.085123Z","iopub.execute_input":"2022-07-05T13:17:13.086211Z","iopub.status.idle":"2022-07-05T13:17:13.095041Z","shell.execute_reply.started":"2022-07-05T13:17:13.086131Z","shell.execute_reply":"2022-07-05T13:17:13.094151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# true_matches = get_true_matches(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:17:13.360524Z","iopub.execute_input":"2022-07-05T13:17:13.360871Z","iopub.status.idle":"2022-07-05T13:17:13.365655Z","shell.execute_reply.started":"2022-07-05T13:17:13.360837Z","shell.execute_reply":"2022-07-05T13:17:13.364605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Matching","metadata":{}},{"cell_type":"markdown","source":"## Train Validation Split","metadata":{}},{"cell_type":"code","source":"def train_(sample_df, model, val):\n    \n    true_matches = get_true_matches(sample_df)\n    sample_arr = create_train_df(sample_df, true_matches = true_matches, model=model, n = N)\n\n    X = sample_arr[:,:-3]\n#     X = pd.DataFrame(X).astype({11 : np.int32, 12 : np.int32})\n    y = sample_arr[:,-3].astype(np.int32)\n    \n    pool = Pool(data=X, label=y)\n    \n#     if not val:\n#         pool.quantize()\n    \n    pairs = sample_arr[:,-2:].astype(np.int32)\n    ids = sample_df.id.values\n    \n    gc.collect()\n    \n    return pool, true_matches, pairs, ids","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:17:14.211217Z","iopub.execute_input":"2022-07-05T13:17:14.211566Z","iopub.status.idle":"2022-07-05T13:17:14.219339Z","shell.execute_reply.started":"2022-07-05T13:17:14.211522Z","shell.execute_reply":"2022-07-05T13:17:14.218168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_val(train):\n\n#     model = models.KeyedVectors.load_word2vec_format(\n#         \"../input/nlpword2vecembeddingspretrained/GoogleNews-vectors-negative300.bin\", binary=True\n#     )\n    model = None\n    \n    train_sorted = train.sort_values(\"point_of_interest\")\n    \n#     df_train, df_val = train_test_split(train, test_size = 0.5, random_state = 42)\n    df_train = train_sorted.iloc[:size]\n    df_val = train_sorted.iloc[size:min(2 * size, len(train))]\n    \n    train_results  = train_(df_train, model, val=False)\n    val_results = train_(df_val, model, val=True)\n    \n    \n    return train_results, val_results\n\nif not is_submission:\n    \n    train_results, val_results = get_train_val(train)\n    \n    train_pool, true_matches_train, pairs_train, ids_train = train_results\n    val_pool, true_matches_val, pairs_val, ids_val = val_results\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:17:14.653516Z","iopub.execute_input":"2022-07-05T13:17:14.654247Z","iopub.status.idle":"2022-07-05T13:33:37.603793Z","shell.execute_reply.started":"2022-07-05T13:17:14.654197Z","shell.execute_reply":"2022-07-05T13:33:37.602499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:33:37.606593Z","iopub.execute_input":"2022-07-05T13:33:37.606999Z","iopub.status.idle":"2022-07-05T13:33:38.126092Z","shell.execute_reply.started":"2022-07-05T13:33:37.60695Z","shell.execute_reply":"2022-07-05T13:33:38.124777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not is_submission:\n    log1 = None\n    gc.collect()\n    log1 = CatBoostClassifier(\n        depth = 9,\n        iterations= 3_000,\n#         learning_rate = 0.05,\n        thread_count = -1,\n        eval_metric='AUC',\n        l2_leaf_reg = 0.2,\n        verbose=200,\n        metric_period=10,\n#         task_type = 'GPU'\n    )\n\n    log1 = log1.fit(\n        train_pool, eval_set=val_pool, use_best_model=True\n    )\n    \n    log1.save_model(\"cat_boost_model_1.cbm\")\n\n    \n    print(\"----------- model 2 ----------------\")\n    \n    \n    log2 = None\n    gc.collect()\n    log2 = CatBoostClassifier(\n        depth = 9,\n        iterations= 3_000,\n        learning_rate = 0.05,\n        thread_count = -1,\n        eval_metric='AUC',\n        l2_leaf_reg = 0.2,\n        verbose=200,\n        metric_period=10,\n#         task_type = 'GPU'\n    )\n\n    log2 = log2.fit(\n        val_pool, eval_set=train_pool, use_best_model=True\n    )\n    \n    log2.save_model(\"cat_boost_model_2.cbm\")\nelse:\n    log1 = CatBoostClassifier()\n    log1.load_model('../input/models-new/cat_boost_model_1.cbm')\n    \n    log2 = CatBoostClassifier()\n    log2.load_model('../input/models-new/cat_boost_model_2.cbm')","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:33:38.127735Z","iopub.execute_input":"2022-07-05T13:33:38.128001Z","iopub.status.idle":"2022-07-05T14:16:55.371742Z","shell.execute_reply.started":"2022-07-05T13:33:38.127969Z","shell.execute_reply":"2022-07-05T14:16:55.370623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, f in enumerate(log1.get_feature_importance()):\n    print(i, f)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T14:20:40.788612Z","iopub.execute_input":"2022-07-05T14:20:40.789023Z","iopub.status.idle":"2022-07-05T14:20:40.885468Z","shell.execute_reply.started":"2022-07-05T14:20:40.788983Z","shell.execute_reply":"2022-07-05T14:20:40.88376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (not is_submission) and (not do_only_model_train):\n    \n    y_pred_all = []\n    \n    for log in [log1, log2]:\n        for p in [train_pool, val_pool]:\n            y_pred_all.append(log.predict_proba(p, verbose=False)[:, 1])\n            \n    y_pred = np.mean([y_pred_all[0], y_pred_all[2]], axis = 0)\n    y_pred_val = np.mean([y_pred_all[1], y_pred_all[3]], axis = 0)\n\n    \n    \n    \n    \n#     from sklearn.metrics import confusion_matrix\n\n#     for b in range(40, 60):\n#         print(b)\n#         print(confusion_matrix(train_pool.get_label(), y_pred > b / 100.0))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-05T14:22:28.999449Z","iopub.execute_input":"2022-07-05T14:22:29.0008Z","iopub.status.idle":"2022-07-05T14:22:56.46721Z","shell.execute_reply.started":"2022-07-05T14:22:29.000735Z","shell.execute_reply":"2022-07-05T14:22:56.465521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.corcoef(y_pred_all[0], y_pred_all[2])","metadata":{"execution":{"iopub.status.busy":"2022-07-05T14:22:56.469199Z","iopub.execute_input":"2022-07-05T14:22:56.469481Z","iopub.status.idle":"2022-07-05T14:22:56.47385Z","shell.execute_reply.started":"2022-07-05T14:22:56.469436Z","shell.execute_reply":"2022-07-05T14:22:56.472566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (not is_submission) and (not do_only_model_train):\n    \n    incorrect_pairs_train = pairs_train[train_pool.get_label() != 1.0 * (y_pred > 0.5)]\n    \n    incorrect_rows_train = []\n    \n    train_sorted = train.sort_values(\"point_of_interest\")\n    \n#     df_train, df_val = train_test_split(train, test_size = 0.5, random_state = 42)\n    df_train = train_sorted.iloc[:size]\n#     df_val = train_sorted.iloc[size:min(2 * size, len(train))]\n    \n    for i, j in incorrect_pairs_train:\n        \n        incorrect_rows_train.append(pd.DataFrame([df_train.iloc[i], df_train.iloc[j]]))","metadata":{"execution":{"iopub.status.busy":"2022-07-05T14:22:56.475192Z","iopub.execute_input":"2022-07-05T14:22:56.475448Z","iopub.status.idle":"2022-07-05T14:23:25.602512Z","shell.execute_reply.started":"2022-07-05T14:22:56.475416Z","shell.execute_reply":"2022-07-05T14:23:25.601489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_predicted_matches_and_score(y_pred, pairs, ids, threshold, true_matches, depth, enh = dict()):\n    \n    predicted_matches = get_matches(y_pred, pairs, ids, threshold, depth)\n    \n    for k, v in enh.items():\n        predicted_matches[k].update(v)\n        \n    score_acc = get_score(true_matches, predicted_matches)\n    print(f'score: {score_acc}')\n    \n    return score_acc","metadata":{"execution":{"iopub.status.busy":"2022-07-05T14:23:25.604382Z","iopub.execute_input":"2022-07-05T14:23:25.604646Z","iopub.status.idle":"2022-07-05T14:23:25.612209Z","shell.execute_reply.started":"2022-07-05T14:23:25.604614Z","shell.execute_reply":"2022-07-05T14:23:25.61093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (not is_submission) and (not do_only_model_train):\n\n    score_acc = get_predicted_matches_and_score(train_pool.get_label(), pairs_train, ids_train, 0.5, true_matches_train, 5)\n    score_acc_val = get_predicted_matches_and_score(val_pool.get_label(), pairs_val, ids_val, 0.5, true_matches_val, 5)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T14:23:25.61397Z","iopub.execute_input":"2022-07-05T14:23:25.61433Z","iopub.status.idle":"2022-07-05T14:23:32.352372Z","shell.execute_reply.started":"2022-07-05T14:23:25.614284Z","shell.execute_reply":"2022-07-05T14:23:32.350229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train = train_sorted.iloc[:size]\n# df_val = train_sorted.iloc[size:min(2 * size, len(train))]\n\n# add_train = get_add_matches(df_train)\n# add_val = get_add_matches(df_val)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T14:23:32.35393Z","iopub.execute_input":"2022-07-05T14:23:32.354184Z","iopub.status.idle":"2022-07-05T14:23:32.359226Z","shell.execute_reply.started":"2022-07-05T14:23:32.354154Z","shell.execute_reply":"2022-07-05T14:23:32.358195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if (not is_submission) and (not do_only_model_train):\n\n#     score_acc = get_predicted_matches_and_score(train_pool.get_label(), pairs_train, ids_train, 0.5, true_matches_train, 5, add_train)\n#     score_acc_val = get_predicted_matches_and_score(val_pool.get_label(), pairs_val, ids_val, 0.5, true_matches_val, 5, add_val)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T14:24:24.727256Z","iopub.execute_input":"2022-07-05T14:24:24.727775Z","iopub.status.idle":"2022-07-05T14:24:24.732243Z","shell.execute_reply.started":"2022-07-05T14:24:24.727731Z","shell.execute_reply":"2022-07-05T14:24:24.731521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_threshold = 0.6\nbest_depth = 5\n\nbest_score = 0.0\n\nif (not is_submission) and (not do_only_model_train):\n    for threshold in range(40, 70, 2):\n        threshold /= 100.0\n        print(f\"{threshold} {best_depth}: -----------\")\n\n        score_acc = get_predicted_matches_and_score(y_pred, pairs_train, ids_train, threshold, true_matches_train, best_depth)\n        score_acc_val = get_predicted_matches_and_score(y_pred_val, pairs_val, ids_val, threshold, true_matches_val, best_depth)\n\n        if score_acc_val > best_score:\n            best_score = score_acc_val\n            best_threshold = threshold\n","metadata":{"execution":{"iopub.status.busy":"2022-07-05T14:24:25.599628Z","iopub.execute_input":"2022-07-05T14:24:25.600163Z","iopub.status.idle":"2022-07-05T14:25:59.554129Z","shell.execute_reply.started":"2022-07-05T14:24:25.600124Z","shell.execute_reply":"2022-07-05T14:25:59.553279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if not is_submission:\n    \n    \n#     train_sorted = train.sort_values(\"point_of_interest\")\n\n    \n#     df_train = train_sorted.iloc[:size]\n#     df_val = train_sorted.iloc[size:min(2 * size, len(train))]\n    \n#     train_check_matches = get_predictions(df_train, log, best_threshold, N)\n#     val_check_matches = get_predictions(df_val, log, best_threshold, N)\n    \n#     score_train_check_acc = get_score(true_matches_train, train_check_matches)\n#     score_val_check_acc = get_score(true_matches_val, val_check_matches)\n\n#     print(score_train_check_acc)\n#     print(score_val_check_acc)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:10:54.96172Z","iopub.status.idle":"2022-07-05T13:10:54.962315Z","shell.execute_reply.started":"2022-07-05T13:10:54.962079Z","shell.execute_reply":"2022-07-05T13:10:54.962112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_threshold","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:10:54.963365Z","iopub.status.idle":"2022-07-05T13:10:54.963889Z","shell.execute_reply.started":"2022-07-05T13:10:54.963649Z","shell.execute_reply":"2022-07-05T13:10:54.963682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_output = pd.DataFrame({\n#     \"id\" : list(predicted_matches.keys()),\n#     \"matches\" : [\" \".join(a) for a in predicted_matches.values()]\n# })","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:10:54.964942Z","iopub.status.idle":"2022-07-05T13:10:54.96544Z","shell.execute_reply.started":"2022-07-05T13:10:54.965227Z","shell.execute_reply":"2022-07-05T13:10:54.965259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('../input/foursquare-location-matching/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:10:54.966414Z","iopub.status.idle":"2022-07-05T13:10:54.966929Z","shell.execute_reply.started":"2022-07-05T13:10:54.966705Z","shell.execute_reply":"2022-07-05T13:10:54.966736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\n\ntest_matches = get_predictions(test, [log1, log2], best_threshold, N)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:10:54.967948Z","iopub.status.idle":"2022-07-05T13:10:54.96844Z","shell.execute_reply.started":"2022-07-05T13:10:54.96823Z","shell.execute_reply":"2022-07-05T13:10:54.968262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_matches = get_matches(y_test, test_arr[:,-2:], test.id.values, best_threshold, best_depth)\n\ntest_submission = pd.DataFrame({\n    \"id\" : list(test_matches.keys()),\n    \"matches\" : [\" \".join(a) for a in test_matches.values()]\n})","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:10:54.969403Z","iopub.status.idle":"2022-07-05T13:10:54.969901Z","shell.execute_reply.started":"2022-07-05T13:10:54.969678Z","shell.execute_reply":"2022-07-05T13:10:54.969721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_submission.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:10:54.970884Z","iopub.status.idle":"2022-07-05T13:10:54.971378Z","shell.execute_reply.started":"2022-07-05T13:10:54.971168Z","shell.execute_reply":"2022-07-05T13:10:54.971199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_submission.to_csv('submission.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-05T13:10:54.972345Z","iopub.status.idle":"2022-07-05T13:10:54.97286Z","shell.execute_reply.started":"2022-07-05T13:10:54.972618Z","shell.execute_reply":"2022-07-05T13:10:54.972649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}