{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Import","metadata":{}},{"cell_type":"code","source":"%%capture\n# install reverse-geocode\n!mkdir -p /tmp/pip/cache/\n!cp ../input/reverse-geocode/reverse_geocode.xyz /tmp/pip/cache/reverse_geocode.tar.gz\n!pip install /tmp/pip/cache/reverse_geocode.tar.gz","metadata":{"papermill":{"duration":36.953941,"end_time":"2022-06-07T00:17:09.827598","exception":false,"start_time":"2022-06-07T00:16:32.873657","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:19:39.450372Z","iopub.execute_input":"2022-07-01T00:19:39.450708Z","iopub.status.idle":"2022-07-01T00:20:13.320193Z","shell.execute_reply.started":"2022-07-01T00:19:39.450606Z","shell.execute_reply":"2022-07-01T00:20:13.31927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearnex import patch_sklearn\npatch_sklearn()\nimport pandas as pd\nimport numpy as np\nfrom tqdm.auto import tqdm\nimport os\nimport gc\nimport random\nfrom sklearn.model_selection import GroupKFold\nimport warnings\nimport pickle\nfrom unidecode import unidecode\nimport reverse_geocode\nimport string\nimport Levenshtein\nimport difflib\nfrom sklearn.neighbors import NearestNeighbors\nfrom transformers import AutoTokenizer\nimport time\nimport joblib\nfrom math import radians","metadata":{"id":"H5QntWoelAkH","outputId":"31efe7df-24ff-40e8-8517-0c7174968413","papermill":{"duration":0.83961,"end_time":"2022-06-07T00:17:11.036906","exception":false,"start_time":"2022-06-07T00:17:10.197296","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:13.322195Z","iopub.execute_input":"2022-07-01T00:20:13.322472Z","iopub.status.idle":"2022-07-01T00:20:21.293339Z","shell.execute_reply.started":"2022-07-01T00:20:13.322432Z","shell.execute_reply":"2022-07-01T00:20:21.29262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained('../input/robertabase/roberta-base')","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:21.294588Z","iopub.execute_input":"2022-07-01T00:20:21.295989Z","iopub.status.idle":"2022-07-01T00:20:21.432818Z","shell.execute_reply.started":"2022-07-01T00:20:21.29595Z","shell.execute_reply":"2022-07-01T00:20:21.432086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Config and Read","metadata":{"papermill":{"duration":0.049616,"end_time":"2022-06-07T00:17:11.135907","exception":false,"start_time":"2022-06-07T00:17:11.086291","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n    seed = 46\n    target = \"point_of_interest\"\n    n_neighbors = 23\n    n_splits = 3\n    threshold = 0.52881\nrandom.seed(CFG.seed)\nos.environ[\"PYTHONHASHSEED\"] = str(CFG.seed)\nnp.random.seed(CFG.seed)","metadata":{"papermill":{"duration":0.059316,"end_time":"2022-06-07T00:17:11.245406","exception":false,"start_time":"2022-06-07T00:17:11.18609","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:21.434941Z","iopub.execute_input":"2022-07-01T00:20:21.435212Z","iopub.status.idle":"2022-07-01T00:20:21.440353Z","shell.execute_reply.started":"2022-07-01T00:20:21.435177Z","shell.execute_reply":"2022-07-01T00:20:21.439488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(\"../input/foursquare-location-matching/test.csv\")\ntest[CFG.target] = \"TEST\"","metadata":{"id":"wz7JepVilAkN","outputId":"0652de28-9bd3-4ab7-c97c-55e6e11935e6","papermill":{"duration":0.078064,"end_time":"2022-06-07T00:17:11.373548","exception":false,"start_time":"2022-06-07T00:17:11.295484","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:21.44193Z","iopub.execute_input":"2022-07-01T00:20:21.44237Z","iopub.status.idle":"2022-07-01T00:20:21.464505Z","shell.execute_reply.started":"2022-07-01T00:20:21.442336Z","shell.execute_reply":"2022-07-01T00:20:21.46391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head(3)","metadata":{"papermill":{"duration":0.08028,"end_time":"2022-06-07T00:17:11.503808","exception":false,"start_time":"2022-06-07T00:17:11.423528","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:21.465473Z","iopub.execute_input":"2022-07-01T00:20:21.465728Z","iopub.status.idle":"2022-07-01T00:20:21.487956Z","shell.execute_reply.started":"2022-07-01T00:20:21.465694Z","shell.execute_reply":"2022-07-01T00:20:21.487319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Clean data","metadata":{"papermill":{"duration":0.050458,"end_time":"2022-06-07T00:17:11.604788","exception":false,"start_time":"2022-06-07T00:17:11.55433","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Get address number to create a new feature later\ntest['number'] = test.address.str.extract('(\\d+)')","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:21.489025Z","iopub.execute_input":"2022-07-01T00:20:21.489324Z","iopub.status.idle":"2022-07-01T00:20:21.494352Z","shell.execute_reply.started":"2022-07-01T00:20:21.489289Z","shell.execute_reply":"2022-07-01T00:20:21.493703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.drop(['state'], axis=1, inplace=True)","metadata":{"papermill":{"duration":0.064729,"end_time":"2022-06-07T00:17:11.71937","exception":false,"start_time":"2022-06-07T00:17:11.654641","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:21.49555Z","iopub.execute_input":"2022-07-01T00:20:21.495972Z","iopub.status.idle":"2022-07-01T00:20:21.511165Z","shell.execute_reply.started":"2022-07-01T00:20:21.495911Z","shell.execute_reply":"2022-07-01T00:20:21.510542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:21.512436Z","iopub.execute_input":"2022-07-01T00:20:21.512788Z","iopub.status.idle":"2022-07-01T00:20:21.68365Z","shell.execute_reply.started":"2022-07-01T00:20:21.512754Z","shell.execute_reply":"2022-07-01T00:20:21.682917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_country(cols):\n    lat = cols[0]\n    long = cols[1]\n    country = cols[2]\n    \n    if country != country: # check if is nan\n        coordinates = (lat, long),\n        result = reverse_geocode.search(coordinates)\n        return result[0]['country_code']\n    return country","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:21.687069Z","iopub.execute_input":"2022-07-01T00:20:21.68727Z","iopub.status.idle":"2022-07-01T00:20:21.693368Z","shell.execute_reply.started":"2022-07-01T00:20:21.687246Z","shell.execute_reply":"2022-07-01T00:20:21.692559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['country'] = test[['latitude','longitude','country']].apply(clean_country, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:21.695204Z","iopub.execute_input":"2022-07-01T00:20:21.695459Z","iopub.status.idle":"2022-07-01T00:20:21.704743Z","shell.execute_reply.started":"2022-07-01T00:20:21.695421Z","shell.execute_reply":"2022-07-01T00:20:21.703978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"china = test.loc[test['country'] == 'CN']['name'].values\njapan = test.loc[test['country'] == 'JP']['name'].values","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:21.706188Z","iopub.execute_input":"2022-07-01T00:20:21.706491Z","iopub.status.idle":"2022-07-01T00:20:21.71706Z","shell.execute_reply.started":"2022-07-01T00:20:21.706464Z","shell.execute_reply":"2022-07-01T00:20:21.716271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def standard_data(df):\n    columns = ['name', 'categories', 'country', 'zip', 'phone', 'url', 'city']\n    \n    for c in columns:        \n        df[c] = df[c].astype(str).str.lower()\n        df[c] = df[c].apply(lambda x: unidecode(x)) # will remove for china and japan later\n        df[c] = df[c].astype(str).str.lower()\n        if c in ['zip', 'phone', 'url']:\n            df[c] = df[c].str.replace('[{}]'.format(string.punctuation), '')\n            df[c] = df[c].str.replace(' ', '')\n        if c == 'url':\n            df[c] = df[c].str.replace('http://', '')\n            df[c] = df[c].str.replace('https://', '')\n            df[c] = df[c].str.replace('http:', '')\n            df[c] = df[c].str.replace('https:', '')\n            df[c] = df[c].str.replace('http', '')\n            df[c] = df[c].str.replace('https', '')\n            df[c] = df[c].str.replace('www.', '')\n            df[c] = df[c].str.replace('www', '')\n        df[c] = df[c].replace('nan', np.nan)\n    return df","metadata":{"papermill":{"duration":0.061251,"end_time":"2022-06-07T00:17:11.832345","exception":false,"start_time":"2022-06-07T00:17:11.771094","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:21.718481Z","iopub.execute_input":"2022-07-01T00:20:21.719037Z","iopub.status.idle":"2022-07-01T00:20:21.73109Z","shell.execute_reply.started":"2022-07-01T00:20:21.718976Z","shell.execute_reply":"2022-07-01T00:20:21.73033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = standard_data(test)","metadata":{"papermill":{"duration":0.071084,"end_time":"2022-06-07T00:17:11.953063","exception":false,"start_time":"2022-06-07T00:17:11.881979","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:21.732351Z","iopub.execute_input":"2022-07-01T00:20:21.732657Z","iopub.status.idle":"2022-07-01T00:20:21.769463Z","shell.execute_reply.started":"2022-07-01T00:20:21.732611Z","shell.execute_reply":"2022-07-01T00:20:21.768328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.loc[test['country'] == 'cn', 'name'] = china\ntest.loc[test['country'] == 'jp', 'name'] = japan\ndel china, japan\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:21.770631Z","iopub.execute_input":"2022-07-01T00:20:21.77086Z","iopub.status.idle":"2022-07-01T00:20:21.933509Z","shell.execute_reply.started":"2022-07-01T00:20:21.77083Z","shell.execute_reply":"2022-07-01T00:20:21.932776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add count features","metadata":{"papermill":{"duration":0.049186,"end_time":"2022-06-07T00:17:14.274486","exception":false,"start_time":"2022-06-07T00:17:14.2253","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test['latitude_round'] = test['latitude'].round(1)\ntest['longitude_round'] = test['longitude'].round(1)\n\ntest['latitude'] = test['latitude'].apply(lambda x: radians(x))\ntest['longitude'] = test['longitude'].apply(lambda x: radians(x))","metadata":{"papermill":{"duration":0.059267,"end_time":"2022-06-07T00:17:14.383303","exception":false,"start_time":"2022-06-07T00:17:14.324036","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:21.934915Z","iopub.execute_input":"2022-07-01T00:20:21.93532Z","iopub.status.idle":"2022-07-01T00:20:21.946719Z","shell.execute_reply.started":"2022-07-01T00:20:21.935282Z","shell.execute_reply":"2022-07-01T00:20:21.945759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['latitude_round_count'] = -99\ntest['longitude_round_count'] = -99\ntest['name_count'] = -99\ntest['country_count'] = -99","metadata":{"papermill":{"duration":0.074659,"end_time":"2022-06-07T00:17:14.517654","exception":false,"start_time":"2022-06-07T00:17:14.442995","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:21.948398Z","iopub.execute_input":"2022-07-01T00:20:21.948809Z","iopub.status.idle":"2022-07-01T00:20:21.958133Z","shell.execute_reply.started":"2022-07-01T00:20:21.948778Z","shell.execute_reply":"2022-07-01T00:20:21.957339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_values(df, col):\n    freq_encode = df[col].value_counts(dropna=True).to_dict()\n    df[col+'_count'] = df[col].map(freq_encode)\n    return df[col+'_count']","metadata":{"papermill":{"duration":0.058028,"end_time":"2022-06-07T00:17:14.625429","exception":false,"start_time":"2022-06-07T00:17:14.567401","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:21.95926Z","iopub.execute_input":"2022-07-01T00:20:21.959575Z","iopub.status.idle":"2022-07-01T00:20:21.968779Z","shell.execute_reply.started":"2022-07-01T00:20:21.95954Z","shell.execute_reply":"2022-07-01T00:20:21.968141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ['country']:\n    test[col+'_count'] = count_values(test, col)\n    test[col+'_count'].fillna(-99, inplace=True)\n    test[col+'_count'] = test[col+'_count'].astype(int)","metadata":{"papermill":{"duration":0.071512,"end_time":"2022-06-07T00:17:14.74658","exception":false,"start_time":"2022-06-07T00:17:14.675068","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:21.969839Z","iopub.execute_input":"2022-07-01T00:20:21.970954Z","iopub.status.idle":"2022-07-01T00:20:21.98296Z","shell.execute_reply.started":"2022-07-01T00:20:21.970902Z","shell.execute_reply":"2022-07-01T00:20:21.98219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:21.984243Z","iopub.execute_input":"2022-07-01T00:20:21.984491Z","iopub.status.idle":"2022-07-01T00:20:22.160208Z","shell.execute_reply.started":"2022-07-01T00:20:21.984459Z","shell.execute_reply":"2022-07-01T00:20:22.159484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Helper functions","metadata":{}},{"cell_type":"code","source":"def add_distance_features(cols, *args):\n    str1 = cols[0]\n    str2 = cols[1]\n    feat = ''.join(args)\n    \n    if str1 == str1 and str2 == str2 and str1 != '' and str2 != '':\n        if feat == 'leven':\n            if str1 == str2:\n                return 0\n            return Levenshtein.distance(str1, str2) # Levenshtein\n        elif feat == 'jaro':\n            if str1 == str2:\n                return 1\n            return Levenshtein.jaro_winkler(str1, str2) # jaro_winkler\n        elif feat == 'lcs':\n            return LCS(str1, str2) # LCS\n        elif feat == 'jaccard_char':\n            if str1 == str2:\n                return 1\n            return calculate_jaccard_char(str1, str2) # jaccard char\n        elif feat == 'jaccard_char_smallest':\n            if str1 == str2:\n                return 1\n            return calculate_jaccard_char_smallest(str1, str2) # jaccard char smallest\n        elif feat == 'jaccard_word':\n            if str1 == str2:\n                return 1\n            return calculate_jaccard_word(str1, str2) # jaccard word\n        elif feat == 'jaccard_word_smallest':\n            if str1 == str2:\n                return 1\n            return calculate_jaccard_word_smallest(str1, str2) # jaccard word smallest\n    return -99","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:22.161353Z","iopub.execute_input":"2022-07-01T00:20:22.161659Z","iopub.status.idle":"2022-07-01T00:20:22.170731Z","shell.execute_reply.started":"2022-07-01T00:20:22.16161Z","shell.execute_reply":"2022-07-01T00:20:22.170019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_jaccard_char(str1, str2):\n    \n    # Combine both tokens to find union.\n    both_tokens = str1 + str2\n    union = set(both_tokens)\n    if len(union) == 0:\n        return 0\n    \n    # Calculate intersection.\n    intersection = set()\n    for w in set(str1):\n        if w in set(str2):\n            intersection.add(w)\n\n    jaccard_score = len(intersection)/len(union)\n    \n    return jaccard_score","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:22.172081Z","iopub.execute_input":"2022-07-01T00:20:22.172555Z","iopub.status.idle":"2022-07-01T00:20:22.185442Z","shell.execute_reply.started":"2022-07-01T00:20:22.172517Z","shell.execute_reply":"2022-07-01T00:20:22.184687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_jaccard_char_smallest(str1, str2):\n    str1 = set(str1)\n    str2 = set(str2)\n    \n    small = min(len(str1), len(str2))\n    if small == 0:\n        return 0\n    \n    # Calculate intersection.\n    intersection = set()\n    for w in str1:\n        if w in str2:\n            intersection.add(w)\n\n    jaccard_score = len(intersection)/small\n    \n    return jaccard_score","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:22.18683Z","iopub.execute_input":"2022-07-01T00:20:22.187151Z","iopub.status.idle":"2022-07-01T00:20:22.195582Z","shell.execute_reply.started":"2022-07-01T00:20:22.187117Z","shell.execute_reply":"2022-07-01T00:20:22.19494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_jaccard_word(str1, str2):\n    \n    # Combine both tokens to find union.\n    words1 = str1.split()\n    words2 = str2.split()\n    union = set(words1 + words2)\n    if len(union) == 0:\n        return 0\n    \n    # Calculate intersection.\n    intersection = set()\n    for word in union:\n        if word in words1 and word in words2:\n            intersection.add(word)\n\n    jaccard_score = len(intersection)/len(union)\n    \n    return jaccard_score","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:22.196579Z","iopub.execute_input":"2022-07-01T00:20:22.198772Z","iopub.status.idle":"2022-07-01T00:20:22.209279Z","shell.execute_reply.started":"2022-07-01T00:20:22.198735Z","shell.execute_reply":"2022-07-01T00:20:22.208515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_jaccard_word_smallest(str1, str2):\n    \n    if str1 == str2:\n        return 1\n    \n    # Combine both tokens to find union.\n    words1 = str1.split()\n    words2 = str2.split()\n    union = set(words1 + words2)\n    small = min(len(set(words1)), len(set(words2)))\n    if small == 0:\n        return 0\n    \n    # Calculate intersection.\n    intersection = set()\n    for word in union:\n        if word in words1 and word in words2:\n            intersection.add(word)\n\n    jaccard_score = len(intersection)/small\n    \n    return jaccard_score","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:22.210691Z","iopub.execute_input":"2022-07-01T00:20:22.210993Z","iopub.status.idle":"2022-07-01T00:20:22.219528Z","shell.execute_reply.started":"2022-07-01T00:20:22.210959Z","shell.execute_reply":"2022-07-01T00:20:22.21874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext Cython","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:22.221721Z","iopub.execute_input":"2022-07-01T00:20:22.222685Z","iopub.status.idle":"2022-07-01T00:20:23.236732Z","shell.execute_reply.started":"2022-07-01T00:20:22.22265Z","shell.execute_reply":"2022-07-01T00:20:23.236029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%cython\nimport numpy as np  # noqa\ncpdef int LCS(str S, str T):\n    cdef int i, j\n    cdef int cost\n    cdef int v1,v2,v3,v4\n    cdef int[:, :] dp = np.zeros((len(S) + 1, len(T) + 1), dtype=np.int32)\n    for i in range(len(S)):\n        for j in range(len(T)):\n            cost = (int)(S[i] == T[j])\n            v1 = dp[i, j] + cost\n            v2 = dp[i + 1, j]\n            v3 = dp[i, j + 1]\n            v4 = dp[i + 1, j + 1]\n            dp[i + 1, j + 1] = max((v1,v2,v3,v4))\n    return dp[len(S)][len(T)]","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:23.239462Z","iopub.execute_input":"2022-07-01T00:20:23.239791Z","iopub.status.idle":"2022-07-01T00:20:29.482286Z","shell.execute_reply.started":"2022-07-01T00:20:23.239761Z","shell.execute_reply":"2022-07-01T00:20:29.481255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from math import sin, cos, sqrt, atan2\n\ndef get_real_distance(cols):\n    R = 6378.0 # radius of earth in km\n    \n    lat1 = cols[0]\n    lon1 = cols[1]\n    lat2 = cols[2]\n    lon2 = cols[3]\n\n    dlon = lon2 - lon1\n    dlat = lat2 - lat1\n\n    a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2\n    c = 2 * atan2(sqrt(a), sqrt(1 - a))\n\n    distance = R * c\n\n    return distance","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:29.488942Z","iopub.execute_input":"2022-07-01T00:20:29.489162Z","iopub.status.idle":"2022-07-01T00:20:29.498186Z","shell.execute_reply.started":"2022-07-01T00:20:29.489135Z","shell.execute_reply":"2022-07-01T00:20:29.497534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Search Candidates","metadata":{"id":"9yJIRkRD3jr-","papermill":{"duration":0.053159,"end_time":"2022-06-07T00:17:14.962604","exception":false,"start_time":"2022-06-07T00:17:14.909445","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if len(test) < CFG.n_neighbors:\n    CFG.n_neighbors = len(test)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:29.501017Z","iopub.execute_input":"2022-07-01T00:20:29.501224Z","iopub.status.idle":"2022-07-01T00:20:29.519414Z","shell.execute_reply.started":"2022-07-01T00:20:29.5012Z","shell.execute_reply":"2022-07-01T00:20:29.518673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_vectors(df, col):\n    \"\"\"Function to search candidates by name vectorized + latitude + longitude\"\"\"\n    encoded_input = tokenizer(df[col].fillna('nan').tolist(), return_tensors='np', padding=True)\n    encoded_input = encoded_input['input_ids']\n    encoded_input = np.c_[encoded_input, df['latitude'].values, df['longitude'].values]\n    normalized_encoded_input = encoded_input/np.linalg.norm(encoded_input)\n    return normalized_encoded_input","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:29.520764Z","iopub.execute_input":"2022-07-01T00:20:29.521067Z","iopub.status.idle":"2022-07-01T00:20:29.52925Z","shell.execute_reply.started":"2022-07-01T00:20:29.520976Z","shell.execute_reply":"2022-07-01T00:20:29.528495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_neighbors_vector(embeddings, n_neighbors, metric, algorithm):\n    \"\"\"Function to search candidates by name vectorized + latitude + longitude\"\"\"\n    matcher = NearestNeighbors(n_neighbors = n_neighbors,\n                       metric = metric,\n                       radius=1,\n                       algorithm=algorithm,\n                       leaf_size=32,\n                       p=2,\n                       n_jobs=-1)\n    \n    matcher.fit(embeddings)\n    distances, indices = matcher.kneighbors(embeddings)\n    return distances, indices","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:29.530362Z","iopub.execute_input":"2022-07-01T00:20:29.530726Z","iopub.status.idle":"2022-07-01T00:20:29.539581Z","shell.execute_reply.started":"2022-07-01T00:20:29.530631Z","shell.execute_reply":"2022-07-01T00:20:29.53886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_neighbors_lat_long(df, n_neighbors, metric, algorithm):\n    \"\"\"Function to search candidates by latitude and longitude\"\"\"\n    matcher = NearestNeighbors(n_neighbors = n_neighbors,\n                       metric = metric,\n                       radius=1,\n                       algorithm=algorithm,\n                       leaf_size=32,\n                       p=2,\n                       n_jobs=-1)\n    matcher.fit(df[['latitude', 'longitude']])\n    distances, indices = matcher.kneighbors(df[['latitude', 'longitude']])\n    return distances, indices","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:29.541042Z","iopub.execute_input":"2022-07-01T00:20:29.541409Z","iopub.status.idle":"2022-07-01T00:20:29.549576Z","shell.execute_reply.started":"2022-07-01T00:20:29.541368Z","shell.execute_reply":"2022-07-01T00:20:29.548873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidate_df = pd.DataFrame()\n\nfor country, country_df in tqdm(test.groupby(\"country\")):\n    dfs = []\n    country_df = country_df.reset_index(drop=True)\n    \n    # name count\n    freq_encode = country_df['name'].value_counts(dropna=True).to_dict()\n    country_df['name_count'] = country_df['name'].map(freq_encode)\n    \n    # latitude_round count\n    freq_encode = country_df['latitude_round'].value_counts(dropna=True).to_dict()\n    country_df['latitude_round_count'] = country_df['latitude_round'].map(freq_encode)\n    \n    # longitude_round count\n    freq_encode = country_df['longitude_round'].value_counts(dropna=True).to_dict()\n    country_df['longitude_round_count'] = country_df['longitude_round'].map(freq_encode)\n    \n    # get neighbors by name vectorized + lat + long\n    embeddings = get_vectors(country_df, 'name')\n    distances, indices = get_neighbors_vector(embeddings, min(len(country_df), CFG.n_neighbors), 'hamming', 'auto')\n\n    # get neighbors by lat + long\n    distances2, indices2 = get_neighbors_lat_long(country_df, min(len(country_df), CFG.n_neighbors), 'manhattan', 'auto')\n    \n    for i in range(min(len(country_df), CFG.n_neighbors)):        \n        # name vectorized + lat + long\n        tmp_df = pd.DataFrame()\n        tmp_df = country_df[[\"id\"]].copy()\n        tmp_df[\"dist\"] = distances[:, i]\n        tmp_df['dist_mean_neighboors'] = distances[:, :].mean()\n        tmp_df[\"dist_type\"] = 0\n        tmp_df[\"country\"] = country\n        tmp_df['country_count'] = country_df['country_count']\n        tmp_df['latitude_round_count'] = country_df['latitude_round_count']\n        tmp_df['latitude_round_count_neighbor'] = country_df['latitude_round_count'].values[indices[:, i]]\n        tmp_df['longitude_round_count'] = country_df['longitude_round_count']\n        tmp_df['longitude_round_count_neighbor'] = country_df['longitude_round_count'].values[indices[:, i]]\n        tmp_df['latitude'] = country_df['latitude']\n        tmp_df['latitude_neighbor'] = country_df['latitude'].values[indices[:, i]]\n        tmp_df['longitude'] = country_df['longitude']\n        tmp_df['longitude_neighbor'] = country_df['longitude'].values[indices[:, i]]\n        tmp_df['id_neighbor'] = country_df['id'].values[indices[:, i]]\n        tmp_df[\"neighbor_nearest\"] = i\n        tmp_df['number'] = country_df['number']\n        tmp_df['number_neighbor'] = country_df['number'].values[indices[:, i]]\n        tmp_df['name'] = country_df['name']\n        tmp_df['name_neighbor'] = country_df['name'].values[indices[:, i]]\n        tmp_df['name_count'] = country_df['name_count']\n        tmp_df['name_count_neighbor'] = country_df['name_count'].values[indices[:, i]]\n        tmp_df['categories'] = country_df['categories']\n        tmp_df['categories_neighbor'] = country_df['categories'].values[indices[:, i]]\n        tmp_df['address'] = country_df['address']\n        tmp_df['address_neighbor'] = country_df['address'].values[indices[:, i]]\n        tmp_df['zip'] = country_df['zip']\n        tmp_df['zip_neighbor'] = country_df['zip'].values[indices[:, i]]\n        tmp_df['phone'] = country_df['phone']\n        tmp_df['phone_neighbor'] = country_df['phone'].values[indices[:, i]]\n        tmp_df['url'] = country_df['url']\n        tmp_df['url_neighbor'] = country_df['url'].values[indices[:, i]]\n        tmp_df['city'] = country_df['city']\n        tmp_df['city_neighbor'] = country_df['city'].values[indices[:, i]]\n        tmp_df['name_jaccard_char'] = tmp_df[['name', 'name_neighbor']].apply(add_distance_features, args='jaccard_char', axis=1)\n        tmp_df['name_jaccard_char_smallest'] = tmp_df[['name', 'name_neighbor']].apply(add_distance_features, args='jaccard_char_smallest', axis=1)\n        tmp_df['name_jaccard_word'] = tmp_df[['name', 'name_neighbor']].apply(add_distance_features, args='jaccard_word', axis=1)\n        tmp_df['name_jaccard_word_smallest'] = tmp_df[['name', 'name_neighbor']].apply(add_distance_features, args='jaccard_word_smallest', axis=1)\n        \n        # lat + long\n        tmp_df2 = pd.DataFrame()\n        tmp_df2 = country_df[[\"id\"]].copy()\n        tmp_df2[\"dist\"] = distances2[:, i]\n        tmp_df2['dist_mean_neighboors'] = distances2[:, :].mean()\n        tmp_df2[\"dist_type\"] = 1\n        tmp_df2[\"country\"] = country\n        tmp_df2['country_count'] = country_df['country_count']\n        tmp_df2['latitude_round_count'] = country_df['latitude_round_count']\n        tmp_df2['latitude_round_count_neighbor'] = country_df['latitude_round_count'].values[indices2[:, i]]\n        tmp_df2['longitude_round_count'] = country_df['longitude_round_count']\n        tmp_df2['longitude_round_count_neighbor'] = country_df['longitude_round_count'].values[indices2[:, i]]\n        tmp_df2['latitude'] = country_df['latitude']\n        tmp_df2['latitude_neighbor'] = country_df['latitude'].values[indices2[:, i]]\n        tmp_df2['longitude'] = country_df['longitude']\n        tmp_df2['longitude_neighbor'] = country_df['longitude'].values[indices2[:, i]]\n        tmp_df2['id_neighbor'] = country_df['id'].values[indices2[:, i]]\n        tmp_df2[\"neighbor_nearest\"] = i\n        tmp_df2['number'] = country_df['number']\n        tmp_df2['number_neighbor'] = country_df['number'].values[indices2[:, i]]\n        tmp_df2['name'] = country_df['name']\n        tmp_df2['name_neighbor'] = country_df['name'].values[indices2[:, i]]\n        tmp_df2['name_count'] = country_df['name_count']\n        tmp_df2['name_count_neighbor'] = country_df['name_count'].values[indices2[:, i]]\n        tmp_df2['categories'] = country_df['categories']\n        tmp_df2['categories_neighbor'] = country_df['categories'].values[indices2[:, i]]\n        tmp_df2['address'] = country_df['address']\n        tmp_df2['address_neighbor'] = country_df['address'].values[indices[:, i]]\n        tmp_df2['zip'] = country_df['zip']\n        tmp_df2['zip_neighbor'] = country_df['zip'].values[indices[:, i]]\n        tmp_df2['phone'] = country_df['phone']\n        tmp_df2['phone_neighbor'] = country_df['phone'].values[indices[:, i]]\n        tmp_df2['url'] = country_df['url']\n        tmp_df2['url_neighbor'] = country_df['url'].values[indices[:, i]]\n        tmp_df2['city'] = country_df['city']\n        tmp_df2['city_neighbor'] = country_df['city'].values[indices[:, i]]\n        tmp_df2['name_jaccard_char'] = tmp_df2[['name', 'name_neighbor']].apply(add_distance_features, args='jaccard_char', axis=1)\n        tmp_df2['name_jaccard_char_smallest'] = tmp_df2[['name', 'name_neighbor']].apply(add_distance_features, args='jaccard_char_smallest', axis=1)\n        tmp_df2['name_jaccard_word'] = tmp_df2[['name', 'name_neighbor']].apply(add_distance_features, args='jaccard_word', axis=1)\n        tmp_df2['name_jaccard_word_smallest'] = tmp_df2[['name', 'name_neighbor']].apply(add_distance_features, args='jaccard_word_smallest', axis=1)\n        \n        # Concat both dataframes\n        tmp_final = pd.concat([tmp_df,tmp_df2])\n        tmp_final.drop_duplicates(subset=['id', 'id_neighbor'], keep='first', inplace=True, ignore_index=False)\n        \n        # Calculate real dist\n        tmp_final['real_dist'] = tmp_final[['latitude','longitude','latitude_neighbor','longitude_neighbor']].apply(get_real_distance, axis=1)\n        tmp_final.drop(['latitude','longitude','latitude_neighbor','longitude_neighbor'], axis=1, inplace=True)\n        gc.collect()\n        \n        # Filter\n        tmp_final = tmp_final[(tmp_final['neighbor_nearest'] <= 1) | (tmp_final['real_dist'] < 15) | (tmp_final['name_jaccard_char'] >= 0.75) | (tmp_final['name_jaccard_word'] >= 0.25)]\n        tmp_final = tmp_final[(tmp_final['neighbor_nearest'] <= 10) | (tmp_final['real_dist'] < 10) | (tmp_final['name_jaccard_char_smallest'] >= 0.9) | (tmp_final['name_jaccard_word'] >= 0.3)]\n        tmp_final = tmp_final[(tmp_final['neighbor_nearest'] <= 20) | (tmp_final['real_dist'] < 1) | (tmp_final['name_jaccard_char_smallest'] >= 0.99) | (tmp_final['name_jaccard_word_smallest'] >= 0.4)]\n        \n        dfs.append(tmp_final)\n        del tmp_df, tmp_df2, tmp_final\n        gc.collect()\n        \n    dfs = pd.concat(dfs)\n    candidate_df = pd.concat([candidate_df, dfs])\n\ndel dfs, tokenizer\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:29.552113Z","iopub.execute_input":"2022-07-01T00:20:29.552314Z","iopub.status.idle":"2022-07-01T00:20:32.971527Z","shell.execute_reply.started":"2022-07-01T00:20:29.552282Z","shell.execute_reply":"2022-07-01T00:20:32.970838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:32.973021Z","iopub.execute_input":"2022-07-01T00:20:32.973478Z","iopub.status.idle":"2022-07-01T00:20:33.168157Z","shell.execute_reply.started":"2022-07-01T00:20:32.973438Z","shell.execute_reply":"2022-07-01T00:20:33.167272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidate_df.drop('country', axis=1, inplace=True)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:33.169899Z","iopub.execute_input":"2022-07-01T00:20:33.170218Z","iopub.status.idle":"2022-07-01T00:20:33.378784Z","shell.execute_reply.started":"2022-07-01T00:20:33.170174Z","shell.execute_reply":"2022-07-01T00:20:33.377844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidate_df.reset_index(drop=True, inplace=True)\ncandidate_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:33.380444Z","iopub.execute_input":"2022-07-01T00:20:33.380713Z","iopub.status.idle":"2022-07-01T00:20:33.39338Z","shell.execute_reply.started":"2022-07-01T00:20:33.380675Z","shell.execute_reply":"2022-07-01T00:20:33.392749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reduce Memory","metadata":{}},{"cell_type":"code","source":"for col in tqdm(candidate_df.columns):\n    if '_count' in col:\n        candidate_df[col].fillna(-99, inplace=True)\n        candidate_df[col] = candidate_df[col].astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:33.39434Z","iopub.execute_input":"2022-07-01T00:20:33.394518Z","iopub.status.idle":"2022-07-01T00:20:33.435654Z","shell.execute_reply.started":"2022-07-01T00:20:33.394496Z","shell.execute_reply":"2022-07-01T00:20:33.435019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df, cols, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in tqdm(cols):\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    if 'int8' not in str(col_type):\n                        df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    if 'int16' not in str(col_type):\n                        df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    if 'int32' not in str(col_type):\n                        df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    if 'int64' not in str(col_type):\n                        df[col] = df[col].astype(np.int64)\n            else:\n                c_prec = df[col].apply(lambda x: np.finfo(x).precision).max()\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max and c_prec == np.finfo(np.float16).precision:\n                    if 'float16' not in str(col_type):\n                        df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max and c_prec == np.finfo(np.float32).precision:\n                    if 'float32' not in str(col_type):\n                        df[col] = df[col].astype(np.float32)\n                else:\n                    if 'float64' not in str(col_type):\n                        df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:33.436881Z","iopub.execute_input":"2022-07-01T00:20:33.43731Z","iopub.status.idle":"2022-07-01T00:20:33.457382Z","shell.execute_reply.started":"2022-07-01T00:20:33.437277Z","shell.execute_reply":"2022-07-01T00:20:33.45667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidate_df = reduce_mem_usage(candidate_df, candidate_df.columns)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:33.458767Z","iopub.execute_input":"2022-07-01T00:20:33.459023Z","iopub.status.idle":"2022-07-01T00:20:33.518891Z","shell.execute_reply.started":"2022-07-01T00:20:33.45899Z","shell.execute_reply":"2022-07-01T00:20:33.518115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:33.520215Z","iopub.execute_input":"2022-07-01T00:20:33.520459Z","iopub.status.idle":"2022-07-01T00:20:33.713857Z","shell.execute_reply.started":"2022-07-01T00:20:33.520426Z","shell.execute_reply":"2022-07-01T00:20:33.713061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Engineering","metadata":{"id":"6k59Vk9d5Pmx","papermill":{"duration":0.050459,"end_time":"2022-06-07T00:17:17.11685","exception":false,"start_time":"2022-06-07T00:17:17.066391","status":"completed"},"tags":[]}},{"cell_type":"code","source":"columns = ['name', 'categories']\nfeats = ['jaccard_char', 'jaccard_char_smallest', 'jaccard_word', 'jaccard_word_smallest',\n         'leven', 'jaro', 'lcs']\n\nfor col in tqdm(columns):\n    for feat in feats:\n        if col == 'name' and 'jaccard' in feat:\n            continue\n        candidate_df[col+'_'+feat] = candidate_df[[col, col+'_neighbor']].apply(add_distance_features, args=feat, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:33.715411Z","iopub.execute_input":"2022-07-01T00:20:33.715818Z","iopub.status.idle":"2022-07-01T00:20:33.770793Z","shell.execute_reply.started":"2022-07-01T00:20:33.715781Z","shell.execute_reply":"2022-07-01T00:20:33.770035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:33.772141Z","iopub.execute_input":"2022-07-01T00:20:33.772385Z","iopub.status.idle":"2022-07-01T00:20:33.965247Z","shell.execute_reply.started":"2022-07-01T00:20:33.772352Z","shell.execute_reply":"2022-07-01T00:20:33.964346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = ['address']\nfeats = ['jaccard_char', 'jaccard_word', 'lcs']\n\nfor col in tqdm(columns):\n    for feat in feats:\n        candidate_df[col+'_'+feat] = candidate_df[[col, col+'_neighbor']].apply(add_distance_features, args=feat, axis=1)\n    candidate_df.drop([col, col+'_neighbor'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:33.966969Z","iopub.execute_input":"2022-07-01T00:20:33.967237Z","iopub.status.idle":"2022-07-01T00:20:34.012349Z","shell.execute_reply.started":"2022-07-01T00:20:33.967204Z","shell.execute_reply":"2022-07-01T00:20:34.011686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:34.013655Z","iopub.execute_input":"2022-07-01T00:20:34.014082Z","iopub.status.idle":"2022-07-01T00:20:34.213408Z","shell.execute_reply.started":"2022-07-01T00:20:34.014045Z","shell.execute_reply":"2022-07-01T00:20:34.212596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = ['zip', 'phone', 'url']\nfeats = ['jaro']\n\nfor col in tqdm(columns):\n    for feat in feats:\n        candidate_df[col+'_'+feat] = candidate_df[[col, col+'_neighbor']].apply(add_distance_features, args=feat, axis=1)\n    candidate_df.drop([col, col+'_neighbor'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:34.214929Z","iopub.execute_input":"2022-07-01T00:20:34.215255Z","iopub.status.idle":"2022-07-01T00:20:34.261204Z","shell.execute_reply.started":"2022-07-01T00:20:34.215214Z","shell.execute_reply":"2022-07-01T00:20:34.259993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:34.262603Z","iopub.execute_input":"2022-07-01T00:20:34.262865Z","iopub.status.idle":"2022-07-01T00:20:34.45663Z","shell.execute_reply.started":"2022-07-01T00:20:34.262832Z","shell.execute_reply":"2022-07-01T00:20:34.455882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = ['city']\nfeats = ['jaccard_char']\n\nfor col in tqdm(columns):\n    print(col+':')\n    for feat in feats:\n        print(feat)\n        candidate_df[col+'_'+feat] = candidate_df[[col, col+'_neighbor']].apply(add_distance_features, args=feat, axis=1)\n    candidate_df.drop([col, col+'_neighbor'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:34.458097Z","iopub.execute_input":"2022-07-01T00:20:34.458524Z","iopub.status.idle":"2022-07-01T00:20:34.50221Z","shell.execute_reply.started":"2022-07-01T00:20:34.458487Z","shell.execute_reply":"2022-07-01T00:20:34.501519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:34.503483Z","iopub.execute_input":"2022-07-01T00:20:34.50388Z","iopub.status.idle":"2022-07-01T00:20:34.701886Z","shell.execute_reply.started":"2022-07-01T00:20:34.503844Z","shell.execute_reply":"2022-07-01T00:20:34.701183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidate_df = reduce_mem_usage(candidate_df, candidate_df.columns)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:34.703393Z","iopub.execute_input":"2022-07-01T00:20:34.703729Z","iopub.status.idle":"2022-07-01T00:20:34.947801Z","shell.execute_reply.started":"2022-07-01T00:20:34.703691Z","shell.execute_reply":"2022-07-01T00:20:34.946992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = ['name', 'categories', 'name_neighbor', 'categories_neighbor']\nfor col in tqdm(columns):\n    candidate_df[col+'_len'] = candidate_df[col].astype(str).map(len) \n    candidate_df.loc[candidate_df[col].isnull(), col+'_len'] = -99\n    candidate_df[col+'_count_word'] = candidate_df[col].astype(str).apply(lambda x: len(x.split()))\n    candidate_df.loc[candidate_df[col].isnull(), col+'_count_word'] = -99\n    candidate_df.drop(col, axis=1, inplace=True)\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:34.949231Z","iopub.execute_input":"2022-07-01T00:20:34.949485Z","iopub.status.idle":"2022-07-01T00:20:35.749009Z","shell.execute_reply.started":"2022-07-01T00:20:34.94945Z","shell.execute_reply":"2022-07-01T00:20:35.748331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidate_df['name_lcs_smallest'] = candidate_df['name_lcs'] / candidate_df[['name_len', 'name_neighbor_len']].min(axis=1)\ncandidate_df['name_lcs_biggest'] = candidate_df['name_lcs'] / candidate_df[['name_len', 'name_neighbor_len']].max(axis=1)\ncandidate_df['categories_lcs_smallest'] = candidate_df['categories_lcs'] / candidate_df[['categories_len', 'categories_neighbor_len']].min(axis=1)\ncandidate_df['categories_lcs_biggest'] = candidate_df['categories_lcs'] / candidate_df[['categories_len', 'categories_neighbor_len']].max(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:35.750059Z","iopub.execute_input":"2022-07-01T00:20:35.750564Z","iopub.status.idle":"2022-07-01T00:20:35.766516Z","shell.execute_reply.started":"2022-07-01T00:20:35.750524Z","shell.execute_reply":"2022-07-01T00:20:35.765886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidate_df['name_leven_biggest'] = candidate_df['name_leven'] / candidate_df[['name_len', 'name_neighbor_len']].max(axis=1)\ncandidate_df['categories_leven_biggest'] = candidate_df['categories_leven'] / candidate_df[['categories_len', 'categories_neighbor_len']].max(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:35.767699Z","iopub.execute_input":"2022-07-01T00:20:35.768165Z","iopub.status.idle":"2022-07-01T00:20:35.77697Z","shell.execute_reply.started":"2022-07-01T00:20:35.768127Z","shell.execute_reply":"2022-07-01T00:20:35.776273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create Feature About Address Number","metadata":{"papermill":{"duration":0.05935,"end_time":"2022-06-07T00:17:20.914854","exception":false,"start_time":"2022-06-07T00:17:20.855504","status":"completed"},"tags":[]}},{"cell_type":"code","source":"candidate_df['same_number'] = candidate_df['number'] == candidate_df['number_neighbor']\ncandidate_df['same_number'] = candidate_df['same_number'].astype(int)","metadata":{"papermill":{"duration":0.067969,"end_time":"2022-06-07T00:17:21.040608","exception":false,"start_time":"2022-06-07T00:17:20.972639","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:35.778028Z","iopub.execute_input":"2022-07-01T00:20:35.778491Z","iopub.status.idle":"2022-07-01T00:20:35.785106Z","shell.execute_reply.started":"2022-07-01T00:20:35.778453Z","shell.execute_reply":"2022-07-01T00:20:35.78433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidate_df.drop(['number', 'number_neighbor'], axis=1, inplace=True)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:35.786199Z","iopub.execute_input":"2022-07-01T00:20:35.786519Z","iopub.status.idle":"2022-07-01T00:20:35.983098Z","shell.execute_reply.started":"2022-07-01T00:20:35.786482Z","shell.execute_reply":"2022-07-01T00:20:35.982238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidate_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:35.984489Z","iopub.execute_input":"2022-07-01T00:20:35.984893Z","iopub.status.idle":"2022-07-01T00:20:35.994182Z","shell.execute_reply.started":"2022-07-01T00:20:35.984855Z","shell.execute_reply":"2022-07-01T00:20:35.993521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Final Reduce Memory","metadata":{}},{"cell_type":"code","source":"candidate_df.replace([np.inf, -np.inf], -99, inplace=True)\ncandidate_df.fillna(-99, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:35.995261Z","iopub.execute_input":"2022-07-01T00:20:35.995933Z","iopub.status.idle":"2022-07-01T00:20:36.005372Z","shell.execute_reply.started":"2022-07-01T00:20:35.995894Z","shell.execute_reply":"2022-07-01T00:20:36.004749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ncandidate_df = reduce_mem_usage(candidate_df, candidate_df.columns)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:36.007572Z","iopub.execute_input":"2022-07-01T00:20:36.008214Z","iopub.status.idle":"2022-07-01T00:20:36.454094Z","shell.execute_reply.started":"2022-07-01T00:20:36.008177Z","shell.execute_reply":"2022-07-01T00:20:36.453399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Delete Not Used Columns","metadata":{"id":"Q3HV5kfs6saZ","papermill":{"duration":0.057177,"end_time":"2022-06-07T00:17:21.155417","exception":false,"start_time":"2022-06-07T00:17:21.09824","status":"completed"},"tags":[]}},{"cell_type":"code","source":"len(candidate_df.columns)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:36.455194Z","iopub.execute_input":"2022-07-01T00:20:36.455905Z","iopub.status.idle":"2022-07-01T00:20:36.461659Z","shell.execute_reply.started":"2022-07-01T00:20:36.455864Z","shell.execute_reply":"2022-07-01T00:20:36.460949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = candidate_df.columns.to_list()\nnot_use = ['id', 'id_neighbor', 'name', 'name_neighbor', 'address', 'address_neighbor', 'city',\n          'city_neighbor', 'country', 'country_neighbor', 'zip', 'zip_neighbor', 'url', 'url_neighbor',\n          'phone', 'phone_neighbor', 'categories', 'categories_neighbor', 'point_of_interest',\n          'point_of_interest_neighbor', 'target', 'number', 'number_neighbor', 'name_gesh', 'categories_gesh']\nfeatures = [item for item in columns if item not in not_use]","metadata":{"id":"GHMG8t1UlAkT","outputId":"9c06b154-1276-4069-d21a-7c48907b98dc","papermill":{"duration":0.066536,"end_time":"2022-06-07T00:17:21.27877","exception":false,"start_time":"2022-06-07T00:17:21.212234","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:36.462866Z","iopub.execute_input":"2022-07-01T00:20:36.463321Z","iopub.status.idle":"2022-07-01T00:20:36.472207Z","shell.execute_reply.started":"2022-07-01T00:20:36.463288Z","shell.execute_reply":"2022-07-01T00:20:36.471542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(features)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:36.473239Z","iopub.execute_input":"2022-07-01T00:20:36.473784Z","iopub.status.idle":"2022-07-01T00:20:36.484199Z","shell.execute_reply.started":"2022-07-01T00:20:36.47375Z","shell.execute_reply":"2022-07-01T00:20:36.483428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidate_df = candidate_df[features + ['id', 'id_neighbor']]\ncandidate_df.reset_index(drop=True, inplace=True)","metadata":{"id":"aKSy9cc3lAkT","outputId":"4d1e8ccb-8d7f-454e-cb15-f1cb8693af74","papermill":{"duration":0.066967,"end_time":"2022-06-07T00:17:21.403988","exception":false,"start_time":"2022-06-07T00:17:21.337021","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:36.485853Z","iopub.execute_input":"2022-07-01T00:20:36.486092Z","iopub.status.idle":"2022-07-01T00:20:36.494955Z","shell.execute_reply.started":"2022-07-01T00:20:36.48606Z","shell.execute_reply":"2022-07-01T00:20:36.49423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:36.496247Z","iopub.execute_input":"2022-07-01T00:20:36.496504Z","iopub.status.idle":"2022-07-01T00:20:36.691418Z","shell.execute_reply.started":"2022-07-01T00:20:36.496471Z","shell.execute_reply":"2022-07-01T00:20:36.69063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inference","metadata":{"papermill":{"duration":0.057256,"end_time":"2022-06-07T00:17:21.518261","exception":false,"start_time":"2022-06-07T00:17:21.461005","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from catboost import CatBoostClassifier\nchunks = 10\nchunk_size = int(len(candidate_df) / chunks)\npred_cat = np.zeros(len(candidate_df))\npred_lgb = np.zeros(len(candidate_df))\n\nfor i in tqdm(range(CFG.n_splits)):\n    \n    # catboost\n    model = CatBoostClassifier()\n    model.load_model(f'../input/modelcatboostfoursquare2/model2/catboost_fold{i}.cbm')\n    for chunk in range(chunks):\n        if chunk < chunks - 1:\n            pred_cat[chunk*chunk_size:chunk*chunk_size+chunk_size] += model.predict_proba(candidate_df[features][chunk*chunk_size:chunk*chunk_size+chunk_size].to_numpy())[:,1] / CFG.n_splits\n        else:\n            pred_cat[chunk*chunk_size:] += model.predict_proba(candidate_df[features][chunk*chunk_size:].to_numpy())[:,1] / CFG.n_splits\n    del model\n    gc.collect()\n    \n    # lgb\n    model = joblib.load(f'../input/modellgbfoursquare29/model29/lgbm_fold{i}.pkl')\n    for chunk in range(chunks):\n        if chunk < chunks - 1:\n            pred_lgb[chunk*chunk_size:chunk*chunk_size+chunk_size] += model.predict(candidate_df[features][chunk*chunk_size:chunk*chunk_size+chunk_size].to_numpy()) / CFG.n_splits\n        else:\n            pred_lgb[chunk*chunk_size:] += model.predict(candidate_df[features][chunk*chunk_size:].to_numpy()) / CFG.n_splits\n    del model\n    gc.collect()\n\ncandidate_df = candidate_df[['id', 'id_neighbor']]\npred = 0.51*pred_cat + 0.49*pred_lgb # original was 0.5 0.5\ndel pred_cat, pred_lgb\ngc.collect()\npred = (pred >= CFG.threshold).astype(int)\ncandidate_df['pred'] = pred","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:36.694374Z","iopub.execute_input":"2022-07-01T00:20:36.694582Z","iopub.status.idle":"2022-07-01T00:20:42.364299Z","shell.execute_reply.started":"2022-07-01T00:20:36.694548Z","shell.execute_reply":"2022-07-01T00:20:42.36359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Set matches","metadata":{}},{"cell_type":"code","source":"test_original = pd.read_csv(\"../input/foursquare-location-matching/test.csv\", usecols=['id'])","metadata":{"papermill":{"duration":0.070158,"end_time":"2022-06-07T00:17:23.453007","exception":false,"start_time":"2022-06-07T00:17:23.382849","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:42.365774Z","iopub.execute_input":"2022-07-01T00:20:42.366168Z","iopub.status.idle":"2022-07-01T00:20:42.373346Z","shell.execute_reply.started":"2022-07-01T00:20:42.366131Z","shell.execute_reply":"2022-07-01T00:20:42.372608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"matches = []\nids_check = []\ncandidate_df_ones = candidate_df[candidate_df['pred'] == 1][['id','id_neighbor']]\nfor id, id_df in tqdm(candidate_df_ones.groupby('id')):\n    match = id_df['id_neighbor'].to_list()\n    if id not in match:\n        match = [id] + match\n    match = ' '.join(match)\n    matches.append(match)\n    ids_check.append(id)\n\ndel candidate_df, candidate_df_ones\ngc.collect()\n\ntest_original[\"matches\"] = test_original[\"id\"]\ntest_original.loc[test_original['id'].isin(ids_check), 'matches'] = matches","metadata":{"papermill":{"duration":0.106748,"end_time":"2022-06-07T00:17:23.617771","exception":false,"start_time":"2022-06-07T00:17:23.511023","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:42.374968Z","iopub.execute_input":"2022-07-01T00:20:42.375371Z","iopub.status.idle":"2022-07-01T00:20:42.649394Z","shell.execute_reply.started":"2022-07-01T00:20:42.375334Z","shell.execute_reply":"2022-07-01T00:20:42.648656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Postproccess","metadata":{}},{"cell_type":"code","source":"def postprocess(df):\n    \"\"\" if A matches B then B matches A\"\"\"\n    id2match = dict(zip(df[\"id\"].values, df[\"matches\"].str.split()))\n\n    for match in tqdm(df[\"matches\"]):\n        match = match.split()\n        if len(match) == 1:        \n            continue\n\n        base = match[0]\n        for m in match[1:]:\n            if not base in id2match[m]:\n                id2match[m].append(base)\n    df[\"matches\"] = df[\"id\"].map(id2match).map(\" \".join)\n    return df \n\ntest_original = postprocess(test_original)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:42.650787Z","iopub.execute_input":"2022-07-01T00:20:42.651029Z","iopub.status.idle":"2022-07-01T00:20:42.688842Z","shell.execute_reply.started":"2022-07-01T00:20:42.650996Z","shell.execute_reply":"2022-07-01T00:20:42.688135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def postprocess2(df):\n    \"\"\" if A matches B and B matches C, then A matchs C and C matches A \"\"\"\n    match_map = df.set_index('id')['matches'].to_dict()\n    matches = []\n    ids_check = []\n    for id, id_df in tqdm(df.groupby('id')):\n        match = id_df['matches'].str.split().iloc[0]\n        new_match = []\n        for match_id in match:\n            if match_id != id:\n                new_match += match_map[match_id].split()\n\n        if len(new_match) > 0:\n            match += new_match\n            match = list(dict.fromkeys(match)) # remove duplicates\n        match = ' '.join(match)\n        matches.append(match)\n        ids_check.append(id)\n\n    df.loc[df['id'].isin(ids_check), 'matches'] = matches\n    return df\n\ntest_original = postprocess2(test_original)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T00:20:42.690356Z","iopub.execute_input":"2022-07-01T00:20:42.690893Z","iopub.status.idle":"2022-07-01T00:20:42.745925Z","shell.execute_reply.started":"2022-07-01T00:20:42.690848Z","shell.execute_reply":"2022-07-01T00:20:42.745138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submit","metadata":{"papermill":{"duration":0.058093,"end_time":"2022-06-07T00:17:23.982251","exception":false,"start_time":"2022-06-07T00:17:23.924158","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ssub = pd.read_csv(\"../input/foursquare-location-matching/sample_submission.csv\")\nssub = ssub.drop(columns=\"matches\")\nssub = ssub.merge(test_original[[\"id\", \"matches\"]], on=\"id\")\nssub.to_csv(\"submission.csv\", index=False)\n\nssub.head()","metadata":{"papermill":{"duration":0.094804,"end_time":"2022-06-07T00:17:24.13712","exception":false,"start_time":"2022-06-07T00:17:24.042316","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-01T00:20:42.747268Z","iopub.execute_input":"2022-07-01T00:20:42.747615Z","iopub.status.idle":"2022-07-01T00:20:42.789256Z","shell.execute_reply.started":"2022-07-01T00:20:42.747576Z","shell.execute_reply":"2022-07-01T00:20:42.788588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"👍","metadata":{}}]}