{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"## Imports\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nimport gc\nimport time\nimport random\nimport Levenshtein\nimport difflib\nimport multiprocessing\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom tqdm.auto import tqdm\ntqdm.pandas()\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.neighbors import NearestNeighbors\n\nfrom catboost import CatBoost, CatBoostRegressor, CatBoostClassifier\nfrom catboost import Pool\nimport pickle","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('../input/sentencetransformer220/sentence-transformers-2.2.0')\nfrom sentence_transformers import SentenceTransformer","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ../input/faissgpu17/faiss_gpu-1.7.0-cp37-cp37m-manylinux2014_x86_64.whl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import faiss\n\n\nTRAIN_FEATURES = [\n                'kdist',\n                'kneighbors',\n                'name_sim',\n                'name_gesh',\n                'name_leven',\n                'name_jaro',\n                'name_lcs',\n                'name_len_diff',\n                'name_nleven',\n                'name_nlcsk',\n                'name_nlcs',\n                'address_sim',\n                'address_gesh',\n                'address_leven',\n                'address_jaro',\n                'address_lcs',\n                'address_len_diff',\n                'address_nleven',\n                'address_nlcsk',\n                'address_nlcs',\n                'city_gesh',\n                'city_leven',\n                'city_jaro',\n                'city_lcs',\n                'city_len_diff',\n                'city_nleven',\n                'city_nlcsk',\n                'city_nlcs',\n                'state_sim',\n                'state_gesh',\n                'state_leven',\n                'state_jaro',\n                'state_lcs',\n                'state_len_diff',\n                'state_nleven',\n                'state_nlcsk',\n                'state_nlcs',\n                'zip_gesh',\n                'zip_leven',\n                'zip_jaro',\n                'zip_lcs',\n                'url_sim',\n                'url_gesh',\n                'url_leven',\n                'url_jaro',\n                'url_lcs',\n                'url_len_diff',\n                'url_nleven',\n                'url_nlcsk',\n                'url_nlcs',\n                'phone_gesh',\n                'phone_leven',\n                'phone_jaro',\n                'phone_lcs',\n                'categories_sim',\n                'categories_gesh',\n                'categories_leven',\n                'categories_jaro',\n                'categories_lcs',\n                'categories_len_diff',\n                'categories_nleven',\n                'categories_nlcsk',\n                'categories_nlcs',\n                'country_sim',\n                'country_gesh',\n                'country_leven',\n                'country_nleven',\n                \n                'text_1',\n                'text_2',\n\n                'category_venn',\n                'haversine_distance',\n\n                #'name_sim_use',\n                #'categories_sim_use',\n                'text_sim_w2v',\n\n                #'name_longest_substr_ratio',\n                #'address_longest_substr_ratio',\n                #'categories_longest_substr_ratio',\n\n                'text_sim_bm25_svd',\n                #'text_sim_mpnet',\n\n                'latitude_round_1',\n                'longitude_round_1',\n                'latitude_round_2',\n                'longitude_round_2',\n                'same_number',  \n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Parameters\nEXP_ID = '067'\n\nNUM_NEIGHBOR = 25\nSEED = 2022\nTHRESHOLD = 0.5\nNUM_SPLIT = 35\nN_FOLD = 3\nfeat_columns = ['name', 'address', 'city', \n            'state', 'zip', 'url', \n           'phone', 'categories', 'country']\nvec_columns = ['name', 'categories', 'address', \n               'state', 'url', 'country']\n\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nseed_everything(SEED)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext Cython","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%cython\ndef LCS(str S, str T):\n    cdef int i, j\n    cdef list dp = [[0] * (len(T) + 1) for _ in range(len(S) + 1)]\n    for i in range(len(S)):\n        for j in range(len(T)):\n            dp[i + 1][j + 1] = max(dp[i][j] + (S[i] == T[j]), dp[i + 1][j], dp[i][j + 1], dp[i + 1][j + 1])\n    return dp[len(S)][len(T)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def post_process(df):\n    id2match = dict(zip(df['id'].values, df['matches'].str.split()))\n\n    for base, match in df[['id', 'matches']].values:\n        match = match.split()\n        if len(match) == 1:        \n            continue\n\n        for m in match:\n            if base not in id2match[m]:\n                id2match[m].append(base)\n    df['matches'] = df['id'].map(id2match).map(' '.join)\n    return df ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_batches(l, n):\n    for i in range(0, len(l), n):\n        yield l[i:i + n]\n        \n        \ndef recall_knn(df, Neighbors = 10):\n    print('Start knn')\n    train_df = []\n    knn = NearestNeighbors(n_neighbors = Neighbors)\n    knn.fit(df[['latitude','longitude']], df.index)\n    dists, nears = knn.kneighbors(df[['latitude','longitude']])\n\n    print('Start faiss')\n    embedder_mpnet = SentenceTransformer('../input/sentence-embedding-models/paraphrase-multilingual-mpnet-base-v2')\n\n    batches = list(get_batches(df['name'].fillna('noname').values, 1000))\n    EMBEDDINGS_MPNET = np.zeros(shape=[len(df['name']), 768], dtype=np.float32)\n    OFFSET = 0\n    for batch in tqdm(batches):\n        n = len(batch)\n        EMBEDDINGS_MPNET[OFFSET:OFFSET + n] = embedder_mpnet.encode(batch, show_progress_bar=False).astype(np.float32)\n        OFFSET += n\n\n    del embedder_mpnet, batches, batch, knn; gc.collect()\n    print(EMBEDDINGS_MPNET.shape)\n\n    Neighbors2 = 25 if len(df) > 256 else 3\n    res = faiss.StandardGpuResources()\n    dim = 768\n    nlist = 1024 if len(df) > 256 else 64\n    M = 32\n    nbits = 8\n    metric = faiss.METRIC_L2\n    ivfpq_config = faiss.GpuIndexIVFPQConfig()\n    ivfpq_config.usePrecomputedTables = True\n\n    faiss_index = faiss.GpuIndexIVFPQ(res, dim, nlist, M, nbits, metric, ivfpq_config)\n    faiss_index.train(EMBEDDINGS_MPNET)\n    faiss_index.add(EMBEDDINGS_MPNET)\n\n    faiss_index.nprobe = 5\n    dists2, nears2 = faiss_index.search(EMBEDDINGS_MPNET, Neighbors2)\n    del faiss_index, EMBEDDINGS_MPNET; gc.collect()\n\n    for k in range(Neighbors):            \n        cur_df = df[['id']]\n        cur_df['match_id'] = df['id'].values[nears[:, k]]\n        cur_df['kdist'] = dists[:, k]\n        cur_df['kneighbors'] = k\n        cur_df = cur_df[cur_df['id']!=cur_df['match_id']]\n        #cur_df = cur_df[(cur_df['id']!=cur_df['match_id'])&(cur_df['kdist']<7.5)]\n        train_df.append(cur_df)\n    del cur_df, dists, nears; gc.collect()\n    \n    for k in range(Neighbors2):  \n        cur_df2 = df[['id']]\n        cur_df2['match_id'] = df['id'].values[nears2[:, k]]\n        cur_df2['kdist'] = dists2[:, k]\n        cur_df2['kneighbors'] = k\n        cur_df2 = cur_df2[cur_df2['id']!=cur_df2['match_id']]\n        #cur_df2 = cur_df2[(cur_df2['id']!=cur_df2['match_id'])&(cur_df2['kdist']<7.5)]\n        train_df.append(cur_df2)\n        \n    del cur_df2, dists2, nears2; gc.collect()\n\n    train_df = pd.concat(train_df)\n    train_df = train_df.drop_duplicates(subset=['id', 'match_id'], keep='first').reset_index(drop=True)\n    train_df = reduce_mem_usage(train_df)\n    \n    train_df['id'] = train_df['id'].astype('object')\n    train_df['match_id'] = train_df['match_id'].astype('object')\n    return train_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_features(df):    \n    for col in tqdm(feat_columns):       \n        if col in vec_columns:\n            tv_fit = tfidf_d[col]\n            indexs = [id2index_d[i] for i in df['id']]\n            match_indexs = [id2index_d[i] for i in df['match_id']]                    \n            df[f'{col}_sim'] = np.array(tv_fit[indexs].multiply(tv_fit[match_indexs]).sum(axis = 1)).ravel()\n        \n        col_values = data.loc[df['id']][col].values.astype(str)\n        matcol_values = data.loc[df['match_id']][col].values.astype(str)\n        \n        geshs = []\n        levens = []\n        jaros = []\n        lcss = []\n        for s, match_s in zip(col_values, matcol_values):\n            if s != 'nan' and match_s != 'nan':                    \n                geshs.append(difflib.SequenceMatcher(None, s, match_s).ratio())\n                levens.append(Levenshtein.distance(s, match_s))\n                jaros.append(Levenshtein.jaro_winkler(s, match_s))\n                lcss.append(LCS(str(s), str(match_s)))\n            else:\n                geshs.append(np.nan)\n                levens.append(np.nan)\n                jaros.append(np.nan)\n                lcss.append(np.nan)\n        \n        df[f'{col}_gesh'] = geshs\n        df[f'{col}_leven'] = levens\n        df[f'{col}_jaro'] = jaros\n        df[f'{col}_lcs'] = lcss\n        \n        if col not in ['phone', 'zip']:\n            df[f'{col}_len'] = list(map(len, col_values))\n            df[f'match_{col}_len'] = list(map(len, matcol_values)) \n            df[f'{col}_len_diff'] = np.abs(df[f'{col}_len'] - df[f'match_{col}_len'])\n            df[f'{col}_nleven'] = df[f'{col}_leven'] / \\\n                                    df[[f'{col}_len', f'match_{col}_len']].max(axis = 1)\n            \n            df[f'{col}_nlcsk'] = df[f'{col}_lcs'] / df[f'match_{col}_len']\n            df[f'{col}_nlcs'] = df[f'{col}_lcs'] / df[f'{col}_len']\n            \n            df = df.drop(f'{col}_len', axis = 1)\n            df = df.drop(f'match_{col}_len', axis = 1)\n            gc.collect()\n            \n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def categorical_similarity(A, B):\n    if not A or not B:\n        return -1\n\n    A = set(str(A).split(\", \"))\n    B = set(str(B).split(\", \"))\n\n    # Find intersection of two sets\n    nominator = A.intersection(B)\n\n    similarity_1 = len(nominator) / len(A)\n    similarity_2 = len(nominator) / len(B)\n\n    return max(similarity_1, similarity_2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numba\nEARTH_RADIUS = 6371\n\n# Numba optimized haversine distance\n@numba.jit(nopython=True)\ndef haversine_np(args):\n    lon1, lat1, lon2, lat2 = args\n    lon1, lat1, lon2, lat2 = map(np.radians, [lon1, lat1, lon2, lat2])\n\n    dlon = lon2 - lon1\n    dlat = lat2 - lat1\n\n    a = np.sin(dlat/2.0)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2.0)**2\n\n    c = 2 * np.arcsin(np.sqrt(a))\n    km = EARTH_RADIUS * c\n    return km\n\n\n# Adds haversine distance between two points\ndef add_haversine_distance(df):\n    df['haversine_distance'] = np.apply_along_axis(\n            haversine_np, 1,\n            df[['longitude_1', 'latitude_1', 'longitude_2', 'latitude_2']].values.astype(np.float32)\n        ).astype(np.float32)\n    return df\n\n\ndef cosine_similarity(a, b):\n    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"longest_substring_columns = [\n    'name',\n    'address',\n    'categories',\n]\n\n# source: https://stackoverflow.com/questions/18715688/find-common-substring-between-two-strings\n@numba.jit(nopython=True, nogil=True, cache=True)\ndef longestSubstringFinder(string1: str, string2: str):\n    answer = 0\n    len1, len2 = len(string1), len(string2)\n\n    for i in range(len1):\n        for j in range(len2):\n            lcs_temp = 0\n            match = 0\n            while ((i+lcs_temp < len1) and (j+lcs_temp<len2) and string1[i+lcs_temp] == string2[j+lcs_temp]):\n                match += 1\n                lcs_temp += 1\n            if match > answer:\n                answer = match\n    return np.uint8(answer)\n\n\n# Longest substring feature\ndef add_longest_substr(df):\n    for col in longest_substring_columns:\n        df[f'{col}_longest_substr'] = df[[f'{col}_1', f'{col}_2']].apply(lambda args: longestSubstringFinder(*args), axis=1, raw=True).astype(np.uint8)\n        df[f'{col}_longest_substr_ratio'] = (\n                (df[f'{col}_longest_substr'] * 2) / (df[f'{col}_1'].apply(len) + df[f'{col}_2'].apply(len))\n            ).astype(np.float32)\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from gensim.models import word2vec\nfrom gensim.models import KeyedVectors\n\n\nclass W2VVectorizer:\n    def __init__(self, sentences, vec_size):\n        self.sentences = sentences.apply(lambda x: x.split())\n        self.vec_size = vec_size\n\n        print('fit models ...')\n        self.model = word2vec.Word2Vec(self.sentences, \n                                       vector_size=self.vec_size,\n                                       min_count=1,\n                                       window=1,\n                                       epochs=100)\n        \n    def vectorize(self, word_list : list) -> np.array:\n        V = []\n        for word in word_list:\n            try:\n                vector = self.model.wv[word]\n            except:\n                vector = [j for j in range(self.vec_size)]\n            V.append(vector)\n        return np.mean(V, 0)        \n\n    def get_vectors(self) -> pd.DataFrame:\n\n        print('get vectors ...')\n        vectors = self.sentences.progress_apply(lambda x: self.vectorize(x))\n        vectors = np.stack(vectors, 0)\n        return vectors","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nimport re\nimport scipy as sp\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.utils.validation import check_is_fitted\nfrom sklearn.feature_extraction.text import _document_frequency\nfrom sklearn.pipeline import make_pipeline, make_union\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.decomposition import TruncatedSVD\n\n\nclass BM25Transformer(BaseEstimator, TransformerMixin):\n    def __init__(self, use_idf=True, k1=2.0, b=0.75):\n        self.use_idf = use_idf\n        self.k1 = k1\n        self.b = b\n\n    def fit(self, X):\n        if not sp.sparse.issparse(X):\n            X = sp.sparse.csc_matrix(X)\n        if self.use_idf:\n            n_samples, n_features = X.shape\n            df = _document_frequency(X)\n            idf = np.log((n_samples - df + 0.5) / (df + 0.5))\n            self._idf_diag = sp.sparse.spdiags(idf, diags=0, m=n_features, n=n_features)\n\n        doc_len = X.sum(axis=1)\n        self._average_document_len = np.average(doc_len)\n\n        return self\n\n    def transform(self, X, copy=True):\n        if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n            X = sp.sparse.csr_matrix(X, copy=copy)\n        else:\n            X = sp.sparse.csr_matrix(X, dtype=np.float, copy=copy)\n\n        n_samples, n_features = X.shape\n        doc_len = X.sum(axis=1)\n        sz = X.indptr[1:] - X.indptr[0:-1]\n        rep = np.repeat(np.asarray(doc_len), sz)\n\n        nom = self.k1 + 1\n        denom = X.data + self.k1 * (1 - self.b + self.b * rep / self._average_document_len)\n        data = X.data * nom / denom\n\n        X = sp.sparse.csr_matrix((data, X.indices, X.indptr), shape=X.shape)\n\n        if self.use_idf:\n            check_is_fitted(self, '_idf_diag', 'idf vector is not fitted')\n\n            expected_n_features = self._idf_diag.shape[0]\n            if n_features != expected_n_features:\n                raise ValueError(\"Input has n_features=%d while the model\"\n                                 \" has been trained with n_features=%d\" % (\n                                     n_features, expected_n_features))\n            X = X * self._idf_diag\n\n        return X ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Dada process\ndata = pd.read_csv('../input/foursquare-location-matching/test.csv')\n\nif len(data) < 25:\n    data = pd.read_csv('../input/foursquare-location-matching/train.csv',\n                      nrows = 256)\n    data = data.drop('point_of_interest', axis = 1)\n    \ndata['text'] = ''\nfor v in vec_columns:\n    data['text'] += data[v].fillna('nan') + ' '","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_2_text_w2v_vector = {k:v for k, v in zip(data['id'], W2VVectorizer(data['text'], vec_size=50).get_vectors())}\ndel W2VVectorizer; gc.collect()\n\n\npipeline = make_pipeline(\n                TfidfVectorizer(max_features=100000),\n                make_union(\n                    TruncatedSVD(n_components=32, random_state=42),\n                    make_pipeline(\n                        BM25Transformer(use_idf=False, k1=2.0, b=0.75),\n                        TruncatedSVD(n_components=32, random_state=42)\n                    ),\n                    n_jobs=1,\n                ),\n             )\n\n\nid_2_text_bm25_svd_vector = {k:v for k, v in zip(data['id'], pipeline.fit_transform(data['text']))}\ndel pipeline, BM25Transformer, BaseEstimator, KeyedVectors, TransformerMixin, TruncatedSVD; gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id2index_d = dict(zip(data['id'].values, data.index))\nid_2_text = {k:v for k, v in zip(data['id'].values, data['text'].values)}\nid_2_cat = {k:v for k, v in zip(data['id'].values, data['categories'].fillna('nocategories').values)}\nid_2_lat = {k:v for k, v in zip(data['id'].values, data['latitude'].values)}\nid_2_lon = {k:v for k, v in zip(data['id'].values, data['longitude'].values)}\nid_2_name = {k:v for k, v in zip(data['id'].values, data['name'].fillna('noname').values)}\nid_2_address = {k:v for k, v in zip(data['id'].values, data['address'].fillna('noaddress').values)}\n\ntfidf_d = {}\nfor col in vec_columns:\n    tfidf = TfidfVectorizer()\n    tv_fit = tfidf.fit_transform(data[col].fillna('nan'))\n    tfidf_d[col] = tv_fit\n    \ndel tv_fit, TfidfVectorizer; gc.collect()\n\n\nout_df = pd.DataFrame()\nout_df['id'] = data['id'].unique().tolist()\nout_df['match_id'] = out_df['id']\n\ntest_data = recall_knn(data, NUM_NEIGHBOR)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del SentenceTransformer, NearestNeighbors, KNeighborsRegressor; gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.set_index('id')\n\n#print('Num of unique id: %s' % test_data['id'].nunique())\n#print('Num of test data: %s' % len(test_data))\n#print(test_data.sample(5))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_cols = ['text_1', 'text_2']\n\n## Prediction\ncount = 0\nstart_row = 0\npred_df = pd.DataFrame()\nunique_id = test_data['id'].unique().tolist()\nnum_split_id = len(unique_id) // NUM_SPLIT\nfor k in range(1, NUM_SPLIT + 1):\n    print('Current split: %s' % k)\n    end_row = start_row + num_split_id\n    if k < NUM_SPLIT:\n        cur_id = unique_id[start_row : end_row]\n        cur_data = test_data[test_data['id'].isin(cur_id)]\n    else:\n        cur_id = unique_id[start_row: ]\n        cur_data = test_data[test_data['id'].isin(cur_id)]\n    \n    # add features & model prediction\n    cur_data = add_features(cur_data)\n    \n    cur_data['latitude_1'] = cur_data['id'].map(id_2_lat)\n    cur_data['latitude_2'] = cur_data['match_id'].map(id_2_lat)\n    cur_data['longitude_1'] = cur_data['id'].map(id_2_lon)\n    cur_data['longitude_2'] = cur_data['match_id'].map(id_2_lon)\n    cur_data['latitude_round_1'] = cur_data['latitude_1'].round(1)\n    cur_data['longitude_round_1'] = cur_data['longitude_1'].round(1)\n    cur_data['latitude_round_2'] = cur_data['latitude_2'].round(1)\n    cur_data['longitude_round_2'] = cur_data['longitude_2'].round(1)\n    cur_data = add_haversine_distance(cur_data)\n    del cur_data['latitude_1'], cur_data['latitude_2'], cur_data['longitude_1'], cur_data['longitude_2']; gc.collect()\n\n    sim = []\n    for nv1, nv2 in zip(cur_data['id'].map(id_2_text_w2v_vector), cur_data['match_id'].map(id_2_text_w2v_vector)):\n        sim.append(cosine_similarity(nv1, nv2))\n    cur_data['text_sim_w2v'] = sim\n\n    sim = []\n    for nv1, nv2 in zip(cur_data['id'].map(id_2_text_bm25_svd_vector), cur_data['match_id'].map(id_2_text_bm25_svd_vector)):\n        sim.append(cosine_similarity(nv1, nv2))\n    cur_data['text_sim_bm25_svd'] = sim\n\n    cur_data['name_1'] = cur_data['id'].map(id_2_name)\n    cur_data['name_2'] = cur_data['match_id'].map(id_2_name)\n    \n    cur_data['categories_1'] = cur_data['id'].map(id_2_cat)\n    cur_data['categories_2'] = cur_data['match_id'].map(id_2_cat)\n    cur_data[\"category_venn\"] = cur_data[[\"categories_1\", \"categories_2\"]] \\\n            .apply(lambda row: categorical_similarity(row.categories_1, row.categories_2),\n                            axis=1)\n    \n    cur_data['address_1'] = cur_data['id'].map(id_2_address)\n    cur_data['address_2'] = cur_data['match_id'].map(id_2_address)\n    cur_data['number_1'] = cur_data['address_1'].str.extract('(\\d+)')\n    cur_data['number_2'] = cur_data['address_2'].str.extract('(\\d+)')\n    cur_data['same_number'] = (cur_data['number_1'] == cur_data['number_2']).astype(int)\n    del cur_data['number_1'], cur_data['number_2']; gc.collect()\n    del cur_data['address_1'], cur_data['address_2']; gc.collect()\n\n    cur_data = reduce_mem_usage(cur_data)\n    cur_data['text_1'] = cur_data['id'].map(id_2_text).astype('object')\n    cur_data['text_2'] = cur_data['match_id'].map(id_2_text).astype('object')\n    \n    cur_data['pred'] = 0\n    with open('../input/foursquare-2022-067/cat_all_train.pkl','rb') as f:\n        cat_model = pickle.load(f)\n\n    cur_data['pred'] = cat_model.predict_proba(Pool(cur_data[TRAIN_FEATURES], text_features=text_cols,))[:, 1]\n        \n    cur_pred_df = cur_data[cur_data['pred'] > 0.5][['id', 'match_id']]\n    pred_df = pd.concat([pred_df, cur_pred_df])\n    \n    start_row = end_row\n    count += len(cur_data)\n\n    del cur_data, cur_pred_df, cat_model, sim; gc.collect()\nprint(count)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data, test_data, unique_id; gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del id2index_d, id_2_cat, id_2_name, id_2_lat, id_2_lon, id_2_address; gc.collect()\ndel id_2_text, id_2_text_w2v_vector, id_2_text_bm25_svd_vector; gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Submission    \nout_df = pd.concat([out_df, pred_df])\ndel pred_df; gc.collect()\n\nout_df = out_df.groupby('id')['match_id'].\\\n                        apply(list).reset_index()\nout_df['matches'] = out_df['match_id'].apply(lambda x: ' '.join(set(x)))\ndel out_df['match_id']; gc.collect()\n\nout_df['matches'] = out_df['matches']+' '+out_df['id']\nout_df['matches'] = out_df['matches'].map(lambda x: ' '.join(set(x.split())))\n\nout_df = post_process(out_df)\n\nout_df[['id', 'matches']].to_csv('submission.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}