{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Foursquare Pipeline","metadata":{}},{"cell_type":"code","source":"!pip install ../input/rapidfuzz2011cp37/jarowinkler-1.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install ../input/rapidfuzz2011cp37/rapidfuzz-2.0.11-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:56:04.098312Z","iopub.execute_input":"2022-07-07T05:56:04.098764Z","iopub.status.idle":"2022-07-07T05:57:07.728774Z","shell.execute_reply.started":"2022-07-07T05:56:04.098676Z","shell.execute_reply":"2022-07-07T05:57:07.727669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nimport sys\nimport pickle\nimport numpy as np\nimport pandas as pd\nfrom unidecode import unidecode\n\ntrain = pd.read_csv(\"../input/foursquare-location-matching/train.csv\", dtype={'phone': str, 'zip': str})\ntest = pd.read_csv(\"../input/foursquare-location-matching/test.csv\", dtype={'phone': str, 'zip': str})\n\n\nif test.shape[0] == 5:\n    test = train.head(2000).copy()\n\ntest[\"name\"] = test[\"name\"].fillna(\"\")\ntest[\"country\"] = test[\"country\"].fillna(\"US\")\ntest[\"categories\"] = test[\"categories\"].fillna(\"\")\ntest['uni_name'] = test[\"name\"].fillna(\"\").astype(str).map(unidecode)\n\ntest.to_feather('test.ftr')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:57:07.731683Z","iopub.execute_input":"2022-07-07T05:57:07.733031Z","iopub.status.idle":"2022-07-07T05:57:17.471324Z","shell.execute_reply.started":"2022-07-07T05:57:07.732967Z","shell.execute_reply":"2022-07-07T05:57:17.470118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transformer Inference","metadata":{}},{"cell_type":"code","source":"%%python\n\nimport gc, sys\nimport pickle\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom unidecode import unidecode\n\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nimport cudf\nimport cupy\nfrom cuml.neighbors import NearestNeighbors as CuNearestNeighbors\nfrom sklearn.neighbors import NearestNeighbors\n\nEXP = \"exp053\"\nN_NEIHGBORS = 128\nSVD_DIM = 32\nSEED = 1006\ncols = [\n    \"id\", \"latitude\", \"longitude\", \"categories\", \"name\",\n    \"address\", \"zip\", \"phone\", \"url\",\n]\n\n\ndef df2dict(df, ctr_samplig_dict, cat2idx0, cat2idx1, map_text_emb, map_svd_emb):\n    result = {\"ids\": []}\n\n    for country, country_df in tqdm(df.groupby(\"country\")):\n        ids = country_df[\"id\"].to_numpy()\n\n        if len(country_df) > N_NEIHGBORS:\n\n            n0, n1 = ctr_samplig_dict.get(country, [128, 0])\n\n            # LatLon base neighbors\n            cu_country_df = cudf.from_pandas(country_df[[\"latitude\", \"longitude\", \"country\"]])\n            knn = CuNearestNeighbors(n_neighbors=N_NEIHGBORS, algorithm=\"brute\", metric=\"haversine\")\n            coords = cupy.deg2rad(cu_country_df[[\"latitude\", \"longitude\"]].values)#.get()\n            knn.fit(coords)\n            dists, nears = knn.kneighbors(coords, return_distance=True)\n            dists = cupy.asnumpy(dists).reshape(-1, N_NEIHGBORS)\n            nears = cupy.asnumpy(nears).reshape(-1, N_NEIHGBORS)\n\n            if n0 != N_NEIHGBORS:\n                # Tfidf Cosine Similality\n                model = TfidfVectorizer(stop_words=None,\n                                    binary=False,\n                                    norm='l2',\n                                    analyzer='char_wb',\n                                    ngram_range=(2, 3),\n                                    min_df=2,\n                                    dtype=np.float32)\n\n                titles = country_df[\"uni_name\"].fillna(\"\").astype(str).tolist()\n                text_embeddings = model.fit_transform(titles)\n\n                knn = CuNearestNeighbors(n_neighbors=N_NEIHGBORS, algorithm=\"brute\", metric=\"cosine\")\n                knn.fit(text_embeddings)\n                name_dists, name_nears = knn.kneighbors(text_embeddings)\n\n                #knn = CuNearestNeighbors(n_neighbors=N_NEIHGBORS, algorithm=\"brute\", metric=\"cosine\")\n                #emb = map_text_emb['uni_name'][country_df['index'].to_numpy()]\n                #knn.fit(emb)\n                #name_dists, name_nears = knn.kneighbors(emb)\n\n                if name_nears.max() >= cu_country_df.shape[0]:\n                    print(f'Use CPU nn on {country} {cu_country_df.shape}')\n                    knn = NearestNeighbors(n_neighbors=N_NEIHGBORS, metric=\"cosine\", n_jobs=-1)\n                    knn.fit(emb)\n                    name_dists, name_nears = knn.kneighbors(emb)\n\n                name_dists = cupy.asnumpy(name_dists).reshape(-1, N_NEIHGBORS)\n                name_nears = cupy.asnumpy(name_nears).reshape(-1, N_NEIHGBORS)\n\n        result[\"ids\"] += ids.tolist()\n        for i, (id, lat, lon, cats, name, address, zipcode, phone, url) in enumerate(\n            country_df[cols].to_numpy()):\n\n            if len(country_df) > N_NEIHGBORS:\n                latlon_nears = [ids[idx] for idx in nears[i][:n0] if ids[idx] != id]\n                if n0 != N_NEIHGBORS:\n                    tfidf_nears = [ids[idx] for idx in name_nears[i][:n1] if ids[idx] != id]\n                    union_nears = list(set(latlon_nears+tfidf_nears))\n                else:\n                    union_nears = latlon_nears\n\n                if len(union_nears) + 1 < N_NEIHGBORS:\n                    add = [ids[idx] for idx in nears[i][n0:n0+N_NEIHGBORS-len(union_nears)-1] if ids[idx]!=id]\n                    union_nears += add\n                    union_nears = list(set(union_nears))\n\n            else:\n                union_nears = [_id for _id in ids if _id != id]\n\n            new_cats0, new_cats1 = [], []\n            for cat in cats.split(\", \"):\n                new_cats0.append(cat2idx0.get(cat, 0))\n                new_cats1.append(cat2idx1.get(cat, 0))\n\n            result[id] = {\n                \"nears\": [id]+union_nears,\n                \"lat\": lat, \"lon\": lon, \"cats0\": new_cats0, \"cats1\": new_cats1,\n                \"name\": name,\n                \"address\": address, \"phone\": phone, \"url\": url, \"zipcode\": zipcode,\n                \"categories\": cats, \"country\": country,\n            }\n\n    # Tfidf\n    for col in [\"uni_name\", \"all\"]:\n        transformed = map_svd_emb[col][df['index'].to_numpy()]\n        for i, id in enumerate(df[\"id\"].to_numpy()):\n            result[id][f\"svd_{col}\"] = transformed[i].tolist()\n\n    return result\n\n\ndf = pd.read_feather(f'test.ftr')\n\nwith open(\"../input/4sq-models/ctr_samplig_dict_n128_rad.pkl\", \"rb\") as pf:\n    ctr_samplig_dict = pickle.load(pf)\n\nwith open(\"../input/4sq-models/cat2idx_fold0.pkl\", \"rb\") as pf:\n    cat2idx0 = pickle.load(pf)\n\nwith open(\"../input/4sq-models/cat2idx_fold1.pkl\", \"rb\") as pf:\n    cat2idx1 = pickle.load(pf)\n\nmap_text_emb ={}\nmap_svd_emb ={}\nfor col in tqdm([\"uni_name\", \"all\"]):\n    if \"all\" in col:\n        df['all'] = [' '.join(row)  for row in df[['name', 'address', 'city', 'state',\n                         'country', 'url', 'phone', 'categories', 'zip']].fillna('').astype(str).values]\n\n    if 'uni_' in col:\n        c = col.split('_')[-1]\n        df[col] = df[c].fillna('').astype(str).map(unidecode)\n\n    model = TfidfVectorizer(\n        stop_words=None, binary=False, norm='l2', analyzer='char_wb',\n        ngram_range=(2, 3), min_df=2, dtype=np.float32\n    )\n\n    titles = df[col].fillna(\"\").astype(str).tolist()\n    text_embeddings = model.fit_transform(titles)\n\n    map_text_emb[col] = text_embeddings\n\n    svd = TruncatedSVD(n_components=SVD_DIM, random_state=SEED)\n    transformed = svd.fit_transform(text_embeddings)\n\n    map_svd_emb[col] = transformed\n\n\ndf_US = df[df[\"country\"]==\"US\"].reset_index()\ninput_dict = df2dict(df_US, ctr_samplig_dict[0], cat2idx0, cat2idx1, map_text_emb, map_svd_emb)\n\nwith open(\"input_dict_US.pkl\", \"wb\") as pf:\n    pickle.dump(input_dict, pf)\n\ndel df_US, input_dict\ngc.collect()\n\n\ndf_TRID = df[df[\"country\"].isin([\"TR\", \"ID\"])].reset_index()\ninput_dict = df2dict(df_TRID, ctr_samplig_dict[0], cat2idx0, cat2idx1, map_text_emb, map_svd_emb)\n\nwith open(\"input_dict_TRID.pkl\", \"wb\") as pf:\n    pickle.dump(input_dict, pf)\n\ndel df_TRID, input_dict\ngc.collect()\n\ndf_other = df[~df[\"country\"].isin([\"US\", \"TR\", \"ID\"])].reset_index()\ninput_dict = df2dict(df_other, ctr_samplig_dict[0], cat2idx0, cat2idx1, map_text_emb, map_svd_emb)\n\nwith open(\"input_dict_other.pkl\", \"wb\") as pf:\n    pickle.dump(input_dict, pf)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:57:17.474647Z","iopub.execute_input":"2022-07-07T05:57:17.475277Z","iopub.status.idle":"2022-07-07T05:57:33.789118Z","shell.execute_reply.started":"2022-07-07T05:57:17.475232Z","shell.execute_reply":"2022-07-07T05:57:33.787963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\n\"\"\" Extract BERT Embed\n\"\"\"\n\nimport gc\nimport os, sys\nimport torch\nimport pickle\nimport pandas as pd\nfrom tqdm import tqdm\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom transformers import AutoModel, AutoTokenizer, AutoConfig\n\n\nclass BertNet(nn.Module):\n    def __init__(self, model_name, config):\n        super().__init__()\n        self.encoder = AutoModel.from_pretrained(model_name, config=config)\n\n    def forward(self, ids, masks):\n        out = self.encoder(ids, attention_mask=masks)[0]\n        masks = masks.unsqueeze(2)\n        out = (masks * out).sum(dim=1) / masks.sum(dim=1)\n        return out\n\nclass BertDataset(Dataset):\n    def __init__(self, df, tokenizer, max_length):\n        super().__init__()\n        self.tokenized = []\n        for name in df[\"name\"].values:\n            bert_sens = tokenizer(\n                name, add_special_tokens=True, max_length=max_length,\n                pad_to_max_length=True, return_attention_mask=True,\n                return_token_type_ids=True, truncation=True\n            )\n            self.tokenized.append(\n                {\"ids\": bert_sens[\"input_ids\"], \"masks\": bert_sens[\"attention_mask\"]}\n            )\n\n    def __len__(self):\n        return len(self.tokenized)\n\n    def __getitem__(self, idx):\n        bert_sens = self.tokenized[idx]\n        ids, masks = (\n            torch.tensor(bert_sens[\"ids\"]).long(),\n            torch.tensor(bert_sens[\"masks\"]).long(),\n        )\n        return {\"ids\": ids, \"masks\": masks}\n\ndef get_bert_embed(df):\n\n    df[\"name\"] = df[\"name\"].fillna(\"unknown\")\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    model_name = \"../input/xlmroberta/xlm-roberta-base\"\n    tokenizer = AutoTokenizer.from_pretrained(model_name)\n    config = AutoConfig.from_pretrained(model_name)\n    model = BertNet(model_name, config).to(device)\n    data_loader = DataLoader(BertDataset(df, tokenizer, max_length=12),\n                        batch_size=128, shuffle=False, drop_last=False, num_workers=2)\n    model.half()\n    model.eval()\n    embeds = []\n    with torch.no_grad():\n        for batch in tqdm(data_loader):\n            ids, masks = (\n                batch[\"ids\"].to(device),\n                batch[\"masks\"].to(device),\n            )\n            embed = model(ids, masks)\n            embeds.append(embed)\n    embeds = torch.cat(embeds, dim=0)\n\n    embed_dict = {id: embed.cpu() for id, embed in zip(df[\"id\"].values, embeds)}\n    return embed_dict\n\n\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\ndf = pd.read_csv(f'../input/foursquare-location-matching/test.csv')\nif df.shape[0] == 5:\n    df = pd.read_csv(f'../input/foursquare-location-matching/train.csv').head(2000)\n\n\ndf[\"name\"] = df[\"name\"].fillna(\"unknown\")\ndf[\"country\"] = df[\"country\"].fillna(\"US\")\n\n\ndf_US = df[df[\"country\"]==\"US\"]\nembed_dict_US = get_bert_embed(df_US)\nwith open(\"bert_embed_dict_US.pkl\", \"wb\") as pf:\n    pickle.dump(embed_dict_US, pf)\n\ndel embed_dict_US, df_US\ngc.collect()\n\n\ndf_TRID = df[df[\"country\"].isin([\"TR\", \"ID\"])]\nembed_dict_TRID = get_bert_embed(df_TRID)\nwith open(\"bert_embed_dict_TRID.pkl\", \"wb\") as pf:\n    pickle.dump(embed_dict_TRID, pf)\n\ndel embed_dict_TRID, df_TRID\ngc.collect()\n\ndf_other = df[~df[\"country\"].isin([\"US\", \"TR\", \"ID\"])]\nembed_dict_other = get_bert_embed(df_other)\nwith open(\"bert_embed_dict_other.pkl\", \"wb\") as pf:\n    pickle.dump(embed_dict_other, pf)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:57:33.792334Z","iopub.execute_input":"2022-07-07T05:57:33.793221Z","iopub.status.idle":"2022-07-07T05:58:23.294224Z","shell.execute_reply.started":"2022-07-07T05:57:33.793182Z","shell.execute_reply":"2022-07-07T05:58:23.293004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nimport sys\nimport re\nimport gc\nimport copy\nimport pickle\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\nimport rapidfuzz\nimport Levenshtein\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\n\n\n\nclass TransformerNet(nn.Module):\n    def __init__(self, in_size, cat_embed_dim, cat_n_unique, embed_dim,\n        n_encoder, n_head, dim_feedforward, dropout, out_size, name_embed_dim,\n        ctr_n_unique, ctr_embed_dim):\n        super().__init__()\n        self.name_fc_embed = nn.Sequential(\n            nn.Linear(768, name_embed_dim),\n            nn.LayerNorm(name_embed_dim),\n            nn.GELU(), nn.Dropout(0.2),\n        )\n        self.cat_embedding = nn.Embedding(cat_n_unique+1, cat_embed_dim, padding_idx=0)\n        self.ctr_embedding = nn.Embedding(ctr_n_unique+1, ctr_embed_dim, padding_idx=0)\n        self.fc_embed = nn.Sequential(\n            nn.Linear(in_size, embed_dim),\n            nn.LayerNorm(embed_dim), nn.GELU(),\n        )\n        self.encoders = nn.ModuleList()\n        for _ in range(n_encoder):\n            self.encoders.append(nn.TransformerEncoderLayer(\n                d_model=embed_dim+name_embed_dim,\n                nhead=n_head,\n                dim_feedforward=dim_feedforward,\n                batch_first=True\n            ))\n        self.dropout = nn.Dropout(dropout)\n        self.head = nn.Linear((embed_dim+name_embed_dim)*2, out_size)\n\n    def forward(self, feat, cat, masks, embed, ctr):\n        name_embed = self.name_fc_embed(embed)\n        ctr_embed = self.ctr_embedding(ctr).squeeze(2)\n        cat_embed_sum = self.cat_embedding(cat[:, :, :-1]).sum(2)\n        sz = cat_embed_sum.size()\n        cat_embed = cat_embed_sum / cat[:, :, [-1]].expand(sz[0], sz[1], sz[2])\n\n        h = torch.cat([feat, cat_embed, ctr_embed], dim=2).float()\n        h = self.fc_embed(h)\n        h = torch.cat([h, name_embed], dim=2)\n\n        for encoder in self.encoders:\n            h = encoder(h)\n        src_h = h[:, [0], :]\n        dst_h = h[:, 1:, :]\n        sz = dst_h.size()\n        diff_h = torch.cat([src_h.expand(sz[0], sz[1], sz[2]), dst_h], dim=2)\n        out = self.head(self.dropout(diff_h))\n\n        return out\n\n\nclass FSQDataset(Dataset):\n    def __init__(self, input_d, max_length, embed_dict, ctr2idx):\n        self.ids = input_d[\"ids\"]\n        self.input_d = input_d\n        self.max_length = max_length\n        self.embed_dict = embed_dict\n        self.ctr2idx = ctr2idx\n\n    def __len__(self):\n        return len(self.ids)\n\n    def __getitem__(self, idx):\n        src_id = self.ids[idx]\n        src_name = self.input_d[src_id][\"name\"]\n        nears = self.input_d[src_id][\"nears\"].copy()\n\n        feats, cats0, cats1, masks, embeds, ctrs = [], [], [], [], [], []\n        for dst_id in nears:\n            feats.append(torch.tensor(\n                [\n                    # lat-lon distance\n                    np.log(calc_distance(\n                        self.input_d[src_id][\"lat\"], self.input_d[src_id][\"lon\"],\n                        self.input_d[dst_id][\"lat\"], self.input_d[dst_id][\"lon\"]\n                    )+1e-8)/10,\n                    (self.input_d[dst_id][\"lat\"]-26.87)/23.14,\n                    (self.input_d[dst_id][\"lon\"]-20.70)/82.67,\n                ] + sum([ # edit distance\n                    calc_dist_feats(self.input_d[src_id][col], self.input_d[dst_id][col], i)\n                    for i, col in enumerate([\"name\", \"address\", \"zipcode\", \"phone\", \"url\", \"categories\"])\n                ], []) + sum([ # svd vector\n                    self.input_d[dst_id][f\"svd_{col}\"]\n                    for col in [\"uni_name\", \"all\"]\n                ], []) + [ # svd vector cosine similality\n                    cos_sim(self.input_d[src_id][f\"svd_{col}\"], self.input_d[dst_id][f\"svd_{col}\"])\n                    for col in [\"uni_name\", \"all\"]\n                ]\n            ))\n\n            cat0 = self.input_d[dst_id][\"cats0\"]\n            cats0.append(\n                torch.tensor(cat0 + [0]*(8-len(cat0)) + [len(cat0)])\n            )\n            cat1 = self.input_d[dst_id][\"cats1\"]\n            cats1.append(\n                torch.tensor(cat1 + [0]*(8-len(cat1)) + [len(cat1)])\n            )\n            masks.append(1)\n            embeds.append(self.embed_dict[dst_id])\n            ctrs.append(torch.tensor([self.ctr2idx.get(self.input_d[dst_id][\"country\"], 0)]))\n\n        for _ in range(self.max_length-sum(masks)):\n            feats.append(torch.tensor([0]*len(feats[0])))\n            cats0.append(torch.tensor([0]*8+[1]))\n            cats1.append(torch.tensor([0]*8+[1]))\n            masks.append(0)\n            nears.append(\"\")\n            embeds.append(torch.zeros(768))\n            ctrs.append(torch.tensor([0]))\n\n        feats = torch.stack(feats).to(torch.float32)[:self.max_length]\n        cats0 = torch.stack(cats0)[:self.max_length]\n        cats1 = torch.stack(cats1)[:self.max_length]\n        masks = torch.tensor(masks).float()[:self.max_length]\n        ids = nears[:self.max_length]\n        embeds = torch.stack(embeds).to(torch.float32)[:self.max_length]\n        ctrs = torch.stack(ctrs)[:self.max_length]\n\n        return feats, cats0, cats1, masks, ids, embeds, ctrs\n\n\ndef calc_dist_feats(str1, str2, i):\n    str1 = np.nan if str1 is None else str1\n    str2 = np.nan if str2 is None else str2\n    if i in [0]:\n        if (str1==str1 and str2==str2):\n            str1, str2 = str1.lower(), str2.lower()\n            #gesh = (difflib.SequenceMatcher(None, str1, str2).ratio() - 0.272) / 0.256\n            leven = (Levenshtein.distance(str1, str2) - 17.37) / 9.6\n            jaro = (Levenshtein.jaro_winkler(str1, str2) - 0.47) / 0.23\n            #lcs = np.log(LCS(str(str1), str(str2))/10+0.05) + 0.903\n            edit_dist0 = (rapidfuzz.string_metric.levenshtein(str1, str2) / 10 - 0.1862) / 0.08398\n            edit_dist1 = (rapidfuzz.fuzz.ratio(str1, str2) / 10 - 0.2328) / 0.13388\n            edit_dist2 = (rapidfuzz.fuzz.partial_ratio(str1, str2) / 10 - 0.3389) / 0.14932\n            edit_dist3 = (rapidfuzz.fuzz.token_set_ratio(str1, str2) / 10 - 0.2777) / 0.13607\n            edit_dist4 = (rapidfuzz.fuzz.token_sort_ratio(str1, str2) / 10 - 0.2751) / 0.1298\n            edit_dist5 = (rapidfuzz.fuzz.token_ratio(str1, str2) / 10 - 0.2781) / 0.13598\n            edit_dist6 = (rapidfuzz.fuzz.partial_token_ratio(str1, str2) / 10 - 0.40518) / 0.16255\n            edit_dist7 = (rapidfuzz.fuzz.WRatio(str1, str2) / 10 - 0.35696) / 0.140198\n            edit_dist8 = (rapidfuzz.fuzz.QRatio(str1, str2) / 10 - 0.2718) / 0.12897\n            r1, r2, r3, r4, r5, r6, r7, r8 = calc_match_word(str1, str2)\n        else:\n            # gesh, leven, jaro, lcs, r1, r2, r3, r4, r5, r6, r7, r8 = -0.5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0\n            leven, jaro, r1, r2, r3, r4, r5, r6, r7, r8 = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0\n            edit_dist0, edit_dist1, edit_dist2, edit_dist3 = 0, 0, 0, 0\n            edit_dist4, edit_dist5, edit_dist6, edit_dist7, edit_dist8 = 0, 0, 0, 0, 0\n        #return [gesh, leven, jaro, lcs, r1, r2, r3, r4, r5, r6, r7, r8]\n        return [\n            leven, jaro, r1, r2, r3, r4, r5, r6, r7, r8,\n            edit_dist0, edit_dist1, edit_dist2, edit_dist3,\n            edit_dist4, edit_dist5, edit_dist6, edit_dist7, edit_dist8\n        ]\n    else:\n        if (str1==str1 and str2==str2):\n            str1, str2 = str1.lower(), str2.lower()\n            leven = (Levenshtein.distance(str1, str2) - 17.37) / 9.6\n            jaro = (Levenshtein.jaro_winkler(str1, str2) - 0.47) / 0.23\n        else:\n            leven, jaro = 0, 0\n        return [leven, jaro]\n\ndef calc_match_word(str1, str2):\n    words_1 = [word for word in re.split('[,.-/ ]', str1) if word != \"\"]\n    words_2 = [word for word in re.split('[,.-/ ]', str2) if word != \"\"]\n    if len(words_1) == 0 or len(words_2) == 0:\n        return 0, 0, 0, 0, 0, 0, 0 ,0\n\n    r1 = len([word for word in words_1 if word in words_2]) / len(words_1)\n    r2 = len([word for word in words_2 if word in words_1]) / len(words_2)\n    r3 = len([word for word in words_1 if word in str2]) / len(words_1)\n    r4 = len([word for word in words_2 if word in str1]) / len(words_2)\n    # 3文字ずつみて一致があるか\n    r5 = len(sum([[word[i:i+3] for i in range(len(word)-2) if word[i:i+3] in str2] for word in words_1], [])) / len(str1) if len(str1) > 2 else 0\n    r6 = len(sum([[word[i:i+3] for i in range(len(word)-2) if word[i:i+3] in str1] for word in words_2], [])) / len(str2) if len(str2) > 2 else 0\n    # 頭文字のみをとってあるか\n    r7 = sum([\n        1 if (words_1[i][0]+words_1[i+1][0]+words_1[i+2][0]) in str2 else 0\n        for i in range(len(words_1)-2)\n    ]) / len(words_1) if len(words_1) > 2 else 0\n    r8 = sum([\n        1 if (words_2[i][0]+words_2[i+1][0]+words_2[i+2][0]) in str1 else 0\n        for i in range(len(words_2)-2)\n    ]) / len(words_2) if len(words_2) > 2 else 0\n    return r1, r2, r3, r4, r5, r6, r7, r8\n\ndef postprocess(df):\n    id2match = dict(zip(df[\"id\"].values, df[\"matches\"].str.split()))\n\n    for match in df[\"matches\"]:\n        match = match.split()\n        if len(match) == 1:\n            continue\n\n        base = match[0]\n        for m in match[1:]:\n            if not base in id2match[m]:\n                id2match[m].append(base)\n    df[\"matches\"] = df[\"id\"].map(id2match).map(\" \".join)\n    return df\n\ndef calc_distance(lat1, lon1, lat2, lon2):\n    lon1, lat1, lon2, lat2 = map(np.radians, [lon1, lat1, lon2, lat2])\n    dlon = lon2 - lon1\n    dlat = lat2 - lat1\n    a = np.sin(dlat / 2.0)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon / 2.0)**2\n    c = 2 * np.arcsin(np.sqrt(a))\n    km = 6367 * c\n    return km\n\ndef cos_sim(v1, v2):\n    sim = np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))\n    return (sim if sim > -1 else 0)\n\n\n\"\"\"\n\"\"\"\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nclass CFG:\n    name = \"trans_exp053\"\n    target = \"point_of_interest\"\n    n_neighbors = 128\n    # Model Params\n    ctr_embed_dim = 32\n    svd_dim = 32\n    name_embed_dim = 128\n    cat_embed_dim = 256\n    embed_dim = 256\n    dim_feedforward = 768\n    n_encoder = 4\n    n_head = 4\n    dropout = 0.2\n    batch_size = 32\n    n_process = 2\n    # 足切りth\n    th = 0.005\n\ndef run_inference(suffix, ckpt0, ckpt1):\n\n    with open(f\"input_dict_{suffix}.pkl\", \"rb\") as pf:\n        id_dict = pickle.load(pf)\n\n    with open(f\"bert_embed_dict_{suffix}.pkl\", \"rb\") as pf:\n        embed_dict = pickle.load(pf)\n\n    with open(\"../input/4sq-models/cat2idx_fold0.pkl\", \"rb\") as pf:\n        cat2idx0 = pickle.load(pf)\n\n    with open(\"../input/4sq-models/cat2idx_fold1.pkl\", \"rb\") as pf:\n        cat2idx1 = pickle.load(pf)\n\n    with open(\"../input/4sq-ctr2idx/ctr2idx.pkl\", \"rb\") as pf:\n        ctr2idx = pickle.load(pf)\n\n    model_0 = TransformerNet(\n        in_size = 3 + 1*19 + 5*2 + 2*1 + 2*CFG.svd_dim + CFG.cat_embed_dim + CFG.ctr_embed_dim,\n        cat_embed_dim = CFG.cat_embed_dim,\n        cat_n_unique = len(cat2idx0),\n        embed_dim = CFG.embed_dim,\n        n_encoder = CFG.n_encoder,\n        n_head = CFG.n_head,\n        dim_feedforward = CFG.dim_feedforward,\n        dropout = CFG.dropout,\n        out_size = 2,\n        name_embed_dim = CFG.name_embed_dim,\n        ctr_n_unique = len(ctr2idx),\n        ctr_embed_dim = CFG.ctr_embed_dim,\n    ).to(device)\n\n    model_1 = TransformerNet(\n        in_size = 3 + 1*19 + 5*2 + 2*1 + 2*CFG.svd_dim + CFG.cat_embed_dim + CFG.ctr_embed_dim,\n        cat_embed_dim = CFG.cat_embed_dim,\n        cat_n_unique = len(cat2idx1),\n        embed_dim = CFG.embed_dim,\n        n_encoder = CFG.n_encoder,\n        n_head = CFG.n_head,\n        dim_feedforward = CFG.dim_feedforward,\n        dropout = CFG.dropout,\n        out_size = 2,\n        name_embed_dim = CFG.name_embed_dim,\n        ctr_n_unique = len(ctr2idx),\n        ctr_embed_dim = CFG.ctr_embed_dim,\n    ).to(device)\n\n    model_0.load_state_dict(torch.load(ckpt0))\n    model_1.load_state_dict(torch.load(ckpt1))\n\n\n    data_loader = DataLoader(FSQDataset(id_dict, CFG.n_neighbors, embed_dict, ctr2idx),\n        batch_size=CFG.batch_size, shuffle=False, drop_last=False, num_workers=2)\n\n    model_0.eval()\n    model_1.eval()\n    preds, probs = [], []\n    with torch.inference_mode():\n        for feat, cat0, cat1, masks, ids, embeds, ctrs in tqdm(data_loader):\n            output0 = model_0(\n                feat.to(device), cat0.to(device), masks.to(device), embeds.to(device), ctrs.to(device))\n            output1 = model_1(\n                feat.to(device), cat1.to(device), masks.to(device), embeds.to(device), ctrs.to(device))\n            output = (F.softmax(output0.cpu(), dim=2) + F.softmax(output1.cpu(), dim=2)) / 2\n\n            # pred = output.argmax(2)\n            pred = output[:, :, 1]\n            ids = np.array(ids).T\n\n            for i in range(len(pred)):\n                preds.append([ids[i][0]]+ids[i][1:][pred[i]>CFG.th].tolist())\n                probs.append([1.0]+pred[i][pred[i]>CFG.th].tolist())\n\n    ids = id_dict[\"ids\"]\n    del id_dict, embed_dict, model_0, model_1, data_loader\n    gc.collect()\n\n    pred_df = pd.DataFrame({\n        \"id\": ids,\n        \"matches\": [\" \".join(pred) for pred in preds],\n        \"probs\": [\" \".join([str(p) for p in prob]) for prob in probs]\n    })\n\n    # pred_df = postprocess(pred_df)\n\n    return pred_df\n\nckpt0 = \"../input/4sq-models/trans_exp053_fold0.pth\"\nckpt1 = \"../input/4sq-models/trans_exp053_fold1.pth\"\n\npred_0 = run_inference(\"US\", ckpt0, ckpt1)\npred_1 = run_inference(\"TRID\", ckpt0, ckpt1)\npred_2 = run_inference(\"other\", ckpt0, ckpt1)\n\nstage_1 = pd.concat([pred_0, pred_1, pred_2], axis=0)\nstage_1.to_csv(\"stage_1.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:58:23.296576Z","iopub.execute_input":"2022-07-07T05:58:23.297234Z","iopub.status.idle":"2022-07-07T05:59:12.431971Z","shell.execute_reply.started":"2022-07-07T05:58:23.297191Z","shell.execute_reply":"2022-07-07T05:59:12.430867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\nimport pandas as pd\n\nstage_1_df = pd.read_csv(\"stage_1.csv\")\npairs = []\nfor src_id, matches, probs in stage_1_df[[\"id\", \"matches\", \"probs\"]].to_numpy():\n    for dst_id, prob in zip(matches.split(), probs.split()):\n        if src_id != dst_id:\n            pairs.append([src_id, dst_id, float(prob)])\n\npair_df = pd.DataFrame(pairs, columns=[\"src_id\", \"dst_id\", \"prob\"])\npair_df.to_feather(\"pair_df.ftr\")","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:59:12.433959Z","iopub.execute_input":"2022-07-07T05:59:12.434403Z","iopub.status.idle":"2022-07-07T05:59:13.042929Z","shell.execute_reply.started":"2022-07-07T05:59:12.434357Z","shell.execute_reply":"2022-07-07T05:59:13.041654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Colum2131 GBDT Stacking","metadata":{}},{"cell_type":"code","source":"%%python\n# ========================================\n# Library\n# ========================================\nimport os\nimport gc\nimport re\nimport sys\nimport json\nimport time\nimport shutil\nimport joblib\nimport random\nimport logging\nimport requests\nfrom ast import literal_eval\nfrom contextlib import contextmanager\nfrom tqdm import tqdm\nfrom pathlib import Path\nfrom glob import glob\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport math\nimport scipy \nimport itertools\n\nimport difflib\nimport Levenshtein\nimport rapidfuzz\nfrom collections import defaultdict\n\nfrom sklearn.model_selection import (\n    StratifiedKFold, \n    KFold, \n    GroupKFold,\n    StratifiedGroupKFold\n)\nfrom sklearn.metrics import (\n    accuracy_score, \n    f1_score,\n    roc_auc_score,\n)\n\nimport lightgbm as lgbm\nimport xgboost as xgb\nimport catboost as cat\n\nimport cupy\nimport cudf\nfrom cuml.neighbors import NearestNeighbors\nfrom cuml.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom cuml import PCA, TSNE, UMAP\nfrom cuml import ForestInference\nfrom cuml.feature_extraction.text import (\n    CountVectorizer,\n    TfidfVectorizer,\n)\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport transformers\nfrom transformers import AutoConfig, AutoModel, AutoTokenizer\nfrom transformers import AdamW, get_linear_schedule_with_warmup\n\n# =====================\n# PRE-PROCESSING\n# =====================\ndef get_basearray_test(cfg, df):\n    cfg.idx2id = df['id'].fillna('').to_numpy()\n    cfg.id2idx = {c: i for i, c in enumerate(cfg.idx2id)}\n    \n    cfg.idx2name = df['name'].fillna('').str.lower().to_numpy()\n    cfg.idx2latitude = df['latitude'].to_numpy()\n    cfg.idx2longitude = df['longitude'].to_numpy()\n    cfg.idx2radlatitude = np.deg2rad(df['latitude'].to_numpy())\n    cfg.idx2radlongitude = np.deg2rad(df['longitude'].to_numpy())\n    \n    cfg.idx2address = df['address'].fillna('').str.lower().to_numpy()\n    cfg.idx2city = df['city'].fillna('').str.lower().to_numpy()\n    cfg.idx2state = df['state'].fillna('').str.lower().to_numpy()\n    cfg.idx2zip = df['zip'].fillna('').str.lower().to_numpy()\n    cfg.idx2country = df['country'].fillna('').str.lower().to_numpy()\n    cfg.idx2url = df['url'].fillna('').str.lower().to_numpy()\n    cfg.idx2phone = df['phone'].fillna('').str.lower().to_numpy()\n    cfg.idx2categories = df['categories'].fillna('').str.lower().to_numpy()\n    return cfg\n\ndef haversine_distance(lat1, lng1, lat2, lng2):\n    dlat = lat2 - lat1\n    dlng = lng2 - lng1\n    a = np.sin(dlat/2)**2 + np.cos(lat1)*np.cos(lat2)*np.sin(dlng/2)**2\n    dist = 2 *  6_371_000 * np.arcsin(np.sqrt(a))\n    return dist\n\ndef trainsform_labelencoding(cfg, df, col, path):\n    encoder = joblib.load(os.path.join(path, f'labelencoder_{col}.joblib'))\n    outputs = encoder.transform(df[[col]]).flatten()\n    return outputs\n\n# =====================\n# FEATURE-ENGINIARING\n# =====================\nclass GBDTDataset(Dataset):\n    def __init__(self, cfg, candidate_df, base_df):\n        self.cfg = cfg\n        self.ids = candidate_df['id'].to_numpy()\n        self.match_ids = candidate_df['match_id'].to_numpy()\n        self.probs = candidate_df['prob'].to_numpy()\n        \n        # label-encoding\n        self.cfg.idx2labelcountry = base_df['label_country'].to_numpy()\n        self.cfg.idx2labelcategories = base_df['label_categories'].to_numpy()\n        \n        # count-tfidf\n        self.idx2tfidfnameword = self.get_tfidfvectorizer(\n            base_df['name'], ngram_range=(1, 1), analyzer='word'\n        ).get()\n        self.idx2tfidfnamechar = self.get_tfidfvectorizer(\n            base_df['name'], ngram_range=(1, 3), analyzer='char'\n        ).get()\n        self.idx2tfidfcategoriesword = self.get_tfidfvectorizer(\n            base_df['categories'].str.replace(',', ''), ngram_range=(1, 1), analyzer='word'\n        ).get()\n        self.idx2tfidfcategorieschar = self.get_tfidfvectorizer(\n            base_df['categories'].str.replace(',', ''), ngram_range=(1, 3), analyzer='char'\n        ).get()\n        self.idx2tfidfaddressword = self.get_tfidfvectorizer(\n            base_df['address'], ngram_range=(1, 1), analyzer='word'\n        ).get()\n        self.idx2tfidfaddresschar = self.get_tfidfvectorizer(\n            base_df['address'], ngram_range=(1, 3), analyzer='char'\n        ).get()\n        self.idx2tfidfphoneword = self.get_tfidfvectorizer(\n            base_df['phone'], ngram_range=(1, 1), analyzer='word'\n        ).get()\n        self.idx2tfidfphonechar = self.get_tfidfvectorizer(\n            base_df['phone'], ngram_range=(1, 3), analyzer='char'\n        ).get()\n        \n        base_df['remove_name'] = remove_number(base_df, 'name')\n        self.idx2tfidfremovenameword = self.get_tfidfvectorizer(\n            base_df['remove_name'], ngram_range=(1, 1), analyzer='word'\n        ).get()\n        self.idx2tfidfremovenamechar = self.get_tfidfvectorizer(\n            base_df['remove_name'], ngram_range=(1, 3), analyzer='char'\n        ).get()\n        \n        base_df['all'] = (\n            base_df['name'] + ' ' + \n            base_df['categories'] + ' ' + \n            base_df['address'] + ' ' + \n            base_df['city'] + ' ' + \n            base_df['state'] + ' ' + \n            base_df['zip'] + ' ' + \n            base_df['url'] + ' ' + \n            base_df['phone']\n        )\n        self.idx2tfidfallword = self.get_tfidfvectorizer(\n            base_df['all'], ngram_range=(1, 1), analyzer='word'\n        ).get()\n        self.idx2tfidfallchar = self.get_tfidfvectorizer(\n            base_df['all'], ngram_range=(1, 3), analyzer='char'\n        ).get()\n        \n        _, self.feature_cols = self.prepare_inputs(\n            self.cfg,\n            cfg.id2idx[self.ids[0]],\n            cfg.id2idx[self.match_ids[0]],\n            self.probs[0],\n            first=True,\n        )\n        \n    def __len__(self):\n        return len(self.ids)\n    \n    def __getitem__(self, index):\n        inputs = self.prepare_inputs(\n            self.cfg,\n            cfg.id2idx[self.ids[index]],\n            cfg.id2idx[self.match_ids[index]],\n            self.probs[index],\n        )\n        inputs = torch.tensor(inputs, dtype=torch.float)\n        return inputs\n        \n    @staticmethod\n    def prepare_inputs(cfg, i, j, prob, first=False):\n        features = []\n        feature_cols = []\n        \n        # base features\n        features.append(i)\n        features.append(j)\n        features.append(cfg.idx2latitude[i])\n        features.append(cfg.idx2longitude[i])\n        features.append(cfg.idx2latitude[j])\n        features.append(cfg.idx2longitude[j])\n        features.append(\n            haversine_distance(\n                cfg.idx2radlatitude[i],\n                cfg.idx2radlongitude[i],\n                cfg.idx2radlatitude[j],\n                cfg.idx2radlongitude[j],\n            )\n        )\n        features.append(cfg.idx2labelcountry[i])\n        features.append(cfg.idx2labelcountry[j])\n        features.append(cfg.idx2labelcategories[i])\n        features.append(cfg.idx2labelcategories[j])\n        features.append(prob)\n        \n        if first:\n            feature_cols.append('id')\n            feature_cols.append('match_id')\n            feature_cols.append('latitude_1')\n            feature_cols.append('longitude_1')\n            feature_cols.append('latitude_2')\n            feature_cols.append('longitude_2')\n            feature_cols.append('haversine_distance')\n            feature_cols.append('country_1')\n            feature_cols.append('country_2')\n            feature_cols.append('categories_1')\n            feature_cols.append('categories_2')\n            feature_cols.append('prob')\n            \n        # edit distance\n        for col, idx2str in [\n            ('name', cfg.idx2name), \n            ('categories', cfg.idx2categories),\n            ('address', cfg.idx2address),\n            ('city', cfg.idx2city),\n            ('state', cfg.idx2state), \n            ('zip', cfg.idx2zip),\n            ('url', cfg.idx2url),\n            ('phone', cfg.idx2phone)]:\n            if idx2str[i]=='' or  idx2str[j]=='':\n                features.append(np.nan)\n                features.append(np.nan)\n                features.append(np.nan)\n                features.append(np.nan)\n                # rapidfuzz\n                features.append(np.nan)\n                features.append(np.nan)\n                features.append(np.nan)\n                features.append(np.nan)\n                features.append(np.nan)\n                features.append(np.nan)\n                features.append(np.nan)\n                features.append(np.nan)\n            else:\n                features.append(\n                    difflib.SequenceMatcher(None, idx2str[i], idx2str[j]).ratio()\n                )\n                features.append(\n                    difflib.SequenceMatcher(None, idx2str[j], idx2str[i]).ratio()\n                )\n                features.append(\n                    1 - Levenshtein.distance(idx2str[i], idx2str[j]) /\\\n                    max(len(idx2str[i]), len(idx2str[j]))\n                )\n                features.append(\n                    Levenshtein.jaro_winkler(idx2str[i], idx2str[j])\n                )\n                # rapidfuzz\n                features.append(rapidfuzz.fuzz.ratio(idx2str[i], idx2str[j])) # simple-ratio\n                features.append(rapidfuzz.fuzz.partial_ratio(idx2str[i], idx2str[j])) # partial-ratio\n                features.append(rapidfuzz.fuzz.token_set_ratio(idx2str[i], idx2str[j])) # token-set-ratio\n                features.append(rapidfuzz.fuzz.token_sort_ratio(idx2str[i], idx2str[j])) # token-sort-ratio\n                features.append(rapidfuzz.fuzz.token_ratio(idx2str[i], idx2str[j])) # token-ratio\n                features.append(rapidfuzz.fuzz.partial_token_ratio(idx2str[i], idx2str[j])) # partial-token-ratio\n                features.append(rapidfuzz.fuzz.WRatio(idx2str[i], idx2str[j])) # wratio\n                features.append(rapidfuzz.fuzz.QRatio(idx2str[i], idx2str[j])) # qratio\n            if first:\n                for feature_col in [\n                    'gesh_12', 'gesh_21', 'leven', 'jaro',\n                    'simple_ratio', 'partial_ratio', 'token_set_ratio', \n                    'token_sort_ratio', 'token_ratio', 'partial_token_ratio',\n                    'wratio', 'qratio']:\n                    feature_cols.append(col + '_' + feature_col)\n            \n        # set\n        for col, idx2str, split in [\n            ('name', cfg.idx2name, ' '),\n            ('categories', cfg.idx2categories, ', ')\n        ]:\n            if idx2str[i]=='' or idx2str[j]=='':\n                features.append(np.nan)\n                features.append(np.nan)\n                features.append(np.nan)\n            else:\n                set_1 = set(idx2str[i].split(split))\n                set_2 = set(idx2str[j].split(split))\n                features.append(len(set_1 & set_2) / (len(set_1 | set_2)))\n                features.append(2*len(set_1 & set_2) / (len(set_1) + len(set_2)))\n                features.append(len(set_1 & set_2) / (min(len(set_1), len(set_2)))) \n            if first:\n                for feature_col in ['jaccard', 'dice', 'sympson']:\n                    feature_cols.append(col + '_' + feature_col)\n            \n        features = np.array(features)\n        if first:\n            return features, feature_cols\n        else:\n            return features\n\n    @staticmethod\n    def get_tfidfvectorizer(series, ngram_range=(1, 1), analyzer='word'):\n        series = cudf.Series(series.fillna('').str.lower())\n        encoder = TfidfVectorizer(ngram_range=ngram_range, analyzer=analyzer, min_df=2)\n        matrix = encoder.fit_transform(series)\n        del series, encoder\n        gc.collect()\n        return matrix\n    \n# =====================\n# INFERENCE\n# =====================\ndef pred_xgboost(X, data_dir, add_suffix=''):\n    models = glob(os.path.join(data_dir, f'xgb_fold*{add_suffix}.model'))\n    models = [ForestInference.load(model, output_class=True, model_type='xgboost') for model in models]\n    preds = np.array([model.predict_proba(X)[:, 1] for model in models])\n    preds = np.mean(preds, axis=0)\n    del models\n    gc.collect()\n    return preds\n\ndef get_similetfidf(matrix, indeies_1, indeies_2):\n    feature = matrix[indeies_1].multiply(matrix[indeies_2]).sum(axis=1).A.ravel()\n    return feature\n\ndef remove_number(df, col):\n    outputs = np.array([re.sub(r'[0-9]+', '', s) for s in df[col].fillna('').to_numpy()])\n    return outputs\n\n\nCOUNTER = 0\nos.makedirs('./feats_colum/', exist_ok=True)\n\ndef batch_inferring(cfg, test_dataset, inputs, outputs, ids, match_ids):\n    test_X = np.concatenate(inputs)\n    test_X = pd.DataFrame(test_X, columns=test_dataset.feature_cols)\n\n    test_X['id'] = test_X['id'].astype(int)\n    test_X['match_id'] = test_X['match_id'].astype(int)\n\n    test_X['simile_tfidfnameword'] = get_similetfidf(\n        test_dataset.idx2tfidfnameword, test_X['id'].tolist(), test_X['match_id'].tolist()\n    ).astype(np.float32)\n    test_X['simile_tfidfnamechar'] = get_similetfidf(\n        test_dataset.idx2tfidfnamechar, test_X['id'].tolist(), test_X['match_id'].tolist()\n    ).astype(np.float32)\n    test_X.loc[test_X['name_gesh_12'].isnull(), 'simile_tfidfnameword'] = np.nan\n    test_X.loc[test_X['name_gesh_12'].isnull(), 'simile_tfidfnamechar'] = np.nan\n    \n    test_X['simile_tfidfcategoriesword'] = get_similetfidf(\n        test_dataset.idx2tfidfcategoriesword, test_X['id'].tolist(), test_X['match_id'].tolist()\n    ).astype(np.float32)\n    test_X['simile_tfidfcategorieschar'] = get_similetfidf(\n        test_dataset.idx2tfidfcategorieschar, test_X['id'].tolist(), test_X['match_id'].tolist()\n    ).astype(np.float32)\n    test_X.loc[test_X['categories_gesh_12'].isnull(), 'simile_tfidfcategoriesword'] = np.nan\n    test_X.loc[test_X['categories_gesh_12'].isnull(), 'simile_tfidfcategorieschar'] = np.nan\n    \n    test_X['simile_tfidfaddressword'] = get_similetfidf(\n        test_dataset.idx2tfidfaddressword, test_X['id'].tolist(), test_X['match_id'].tolist()\n    ).astype(np.float32)\n    test_X['simile_tfidfaddresschar'] = get_similetfidf(\n        test_dataset.idx2tfidfaddresschar, test_X['id'].tolist(), test_X['match_id'].tolist()\n    ).astype(np.float32)\n    test_X.loc[test_X['address_gesh_12'].isnull(), 'simile_tfidfaddressword'] = np.nan\n    test_X.loc[test_X['address_gesh_12'].isnull(), 'simile_tfidfaddresschar'] = np.nan\n    \n    test_X['simile_tfidfphoneword'] = get_similetfidf(\n        test_dataset.idx2tfidfphoneword, test_X['id'].tolist(), test_X['match_id'].tolist()\n    ).astype(np.float32)\n    test_X['simile_tfidfphonechar'] = get_similetfidf(\n        test_dataset.idx2tfidfphonechar, test_X['id'].tolist(), test_X['match_id'].tolist()\n    ).astype(np.float32)\n    test_X.loc[test_X['phone_gesh_12'].isnull(), 'simile_tfidfphoneword'] = np.nan\n    test_X.loc[test_X['phone_gesh_12'].isnull(), 'simile_tfidfphonechar'] = np.nan\n\n    test_X['simile_tfidfremovenameword'] = get_similetfidf(\n        test_dataset.idx2tfidfremovenameword, test_X['id'].tolist(), test_X['match_id'].tolist()\n    ).astype(np.float32)\n    test_X['simile_tfidfremovenamechar'] = get_similetfidf(\n        test_dataset.idx2tfidfremovenamechar, test_X['id'].tolist(), test_X['match_id'].tolist()\n    ).astype(np.float32)\n    test_X.loc[test_X['name_gesh_12'].isnull(), 'simile_tfidfremovenameword'] = np.nan\n    test_X.loc[test_X['name_gesh_12'].isnull(), 'simile_tfidfremovenamechar'] = np.nan\n    \n    test_X['simile_tfidfallword'] = get_similetfidf(\n        test_dataset.idx2tfidfallword, test_X['id'].tolist(), test_X['match_id'].tolist()\n    ).astype(np.float32)\n    test_X['simile_tfidfallchar'] = get_similetfidf(\n        test_dataset.idx2tfidfallchar, test_X['id'].tolist(), test_X['match_id'].tolist()\n    ).astype(np.float32)\n\n    test_X['id'] = test_X['id'].map(lambda x: test_dataset.cfg.idx2id[x])\n    test_X['match_id'] = test_X['match_id'].map(lambda x: test_dataset.cfg.idx2id[x])\n    ids.append(test_X['id'].to_numpy())\n    match_ids.append(test_X['match_id'].to_numpy())\n    \n    #### TKM\n    global COUNTER\n    test_X.rename({'id': 'src_id', 'match_id': 'dst_id'}, axis=1).to_feather(f'./feats_colum/feat_colum_{COUNTER}.ftr')\n    COUNTER += 1\n    ####\n    \n    test_X = test_X.drop(columns=['id', 'match_id'])\n    outputs.append(pred_xgboost(test_X, cfg.model_path, '_1st'))\n    del test_X\n    gc.collect()\n    return outputs, ids, match_ids\n\ndef inferring(test_dataset, batch_size=32, max_size=1_000_000):\n    test_loader = DataLoader(\n        dataset=test_dataset,\n        batch_size=batch_size,\n        shuffle=False,\n        pin_memory=True,\n        drop_last=False,\n        num_workers=1,\n    )\n    count = 0\n    inputs = []\n    outputs = []\n    ids = []\n    match_ids = []\n    for step, (batch_inputs) in tqdm(enumerate(test_loader), total=len(test_loader)):\n        inputs.append(batch_inputs.detach().numpy())\n        count += 1\n        if count*batch_size > max_size:\n            outputs, ids, match_ids = batch_inferring(\n                cfg, test_dataset, inputs, outputs, ids, match_ids\n            )\n            del inputs\n            gc.collect()\n            \n            # init\n            count = 0\n            inputs = []\n\n    if len(inputs) > 0:\n        outputs, ids, match_ids = batch_inferring(\n            cfg, test_dataset, inputs, outputs, ids, match_ids\n        )\n        del inputs\n        gc.collect()\n\n    outputs = np.concatenate(outputs)\n    ids = np.concatenate(ids)\n    match_ids = np.concatenate(match_ids)\n\n    output_df = pd.DataFrame({\n        'id': ids,\n        'match_id': match_ids,\n        'pred': outputs\n    })\n    del outputs, ids, match_ids\n    gc.collect()\n    return output_df\n\n\n\nclass Config:\n    n_neighbors = 300\n    model_path = '../input/4sq-exp102-xgb-053-0005-uf-09/model'\n    threshold = 0.5\n    \ncfg = Config\ntest = pd.read_feather(f'test.ftr')\ncandidate_df = pd.read_feather(\"pair_df.ftr\")\ncandidate_df = candidate_df.rename(columns={'src_id': 'id', 'dst_id': 'match_id'})\n\ncfg = get_basearray_test(cfg, test)\n\n\nclass UnionFind():\n    def __init__(self, n):\n        self.n = n\n        self.parents = [-1] * n\n\n    def find(self, x):\n        if self.parents[x] < 0:\n            return x\n        else:\n            self.parents[x] = self.find(self.parents[x])\n            return self.parents[x]\n\n    def union(self, x, y):\n        x = self.find(x)\n        y = self.find(y)\n        if x == y:\n            return\n        if self.parents[x] > self.parents[y]:\n            x, y = y, x\n        self.parents[x] += self.parents[y]\n        self.parents[y] = x\n        \ndef candidate_unionfind(candidate_df, base_df, threshold=0.5):\n    id2num = {id_str: i for i, id_str in enumerate(base_df['id'].to_numpy())}\n    id2num_rev = {v: k for k, v in id2num.items()}\n    \n    uf = UnionFind(n=len(base_df)+1)\n    tmp_candidate_df = candidate_df[candidate_df['prob']>threshold]\n    for id_str, match_id_str in zip(tmp_candidate_df['id'].tolist(), tmp_candidate_df['match_id'].tolist()):\n        uf.union(id2num[id_str], id2num[match_id_str])\n\n    groups = defaultdict(list)\n    for id_str in base_df['id'].to_numpy():\n        groups[uf.find(id2num[id_str])].append(id_str)\n        \n    outputs = {'id': [], 'match_id': []}\n    for id_str in tqdm(base_df['id'].to_numpy()):\n        for match_id_str in groups[uf.find(id2num[id_str])]:\n            if id_str == match_id_str: continue\n            outputs['id'].append(id_str)\n            outputs['match_id'].append(match_id_str)\n    outputs = pd.DataFrame(outputs)  \n    return outputs\n\ngroups = candidate_unionfind(candidate_df, test, threshold=0.9)\ncandidate_df = candidate_df.merge(groups, on=['id', 'match_id'], how='outer')\ndel groups\ngc.collect()\n\n\nfor col in ['country', 'categories']:\n    test[f'label_{col}'] = trainsform_labelencoding(\n        cfg, test, col, cfg.model_path\n    )\ntest_dataset = GBDTDataset(cfg, candidate_df, test)\ndel candidate_df\ngc.collect()\n\ntest_f = inferring(test_dataset, batch_size=128, max_size=250_000)\ntest_f.to_feather('stacking_colum2131_gbdt.feather')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:59:13.045072Z","iopub.execute_input":"2022-07-07T05:59:13.045464Z","iopub.status.idle":"2022-07-07T06:00:28.255953Z","shell.execute_reply.started":"2022-07-07T05:59:13.045424Z","shell.execute_reply":"2022-07-07T06:00:28.254812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\nimport numpy as np\nimport pandas as pd\n\ntest = pd.read_feather(f'test.ftr')\ntest_f = pd.read_feather(\"stacking_colum2131_gbdt.feather\")\ntest_f.loc[:, ['id', 'match_id']] = np.sort(test_f[['id', 'match_id']].values, axis=1)\ntest_f = test_f.groupby(['id', 'match_id'])['pred'].mean().reset_index()\ntest_f.rename({'id': 'src_id', 'match_id': 'dst_id'}, axis=1).to_feather('pred_colum_gbdt.ftr')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:00:28.257441Z","iopub.execute_input":"2022-07-07T06:00:28.257775Z","iopub.status.idle":"2022-07-07T06:00:28.909783Z","shell.execute_reply.started":"2022-07-07T06:00:28.257742Z","shell.execute_reply":"2022-07-07T06:00:28.90841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Colum2131 BERT (xlm-roberta-base) Stacking","metadata":{}},{"cell_type":"code","source":"%%python\n# ========================================\n# Library\n# ========================================\nimport os\nimport gc\nimport re\nimport sys\nimport json\nimport time\nimport shutil\nimport joblib\nimport random\nimport logging\nimport requests\nfrom ast import literal_eval\nfrom contextlib import contextmanager\nfrom tqdm import tqdm\nfrom pathlib import Path\nfrom glob import glob\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport math\nimport scipy \nimport itertools\n\nimport difflib\nimport Levenshtein\nimport rapidfuzz\nfrom collections import defaultdict\n\nfrom sklearn.model_selection import (\n    StratifiedKFold, \n    KFold, \n    GroupKFold,\n    StratifiedGroupKFold\n)\nfrom sklearn.metrics import (\n    accuracy_score, \n    f1_score,\n    roc_auc_score,\n)\n\nimport lightgbm as lgbm\nimport xgboost as xgb\nimport catboost as cat\n\nimport cupy\nimport cudf\nfrom cuml.neighbors import NearestNeighbors\nfrom cuml.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom cuml import PCA, TSNE, UMAP\nfrom cuml import ForestInference\nfrom cuml.feature_extraction.text import (\n    CountVectorizer,\n    TfidfVectorizer,\n)\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport transformers\nfrom transformers import AutoConfig, AutoModel, AutoTokenizer\nfrom transformers import AdamW, get_linear_schedule_with_warmup\n\n# =====================\n# PRE-PROCESSING\n# =====================\ndef get_basearray_test(cfg, df):\n    cfg.idx2id = df['id'].fillna('').to_numpy()\n    cfg.id2idx = {c: i for i, c in enumerate(cfg.idx2id)}\n    \n    cfg.idx2name = df['name'].fillna('').str.lower().to_numpy()\n    cfg.idx2latitude = df['latitude'].to_numpy()\n    cfg.idx2longitude = df['longitude'].to_numpy()\n    cfg.idx2radlatitude = np.deg2rad(df['latitude'].to_numpy())\n    cfg.idx2radlongitude = np.deg2rad(df['longitude'].to_numpy())\n    \n    cfg.idx2address = df['address'].fillna('').str.lower().to_numpy()\n    cfg.idx2city = df['city'].fillna('').str.lower().to_numpy()\n    cfg.idx2state = df['state'].fillna('').str.lower().to_numpy()\n    cfg.idx2zip = df['zip'].fillna('').str.lower().to_numpy()\n    cfg.idx2country = df['country'].fillna('').str.lower().to_numpy()\n    cfg.idx2url = df['url'].fillna('').str.lower().to_numpy()\n    cfg.idx2phone = df['phone'].fillna('').str.lower().to_numpy()\n    cfg.idx2categories = df['categories'].fillna('').str.lower().to_numpy()\n    return cfg\n\ndef collatte(inputs, labels=None):\n    mask_len = int(inputs[\"attention_mask\"].sum(axis=1).max())\n    if not labels is None:\n        outputs = {\n            \"input_ids\" : inputs['input_ids'][:,:mask_len],\n            \"attention_mask\" : inputs['attention_mask'][:,:mask_len],\n        }\n        for k, v in inputs.items():\n            if not k in [\"input_ids\", \"attention_mask\"]:\n                outputs[k] = v\n        labels = labels[:,:mask_len]\n        return outputs, labels, mask_len\n    else:\n        outputs = {\n            \"input_ids\" : inputs['input_ids'][:,:mask_len],\n            \"attention_mask\" : inputs['attention_mask'][:,:mask_len],\n        }\n        for k, v in inputs.items():\n            if not k in [\"input_ids\", \"attention_mask\"]:\n                outputs[k] = v\n        return outputs, mask_len\n\n# =====================\n# FEATURE-ENGINIARING\n# ====================\ndef get_text(cfg, i, j):\n    text = cfg.idx2country[i]+cfg.sep+\\\n            cfg.idx2name[i]+cfg.sep+cfg.idx2categories[i]+cfg.sep+cfg.idx2address[i]+\\\n            cfg.sep + cfg.sep +\\\n            cfg.idx2name[j]+cfg.sep+cfg.idx2categories[j]+cfg.sep+cfg.idx2address[j]\n    return text\n\ndef get_text_pandas(cfg, id_1, id_2):\n    i = cfg.id2idx[id_1]\n    j = cfg.id2idx[id_2]\n    text = get_text(cfg, i, j)\n    return text\n\ndef haversine_distance(lat1, lng1, lat2, lng2):\n    dlat = lat2 - lat1\n    dlng = lng2 - lng1\n    a = np.sin(dlat/2)**2 + np.cos(lat1)*np.cos(lat2)*np.sin(dlng/2)**2\n    dist = 2 * 6_371 * np.arcsin(np.sqrt(a))\n    return dist\n\ndef get_haversine_pandas(cfg, id_1, id_2):\n    i = cfg.id2idx[id_1]\n    j = cfg.id2idx[id_2]\n    distance = haversine_distance(\n        cfg.idx2radlatitude[i],\n        cfg.idx2radlongitude[i],\n        cfg.idx2radlatitude[j],\n        cfg.idx2radlongitude[j],\n    )\n    return distance\n\n# ====================\n# BERT-INFERENCE\n# ====================\nclass TestDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        self.texts = df['text'].to_numpy()\n        self.num_features = df[cfg.num_cols].to_numpy()\n\n    def __len__(self):\n        return len(self.texts)\n\n    def __getitem__(self, index):\n        inputs = self.prepare_input(self.cfg, self.texts[index], self.num_features[index])\n        return inputs\n    \n    @staticmethod\n    def prepare_input(cfg, text, num_features):\n        inputs = cfg.tokenizer(\n            text,\n            add_special_tokens=True,\n            max_length=cfg.max_len,\n            padding=\"max_length\",\n            return_offsets_mapping=False\n        )\n        for k, v in inputs.items():\n            inputs[k] = torch.tensor(v, dtype=torch.long)\n        inputs['num_features'] = torch.tensor(num_features, dtype=torch.float)\n        return inputs\n\nclass CustomModel(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n        self.config = AutoConfig.from_pretrained(\n            cfg.MODEL_PATH,\n            output_hidden_states=True\n        )\n        self.config.attention_probs_dropout_prob = 0.0\n        self.config.hidden_dropout_prob = 0.0\n        self.backbone = AutoModel.from_config(config=self.config)\n        self.linear1 = nn.Sequential(\n            nn.Linear(self.config.hidden_size+len(cfg.num_cols), 1024),\n            nn.SELU(),\n            nn.Linear(1024, 1)\n        )\n        \n    def forward(self, inputs, labels=None):\n        outputs = self.backbone(\n            inputs['input_ids'],\n            attention_mask=inputs['attention_mask']\n        )[\"last_hidden_state\"]\n        outputs = outputs[:, 0, :]\n        outputs = torch.cat((\n            outputs,\n            inputs['num_features']\n        ), dim=1)\n        logits = self.linear1(outputs).flatten()\n        return logits\n\ndef inferring_bert(cfg, test, custom_model):\n    print('\\n'.join(cfg.model_weights))\n    sub_pred = np.zeros(len(test))\n    for model_weight in cfg.model_weights:\n        # dataset, dataloader\n        test_dataset = TestDataset(cfg, test)\n        test_loader = DataLoader(\n            dataset=test_dataset, \n            batch_size=cfg.batch_size, \n            shuffle=False,\n            pin_memory=True\n        )\n        model = custom_model(cfg)\n        model.load_state_dict(torch.load(model_weight))\n        model = model.to(cfg.device)\n\n        model.eval()\n        tmp_pred = []\n        with torch.no_grad():\n            for inputs in tqdm(test_loader, total=len(test_loader)):\n                inputs, max_len = collatte(inputs)\n                for k, v in inputs.items():\n                    inputs[k] = v.to(cfg.device)\n                with autocast():\n                    output = model(inputs)\n                output = output.sigmoid().detach().cpu().numpy()      \n                tmp_pred.append(output)\n        sub_pred += np.concatenate(tmp_pred) / len(cfg.model_weights)\n        del model\n        torch.cuda.empty_cache()\n        gc.collect()\n    return sub_pred\n\n\n\nclass Config:\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    max_len = 512\n    batch_size = 128\n    MODEL_PATH = '../input/exp116-colum-roberta-base-full'\n    \n\ncfg = Config\ntest = pd.read_feather(f'test.ftr')\ncfg = get_basearray_test(cfg, test)\ncandidate_df = pd.read_feather(\"pair_df.ftr\")\ncandidate_df = candidate_df.rename(\n    columns={'src_id': 'id', 'dst_id': 'match_id'}\n)\ncfg.tokenizer = AutoTokenizer.from_pretrained(cfg.MODEL_PATH)\ncfg.sep = cfg.tokenizer.sep_token\nprint(cfg.sep)\n\n\nclass UnionFind():\n    def __init__(self, n):\n        self.n = n\n        self.parents = [-1] * n\n\n    def find(self, x):\n        if self.parents[x] < 0:\n            return x\n        else:\n            self.parents[x] = self.find(self.parents[x])\n            return self.parents[x]\n\n    def union(self, x, y):\n        x = self.find(x)\n        y = self.find(y)\n        if x == y:\n            return\n        if self.parents[x] > self.parents[y]:\n            x, y = y, x\n        self.parents[x] += self.parents[y]\n        self.parents[y] = x\n        \ndef candidate_unionfind(candidate_df, base_df, threshold=0.5):\n    id2num = {id_str: i for i, id_str in enumerate(base_df['id'].to_numpy())}\n    id2num_rev = {v: k for k, v in id2num.items()}\n    \n    uf = UnionFind(n=len(base_df)+1)\n    tmp_candidate_df = candidate_df[candidate_df['prob']>threshold]\n    for id_str, match_id_str in zip(tmp_candidate_df['id'].tolist(), tmp_candidate_df['match_id'].tolist()):\n        uf.union(id2num[id_str], id2num[match_id_str])\n\n    groups = defaultdict(list)\n    for id_str in base_df['id'].to_numpy():\n        groups[uf.find(id2num[id_str])].append(id_str)\n        \n    outputs = {'id': [], 'match_id': []}\n    for id_str in tqdm(base_df['id'].to_numpy()):\n        for match_id_str in groups[uf.find(id2num[id_str])]:\n            if id_str == match_id_str: continue\n            outputs['id'].append(id_str)\n            outputs['match_id'].append(match_id_str)\n    outputs = pd.DataFrame(outputs)  \n    return outputs\n\n\ngroups = candidate_unionfind(candidate_df, test, threshold=0.9)\ncandidate_df = candidate_df.merge(groups, on=['id', 'match_id'], how='outer')\ncandidate_df['prob'] = candidate_df['prob'].fillna(-1)\ndel groups\ngc.collect()\n\ncfg.model_weights = ['../input/exp116-colum-roberta-base-full/full.pth']\ncfg.num_cols = ['prob', 'haversine_distance']\ncandidate_df['text'] = np.vectorize(get_text_pandas)(\n    cfg, candidate_df['id'], candidate_df['match_id']\n)\ncandidate_df['haversine_distance'] = np.vectorize(get_haversine_pandas)(\n    cfg, candidate_df['id'], candidate_df['match_id']\n)\ncandidate_df['haversine_distance'] = np.log1p(candidate_df['haversine_distance'])\n\n\ncandidate_df['text_len'] = candidate_df['text'].str.len()\ncandidate_df = candidate_df.sort_values('text_len')\ncandidate_df = candidate_df.reset_index()\n\ncandidate_df['pred'] = inferring_bert(cfg, candidate_df, CustomModel)\ncandidate_df = candidate_df.set_index('index')\ncandidate_df = candidate_df.sort_index()\n\ncandidate_df[['id', 'match_id', 'pred']].reset_index(drop=True).to_feather('stacking_colum2131_xlm_roberta_base.feather')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:00:28.911919Z","iopub.execute_input":"2022-07-07T06:00:28.912363Z","iopub.status.idle":"2022-07-07T06:01:05.488909Z","shell.execute_reply.started":"2022-07-07T06:00:28.912315Z","shell.execute_reply":"2022-07-07T06:01:05.487869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\nimport numpy as np\nimport pandas as pd\n\ntest = pd.read_feather(f'test.ftr')\ntest_f = pd.read_feather('stacking_colum2131_xlm_roberta_base.feather')\ntest_f.loc[:, ['id', 'match_id']] = np.sort(test_f[['id', 'match_id']].values, axis=1)\ntest_f = test_f.groupby(['id', 'match_id'])['pred'].mean().reset_index()\ntest_f.rename({'id': 'src_id', 'match_id': 'dst_id'}, axis=1).to_feather('pred_colum_bert.ftr')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:01:05.495729Z","iopub.execute_input":"2022-07-07T06:01:05.496256Z","iopub.status.idle":"2022-07-07T06:01:06.247175Z","shell.execute_reply.started":"2022-07-07T06:01:05.496223Z","shell.execute_reply":"2022-07-07T06:01:06.245986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## yabea BERT (mluke-base) Stacking","metadata":{}},{"cell_type":"code","source":"%%python\nimport pickle\n\nimport numpy as np\nimport pandas as pd\nfrom munch import Munch\n\nfrom transformers import AutoTokenizer\n\n\ndef get_text(base, i, j, sep):\n    text = (\n        base.idx2country[i]\n        + sep\n        + base.idx2name[i]\n        + sep\n        + base.idx2categories[i]\n        + sep\n        + base.idx2address[i]\n        + sep\n        + base.idx2name[j]\n        + sep\n        + base.idx2categories[j]\n        + sep\n        + base.idx2address[j]\n    )\n    return text\n\n\ndef get_text_pandas(base, id_i, id_j, sep):\n    i = base.id2idx[id_i]\n    j = base.id2idx[id_j]\n    text = get_text(base, i, j, sep)\n    return text\n\n\ndef get_d_lat_pandas(base, id_i, id_j):\n    i = base.id2idx[id_i]\n    j = base.id2idx[id_j]\n    d_lat = abs(base.idx2radlat[i] - base.idx2radlat[j])\n    return d_lat\n\n\ndef get_d_long_pandas(base, id_i, id_j):\n    i = base.id2idx[id_i]\n    j = base.id2idx[id_j]\n    d_long = abs(base.idx2radlong[i] - base.idx2radlong[j])\n    return d_long\n\n\noof = pd.read_feather(\"pair_df.ftr\")\noof = oof[oof[\"prob\"] > 0.005]\noof = oof[oof[\"dst_id\"] != \"\"]\noof = oof.reset_index(drop=True)\n\ntmp = oof.rename({\"src_id\": \"dst_id\", \"dst_id\": \"src_id\", \"prob\": \"rev_prob\"}, axis=1)\noof = oof.merge(tmp, on=[\"src_id\", \"dst_id\"], how=\"outer\")\noof[\"prob\"] = oof[\"prob\"].fillna(oof[\"rev_prob\"])\noof = oof.drop([\"rev_prob\"], axis=1)\n\ndf = pd.read_feather(\"test.ftr\")\n\nbasearray = Munch()\nbasearray.id2idx = {id_str: i for i, id_str in enumerate(df[\"id\"].to_numpy())}\nbasearray.idx2name = df[\"name\"].fillna(\"\").str.lower().to_numpy()\nbasearray.idx2categories = df[\"categories\"].fillna(\"\").str.lower().to_numpy()\nbasearray.idx2country = df[\"country\"].fillna(\"\").str.lower().to_numpy()\nbasearray.idx2address = df[\"address\"].fillna(\"\").str.lower().to_numpy()\n\nsep = AutoTokenizer.from_pretrained(\"../input/4sq-exp412-yabea-mluke-base-all\").sep_token\n\noof[\"text\"] = np.vectorize(get_text_pandas)(basearray, oof[\"src_id\"], oof[\"dst_id\"], sep)\noof[\"len_text\"] = oof[\"text\"].map(len)\n\nbasearray.idx2radlat = np.deg2rad(df[\"latitude\"].to_numpy())\nbasearray.idx2radlong = np.deg2rad(df[\"longitude\"].to_numpy())\n\noof[\"d_lat\"] = np.vectorize(get_d_lat_pandas)(basearray, oof[\"src_id\"], oof[\"dst_id\"])\noof[\"d_long\"] = np.vectorize(get_d_long_pandas)(basearray, oof[\"src_id\"], oof[\"dst_id\"])\n\noof.to_feather(\"/tmp/yabea_bert_stacking_feat.ftr\")","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:01:06.249268Z","iopub.execute_input":"2022-07-07T06:01:06.249676Z","iopub.status.idle":"2022-07-07T06:01:27.920934Z","shell.execute_reply.started":"2022-07-07T06:01:06.249634Z","shell.execute_reply":"2022-07-07T06:01:27.919447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\nimport pandas as pd\nimport numpy as np\nfrom tqdm.auto import tqdm\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom transformers import AutoTokenizer, AutoConfig, AutoModel\n\n\n# ==================================================\n# SETUP\n# ==================================================\n\n\nclass CFG:\n    max_len = 256\n    test_batch_size = 128\n    model_path = \"../input/4sq-exp412-yabea-mluke-base-all\"\n    model_weight_path = \"../input/4sq-exp412-yabea-mluke-base-all/all.pth\"\n    num_features = [\"prob\", \"d_lat\", \"d_long\"]\n\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n\n# ==================================================\n# FUNCTIONS\n# ==================================================\n\n\nclass TestBERTDataset(Dataset):\n    def __init__(self, df, tokenizer):\n        self.texts = df[\"text\"].to_numpy()\n        self.num_features = df[CFG.num_features].to_numpy()\n        self.tokenizer = tokenizer\n\n    def __len__(self):\n        return len(self.texts)\n\n    def __getitem__(self, index):\n        inputs = self.__prepare_input(self.texts[index], self.num_features[index])\n        return inputs\n\n    def __prepare_input(self, text, num_features):\n        inputs = self.tokenizer(\n            text,\n            add_special_tokens=True,\n            max_length=CFG.max_len,\n            padding=\"max_length\",\n            truncation=\"longest_first\",\n        )\n        for k, v in inputs.items():\n            inputs[k] = torch.tensor(v, dtype=torch.long)\n        inputs[\"num_features\"] = torch.tensor(num_features, dtype=torch.float)\n        return inputs\n\n\nclass CustomModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.config = AutoConfig.from_pretrained(CFG.model_path, output_hidden_states=True)\n        self.config.attention_probs_dropout_prob = 0.0\n        self.config.hidden_dropout_prob = 0.0\n        self.backbone = AutoModel.from_config(config=self.config)\n        self.linear1 = nn.Sequential(\n            nn.Linear(self.config.hidden_size + len(CFG.num_features), 1024),\n            nn.SELU(),\n            nn.Linear(1024, 1),\n        )\n\n    def forward(self, inputs):\n        outputs = self.backbone(inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"])[\"last_hidden_state\"]\n        outputs = outputs[:, 0, :]\n        outputs = torch.cat((outputs, inputs[\"num_features\"]), dim=1)\n        logits = self.linear1(outputs).flatten()\n        return logits\n\n    \ndef collatte(inputs):\n    mask_len = int(inputs[\"attention_mask\"].sum(axis=1).max())\n    outputs = {\n        \"input_ids\": inputs[\"input_ids\"][:, :mask_len],\n        \"attention_mask\": inputs[\"attention_mask\"][:, :mask_len],\n    }\n    for k, v in inputs.items():\n        if k not in [\"input_ids\", \"attention_mask\"]:\n            outputs[k] = v\n    return outputs\n\n\n# ==================================================\n# MAIN\n# ==================================================\n\n\noof = pd.read_feather(\"/tmp/yabea_bert_stacking_feat.ftr\")\n\n# uniform length batching\noof = oof.sort_values(by=\"len_text\")\noof = oof.reset_index(drop=True)\n\ntokenizer = AutoTokenizer.from_pretrained(CFG.model_path)\n\ntest_dataset = TestBERTDataset(oof, tokenizer)\ntest_loader = DataLoader(\n    dataset=test_dataset, batch_size=CFG.test_batch_size, shuffle=False, pin_memory=False, drop_last=False\n)\n\nmodel = CustomModel().to(device)\nmodel.load_state_dict(torch.load(CFG.model_weight_path))\nmodel.eval()\n\npreds = []\nwith torch.inference_mode():\n    with tqdm(test_loader, total=len(test_loader)) as pbar:\n        for inputs in pbar:\n            inputs = collatte(inputs)\n            for k, v in inputs.items():\n                inputs[k] = v.to(device)\n            output = model(inputs)\n            output = output.sigmoid().detach().cpu().numpy()\n            preds.append(output)\n\npreds = np.concatenate(preds)\noof[\"pred\"] = preds\noof[[\"src_id\", \"dst_id\", \"pred\"]].to_feather(\"pred_yabea_bert.ftr\")","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:01:27.922952Z","iopub.execute_input":"2022-07-07T06:01:27.923362Z","iopub.status.idle":"2022-07-07T06:02:26.692854Z","shell.execute_reply.started":"2022-07-07T06:01:27.923326Z","shell.execute_reply":"2022-07-07T06:02:26.691648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ria part","metadata":{}},{"cell_type":"code","source":"%%python\nimport pandas as pd\n\nstage_1_df = pd.read_csv(\"stage_1.csv\")\npairs = []\nfor src_id, matches, probs in stage_1_df[[\"id\", \"matches\", \"probs\"]].to_numpy():\n    for dst_id, prob in zip(matches.split(), probs.split()):\n        if src_id != dst_id and float(prob) > 0.005:\n            pairs.append([src_id, dst_id, float(prob)])\n    \npair_df = pd.DataFrame(pairs, columns=[\"src_id\", \"dst_id\", \"prob\"])\npair_df = pair_df[(pair_df['dst_id'].notnull())]\npair_df = pair_df[(pair_df['dst_id']!='')]\npair_df = pair_df.reset_index(drop=True)\n\npair_df.to_feather(\"pair_df.ftr\")","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:02:26.696411Z","iopub.execute_input":"2022-07-07T06:02:26.696778Z","iopub.status.idle":"2022-07-07T06:02:27.339155Z","shell.execute_reply.started":"2022-07-07T06:02:26.696742Z","shell.execute_reply":"2022-07-07T06:02:27.337861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 接続部分\nCANDIDATE_PATH='pair_df.ftr'\n\nimport os\nIS_IN_KAGGLE = 'KAGGLE_URL_BASE' in os.environ\nn_line_test = sum(1 for line in open('../input/foursquare-location-matching/test.csv'))\nif IS_IN_KAGGLE & (n_line_test < 10):\n    !head -n 5000 ../input/foursquare-location-matching/train.csv > less_train.csv\n    \n    \nfrom datetime import datetime\n\nimport os\n\nIS_IN_KAGGLE = 'KAGGLE_URL_BASE' in os.environ\nn_line_test = sum(1 for line in open('../input/foursquare-location-matching/test.csv'))\n\nIS_SUBMISSION = True\nUSE_FIL = True\nIS_DEBUG = False\n\nif IS_SUBMISSION & (n_line_test < 10):\n    INPUT_FILE = 'less_train.csv'\nelse:\n    INPUT_FILE = '../input/foursquare-location-matching/test.csv'\n#     INPUT_FILE = '../input/foursquare-location-matching/train.csv'\n\n# FITTED_ENCODER_PATH='outputs/n04_01_06_get_label_encoding/001/2022-06-09--18-13-28_word_level/result.pickle'\nFITTED_ENCODER_PATH='../input/4sq-cat-encoder/2022-06-09--18-13-28_word_level/result.pickle'\n\n\n#TODO change in kaggle\nMODEL_DIR = '../input/4sq-model-trans-candidate053-1/model_172'\n\n\nIS_DEBUG_FLAG = '--is_debug' if IS_DEBUG else ''\n\n\nMATCH_THRESHOLD = 0.5\n# POST_PROCESS_TYPE = 'unionfind'\nPOST_PROCESS_TYPE = 'inter_connect_proba'\n# POST_PROCESS_TYPE = 'none'\n# MERGE_BEFORE_PRED = '--merge_pair_in_pred'\nMERGE_BEFORE_PRED = ''\n\nUSE_FIL_STR = '--use_forest_inference' if USE_FIL else ''\n\nIS_IN_KAGGLE, INPUT_FILE, MODEL_DIR, IS_DEBUG_FLAG\n\n\nif IS_IN_KAGGLE:\n    !cp -r ../input/4sq-code-no-tkm-feat/* ./\n    !ls .\n\n    \nts = datetime.now().strftime(\"%Y-%m-%d--%H-%M-%S\")\nOUTPUT_BASE = f'outputs/n04_05_make_submission/{ts}'\nfinal_out_dir = './' if IS_SUBMISSION else f'{OUTPUT_BASE}/submission'\nOUTPUT_BASE, final_out_dir\n\nout_cat_encode = f'{OUTPUT_BASE}/cat_encode'\n!echo python ./n04_01_06_get_label_encoding.py --data_csv={INPUT_FILE} --is_test \\\n    --out_dir={out_cat_encode} $IS_DEBUG_FLAG --fitted_encoder_path=$FITTED_ENCODER_PATH\n!python ./n04_01_06_get_label_encoding.py --data_csv={INPUT_FILE} --is_test \\\n    --out_dir={out_cat_encode} $IS_DEBUG_FLAG --fitted_encoder_path=$FITTED_ENCODER_PATH\n\n# 前処理\nout_cand_bert = f'{OUTPUT_BASE}/cand_bert'\n\n!echo python n04_12_01_bert_pair2pairs.py --data_csv={INPUT_FILE} \\\n    --is_test $IS_DEBUG_FLAG --cand_dict_path=$CANDIDATE_PATH \\\n    --out_dir=$out_cand_bert --filter_thresh=0.005 --use_uf_filter\n\n!python n04_12_01_bert_pair2pairs.py --data_csv={INPUT_FILE} \\\n    --is_test $IS_DEBUG_FLAG --cand_dict_path=$CANDIDATE_PATH \\\n    --out_dir=$out_cand_bert --filter_thresh=0.005 --use_uf_filter\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:02:27.341394Z","iopub.execute_input":"2022-07-07T06:02:27.341799Z","iopub.status.idle":"2022-07-07T06:02:53.062764Z","shell.execute_reply.started":"2022-07-07T06:02:27.341758Z","shell.execute_reply":"2022-07-07T06:02:53.061738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#推論\nout03_predict_bert = f'{OUTPUT_BASE}/03_bert'\n\n!sed -i -e \"s/preds = \\[\\]/preds = \\[\\]\\n    df_pairs_with_feat[['id_1', 'id_2'] + feat_cols].to_feather('feat_ria.ftr')/g\" ./n04_03_predict_with_lbg_model_plain.py\n\n!echo  python ./n04_03_predict_with_lbg_model_plain.py --pairs_dir={out_cand_bert} \\\n    --model_dir={MODEL_DIR} --out_dir={out03_predict_bert} $USE_FIL_STR --cat_encoder_dir=$out_cat_encode\n!python ./n04_03_predict_with_lbg_model_plain.py --pairs_dir={out_cand_bert} \\\n    --model_dir={MODEL_DIR} --out_dir={out03_predict_bert} $USE_FIL_STR --cat_encoder_dir=$out_cat_encode","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:02:53.064431Z","iopub.execute_input":"2022-07-07T06:02:53.064851Z","iopub.status.idle":"2022-07-07T06:05:30.641309Z","shell.execute_reply.started":"2022-07-07T06:02:53.064805Z","shell.execute_reply":"2022-07-07T06:05:30.640002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pickle\n\nwith open(out03_predict_bert + '/result.pickle', 'rb') as f:\n    pred_result = pickle.load(f)\n    preds = pred_result['preds']\n    pred_args = pred_result['args']\n    df_ids = pred_result['df_ids']\n\nmean_pred = np.stack(preds).mean(axis=0)\nmean_pred = mean_pred[:, 1]\n\ndf_ids['pred'] = mean_pred\n\ndf_ids.rename({'id_1': 'src_id', 'id_2': 'dst_id'}, axis=1).to_feather('pred_ria.ftr')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:05:30.643693Z","iopub.execute_input":"2022-07-07T06:05:30.644512Z","iopub.status.idle":"2022-07-07T06:05:30.671877Z","shell.execute_reply.started":"2022-07-07T06:05:30.644458Z","shell.execute_reply":"2022-07-07T06:05:30.670883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TKM stacking","metadata":{}},{"cell_type":"code","source":"%%python\nimport pandas as pd\nimport lightgbm as lgb\nimport treelite\nfrom cuml import ForestInference\nimport glob\n\nlist_clf = []\nmodel_path = '../input/lib-4square/results_train50_noagg3/'\n\npath = f'{model_path}/all_data_clf.lgb'\nclf = lgb.Booster(model_file=path)\nfeature_name = clf.feature_name()\nclf = ForestInference()\nclf.load_from_treelite_model(treelite.Model.load(path, model_format='lightgbm'))\n\n\ndf_ria = pd.read_feather('feat_ria.ftr')\ndf_ria.rename(columns={'id_1': 'src_id', 'id_2': 'dst_id'}, inplace=True, copy=False)\ndf_ria.drop_duplicates(subset=['src_id', 'dst_id'], inplace=True)\ndf_ria.set_index(['src_id', 'dst_id'], inplace=True)\ndf_ria = df_ria[[c for c in df_ria.columns if c in feature_name]]\n\n\n\nfrom tqdm.auto import tqdm\nret = []\n\nfor path in tqdm(glob.glob('./feats_colum/feat_colum_*.ftr')):\n    df_col = pd.read_feather(path)\n    df_col.drop_duplicates(subset=['src_id', 'dst_id'], inplace=True)\n    df_col.set_index(['src_id', 'dst_id'], inplace=True)\n    \n    df_col = df_col[[c for c in df_col.columns if c in feature_name]]\n    df_col['prob'] = df_col['prob'].fillna(0.85)\n    tmp = df_ria.reindex(df_col.index.values)[[c for c in df_ria.columns if c not in df_col.columns]]\n    feats = pd.concat([df_col, tmp], axis=1)[feature_name].fillna(-1).values\n    \n    pred = clf.predict(feats)\n        \n    tmp = df_col[[]].copy()\n    tmp['pred'] = pred\n    tmp = tmp.reset_index()\n    ret.append(tmp)\n    \n\ndf_ret = pd.concat(ret, axis=0, ignore_index=True)\n\ndf_ret = df_ret.append(df_ret.rename({'src_id': 'dst_id', 'dst_id': 'src_id'}, axis=1)[['src_id', 'dst_id', 'pred']])\ndf_ret = df_ret.groupby(['src_id', 'dst_id'], as_index=False)[['pred']].mean()\ndf_ret.to_feather('df_tkm_lgb.ftr')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:05:30.674256Z","iopub.execute_input":"2022-07-07T06:05:30.675066Z","iopub.status.idle":"2022-07-07T06:05:48.790171Z","shell.execute_reply.started":"2022-07-07T06:05:30.675017Z","shell.execute_reply":"2022-07-07T06:05:48.78899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble","metadata":{}},{"cell_type":"code","source":"%%python\n\nimport gc\nimport pandas as pd\nimport numpy as np\n\n# [1]\ndf_col_gbdt = pd.read_feather('pred_colum_gbdt.ftr').rename(columns={'pred': 'pred_colum2131_xgb'})\n# [2]\ndf_col_bert = pd.read_feather('pred_colum_bert.ftr').rename(columns={'pred': 'pred_colum2131_bert'})\n# [3]\ndf_ria = pd.read_feather('pred_ria.ftr').rename(columns={'pred': 'pred_ria_lgbm'})\ndf_ria.loc[:, ['src_id', 'dst_id']] = np.sort(df_ria[['src_id', 'dst_id']].values, axis=1)\ndf_ria = df_ria.groupby(['src_id', 'dst_id'])['pred_ria_lgbm'].mean().reset_index()\n# [4]\ndf_yab_bert = pd.read_feather('pred_yabea_bert.ftr').rename(columns={'pred': 'pred_yabea_bert'})\ndf_yab_bert.loc[:, ['src_id', 'dst_id']] = np.sort(df_yab_bert[['src_id', 'dst_id']].values, axis=1)\ndf_yab_bert = df_yab_bert.groupby(['src_id', 'dst_id'])['pred_yabea_bert'].mean().reset_index()\n# [5]\ndf_tkm_gbdt = pd.read_feather('df_tkm_lgb.ftr').rename(columns={'pred': 'pred_tkm_lgbm'})\ndf_tkm_gbdt.loc[:, ['src_id', 'dst_id']] = np.sort(df_tkm_gbdt[['src_id', 'dst_id']].values, axis=1)\ndf_tkm_gbdt = df_tkm_gbdt.groupby(['src_id', 'dst_id'])['pred_tkm_lgbm'].mean().reset_index()\n\n# merge\ndf_ensemble = df_col_gbdt.merge(df_col_bert, on=['src_id', 'dst_id'], how='outer')\ndf_ensemble = df_ensemble.merge(df_ria, on=['src_id', 'dst_id'], how='outer')\ndf_ensemble = df_ensemble.merge(df_yab_bert, on=['src_id', 'dst_id'], how='outer')\ndf_ensemble = df_ensemble.merge(df_tkm_gbdt, on=['src_id', 'dst_id'], how='outer')\n\n# blend\nweights = [1/6, 1/4, 1/6, 1/4, 1/6]\nprint(sum(weights))\npred_arr = df_ensemble[['pred_colum2131_xgb', 'pred_colum2131_bert', 'pred_ria_lgbm', 'pred_yabea_bert', 'pred_tkm_lgbm']].to_numpy()\nweight_arr = np.zeros_like(pred_arr)\nfor i, weight in enumerate(weights):\n    weight_arr[:, i] = weight\nweight_arr[np.isnan(pred_arr)] = 0\n\nweight_arr /= weight_arr.sum(axis=1).reshape(-1, 1)\ndf_ensemble['pred'] = np.nansum(pred_arr * weight_arr, axis=1)\n\ndf_ensemble.to_feather(\"df_ensemble.ftr\")\n\ndf_ensemble[\"pred_diff_sorted\"] = np.abs(df_ensemble[\"pred\"] - 0.5)\n\nn = 450000\nif len(df_ensemble) > n:\n    df_selected = df_ensemble.sort_values(\"pred_diff_sorted\").head(n)[[\"src_id\", \"dst_id\"]]\nelse:\n    df_selected = df_ensemble.sort_values(\"pred_diff_sorted\").head(100)[[\"src_id\", \"dst_id\"]]\n    \npair_df = pd.read_feather(\"pair_df.ftr\")\n\ndf_selected = df_selected.merge(pair_df, on=[\"src_id\", \"dst_id\"], how=\"left\")\ndf_selected = df_selected.dropna(subset=[\"prob\"]).reset_index(drop=True)\ndf_selected.to_feather(\"selected_pair_df.ftr\")","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:05:48.792317Z","iopub.execute_input":"2022-07-07T06:05:48.792781Z","iopub.status.idle":"2022-07-07T06:05:49.529878Z","shell.execute_reply.started":"2022-07-07T06:05:48.792716Z","shell.execute_reply":"2022-07-07T06:05:49.528859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## xlm-roberta-large","metadata":{}},{"cell_type":"code","source":"%%python\n# ========================================\n# Library\n# ========================================\nimport os\nimport gc\nimport re\nimport sys\nimport json\nimport time\nimport shutil\nimport joblib\nimport random\nimport logging\nimport requests\nfrom ast import literal_eval\nfrom contextlib import contextmanager\nfrom tqdm import tqdm\nfrom pathlib import Path\nfrom glob import glob\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport math\nimport scipy \nimport itertools\n\nimport difflib\nimport Levenshtein\nimport rapidfuzz\nfrom collections import defaultdict\n\nfrom sklearn.model_selection import (\n    StratifiedKFold, \n    KFold, \n    GroupKFold,\n    StratifiedGroupKFold\n)\nfrom sklearn.metrics import (\n    accuracy_score, \n    f1_score,\n    roc_auc_score,\n)\n\nimport lightgbm as lgbm\nimport xgboost as xgb\nimport catboost as cat\n\nimport cupy\nimport cudf\nfrom cuml.neighbors import NearestNeighbors\nfrom cuml.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom cuml import PCA, TSNE, UMAP\nfrom cuml import ForestInference\nfrom cuml.feature_extraction.text import (\n    CountVectorizer,\n    TfidfVectorizer,\n)\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport transformers\nfrom transformers import AutoConfig, AutoModel, AutoTokenizer\nfrom transformers import AdamW, get_linear_schedule_with_warmup\n\n# =====================\n# PRE-PROCESSING\n# =====================\ndef get_basearray_test(cfg, df):\n    cfg.idx2id = df['id'].fillna('').to_numpy()\n    cfg.id2idx = {c: i for i, c in enumerate(cfg.idx2id)}\n    \n    cfg.idx2name = df['name'].fillna('').str.lower().to_numpy()\n    cfg.idx2latitude = df['latitude'].to_numpy()\n    cfg.idx2longitude = df['longitude'].to_numpy()\n    cfg.idx2radlatitude = np.deg2rad(df['latitude'].to_numpy())\n    cfg.idx2radlongitude = np.deg2rad(df['longitude'].to_numpy())\n    \n    cfg.idx2address = df['address'].fillna('').str.lower().to_numpy()\n    cfg.idx2city = df['city'].fillna('').str.lower().to_numpy()\n    cfg.idx2state = df['state'].fillna('').str.lower().to_numpy()\n    cfg.idx2zip = df['zip'].fillna('').str.lower().to_numpy()\n    cfg.idx2country = df['country'].fillna('').str.lower().to_numpy()\n    cfg.idx2url = df['url'].fillna('').str.lower().to_numpy()\n    cfg.idx2phone = df['phone'].fillna('').str.lower().to_numpy()\n    cfg.idx2categories = df['categories'].fillna('').str.lower().to_numpy()\n    return cfg\n\ndef collatte(inputs, labels=None):\n    mask_len = int(inputs[\"attention_mask\"].sum(axis=1).max())\n    if not labels is None:\n        outputs = {\n            \"input_ids\" : inputs['input_ids'][:,:mask_len],\n            \"attention_mask\" : inputs['attention_mask'][:,:mask_len],\n        }\n        for k, v in inputs.items():\n            if not k in [\"input_ids\", \"attention_mask\"]:\n                outputs[k] = v\n        labels = labels[:,:mask_len]\n        return outputs, labels, mask_len\n    else:\n        outputs = {\n            \"input_ids\" : inputs['input_ids'][:,:mask_len],\n            \"attention_mask\" : inputs['attention_mask'][:,:mask_len],\n        }\n        for k, v in inputs.items():\n            if not k in [\"input_ids\", \"attention_mask\"]:\n                outputs[k] = v\n        return outputs, mask_len\n\n# =====================\n# FEATURE-ENGINIARING\n# ====================\ndef get_text(cfg, i, j):\n    text = cfg.idx2country[i]+cfg.sep+\\\n            cfg.idx2name[i]+cfg.sep+cfg.idx2categories[i]+cfg.sep+cfg.idx2address[i]+\\\n            cfg.sep + cfg.sep +\\\n            cfg.idx2name[j]+cfg.sep+cfg.idx2categories[j]+cfg.sep+cfg.idx2address[j]\n    return text\n\ndef get_text_pandas(cfg, id_1, id_2):\n    i = cfg.id2idx[id_1]\n    j = cfg.id2idx[id_2]\n    text = get_text(cfg, i, j)\n    return text\n\ndef haversine_distance(lat1, lng1, lat2, lng2):\n    dlat = lat2 - lat1\n    dlng = lng2 - lng1\n    a = np.sin(dlat/2)**2 + np.cos(lat1)*np.cos(lat2)*np.sin(dlng/2)**2\n    dist = 2 * 6_371 * np.arcsin(np.sqrt(a))\n    return dist\n\ndef get_haversine_pandas(cfg, id_1, id_2):\n    i = cfg.id2idx[id_1]\n    j = cfg.id2idx[id_2]\n    distance = haversine_distance(\n        cfg.idx2radlatitude[i],\n        cfg.idx2radlongitude[i],\n        cfg.idx2radlatitude[j],\n        cfg.idx2radlongitude[j],\n    )\n    return distance\n\n# ====================\n# BERT-INFERENCE\n# ====================\nclass TestDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        self.texts = df['text'].to_numpy()\n        self.num_features = df[cfg.num_cols].to_numpy()\n\n    def __len__(self):\n        return len(self.texts)\n\n    def __getitem__(self, index):\n        inputs = self.prepare_input(self.cfg, self.texts[index], self.num_features[index])\n        return inputs\n    \n    @staticmethod\n    def prepare_input(cfg, text, num_features):\n        inputs = cfg.tokenizer(\n            text,\n            add_special_tokens=True,\n            max_length=cfg.max_len,\n            padding=\"max_length\",\n            return_offsets_mapping=False\n        )\n        for k, v in inputs.items():\n            inputs[k] = torch.tensor(v, dtype=torch.long)\n        inputs['num_features'] = torch.tensor(num_features, dtype=torch.float)\n        return inputs\n\nclass CustomModel(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n        self.config = AutoConfig.from_pretrained(\n            cfg.MODEL_PATH,\n            output_hidden_states=True\n        )\n        self.config.attention_probs_dropout_prob = 0.0\n        self.config.hidden_dropout_prob = 0.0\n        self.backbone = AutoModel.from_config(config=self.config)\n        self.linear1 = nn.Sequential(\n            nn.Linear(self.config.hidden_size+len(cfg.num_cols), 1024),\n            nn.SELU(),\n            nn.Linear(1024, 1)\n        )\n        \n    def forward(self, inputs, labels=None):\n        outputs = self.backbone(\n            inputs['input_ids'],\n            attention_mask=inputs['attention_mask']\n        )[\"last_hidden_state\"]\n        outputs = outputs[:, 0, :]\n        outputs = torch.cat((\n            outputs,\n            inputs['num_features']\n        ), dim=1)\n        logits = self.linear1(outputs).flatten()\n        return logits\n\ndef inferring_bert(cfg, test, custom_model):\n    print('\\n'.join(cfg.model_weights))\n    sub_pred = np.zeros(len(test))\n    for model_weight in cfg.model_weights:\n        # dataset, dataloader\n        test_dataset = TestDataset(cfg, test)\n        test_loader = DataLoader(\n            dataset=test_dataset, \n            batch_size=cfg.batch_size, \n            shuffle=False,\n            pin_memory=True\n        )\n        model = custom_model(cfg)\n        model.load_state_dict(torch.load(model_weight))\n        model = model.to(cfg.device)\n\n        model.eval()\n        tmp_pred = []\n        with torch.no_grad():\n            for inputs in tqdm(test_loader, total=len(test_loader)):\n                inputs, max_len = collatte(inputs)\n                for k, v in inputs.items():\n                    inputs[k] = v.to(cfg.device)\n                with autocast():\n                    output = model(inputs)\n                output = output.sigmoid().detach().cpu().numpy()      \n                tmp_pred.append(output)\n        sub_pred += np.concatenate(tmp_pred) / len(cfg.model_weights)\n        del model\n        torch.cuda.empty_cache()\n        gc.collect()\n    return sub_pred\n\n\n\nclass Config:\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    max_len = 512\n    batch_size = 24\n    MODEL_PATH = '../input/exp117-colum-roberta-large-full'\n    \n\ncfg = Config\ntest = pd.read_feather(f'test.ftr')\ncfg = get_basearray_test(cfg, test)\ncandidate_df = pd.read_feather(\"selected_pair_df.ftr\")\ncandidate_df = candidate_df.rename(\n    columns={'src_id': 'id', 'dst_id': 'match_id'}\n)\ncfg.tokenizer = AutoTokenizer.from_pretrained(cfg.MODEL_PATH)\ncfg.sep = cfg.tokenizer.sep_token\nprint(cfg.sep)\n\nclass UnionFind():\n    def __init__(self, n):\n        self.n = n\n        self.parents = [-1] * n\n\n    def find(self, x):\n        if self.parents[x] < 0:\n            return x\n        else:\n            self.parents[x] = self.find(self.parents[x])\n            return self.parents[x]\n\n    def union(self, x, y):\n        x = self.find(x)\n        y = self.find(y)\n        if x == y:\n            return\n        if self.parents[x] > self.parents[y]:\n            x, y = y, x\n        self.parents[x] += self.parents[y]\n        self.parents[y] = x\n        \ndef candidate_unionfind(candidate_df, base_df, threshold=0.5):\n    id2num = {id_str: i for i, id_str in enumerate(base_df['id'].to_numpy())}\n    id2num_rev = {v: k for k, v in id2num.items()}\n    \n    uf = UnionFind(n=len(base_df)+1)\n    tmp_candidate_df = candidate_df[candidate_df['prob']>threshold]\n    for id_str, match_id_str in zip(tmp_candidate_df['id'].tolist(), tmp_candidate_df['match_id'].tolist()):\n        uf.union(id2num[id_str], id2num[match_id_str])\n\n    groups = defaultdict(list)\n    for id_str in base_df['id'].to_numpy():\n        groups[uf.find(id2num[id_str])].append(id_str)\n        \n    outputs = {'id': [], 'match_id': []}\n    for id_str in tqdm(base_df['id'].to_numpy()):\n        for match_id_str in groups[uf.find(id2num[id_str])]:\n            if id_str == match_id_str: continue\n            outputs['id'].append(id_str)\n            outputs['match_id'].append(match_id_str)\n    outputs = pd.DataFrame(outputs)  \n    return outputs\n\n\n#groups = candidate_unionfind(candidate_df, test, threshold=0.9)\n#candidate_df = candidate_df.merge(groups, on=['id', 'match_id'], how='outer')\n#candidate_df['prob'] = candidate_df['prob'].fillna(-1)\n#del groups\n#gc.collect()\n\ncfg.model_weights = ['../input/exp117-colum-roberta-large-full/full.pth']\ncfg.num_cols = ['prob', 'haversine_distance']\ncandidate_df['text'] = np.vectorize(get_text_pandas)(\n    cfg, candidate_df['id'], candidate_df['match_id']\n)\ncandidate_df['haversine_distance'] = np.vectorize(get_haversine_pandas)(\n    cfg, candidate_df['id'], candidate_df['match_id']\n)\ncandidate_df['haversine_distance'] = np.log1p(candidate_df['haversine_distance'])\n\ncandidate_df['text_len'] = candidate_df['text'].str.len()\ncandidate_df = candidate_df.sort_values('text_len')\ncandidate_df = candidate_df.reset_index()\n\ncandidate_df['pred'] = inferring_bert(cfg, candidate_df, CustomModel)\ncandidate_df = candidate_df.set_index('index')\ncandidate_df = candidate_df.sort_index()\n\ncandidate_df[['id', 'match_id', 'pred']].reset_index(drop=True).to_feather('stacking_colum2131_xlm_roberta_large.feather')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:05:49.534634Z","iopub.execute_input":"2022-07-07T06:05:49.535375Z","iopub.status.idle":"2022-07-07T06:06:41.209421Z","shell.execute_reply.started":"2022-07-07T06:05:49.535334Z","shell.execute_reply":"2022-07-07T06:06:41.208288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\nimport numpy as np\nimport pandas as pd\n\ntest = pd.read_feather(f'test.ftr')\ntest_f = pd.read_feather('stacking_colum2131_xlm_roberta_large.feather')\ntest_f.loc[:, ['id', 'match_id']] = np.sort(test_f[['id', 'match_id']].values, axis=1)\ntest_f = test_f.groupby(['id', 'match_id'])['pred'].mean().reset_index()\ntest_f.rename({'id': 'src_id', 'match_id': 'dst_id'}, axis=1).to_feather('pred_colum_bert_large.ftr')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:06:41.211394Z","iopub.execute_input":"2022-07-07T06:06:41.21186Z","iopub.status.idle":"2022-07-07T06:06:41.826944Z","shell.execute_reply.started":"2022-07-07T06:06:41.211812Z","shell.execute_reply":"2022-07-07T06:06:41.82573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## mluke-large","metadata":{}},{"cell_type":"code","source":"%%python\nimport pickle\n\nimport numpy as np\nimport pandas as pd\nfrom munch import Munch\n\nfrom transformers import AutoTokenizer\n\n\ndef get_text(base, i, j, sep):\n    text = (\n        base.idx2country[i]\n        + sep\n        + base.idx2name[i]\n        + sep\n        + base.idx2categories[i]\n        + sep\n        + base.idx2address[i]\n        + sep\n        + base.idx2name[j]\n        + sep\n        + base.idx2categories[j]\n        + sep\n        + base.idx2address[j]\n    )\n    return text\n\n\ndef get_text_pandas(base, id_i, id_j, sep):\n    i = base.id2idx[id_i]\n    j = base.id2idx[id_j]\n    text = get_text(base, i, j, sep)\n    return text\n\n\ndef get_d_lat_pandas(base, id_i, id_j):\n    i = base.id2idx[id_i]\n    j = base.id2idx[id_j]\n    d_lat = abs(base.idx2radlat[i] - base.idx2radlat[j])\n    return d_lat\n\n\ndef get_d_long_pandas(base, id_i, id_j):\n    i = base.id2idx[id_i]\n    j = base.id2idx[id_j]\n    d_long = abs(base.idx2radlong[i] - base.idx2radlong[j])\n    return d_long\n\n\noof = pd.read_feather(\"selected_pair_df.ftr\")\n\ntmp = oof.rename({\"src_id\": \"dst_id\", \"dst_id\": \"src_id\", \"prob\": \"rev_prob\"}, axis=1)\noof = oof.merge(tmp, on=[\"src_id\", \"dst_id\"], how=\"outer\")\noof[\"prob\"] = oof[\"prob\"].fillna(oof[\"rev_prob\"])\noof = oof.drop([\"rev_prob\"], axis=1)\n\ndf = pd.read_feather(\"test.ftr\")\n\nbasearray = Munch()\nbasearray.id2idx = {id_str: i for i, id_str in enumerate(df[\"id\"].to_numpy())}\nbasearray.idx2name = df[\"name\"].fillna(\"\").str.lower().to_numpy()\nbasearray.idx2categories = df[\"categories\"].fillna(\"\").str.lower().to_numpy()\nbasearray.idx2country = df[\"country\"].fillna(\"\").str.lower().to_numpy()\nbasearray.idx2address = df[\"address\"].fillna(\"\").str.lower().to_numpy()\n\nsep = AutoTokenizer.from_pretrained(\"../input/4sq-exp413-yabea-mluke-large-all\").sep_token\n\noof[\"text\"] = np.vectorize(get_text_pandas)(basearray, oof[\"src_id\"], oof[\"dst_id\"], sep)\noof[\"len_text\"] = oof[\"text\"].map(len)\n\nbasearray.idx2radlat = np.deg2rad(df[\"latitude\"].to_numpy())\nbasearray.idx2radlong = np.deg2rad(df[\"longitude\"].to_numpy())\n\noof[\"d_lat\"] = np.vectorize(get_d_lat_pandas)(basearray, oof[\"src_id\"], oof[\"dst_id\"])\noof[\"d_long\"] = np.vectorize(get_d_long_pandas)(basearray, oof[\"src_id\"], oof[\"dst_id\"])\n\noof.to_feather(\"/tmp/yabea_bert_stacking_feat_selected.ftr\")","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:06:41.831201Z","iopub.execute_input":"2022-07-07T06:06:41.831551Z","iopub.status.idle":"2022-07-07T06:07:04.800583Z","shell.execute_reply.started":"2022-07-07T06:06:41.831517Z","shell.execute_reply":"2022-07-07T06:07:04.799246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\nimport pandas as pd\nimport numpy as np\nfrom tqdm.auto import tqdm\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom transformers import AutoTokenizer, AutoConfig, AutoModel\n\n\n# ==================================================\n# SETUP\n# ==================================================\n\n\nclass CFG:\n    max_len = 256\n    test_batch_size = 24\n    model_path = \"../input/4sq-exp413-yabea-mluke-large-all\"\n    model_weight_path = \"../input/4sq-exp413-yabea-mluke-large-all/all.pth\"\n    num_features = [\"prob\", \"d_lat\", \"d_long\"]\n\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n\n# ==================================================\n# FUNCTIONS\n# ==================================================\n\n\nclass TestBERTDataset(Dataset):\n    def __init__(self, df, tokenizer):\n        self.texts = df[\"text\"].to_numpy()\n        self.num_features = df[CFG.num_features].to_numpy()\n        self.tokenizer = tokenizer\n\n    def __len__(self):\n        return len(self.texts)\n\n    def __getitem__(self, index):\n        inputs = self.__prepare_input(self.texts[index], self.num_features[index])\n        return inputs\n\n    def __prepare_input(self, text, num_features):\n        inputs = self.tokenizer(\n            text,\n            add_special_tokens=True,\n            max_length=CFG.max_len,\n            padding=\"max_length\",\n            truncation=\"longest_first\",\n        )\n        for k, v in inputs.items():\n            inputs[k] = torch.tensor(v, dtype=torch.long)\n        inputs[\"num_features\"] = torch.tensor(num_features, dtype=torch.float)\n        return inputs\n\n\nclass CustomModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.config = AutoConfig.from_pretrained(CFG.model_path, output_hidden_states=True)\n        self.config.attention_probs_dropout_prob = 0.0\n        self.config.hidden_dropout_prob = 0.0\n        self.backbone = AutoModel.from_config(config=self.config)\n        self.linear1 = nn.Sequential(\n            nn.Linear(self.config.hidden_size + len(CFG.num_features), 1024),\n            nn.SELU(),\n            nn.Linear(1024, 1),\n        )\n\n    def forward(self, inputs):\n        outputs = self.backbone(inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"])[\"last_hidden_state\"]\n        outputs = outputs[:, 0, :]\n        outputs = torch.cat((outputs, inputs[\"num_features\"]), dim=1)\n        logits = self.linear1(outputs).flatten()\n        return logits\n\n    \ndef collatte(inputs):\n    mask_len = int(inputs[\"attention_mask\"].sum(axis=1).max())\n    outputs = {\n        \"input_ids\": inputs[\"input_ids\"][:, :mask_len],\n        \"attention_mask\": inputs[\"attention_mask\"][:, :mask_len],\n    }\n    for k, v in inputs.items():\n        if k not in [\"input_ids\", \"attention_mask\"]:\n            outputs[k] = v\n    return outputs\n\n\n# ==================================================\n# MAIN\n# ==================================================\n\n\noof = pd.read_feather(\"/tmp/yabea_bert_stacking_feat_selected.ftr\")\n\n# uniform length batching\noof = oof.sort_values(by=\"len_text\")\noof = oof.reset_index(drop=True)\n\ntokenizer = AutoTokenizer.from_pretrained(CFG.model_path)\n\ntest_dataset = TestBERTDataset(oof, tokenizer)\ntest_loader = DataLoader(\n    dataset=test_dataset, batch_size=CFG.test_batch_size, shuffle=False, pin_memory=False, drop_last=False\n)\n\nmodel = CustomModel().to(device)\nmodel.load_state_dict(torch.load(CFG.model_weight_path))\nmodel.eval()\n\npreds = []\nwith torch.inference_mode():\n    with tqdm(test_loader, total=len(test_loader)) as pbar:\n        for inputs in pbar:\n            inputs = collatte(inputs)\n            for k, v in inputs.items():\n                inputs[k] = v.to(device)\n            output = model(inputs)\n            output = output.sigmoid().detach().cpu().numpy()\n            preds.append(output)\n\npreds = np.concatenate(preds)\noof[\"pred\"] = preds\noof[[\"src_id\", \"dst_id\", \"pred\"]].to_feather(\"/tmp/stacking_yabea_bert_large.ftr\")","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:07:04.802268Z","iopub.execute_input":"2022-07-07T06:07:04.803142Z","iopub.status.idle":"2022-07-07T06:08:17.951895Z","shell.execute_reply.started":"2022-07-07T06:07:04.8031Z","shell.execute_reply":"2022-07-07T06:08:17.950468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\nimport numpy as np\nimport pandas as pd\n\ntest_f = pd.read_feather('/tmp/stacking_yabea_bert_large.ftr')\ntest_f.loc[:, ['src_id', 'dst_id']] = np.sort(test_f[['src_id', 'dst_id']].values, axis=1)\ntest_f = test_f.groupby(['src_id', 'dst_id'])['pred'].mean().reset_index()\ntest_f.to_feather('pred_yabea_bert_large.ftr')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Re-Ensemble","metadata":{}},{"cell_type":"code","source":"#%%python\n\nimport gc\nimport pandas as pd\nimport numpy as np\n\ndf_ensemble = pd.read_feather('df_ensemble.ftr').rename(columns={'pred': 'pred_ensemble'})\ndf_rob_large = pd.read_feather('pred_colum_bert_large.ftr').rename(columns={'pred': 'pred_rob_large'})\ndf_mlu_large = pd.read_feather('pred_yabea_bert_large.ftr').rename(columns={'pred': 'pred_mlu_large'})\n\n# merge\ndf_ensemble = df_ensemble.merge(df_rob_large, on=['src_id', 'dst_id'], how='outer')\ndf_ensemble = df_ensemble.merge(df_mlu_large, on=['src_id', 'dst_id'], how='outer')\n\n# blend\nweights = [0.6, 0.2, 0.2]\npred_arr = df_ensemble[['pred_ensemble', 'pred_rob_large', 'pred_mlu_large']].to_numpy()\nweight_arr = np.zeros_like(pred_arr)\nfor i, weight in enumerate(weights):\n    weight_arr[:, i] = weight\nweight_arr[np.isnan(pred_arr)] = 0\n\nweight_arr /= weight_arr.sum(axis=1).reshape(-1, 1)\ndf_ensemble['pred'] = np.nansum(pred_arr * weight_arr, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:08:17.954266Z","iopub.execute_input":"2022-07-07T06:08:17.95475Z","iopub.status.idle":"2022-07-07T06:08:17.991784Z","shell.execute_reply.started":"2022-07-07T06:08:17.954693Z","shell.execute_reply":"2022-07-07T06:08:17.990891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_feather(f'test.ftr')\n\nsub_dict = dict()\nfor id_str in test['id'].to_numpy():\n    sub_dict[id_str] = set([id_str])\n\nid2match = df_ensemble[df_ensemble['pred']>0.5].groupby('src_id')['dst_id'].apply(list).to_dict()\nfor id_str in df_ensemble.loc[df_ensemble['pred']>0.5, 'src_id'].to_numpy():\n    sub_dict[id_str] |= set(id2match[id_str])\n    for match_id_str in id2match[id_str]:\n        sub_dict[match_id_str] |= set([id_str])\n\nsub = pd.DataFrame({\n    'id': sub_dict.keys(),\n    'matches': sub_dict.values()\n})\nsub['matches'] = sub['matches'].map(lambda x: ' '.join(x))\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:08:17.99339Z","iopub.execute_input":"2022-07-07T06:08:17.993847Z","iopub.status.idle":"2022-07-07T06:08:18.033258Z","shell.execute_reply.started":"2022-07-07T06:08:17.993802Z","shell.execute_reply":"2022-07-07T06:08:18.032284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head -n 6 submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-07-07T06:08:18.034925Z","iopub.execute_input":"2022-07-07T06:08:18.035343Z","iopub.status.idle":"2022-07-07T06:08:18.793108Z","shell.execute_reply.started":"2022-07-07T06:08:18.035301Z","shell.execute_reply":"2022-07-07T06:08:18.791968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}