{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%capture\n!pip install ../input/faiss-163/faiss_gpu-1.6.3-cp37-cp37m-manylinux2010_x86_64.whl\n!pip install ../input/shopee-libs/editdistance-0.5.3-cp37-cp37m-manylinux1_x86_64.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-10T13:39:11.812673Z","iopub.execute_input":"2022-08-10T13:39:11.813198Z","iopub.status.idle":"2022-08-10T13:40:06.426036Z","shell.execute_reply.started":"2022-08-10T13:39:11.813087Z","shell.execute_reply":"2022-08-10T13:40:06.425002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time    # this is an inbuilt library in Python\nstart_time = time.time()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T13:40:06.428593Z","iopub.execute_input":"2022-08-10T13:40:06.429154Z","iopub.status.idle":"2022-08-10T13:40:06.434889Z","shell.execute_reply.started":"2022-08-10T13:40:06.429112Z","shell.execute_reply":"2022-08-10T13:40:06.434135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile lyk_config.py\n\nk = 50\nconf_th = 0.7\n\nimport pandas as pd\nfrom pathlib import Path\n\nDEBUG = len(pd.read_csv('../input/testdata/test.csv')) == 3\n\ndef load_data():\n    if DEBUG:\n        nrows = 1000\n        df = pd.read_csv('../input/shopee-product-matching/train.csv', nrows=nrows, usecols=['posting_id', 'image', 'title'])\n#         nrows = None\n#         df = pd.read_csv('../input/shopee-product-matching/train.csv', nrows=nrows, usecols=['posting_id', 'image', 'title']).append(\n#              pd.read_csv('../input/shopee-product-matching/train.csv', nrows=nrows, usecols=['posting_id', 'image', 'title'])).reset_index(drop=True)\n        img_dir = Path('../input/shopee-product-matching/train_images/')\n    else:\n        nrows = 1000\n        df = pd.read_csv('../input/testdata6/test6.csv', nrows=nrows, usecols=['posting_id', 'image', 'title'])\n        img_dir = Path('../input/testdata6/test_images/test_images/')\n    return df, img_dir","metadata":{"execution":{"iopub.status.busy":"2022-08-10T13:40:06.438113Z","iopub.execute_input":"2022-08-10T13:40:06.438395Z","iopub.status.idle":"2022-08-10T13:40:06.447719Z","shell.execute_reply.started":"2022-08-10T13:40:06.438370Z","shell.execute_reply":"2022-08-10T13:40:06.446973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image similarity, Multi-modal similarity","metadata":{}},{"cell_type":"code","source":"%%python\nfrom lyk_config import k, conf_th, DEBUG, load_data\n\nimport sys\nsys.path.append('../input/timm045/')\nimport timm\nimport torchvision.transforms\nfrom torchvision import datasets, transforms\nimport PIL\n\nfrom itertools import zip_longest\nimport json\nimport math\nimport gc\nimport os\nfrom pathlib import Path\n\nimport faiss\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision.io import read_image\nfrom torchvision.transforms import Resize, RandomHorizontalFlip, ColorJitter, Normalize, Compose, RandomResizedCrop, CenterCrop, ToTensor\n\nfrom tqdm import tqdm\nfrom PIL import Image\nimport joblib\nfrom scipy.sparse import hstack, vstack, csc_matrix, csr_matrix\nimport editdistance\nimport networkx as nx\nfrom transformers import BertConfig, BertModel, BertTokenizerFast\n\nNUM_CLASSES = 11014\nNUM_WORKERS = 2\nSEED = 0\n\n\ndef gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n\n    \nclass ShopeeNet(nn.Module):\n\n    def __init__(self,\n                 backbone,\n                 num_classes,\n                 fc_dim=512,\n                 s=30, margin=0.5, p=3):\n        super(ShopeeNet, self).__init__()\n\n        self.backbone = backbone\n        self.backbone.reset_classifier(num_classes=0)  # remove classifier\n\n        self.fc = nn.Linear(self.backbone.num_features, fc_dim)\n        self.bn = nn.BatchNorm1d(fc_dim)\n        self._init_params()\n        self.p = p\n\n    def _init_params(self):\n        nn.init.xavier_normal_(self.fc.weight)\n        nn.init.constant_(self.fc.bias, 0)\n        nn.init.constant_(self.bn.weight, 1)\n        nn.init.constant_(self.bn.bias, 0)\n\n    def extract_feat(self, x):\n        batch_size = x.shape[0]\n        x = self.backbone.forward_features(x)\n        if isinstance(x, tuple):\n            x = (x[0] + x[1]) / 2\n            x = self.bn(x)\n        else:\n            x = gem(x, p=self.p).view(batch_size, -1)\n            x = self.fc(x)\n            x = self.bn(x)\n        return x\n\n    def forward(self, x, label):\n        feat = self.extract_feat(x)\n        x = self.loss_module(feat, label)\n        return x, feat\n\n\nclass ShopeeDataset(Dataset):\n\n    def __init__(self, df, img_dir, transform=None):\n        self.df = df\n        self.img_dir = img_dir\n        self.transform = transform\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        img_loc = str(self.img_dir / row['image'])\n        #image = PIL.Image.open(img_loc).convert(\"RGB\")\n        img = read_image(img_loc)\n        _, h, w = img.shape\n        st_size = (self.img_dir / row['image']).stat().st_size\n        if self.transform is not None:\n            img = self.transform(img)\n\n        return img, row['title'], h, w, st_size\n\n    def __len__(self):\n        return len(self.df)\n\n\nclass MultiModalNet(nn.Module):\n\n    def __init__(self,\n                 backbone,\n                 bert_model,\n                 num_classes,\n                 tokenizer,\n                 max_len=32,\n                 fc_dim=512,\n                 s=30, margin=0.5, p=3, loss='ArcMarginProduct'):\n        super().__init__()\n\n        self.backbone = backbone\n        self.backbone.reset_classifier(num_classes=0)  # remove classifier\n\n        self.bert_model = bert_model\n        self.tokenizer = tokenizer\n        self.max_len = max_len\n        self.fc = nn.Linear(self.bert_model.config.hidden_size + self.backbone.num_features, fc_dim)\n        self.bn = nn.BatchNorm1d(fc_dim)\n        self._init_params()\n        self.p = p\n\n    def _init_params(self):\n        nn.init.xavier_normal_(self.fc.weight)\n        nn.init.constant_(self.fc.bias, 0)\n        nn.init.constant_(self.bn.weight, 1)\n        nn.init.constant_(self.bn.bias, 0)\n\n    def extract_feat(self, img, title):\n        batch_size = img.shape[0]\n        img = self.backbone.forward_features(img)\n        img = gem(img, p=self.p).view(batch_size, -1)\n\n        tokenizer_output = self.tokenizer(title, truncation=True, padding=True, max_length=self.max_len)\n        input_ids = torch.LongTensor(tokenizer_output['input_ids']).to('cuda')\n        token_type_ids = torch.LongTensor(tokenizer_output['token_type_ids']).to('cuda')\n        attention_mask = torch.LongTensor(tokenizer_output['attention_mask']).to('cuda')\n        title = self.bert_model(input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)\n        # x = x.last_hidden_state.sum(dim=1) / attention_mask.sum(dim=1, keepdims=True)\n        title = title.last_hidden_state.mean(dim=1)\n\n        x = torch.cat([img, title], dim=1)\n        x = self.fc(x)\n        x = self.bn(x)\n        return x\n\n\n####\n\ndf, img_dir = load_data()\n    \n###\n\ncheckpoint1 = torch.load('../input/shopee/v45.pth')\ncheckpoint2 = torch.load('../input/shopee/v34.pth')\ncheckpoint3 = torch.load('../input/shopee/v79.pth')\nparams1 = checkpoint1['params']\nparams2 = checkpoint2['params']\nparams3 = checkpoint3['params']\n\ntransform = Compose([\n    Resize(size=params1['test_size'] + 32, interpolation=Image.BICUBIC),\n    CenterCrop((params1['test_size'], params1['test_size'])),\n    Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\ndataset = ShopeeDataset(df=df, img_dir=img_dir, transform=None)\ndata_loader = DataLoader(dataset, batch_size=8, shuffle=False,\n                         drop_last=False, pin_memory=True, num_workers=NUM_WORKERS, collate_fn=lambda x: x)\n\nbackbone = timm.create_model(model_name=params1['backbone'], pretrained=False)\nmodel1 = ShopeeNet(backbone, num_classes=0, fc_dim=params1['fc_dim'])\nmodel1 = model1.to('cuda')\nmodel1.load_state_dict(checkpoint1['model'], strict=False)\nmodel1.train(False)\nmodel1.p = params1['p_eval']\n\nbackbone = timm.create_model(model_name=params2['backbone'], pretrained=False)\nmodel2 = ShopeeNet(backbone, num_classes=0, fc_dim=params2['fc_dim'])\nmodel2 = model2.to('cuda')\nmodel2.load_state_dict(checkpoint2['model'], strict=False)\nmodel2.train(False)\nmodel2.p = params2['p_eval']\n\nbackbone = timm.create_model(model_name=params3['backbone'], pretrained=False)\ntokenizer = BertTokenizerFast(vocab_file='../input/bert-indo/vocab.txt')\nbert_config = BertConfig.from_json_file('../input/bert-indo/config.json')\nbert_model = BertModel(bert_config)\nmodel3 = MultiModalNet(backbone, bert_model, num_classes=0, tokenizer=tokenizer, max_len=params3['max_len'],\n                       fc_dim=params3['fc_dim'], s=params3['s'], margin=params3['margin'], loss=params3['loss'])\nmodel3 = model3.to('cuda')\nmodel3.load_state_dict(checkpoint3['model'], strict=False)\nmodel3.train(False)\nmodel3.p = params3['p_eval']\n\nimg_feats1 = []\nimg_feats2 = []\nmm_feats = []\nimg_hs = []\nimg_ws = []\nst_sizes = []\nfor batch in tqdm(data_loader, total=len(data_loader), miniters=None, ncols=55):\n    img, title, h, w, st_size = list(zip(*batch))\n    img = torch.cat([transform(x.to('cuda').float() / 255)[None] for x in img], axis=0)\n    title = list(title)\n    with torch.no_grad():\n        feats_minibatch1 = model1.extract_feat(img)\n        img_feats1.append(feats_minibatch1.cpu().numpy())\n        feats_minibatch2 = model2.extract_feat(img)\n        img_feats2.append(feats_minibatch2.cpu().numpy())\n        feats_minibatch3 = model3.extract_feat(img, title)\n        mm_feats.append(feats_minibatch3.cpu().numpy())\n    img_hs.extend(list(h))\n    img_ws.extend(list(w))\n    st_sizes.extend(list(st_size))\n\nimg_feats1 = np.concatenate(img_feats1)\nimg_feats1 /= np.linalg.norm(img_feats1, 2, axis=1, keepdims=True)\nimg_feats2 = np.concatenate(img_feats2)\nimg_feats2 /= np.linalg.norm(img_feats2, 2, axis=1, keepdims=True)\nmm_feats = np.concatenate(mm_feats)\nmm_feats /= np.linalg.norm(mm_feats, 2, axis=1, keepdims=True)\n\nnp.save('/tmp/img_feats1', img_feats1)\nnp.save('/tmp/img_feats2', img_feats2)\n\nimg_feats = np.concatenate([\n    img_feats1 * 1.0,\n    img_feats2 * 1.0,\n], axis=1)\nimg_feats /= np.linalg.norm(img_feats, 2, axis=1, keepdims=True)\n###\n\nnp.save('/tmp/img_feats', img_feats)\n\nres = faiss.StandardGpuResources()\nindex_img = faiss.IndexFlatIP(params1['fc_dim'] + params2['fc_dim'])\nindex_img = faiss.index_cpu_to_gpu(res, 0, index_img)\nindex_img.add(img_feats)\nsimilarities_img, indexes_img = index_img.search(img_feats, k)\n\n\njoblib.dump([similarities_img, indexes_img], '/tmp/lyk_img_data.pkl')\njoblib.dump([st_sizes, img_hs, img_ws], '/tmp/lyk_img_meta_data.pkl')\n\nres = faiss.StandardGpuResources()\nindex_mm = faiss.IndexFlatIP(params3['fc_dim'])\nindex_mm = faiss.index_cpu_to_gpu(res, 0, index_mm)\nindex_mm.add(mm_feats)\nsimilarities_mm, indexes_mm = index_mm.search(mm_feats, k)\n\njoblib.dump([similarities_mm, indexes_mm], '/tmp/lyk_mm_data.pkl')\n\n### for TKM\nnp.save('/tmp/mm_feats', mm_feats)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T13:40:06.451169Z","iopub.execute_input":"2022-08-10T13:40:06.451411Z","iopub.status.idle":"2022-08-10T13:42:05.962829Z","shell.execute_reply.started":"2022-08-10T13:40:06.451388Z","shell.execute_reply":"2022-08-10T13:42:05.962028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### image QE","metadata":{}},{"cell_type":"code","source":"%%python\nimport gc\nimport numpy as np\nimport faiss\n\ndef query_expansion(feats, sims, topk_idx, alpha=0.5, k=2):\n    weights = np.expand_dims(sims[:, :k] ** alpha, axis=-1).astype(np.float32)\n    feats = (feats[topk_idx[:, :k]] * weights).sum(axis=1)\n    return feats\n\nimg_feats = np.load('/tmp/img_feats.npy')\n\nres = faiss.StandardGpuResources()\nindex_img = faiss.IndexFlatIP(img_feats.shape[1])\nindex_img = faiss.index_cpu_to_gpu(res, 0, index_img)\nindex_img.add(img_feats)\nimg_D, img_I = index_img.search(img_feats, 60)\n\nnp.save('/tmp/img_D', img_D)\nnp.save('/tmp/img_I', img_I)\n\nimg_feats_qe = query_expansion(img_feats, img_D, img_I)\nimg_feats_qe /= np.linalg.norm(img_feats_qe, 2, axis=1, keepdims=True)\n\nimg_feats = np.hstack([img_feats, img_feats_qe])\nimg_feats /= np.linalg.norm(img_feats, axis=1).reshape((-1, 1))\n\nindex = faiss.IndexFlatIP(img_feats.shape[1])\nres = faiss.StandardGpuResources()\nindex = faiss.index_cpu_to_gpu(res, 0, index)\n\nindex.add(img_feats)\nimg_D, img_I = index.search(img_feats, 60)\n\nnp.save('/tmp/img_D_qe', img_D)\nnp.save('/tmp/img_I_qe', img_I)\n\nprint('end')","metadata":{"execution":{"iopub.status.busy":"2022-08-10T13:42:05.966283Z","iopub.execute_input":"2022-08-10T13:42:05.966557Z","iopub.status.idle":"2022-08-10T13:42:07.076233Z","shell.execute_reply.started":"2022-08-10T13:42:05.966527Z","shell.execute_reply":"2022-08-10T13:42:07.075196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Multi-modal QE","metadata":{}},{"cell_type":"code","source":"%%python\nimport gc\nimport numpy as np\nimport faiss\n\ndef query_expansion(feats, sims, topk_idx, alpha=0.5, k=2):\n    weights = np.expand_dims(sims[:, :k] ** alpha, axis=-1).astype(np.float32)\n    feats = (feats[topk_idx[:, :k]] * weights).sum(axis=1)\n    return feats\n\nmm_feats = np.load('/tmp/mm_feats.npy')\n\nres = faiss.StandardGpuResources()\nindex_mm = faiss.IndexFlatIP(mm_feats.shape[1])\nindex_mm = faiss.index_cpu_to_gpu(res, 0, index_mm)\nindex_mm.add(mm_feats)\nmm_D, mm_I = index_mm.search(mm_feats, 60)\n\nnp.save('/tmp/mut_D', mm_D)\nnp.save('/tmp/mut_I', mm_I)\n\nmm_feats_qe = query_expansion(mm_feats, mm_D, mm_I)\nmm_feats_qe /= np.linalg.norm(mm_feats_qe, 2, axis=1, keepdims=True)\n\nmm_feats = np.hstack([mm_feats, mm_feats_qe])\nmm_feats /= np.linalg.norm(mm_feats, axis=1).reshape((-1, 1))\n\nindex = faiss.IndexFlatIP(mm_feats.shape[1])\nres = faiss.StandardGpuResources()\nindex = faiss.index_cpu_to_gpu(res, 0, index)\n\nindex.add(mm_feats)\nmm_D, mm_I = index.search(mm_feats, 60)\n\nnp.save('/tmp/mut_D_qe', mm_D)\nnp.save('/tmp/mut_I_qe', mm_I)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T13:42:07.078167Z","iopub.execute_input":"2022-08-10T13:42:07.078447Z","iopub.status.idle":"2022-08-10T13:42:08.128059Z","shell.execute_reply.started":"2022-08-10T13:42:07.078417Z","shell.execute_reply":"2022-08-10T13:42:08.127035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# BERT similarity","metadata":{}},{"cell_type":"code","source":"%%python\nfrom lyk_config import k, conf_th, DEBUG, load_data\nimport sys\nsys.path.append('../input/timm045/')\nimport timm\n\nfrom itertools import zip_longest\nimport json\nimport math\nimport gc\nimport os\nfrom pathlib import Path\n\nimport faiss\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision.io import read_image\nfrom torchvision.transforms import Resize, RandomHorizontalFlip, ColorJitter, Normalize, Compose, RandomResizedCrop, CenterCrop, ToTensor\n\nfrom tqdm import tqdm\nfrom PIL import Image\nimport joblib\nfrom scipy.sparse import hstack, vstack, csc_matrix, csr_matrix\nimport editdistance\nimport networkx as nx\n\nfrom transformers import BertConfig, BertModel, BertTokenizerFast\n\nNUM_CLASSES = 11014\nNUM_WORKERS = 2\nSEED = 0\n\n\ndef gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n\n\nclass BertNet(nn.Module):\n\n    def __init__(self,\n                 bert_model,\n                 num_classes,\n                 tokenizer,\n                 max_len=32,\n                 fc_dim=512,\n                 simple_mean=True,\n                 s=30, margin=0.5, p=3, loss='ArcMarginProduct'):\n        super().__init__()\n\n        self.bert_model = bert_model\n        self.tokenizer = tokenizer\n        self.max_len = max_len\n        self.fc = nn.Linear(self.bert_model.config.hidden_size, fc_dim)\n        self.bn = nn.BatchNorm1d(fc_dim)\n        self._init_params()\n        self.p = p\n        self.simple_mean = simple_mean\n\n    def _init_params(self):\n        nn.init.xavier_normal_(self.fc.weight)\n        nn.init.constant_(self.fc.bias, 0)\n        nn.init.constant_(self.bn.weight, 1)\n        nn.init.constant_(self.bn.bias, 0)\n\n    def extract_feat(self, x):\n        tokenizer_output = self.tokenizer(x, truncation=True, padding=True, max_length=self.max_len)\n        if 'token_type_ids' in tokenizer_output:\n            input_ids = torch.LongTensor(tokenizer_output['input_ids']).to('cuda')\n            token_type_ids = torch.LongTensor(tokenizer_output['token_type_ids']).to('cuda')\n            attention_mask = torch.LongTensor(tokenizer_output['attention_mask']).to('cuda')\n            x = self.bert_model(input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)\n        else:\n            input_ids = torch.LongTensor(tokenizer_output['input_ids']).to('cuda')\n            attention_mask = torch.LongTensor(tokenizer_output['attention_mask']).to('cuda')\n            x = self.bert_model(input_ids=input_ids, attention_mask=attention_mask)\n        if self.simple_mean:\n            x = x.last_hidden_state.mean(dim=1)\n        else:\n            x = torch.sum(x.last_hidden_state * attention_mask.unsqueeze(-1), dim=1) / attention_mask.sum(dim=1, keepdims=True)\n        x = self.fc(x)\n        x = self.bn(x)\n        return x\n\n\nclass BertDataset(Dataset):\n\n    def __init__(self, df):\n        self.df = df\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n\n        if 'y' in row.keys():\n            target = torch.tensor(row['y'], dtype=torch.long)\n            return row['title'], target\n        else:\n            return row['title']\n\n    def __len__(self):\n        return len(self.df)\n\ndf, img_dir = load_data()\n\ncheckpoint = torch.load('../input/shopee/v75.pth')\ncheckpoint2 = torch.load('../input/shopee/v102.pth')\ncheckpoint3 = torch.load('../input/shopee/v103.pth')\n\nparams_bert = checkpoint['params']\nparams_bert2 = checkpoint2['params']\nparams_bert3 = checkpoint3['params']\n\ndatasets = {\n    'valid': BertDataset(df=df)\n}\ndata_loaders = {\n    'valid': DataLoader(datasets['valid'], batch_size=params_bert['batch_size'] * 2, shuffle=False,\n                        drop_last=False, pin_memory=True, num_workers=NUM_WORKERS)\n}\n\ntokenizer = BertTokenizerFast(vocab_file='../input/bert-indo/vocab.txt')\nbert_config = BertConfig.from_json_file('../input/bert-indo/config.json')\nbert_model = BertModel(bert_config)\nmodel = BertNet(bert_model, num_classes=0, tokenizer=tokenizer, max_len=params_bert['max_len'], simple_mean=True,\n                fc_dim=params_bert['fc_dim'], s=params_bert['s'], margin=params_bert['margin'], loss=params_bert['loss'])\n\nmodel = model.to('cuda')\nmodel.load_state_dict(checkpoint['model'], strict=False)\nmodel.train(False)\n\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig\n\nmodel_name = params_bert2['model_name']\ntokenizer = AutoTokenizer.from_pretrained('../input/bertmultilingual/')\nbert_config = AutoConfig.from_pretrained('../input/bertmultilingual/')\nbert_model = AutoModel.from_config(bert_config)\nmodel2 = BertNet(bert_model, num_classes=0, tokenizer=tokenizer, max_len=params_bert['max_len'], simple_mean=False,\n                 fc_dim=params_bert['fc_dim'], s=params_bert['s'], margin=params_bert['margin'], loss=params_bert['loss'])\nmodel2 = model2.to('cuda')\nmodel2.load_state_dict(checkpoint2['model'], strict=False)\nmodel2.train(False)\n\n#########\n\nmodel_name = params_bert3['model_name']\ntokenizer = AutoTokenizer.from_pretrained('../input/bertxlm/')\nbert_config = AutoConfig.from_pretrained('../input/bertxlm/')\nbert_model = AutoModel.from_config(bert_config)\nmodel3 = BertNet(bert_model, num_classes=0, tokenizer=tokenizer, max_len=params_bert3['max_len'], simple_mean=False,\n                 fc_dim=params_bert3['fc_dim'], s=params_bert3['s'], margin=params_bert3['margin'], loss=params_bert3['loss'])\nmodel3 = model3.to('cuda')\nmodel3.load_state_dict(checkpoint3['model'], strict=False)\nmodel3.train(False)\n\nbert_feats1 = []\nbert_feats2 = []\nbert_feats3 = []\nfor i, title in tqdm(enumerate(data_loaders['valid']),\n                     total=len(data_loaders['valid']), miniters=None, ncols=55):\n    with torch.no_grad():\n        bert_feats_minibatch = model.extract_feat(title)\n        bert_feats1.append(bert_feats_minibatch.cpu().numpy())\n        bert_feats_minibatch = model2.extract_feat(title)\n        bert_feats2.append(bert_feats_minibatch.cpu().numpy())\n        bert_feats_minibatch = model3.extract_feat(title)\n        bert_feats3.append(bert_feats_minibatch.cpu().numpy())\n\nbert_feats1 = np.concatenate(bert_feats1)\nbert_feats1 /= np.linalg.norm(bert_feats1, 2, axis=1, keepdims=True)\nbert_feats2 = np.concatenate(bert_feats2)\nbert_feats2 /= np.linalg.norm(bert_feats2, 2, axis=1, keepdims=True)\nbert_feats3 = np.concatenate(bert_feats3)\nbert_feats3 /= np.linalg.norm(bert_feats3, 2, axis=1, keepdims=True)\n\nbert_feats = np.concatenate([bert_feats1, bert_feats2], axis=1)\nbert_feats /= np.linalg.norm(bert_feats, 2, axis=1, keepdims=True)\n\nres = faiss.StandardGpuResources()\nindex_bert = faiss.IndexFlatIP(params_bert['fc_dim'])\nindex_bert = faiss.index_cpu_to_gpu(res, 0, index_bert)\nindex_bert.add(bert_feats1)\nsimilarities_bert, indexes_bert = index_bert.search(bert_feats1, k)\n\nnp.save('/tmp/bert_feats1', bert_feats1)\nnp.save('/tmp/bert_feats2', bert_feats2)\nnp.save('/tmp/bert_feats3', bert_feats3)\n\nbert_feats = np.concatenate([bert_feats1, bert_feats2, bert_feats3], axis=1)\nbert_feats /= np.linalg.norm(bert_feats, 2, axis=1, keepdims=True)\n\nnp.save('/tmp/bert_feats', bert_feats)\n\njoblib.dump([similarities_bert, indexes_bert], '/tmp/lyk_bert_data.pkl')","metadata":{"execution":{"iopub.status.busy":"2022-08-10T13:42:08.131174Z","iopub.execute_input":"2022-08-10T13:42:08.131589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Bert QE","metadata":{}},{"cell_type":"code","source":"%%python\nimport gc\nimport numpy as np\nimport faiss\n\ndef query_expansion(feats, sims, topk_idx, alpha=0.5, k=2):\n    weights = np.expand_dims(sims[:, :k] ** alpha, axis=-1).astype(np.float32)\n    feats = (feats[topk_idx[:, :k]] * weights).sum(axis=1)\n    return feats\n\nbrt_feats = np.load('/tmp/bert_feats.npy')\n\nres = faiss.StandardGpuResources()\nindex_brt = faiss.IndexFlatIP(brt_feats.shape[1])\nindex_brt = faiss.index_cpu_to_gpu(res, 0, index_brt)\nindex_brt.add(brt_feats)\nbrt_D, brt_I = index_brt.search(brt_feats, 60)\n\nnp.save('/tmp/brt_D', brt_D)\nnp.save('/tmp/brt_I', brt_I)\n\ndel index_brt\ngc.collect()\n\nbrt_feats_qe = query_expansion(brt_feats, brt_D, brt_I)\nbrt_feats_qe /= np.linalg.norm(brt_feats_qe, 2, axis=1, keepdims=True)\n\nbrt_feats = np.hstack([brt_feats, brt_feats_qe])\nbrt_feats /= np.linalg.norm(brt_feats, axis=1).reshape((-1, 1))\n\nindex = faiss.IndexFlatIP(brt_feats.shape[1])\nres = faiss.StandardGpuResources()\nindex = faiss.index_cpu_to_gpu(res, 0, index)\n\nindex.add(brt_feats)\nbrt_D, brt_I = index.search(brt_feats, 60)\n\nnp.save('/tmp/brt_D_qe', brt_D)\nnp.save('/tmp/brt_I_qe', brt_I)\nprint('end')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image & BERT similarity","metadata":{}},{"cell_type":"code","source":"%%python\nimport gc\nimport numpy as np\nimport faiss\n\ndef query_expansion(feats, sims, topk_idx, alpha=0.5, k=2):\n    weights = np.expand_dims(sims[:, :k] ** alpha, axis=-1).astype(np.float32)\n    feats = (feats[topk_idx[:, :k]] * weights).sum(axis=1)\n    return feats\n\n\nfeats_bert = np.load('/tmp/bert_feats.npy')\nfeats_img = np.load('/tmp/img_feats.npy')\n\nbth_feats = np.hstack([feats_bert, feats_img])\nbth_feats /= np.linalg.norm(bth_feats, 2, axis=1, keepdims=True)\n\nprint(bth_feats.shape)\n\nres = faiss.StandardGpuResources()\nindex = faiss.IndexFlatIP(bth_feats.shape[1])\nindex = faiss.index_cpu_to_gpu(res, 0, index)\nindex.add(bth_feats)\n\nbth_D, bth_I = index.search(bth_feats, 60)\nnp.save('/tmp/bth_D', bth_D)\nnp.save('/tmp/bth_I', bth_I)\n\ndel index\ngc.collect()\n\nbth_feats_qe = query_expansion(bth_feats, bth_D, bth_I)\nbth_feats_qe /= np.linalg.norm(bth_feats_qe, 2, axis=1, keepdims=True)\n\nbth_feats = np.hstack([bth_feats, bth_feats_qe])\nbth_feats /= np.linalg.norm(bth_feats, axis=1).reshape((-1, 1))\n\nindex = faiss.IndexFlatIP(bth_feats.shape[1])\nres = faiss.StandardGpuResources()\nindex = faiss.index_cpu_to_gpu(res, 0, index)\n\nindex.add(bth_feats)\nbth_D, bth_I = index.search(bth_feats, 60)\n\nnp.save('/tmp/bth_D_qe', bth_D)\nnp.save('/tmp/bth_I_qe', bth_I)\nprint('end')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# lyakaap Side (GCN)","metadata":{}},{"cell_type":"code","source":"%%python\nfrom lyk_config import k, conf_th, DEBUG, load_data\nimport sys\nsys.path.append('../input/timm045/')\nimport timm\n\nfrom itertools import zip_longest\nimport json\nimport math\nimport gc\nimport os\nfrom pathlib import Path\n\nimport faiss\nimport numpy as np\nimport cupy as cp\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom tqdm import tqdm\nfrom PIL import Image\nimport joblib\nimport lightgbm as lgb\nfrom scipy.sparse import hstack, vstack, csc_matrix, csr_matrix\nimport editdistance\nimport networkx as nx\n\nimport string\nimport nltk\nfrom nltk.tokenize.treebank import TreebankWordTokenizer\nfrom nltk.tokenize import TweetTokenizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nNUM_CLASSES = 11014\nNUM_WORKERS = 2\nSEED = 0\n\nclass GraphDataset(Dataset):\n\n    def __init__(self, feats=None, labels=None, weights=None, pair_tuples=None, k=50, top_neighbors=None):\n        self.feats = feats\n        self.labels = labels\n        self.weights = weights\n        self.pair_tuples = pair_tuples\n        self.k = k\n        self.top_neighbors = top_neighbors\n\n    def __getitem__(self, index):\n        i, j = self.pair_tuples[index]\n        feat = torch.FloatTensor(self.feats[i][j])\n\n        padding_i = [[0] * feat.shape[0]] * (self.k - len(self.top_neighbors[i]))\n        neighbor_feats_i = torch.FloatTensor([\n            self.feats[i][neighbor]\n            for neighbor in self.top_neighbors[i]\n        ] + padding_i)\n        padding_j = [[0] * feat.shape[0]] * (self.k - len(self.top_neighbors[j]))\n        neighbor_feats_j = torch.FloatTensor([\n            self.feats[j][neighbor]\n            for neighbor in self.top_neighbors[j]\n        ] + padding_j)\n        neighbor_feats = torch.cat([feat.unsqueeze(0), neighbor_feats_i, neighbor_feats_j], dim=0)\n\n        outputs = (feat, neighbor_feats)\n        if self.labels is not None:\n            outputs += (self.labels[i] == self.labels[j],)\n        if self.weights is not None:\n            outputs += (self.weights[i],)\n\n        return outputs\n\n    def __len__(self):\n        return len(self.pair_tuples)\n\n\nclass GraphAttentionLayer(nn.Module):\n\n    def __init__(self, in_features, out_features, dropout=0.6, alpha=0.2, concat=True):\n        super().__init__()\n        self.dropout = dropout\n        self.in_features = in_features\n        self.out_features = out_features\n        self.alpha = alpha\n        self.concat = concat\n\n        self.W = nn.Parameter(torch.empty(size=(in_features, out_features)))\n        nn.init.xavier_uniform_(self.W.data, gain=1.414)\n        self.a = nn.Parameter(torch.empty(size=(2 * out_features, 1)))\n        nn.init.xavier_uniform_(self.a.data, gain=1.414)\n\n        self.leakyrelu = nn.LeakyReLU(self.alpha)\n\n    def forward(self, h):\n        Wh = h @ self.W  # h.shape: (B, N, in_features), Wh.shape: (B, N, out_features)\n        a_input = self._prepare_attentional_mechanism_input(Wh)\n        e = self.leakyrelu(torch.matmul(a_input, self.a).squeeze(3))\n\n        attention = F.softmax(e, dim=1)\n        attention = F.dropout(attention, self.dropout, training=self.training)\n        h_prime = torch.bmm(attention, Wh)\n\n        if self.concat:\n            return F.elu(h_prime)\n        else:\n            return h_prime\n\n    def _prepare_attentional_mechanism_input(self, Wh):\n        B, N, D = Wh.shape\n\n        Wh_repeated_in_chunks = Wh.repeat_interleave(N, dim=1)\n        Wh_repeated_alternating = Wh.repeat(1, N, 1)\n\n        all_combinations_matrix = torch.cat([Wh_repeated_in_chunks, Wh_repeated_alternating], dim=2)\n        return all_combinations_matrix.view(-1, N, N, 2 * D)\n\n    def __repr__(self):\n        return self.__class__.__name__ + ' (' + str(self.in_features) + ' -> ' + str(self.out_features) + ')'\n\n\nclass GATPairClassifier(nn.Module):\n    def __init__(self, nfeat, nhid=8, nclass=1, dropout=0.6, alpha=0.2, nheads=8, pooling='first'):\n        super().__init__()\n        self.dropout = dropout\n        self.pooling = pooling\n\n        self.attentions = [GraphAttentionLayer(nfeat, nhid, dropout=dropout, alpha=alpha, concat=True) for _ in range(nheads)]\n        for i, attention in enumerate(self.attentions):\n            self.add_module('attention_{}'.format(i), attention)\n\n        self.out_att = GraphAttentionLayer(nhid * nheads, nhid, dropout=dropout, alpha=alpha, concat=False)\n\n        self.classifier = nn.Sequential(\n            nn.Linear(nfeat + nhid, nhid),\n            nn.PReLU(),\n            nn.BatchNorm1d(nhid),\n            nn.Linear(nhid, nclass),\n        )\n\n    def forward_gat(self, x):\n        x = F.dropout(x, self.dropout, training=self.training)\n        x = torch.cat([att(x) for att in self.attentions], dim=2)\n        x = F.dropout(x, self.dropout, training=self.training)\n        x = F.elu(self.out_att(x))\n        if self.pooling == 'first':\n            return x[:, 0]\n        elif self.pooling == 'mean':\n            return x.mean(dim=1)\n\n    def forward(self, feats, neighbor_feats):\n        gat_feats = self.forward_gat(neighbor_feats)\n        cat_feats = torch.cat([feats, gat_feats], dim=1)\n        return self.classifier(cat_feats).squeeze(1)\n\n\nimport time\nfrom contextlib import contextmanager\nfrom collections import defaultdict\nmap_used_time = defaultdict(float)\n@contextmanager\ndef timer(title):\n    t0 = time.time()\n    yield\n    tt = time.time() - t0\n    map_used_time[title] += tt\n    print(\"  {} - done in {:.5f}s\".format(title, tt))\n\n\ndf, img_dir = load_data()\n\nstop_words = set([\n    'promo','diskon','baik','terbaik', 'murah',\n    'termurah', 'harga', 'price', 'best', 'seller',\n    'bestseller', 'ready', 'stock', 'stok', 'limited',\n    'bagus', 'kualitas', 'berkualitas', 'hari', 'ini',\n    'jadi', 'gratis',\n])\n\n\ntitles = [\n    title.translate(str.maketrans({_: ' ' for _ in string.punctuation}))\n    for title in df['title'].str.lower().values\n]\n\ntokenizer = TweetTokenizer()\ntfidf_vectorizer = TfidfVectorizer(stop_words=stop_words, \n                                   binary=True, \n                                   min_df=2, \n                                   token_pattern='(?u)\\\\b\\\\w+\\\\b', \n                                   tokenizer=tokenizer.tokenize,\n                                   dtype=np.float32,\n                                   norm='l2')\ntfidf_feats = tfidf_vectorizer.fit_transform(titles)\nsimmat_tfidf = tfidf_feats @ tfidf_feats.T\n\nwith timer('load'):\n    st_sizes, img_hs, img_ws = joblib.load('/tmp/lyk_img_meta_data.pkl')\n    similarities_img = np.load('/tmp/img_D_qe.npy')[:, :k]\n    indexes_img = np.load('/tmp/img_I_qe.npy')[:, :k]\n\n    similarities_bert = np.load('/tmp/brt_D_qe.npy')[:, :k]\n    indexes_bert = np.load('/tmp/brt_I_qe.npy')[:, :k]\n\n    similarities_mm = np.load('/tmp/mut_D_qe.npy')[:, :k]\n    indexes_mm = np.load('/tmp/mut_I_qe.npy')[:, :k]\n    \n    row = indexes_bert.ravel()\n    col = np.arange(len(indexes_bert)).repeat(k)\n    data = similarities_bert.ravel()\n    simmat_bert = {(i, j): d for i, j, d in zip(col, row, data)}\n\n    row = indexes_img.ravel()\n    col = np.arange(len(indexes_img)).repeat(k)\n    data = similarities_img.ravel()\n    simmat_img = {(i, j): d for i, j, d in zip(col, row, data)}\n\n    row = indexes_mm.ravel()\n    col = np.arange(len(indexes_mm)).repeat(k)\n    data = similarities_mm.ravel()\n    simmat_mm = {(i, j): d for i, j, d in zip(col, row, data)}\n\ndel row, col, data\ngc.collect()\n\nckpt = torch.load('../input/shopee-meta-models/v135.pth')\nparams = ckpt['params']\n\ntop_neighbors = defaultdict(list)\nfeats = defaultdict(lambda: defaultdict())\n\npair_tuples = []\nfor i in tqdm(range(len(df))):\n    right_indexes = set(indexes_img[i, :k].tolist() + indexes_bert[i, :k].tolist())\n    right_indexes.remove(i)  # remove self\n\n    right_indexes = list(right_indexes)\n    scores = {}\n    for j in right_indexes:\n        pair_tuples.append((i, j))\n\n        sim_img = simmat_img.get((i, j), 0)\n        sim_bert = simmat_bert.get((i, j), 0)\n        sim_mm = simmat_mm.get((i, j), 0)\n        sim_tfidf = simmat_tfidf[i, j]\n        if sim_img == 0 and sim_bert == 0:\n            continue\n\n        feats[i][j] = [\n            sim_img,\n            sim_tfidf,\n            sim_bert,\n            sim_mm,\n        ]\n        scores[j] = sim_img + sim_tfidf + sim_bert + sim_mm\n\n    top_neighbors[i] = sorted(right_indexes, key=lambda x: scores[x], reverse=True)[:params['k']]\n\ndataset = GraphDataset(\n    feats=feats,\n    pair_tuples=pair_tuples,\n    k=params['k'],\n    top_neighbors=top_neighbors,\n)\nloader = DataLoader(dataset, batch_size=2 ** 12, shuffle=False, drop_last=False, num_workers=2, pin_memory=True)\n\ngat = GATPairClassifier(nfeat=len(feats[i][j]), nhid=params['nhid'],\n                        dropout=params['dropout'], nheads=params['nheads'], pooling=params['pooling'])\ngat.to('cuda').eval()\ngat.load_state_dict(ckpt['model'])\n\ndel tfidf_feats\ngc.collect()\n###\n\npreds = []\nfor feats, neighbor_feats in tqdm(loader, desc='predict', leave=False):\n    feats = feats.to('cuda', non_blocking=True)\n    neighbor_feats = neighbor_feats.to('cuda', non_blocking=True)\n    with torch.no_grad():\n        pred = gat(feats, neighbor_feats).sigmoid().detach().cpu().numpy().tolist()\n        preds.extend(pred)\n\nconf_th_gcn = 0.3\ndf_pair = pd.DataFrame()\ncol, row = list(zip(*pair_tuples))\ndf_pair['i'] = col\ndf_pair['j'] = row\n\ndf_pair['posting_id'] = df['posting_id'].values[df_pair['i'].values]\ndf_pair['posting_id_target'] = df['posting_id'].values[df_pair['j'].values]\n\ndf_pair = df_pair[['posting_id', 'posting_id_target']]\ndf_pair['pred'] = preds\ndf_pair['pred'] -= conf_th_gcn\n\ndf_pair.to_pickle('submission_lyak_gcn.pkl')\ndf_pair","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# lyakaap Side (LGB)","metadata":{}},{"cell_type":"code","source":"%%python\nfrom lyk_config import k, conf_th, DEBUG, load_data\nimport sys\nsys.path.append('../input/timm045/')\nimport timm\n\nfrom itertools import zip_longest\nimport json\nimport math\nimport gc\nimport os\nfrom pathlib import Path\n\nimport faiss\nimport numpy as np\nimport cupy as cp\nimport pandas as pd\n\nfrom tqdm import tqdm\nfrom PIL import Image\nimport joblib\nimport lightgbm as lgb\nfrom scipy.sparse import hstack, vstack, csc_matrix, csr_matrix\nimport editdistance\nimport networkx as nx\n\nimport string\nimport nltk\nfrom nltk.tokenize.treebank import TreebankWordTokenizer\nfrom nltk.tokenize import TweetTokenizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nNUM_CLASSES = 11014\nNUM_WORKERS = 2\nSEED = 0\n\n###\nimport time\nfrom contextlib import contextmanager\nfrom collections import defaultdict\nmap_used_time = defaultdict(float)\n@contextmanager\ndef timer(title):\n    t0 = time.time()\n    yield\n    tt = time.time() - t0\n    map_used_time[title] += tt\n    print(\"  {} - done in {:.5f}s\".format(title, tt))\n\n\ndf, img_dir = load_data()\n\nstop_words = set([\n    'promo','diskon','baik','terbaik', 'murah',\n    'termurah', 'harga', 'price', 'best', 'seller',\n    'bestseller', 'ready', 'stock', 'stok', 'limited',\n    'bagus', 'kualitas', 'berkualitas', 'hari', 'ini',\n    'jadi', 'gratis',\n])\n\ntitles = [\n    title.translate(str.maketrans({_: ' ' for _ in string.punctuation}))\n    for title in df['title'].str.lower().values\n]\n\ntokenizer = TweetTokenizer()\ntfidf_vectorizer = TfidfVectorizer(stop_words=stop_words, \n                                   binary=True, \n                                   min_df=2, \n                                   token_pattern='(?u)\\\\b\\\\w+\\\\b', \n                                   tokenizer=tokenizer.tokenize,\n                                   dtype=np.float32,\n                                   norm='l2')\ntfidf_feats = tfidf_vectorizer.fit_transform(titles)\n\nwith timer('load'):\n    similarities_bert, indexes_bert = joblib.load('/tmp/lyk_bert_data.pkl')\n    similarities_img, indexes_img = joblib.load('/tmp/lyk_img_data.pkl')\n    st_sizes, img_hs, img_ws = joblib.load('/tmp/lyk_img_meta_data.pkl')\n    similarities_mm, indexes_mm = joblib.load('/tmp/lyk_mm_data.pkl')\n    \n    row = indexes_bert.ravel()\n    col = np.arange(len(indexes_bert)).repeat(k)\n    data = similarities_bert.ravel()\n    simmat_bert = {(i, j): d for i, j, d in zip(col, row, data)}\n\n    row = indexes_img.ravel()\n    col = np.arange(len(indexes_img)).repeat(k)\n    data = similarities_img.ravel()\n    simmat_img = {(i, j): d for i, j, d in zip(col, row, data)}\n\n    row = indexes_mm.ravel()\n    col = np.arange(len(indexes_mm)).repeat(k)\n    data = similarities_mm.ravel()\n    simmat_mm = {(i, j): d for i, j, d in zip(col, row, data)}\n\ndel row, col, data\ngc.collect()\n\nmean_sim_img_top5 = similarities_img[:, :5].mean(1)\nmean_sim_bert_top5 = similarities_bert[:, :5].mean(1)\nmean_mean_sim_img_top5 = mean_sim_img_top5[indexes_img[:, :5]].mean(1)\nmean_mean_sim_bert_top5 = mean_sim_bert_top5[indexes_bert[:, :5]].mean(1)\n\nmean_sim_img_top5 = (mean_sim_img_top5 - mean_sim_img_top5.mean()) / mean_sim_img_top5.std()\nmean_sim_bert_top5 = (mean_sim_bert_top5 - mean_sim_bert_top5.mean()) / mean_sim_bert_top5.std()\nmean_mean_sim_img_top5 = (mean_mean_sim_img_top5 - mean_mean_sim_img_top5.mean()) / mean_mean_sim_img_top5.std()\nmean_mean_sim_bert_top5 = (mean_mean_sim_bert_top5 - mean_mean_sim_bert_top5.mean()) / mean_mean_sim_bert_top5.std()\n\nmean_sim_img_top15 = similarities_img[:, :15].mean(1)\nmean_sim_bert_top15 = similarities_bert[:, :15].mean(1)\nmean_sim_img_top15 = (mean_sim_img_top15 - mean_sim_img_top15.mean()) / mean_sim_img_top15.std()\nmean_sim_bert_top15 = (mean_sim_bert_top15 - mean_sim_bert_top15.mean()) / mean_sim_bert_top15.std()\n\nmean_sim_img_top30 = similarities_img[:, :30].mean(1)\nmean_sim_bert_top30 = similarities_bert[:, :30].mean(1)\nmean_sim_img_top30 = (mean_sim_img_top30 - mean_sim_img_top30.mean()) / mean_sim_img_top30.std()\nmean_sim_bert_top30 = (mean_sim_bert_top30 - mean_sim_bert_top30.mean()) / mean_sim_bert_top30.std()\n\nmean_sim_mm_top5 = similarities_mm[:, :5].mean(1)\nmean_mean_sim_mm_top5 = mean_sim_mm_top5[indexes_mm[:, :5]].mean(1)\n\nmean_sim_mm_top5 = (mean_sim_mm_top5 - mean_sim_mm_top5.mean()) / mean_sim_mm_top5.std()\nmean_mean_sim_mm_top5 = (mean_mean_sim_mm_top5 - mean_mean_sim_mm_top5.mean()) / mean_mean_sim_mm_top5.std()\n\nmean_sim_mm_top15 = similarities_mm[:, :15].mean(1)\nmean_sim_mm_top15 = (mean_sim_mm_top15 - mean_sim_mm_top15.mean()) / mean_sim_mm_top15.std()\n\nmean_sim_mm_top30 = similarities_mm[:, :30].mean(1)\nmean_sim_mm_top30 = (mean_sim_mm_top30 - mean_sim_mm_top30.mean()) / mean_sim_mm_top30.std()\n\nrow_titles = df['title'].values\nposting_ids = df['posting_id'].values\n\ntmp_dir = Path('/tmp/rows')\ntmp_dir.mkdir(exist_ok=True, parents=True)\n\nrows = []\nfor i in tqdm(range(len(df))):\n    right_indexes = set(indexes_img[i].tolist() + indexes_bert[i].tolist())\n\n    for _, j in enumerate(right_indexes):\n        if i == j:\n            continue\n        sim_img = simmat_img.get((i, j), 0)\n        sim_bert = simmat_bert.get((i, j), 0)\n        sim_mm = simmat_mm.get((i, j), 0)\n        if sim_img == 0 and sim_bert == 0:\n            continue\n\n        rows.append({\n            'i': i,\n            'j': j,\n            'posting_id': posting_ids[i],\n            'posting_id_target': posting_ids[j],\n            'sim_img': sim_img,\n            'sim_bert': sim_bert,\n            'sim_mm': sim_mm,\n            'edit_distance': editdistance.eval(titles[i], titles[j]),\n            'title_len': len(row_titles[i]),\n            'title_len_target': len(row_titles[j]),\n            'title_num_words': len(row_titles[i].split()),\n            'title_num_words_target': len(row_titles[j].split()),\n            'mean_sim_img_top5': mean_sim_img_top5[i],\n            'mean_sim_img_target_top5': mean_sim_img_top5[j],\n            'mean_sim_bert_top5': mean_sim_bert_top5[i],\n            'mean_sim_bert_target_top5': mean_sim_bert_top5[j],\n            'mean_sim_img_top15': mean_sim_img_top15[i],\n            'mean_sim_img_target_top15': mean_sim_img_top15[j],\n            'mean_sim_bert_top15': mean_sim_bert_top15[i],\n            'mean_sim_bert_target_top15': mean_sim_bert_top15[j],\n            'mean_sim_img_top30': mean_sim_img_top30[i],\n            'mean_sim_img_target_top30': mean_sim_img_top30[j],\n            'mean_sim_bert_top30': mean_sim_bert_top30[i],\n            'mean_sim_bert_target_top30': mean_sim_bert_top30[j],\n            'st_size': st_sizes[i],\n            'st_size_target': st_sizes[j],\n            'wxh/st_size': img_ws[i] * img_hs[i] / st_sizes[i],\n            'wxh/st_size_target': img_ws[j] * img_hs[j] / st_sizes[j],\n            'mean_mean_sim_img_top5': mean_mean_sim_img_top5[i],\n            'mean_mean_sim_img_target_top5': mean_mean_sim_img_top5[j],\n            'mean_mean_sim_bert_top5': mean_mean_sim_bert_top5[i],\n            'mean_mean_sim_bert_target_top5': mean_mean_sim_bert_top5[j],\n            'mean_sim_mm_top5': mean_sim_mm_top5[i],\n            'mean_sim_mm_target_top5': mean_sim_mm_top5[j],\n            'mean_sim_mm_top15': mean_sim_mm_top15[i],\n            'mean_sim_mm_target_top15': mean_sim_mm_top15[j],\n            'mean_sim_mm_top30': mean_sim_mm_top30[i],\n            'mean_sim_mm_target_top30': mean_sim_mm_top30[j],\n            'mean_mean_sim_mm_top5': mean_mean_sim_mm_top5[i],\n            'mean_mean_sim_mm_target_top5': mean_mean_sim_mm_top5[j],\n        })\n\n    if i % 10000 == 9999 or i == len(df) - 1:\n        tmp_df = pd.DataFrame(rows)\n        for col in tmp_df.columns:\n            if tmp_df[col].dtype == 'float64':\n                tmp_df[col] = tmp_df[col].astype('float32')\n            elif tmp_df[col].dtype == 'int64':\n                tmp_df[col] = tmp_df[col].astype('int32')\n        tmp_df.to_feather(tmp_dir / f'{i}.feather')\n        rows = []\n\ndf.drop(['image', 'title'], axis=1, inplace=True)\ndel (\n    mean_sim_img_top5, mean_sim_img_top15, mean_sim_img_top30, mean_mean_sim_img_top5,\n    mean_sim_bert_top5, mean_sim_bert_top15, mean_sim_bert_top30, mean_mean_sim_bert_top5,\n    mean_sim_mm_top5, mean_sim_mm_top15, mean_sim_mm_top30, mean_mean_sim_mm_top5,\n    simmat_img, simmat_bert, simmat_mm,\n    similarities_img, indexes_img,\n    similarities_bert, indexes_bert,\n    similarities_mm, indexes_mm,\n)\ngc.collect()\nwith timer('to_frame'):\n    df_pair = pd.concat([pd.read_feather(path) for path in tmp_dir.glob('**/*.feather')], axis=0).reset_index(drop=True)\ndel rows\ngc.collect()\n\nwith timer('sim_tfidf'):\n    df_pair['sim_tfidf'] = tfidf_feats[df_pair['i'].values].multiply(tfidf_feats[df_pair['j'].values]).sum(axis=1)\ndf_pair['title_len_diff'] = np.abs(df_pair['title_len'] - df_pair['title_len_target'])\ndf_pair['title_num_words_diff'] = np.abs(df_pair['title_num_words'] - df_pair['title_num_words_target'])\n\ndel tfidf_feats\ngc.collect()\n###\n\nfrom cuml import ForestInference\nimport treelite\nlist_clf = []\nfor clf in joblib.load('../input/shopee/boosters_v34_v45_mm.pickle'):\n    clf.save_model('/tmp/tmp.lgb')\n    fi = ForestInference()\n    fi.load_from_treelite_model(treelite.Model.load('/tmp/tmp.lgb', model_format='lightgbm'))\n    list_clf.append(fi)\n\nX = df_pair[[\n    'sim_img', 'sim_tfidf', 'sim_bert', 'sim_mm', 'edit_distance',\n    'title_len', 'title_len_target', 'title_len_diff',\n    'title_num_words', 'title_num_words_target', 'title_num_words_diff',\n    'mean_sim_img_top5', 'mean_sim_img_target_top5',\n    'mean_sim_bert_top5', 'mean_sim_bert_target_top5',\n    'mean_sim_mm_top5', 'mean_sim_mm_target_top5',\n    'mean_sim_img_top15', 'mean_sim_img_target_top15',\n    'mean_sim_bert_top15', 'mean_sim_bert_target_top15',\n    'mean_sim_mm_top15', 'mean_sim_mm_target_top15',\n    'mean_sim_img_top30', 'mean_sim_img_target_top30',\n    'mean_sim_bert_top30', 'mean_sim_bert_target_top30',\n    'mean_sim_mm_top30', 'mean_sim_mm_target_top30',\n    'st_size', 'st_size_target',\n    'wxh/st_size', 'wxh/st_size_target',\n    'mean_mean_sim_img_top5', 'mean_mean_sim_img_target_top5',\n    'mean_mean_sim_bert_top5', 'mean_mean_sim_bert_target_top5',\n    'mean_mean_sim_mm_top5', 'mean_mean_sim_mm_target_top5',\n]]\n\n## passing as cupy array might be able to avoid multipy copy to GPU.\nX = cp.asarray(X[clf.feature_name()].values.astype(np.float32))\ndf_pair = df_pair[['posting_id', 'posting_id_target']]\n\ngc.collect()\nwith timer('predict'):\n    df_pair['pred'] = np.mean([clf.predict(X).get() for clf in list_clf], axis=0) - conf_th\n\ndf_pair.to_pickle('submission_lyak.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TKM side","metadata":{}},{"cell_type":"code","source":"%%bash\npip install ../input/shopee-libs/imagesize-1.2.0-py2.py3-none-any.whl \\\n../input/shopee-libs/PyStemmer-2.0.1/dist/PyStemmer-2.0.1.tar","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n!cp ../input/rapids/rapids.0.18.0 /opt/conda/envs/rapids.tar.gz\n!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz > /dev/null\nsys.path = [\"/opt/conda/envs/rapids/lib/python3.7/site-packages\"] + sys.path\nsys.path = [\"/opt/conda/envs/rapids/lib/python3.7\"] + sys.path\nsys.path = [\"/opt/conda/envs/rapids/lib\"] + sys.path \n!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\nimport pandas as pd\nimport numpy as np\nimport sys\nimport ast\nimport os\nimport time\nimport cv2\nimport PIL.Image\nimport random\nimport joblib\n\nfrom multiprocessing import Pool\nfrom sklearn.metrics import accuracy_score\n\nimport langid\nimport Levenshtein\n\n#import albumentations\n#from albumentations import *\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport gc\nfrom sklearn.metrics import roc_auc_score\nfrom warnings import filterwarnings\n\nfrom contextlib import contextmanager\nfrom collections import defaultdict\nmap_used_time = defaultdict(float)\n@contextmanager\ndef timer(title):\n    t0 = time.time()\n    yield\n    tt = time.time() - t0\n    map_used_time[title] += tt\n    print(\"  {} - done in {:.5f}s\".format(title, tt))\n\n\nfilterwarnings(\"ignore\")\n\n\n###\n\nimport imagesize\nimport Stemmer\nstemmer = Stemmer.Stemmer('indonesian')\nDEBUG = len(pd.read_csv('../input/testdata6/test6.csv')) == 3\n\n\nif DEBUG:\n    data_dir = '../input/shopee-product-matching/train_images/'\nelse:\n    data_dir = '../input/testdata6/testimages/testimages/'\n    \n###\n\nif DEBUG:\n    if 1:\n        nrows = 1000\n        df_test = pd.read_csv('../input/shopee-libs/train_newfold_stmmedid.csv', nrows=nrows)\n    else:\n        df_test = pd.read_csv('../input/shopee-libs/train_newfold_stmmedid.csv').append(\n            pd.read_csv('../input/shopee-libs/train_newfold_stmmedid.csv'), ignore_index=True\n        )\n        nrows = 1000\n        df_test = pd.read_csv('../input/shopee-libs/train_newfold_stmmedid.csv', nrows=nrows)\n    \n    label_groups = np.sort(df_test['label_group'].unique())\n    map_label2id = {g: i for i, g in enumerate(label_groups)}\n    df_test['label'] = df_test['label_group'].map(map_label2id)\n    df_test['file_path'] = df_test.image.apply(lambda x: os.path.join(data_dir, f'{x}'))\nelse:\n    df_test = pd.read_csv('../input/testdata6/test6.csv', nrows=1000)\n    df_test['file_path'] = df_test.image.apply(lambda x: os.path.join(data_dir, f'{x}'))\n\n    titles = df_test['title'].str.lower().values\n\n    with timer('get lang'):\n        df_test['lang'] = [langid.classify(t)[0] for t in tqdm(titles)]\n        list_lang = df_test['lang'].values\n    with timer('lemmatize'):\n        titles = np.array([t.encode('ascii').decode('unicode-escape').encode('ascii', 'replace').decode('ascii').replace('?', ' ') for t in titles])\n        titles = [' '.join(stemmer.stemWords(t.split())) if list_lang[i] in {'id', 'ms'} else t for i, t in enumerate(tqdm(titles))]\n        df_test['title'] = titles\n\nwith timer('get image size'):\n    st_sizes, img_hs, img_ws = joblib.load('/tmp/lyk_img_meta_data.pkl')\n    df_test['width'] = img_ws\n    df_test['hight'] = img_hs\n    df_test['st_size'] = st_sizes\n    df_test['wxh/st_size'] = df_test['width'] * df_test['hight'] / df_test['st_size']\n\ndf_test.to_pickle('/tmp/df_test_tkm.pkl')\n###\n\nK = min(60, df_test.shape[0])\n\n###\nprint('Computing text embeddings...')\nimport cupy as cp\nimport pickle\nimport gc\nfrom cuml.feature_extraction.text import TfidfVectorizer\nimport cudf\n\nmodel = TfidfVectorizer(stop_words=None, \n                        binary=True, \n                        max_features=100000,\n                        max_df=0.3,\n                        min_df=2,\n                        dtype=np.float32)\n\nwith timer('tfidf fit'):\n    titles = pd.read_csv('../input/shopee-libs/train_newfold_stmmedid.csv', \n                         usecols=['title'])['title'].values.tolist()\n    test_titles = df_test.title.values.tolist()\n    titles += test_titles\n    model.fit(cudf.Series(titles))\n    text_embeddings = model.transform(cudf.Series(test_titles))\n    print('text embeddings shape',text_embeddings.shape)\n\nwith timer('tfidf pred'):\n    CHUNK = 1024*4\n    print('Finding similar titles...')\n    text_D = np.zeros((df_test.shape[0], K), dtype=np.float32)\n    text_I = np.zeros((df_test.shape[0], K), dtype=np.int32)\n\n\n    CTS = text_embeddings.shape[0]//CHUNK\n    if  text_embeddings.shape[0]%CHUNK!=0: CTS += 1\n    cnt = 0\n    for j in range( CTS ):\n\n        a = j*CHUNK\n        b = (j+1)*CHUNK\n        b = min(b, text_embeddings.shape[0])\n        print('chunk',a,'to',b, text_embeddings.shape[0])\n\n        #COSINE SIMILARITY DISTANCE\n        cts = (text_embeddings * text_embeddings[a:b].T).T.toarray()\n        indices = cp.argsort(cts, axis=1)\n\n        for k in range(b-a):\n            idx = indices[k][::-1]\n            text_I[cnt] = idx[:K].get()\n            text_D[cnt] = cts[k, idx[:K]].get()\n            cnt += 1\n\ndel text_embeddings, indices, cts\ngc.collect()\n###\n\nimg_D = np.load('/tmp/img_D_qe.npy')\nimg_I = np.load('/tmp/img_I_qe.npy')\n\n###\n\nbert_D = np.load('/tmp/brt_D_qe.npy')\nbert_I = np.load('/tmp/brt_I_qe.npy')\n\n###\n\nbth_D = np.load('/tmp/bth_D_qe.npy')\nbth_I = np.load('/tmp/bth_I_qe.npy')\n###\n\nmut_D = np.load('/tmp/mut_D_qe.npy')\nmut_I = np.load('/tmp/mut_I_qe.npy')\n###\n\nmap_col2id = {}\n###\n\nimport langid\nimport Levenshtein\ntitles = df_test['title'].values\ntitles_set = [set(t) for t in titles]\nlangs = df_test['lang'].values\nst_size = df_test['st_size'].values\nwh_st_size = df_test['wxh/st_size'].values\n###\n\nnumset = set('0123456789')\n\n###\ntext_D = np.array(text_D)\ntxt_cnt_all = np.vstack([(text_D > t).sum(axis=1) for t in [0.9, 0.8, 0.7, 0.6, 0.5]]).T\ntxt_avg_raw_all = text_D.mean(axis=1)\ntxt_avg_all = (txt_avg_raw_all - txt_avg_raw_all.mean()) / txt_avg_raw_all.std()\ntxt_std_all = text_D.std(axis=1)\n\ntxt_avg_5_all = text_D[:, :5].mean(axis=1)\ntxt_avg_10_all = text_D[:, :10].mean(axis=1)\ntxt_avg_15_all = text_D[:, :15].mean(axis=1)\ntxt_avg_30_all = text_D[:, :30].mean(axis=1)\n\ntxt_avg_5_all = (txt_avg_5_all - txt_avg_5_all.mean()) / txt_avg_5_all.std()\ntxt_avg_10_all = (txt_avg_10_all - txt_avg_10_all.mean()) / txt_avg_10_all.std()\ntxt_avg_15_all = (txt_avg_15_all - txt_avg_15_all.mean()) / txt_avg_15_all.std()\ntxt_avg_30_all = (txt_avg_30_all - txt_avg_30_all.mean()) / txt_avg_30_all.std()\n    \n###\nbrt_cnt_all = np.vstack([(bert_D > t).sum(axis=1) for t in [0.9, 0.8, 0.7, 0.6, 0.5]]).T\nbrt_avg_raw_all = bert_D.mean(axis=1)\nbrt_avg_all = (brt_avg_raw_all - brt_avg_raw_all.mean()) / brt_avg_raw_all.std()\nbrt_std_all = bert_D.std(axis=1)\n\nbrt_avg_5_all = bert_D[:, :5].mean(axis=1)\nbrt_avg_10_all = bert_D[:, :10].mean(axis=1)\nbrt_avg_15_all = bert_D[:, :15].mean(axis=1)\nbrt_avg_30_all = bert_D[:, :30].mean(axis=1)\n\nbrt_avg_5_all = (brt_avg_5_all - brt_avg_5_all.mean()) / brt_avg_5_all.std()\nbrt_avg_10_all = (brt_avg_10_all - brt_avg_10_all.mean()) / brt_avg_10_all.std()\nbrt_avg_15_all = (brt_avg_15_all - brt_avg_15_all.mean()) / brt_avg_15_all.std()\nbrt_avg_30_all = (brt_avg_30_all - brt_avg_30_all.mean()) / brt_avg_30_all.std()\n\n###\nbth_cnt_all = np.vstack([(bth_D > t).sum(axis=1) for t in [0.9, 0.8, 0.7, 0.6, 0.5]]).T\nbth_avg_raw_all = bth_D.mean(axis=1)\nbth_avg_all = (bth_avg_raw_all - bth_avg_raw_all.mean()) / bth_avg_raw_all.std()\nbth_std_all = bth_D.std(axis=1)\n\nbth_avg_5_all = bth_D[:, :5].mean(axis=1)\nbth_avg_10_all = bth_D[:, :10].mean(axis=1)\nbth_avg_15_all = bth_D[:, :15].mean(axis=1)\nbth_avg_30_all = bth_D[:, :30].mean(axis=1)\n\nbth_avg_5_all = (bth_avg_5_all - bth_avg_5_all.mean()) / bth_avg_5_all.std()\nbth_avg_10_all = (bth_avg_10_all - bth_avg_10_all.mean()) / bth_avg_10_all.std()\nbth_avg_15_all = (bth_avg_15_all - bth_avg_15_all.mean()) / bth_avg_15_all.std()\nbth_avg_30_all = (bth_avg_30_all - bth_avg_30_all.mean()) / bth_avg_30_all.std()\n        \n###\nmut_cnt_all = np.vstack([(mut_D > t).sum(axis=1) for t in [0.9, 0.8, 0.7, 0.6, 0.5]]).T\nmut_avg_raw_all = mut_D.mean(axis=1)\nmut_avg_all = (mut_avg_raw_all - mut_avg_raw_all.mean()) / mut_avg_raw_all.std()\nmut_std_all = mut_D.std(axis=1)\n\nmut_avg_5_all = mut_D[:, :5].mean(axis=1)\nmut_avg_10_all = mut_D[:, :10].mean(axis=1)\nmut_avg_15_all = mut_D[:, :15].mean(axis=1)\nmut_avg_30_all = mut_D[:, :30].mean(axis=1)\n\nmut_avg_5_all = (mut_avg_5_all - mut_avg_5_all.mean()) / mut_avg_5_all.std()\nmut_avg_10_all = (mut_avg_10_all - mut_avg_10_all.mean()) / mut_avg_10_all.std()\nmut_avg_15_all = (mut_avg_15_all - mut_avg_15_all.mean()) / mut_avg_15_all.std()\nmut_avg_30_all = (mut_avg_30_all - mut_avg_30_all.mean()) / mut_avg_30_all.std()\n        \n###\nimg_cnt_all = np.vstack([(img_D > t).sum(axis=1) for t in [0.9, 0.8, 0.7, 0.6, 0.5]]).T\nimg_avg_raw_all = img_D.mean(axis=1)\nimg_avg_all = (img_avg_raw_all - img_avg_raw_all.mean()) / img_avg_raw_all.std()\nimg_std_all = img_D.std(axis=1)\n\nimg_avg_5_all = img_D[:, :5].mean(axis=1)\nimg_avg_10_all = img_D[:, :10].mean(axis=1)\nimg_avg_15_all = img_D[:, :15].mean(axis=1)\nimg_avg_30_all = img_D[:, :30].mean(axis=1)\n\nimg_avg_5_all = (img_avg_5_all - img_avg_5_all.mean()) / img_avg_5_all.std()\nimg_avg_10_all = (img_avg_10_all - img_avg_10_all.mean()) / img_avg_10_all.std()\nimg_avg_15_all = (img_avg_15_all - img_avg_15_all.mean()) / img_avg_15_all.std()\nimg_avg_30_all = (img_avg_30_all - img_avg_30_all.mean()) / img_avg_30_all.std()\n\nwidth_hight = df_test[['width', 'hight']].values\n\nlist_pred_id = [[] for _ in range(df_test.shape[0])]\n\nindices = df_test.index.values\n\nptr = 0\nall_feat = np.memmap('/tmp/tkm_feat.dat', dtype='float32', mode='w+', shape=(df_test.shape[0] * 60 * 5, 150), order='F')\n\nfeat = np.zeros((60 * 5, 150), dtype='float32')\n\nlist_idx = []\nlist_idx2 = []\nlist_feats = []\nfor i in tqdm(indices):\n    img_d = img_D[i]\n    img_i = img_I[i]\n\n    img_cnt = img_cnt_all[i]\n    img_avg = img_avg_all[i]\n    img_std = img_std_all[i]\n\n    img_width ,img_hight = width_hight[i]\n\n    ###\n    txt_d = text_D[i]\n    txt_i = text_I[i]\n\n    txt_cnt = txt_cnt_all[i]\n    txt_avg = txt_avg_all[i]\n    txt_std = txt_std_all[i]\n\n    txt_set = set(titles[i])\n    ###\n    brt_d = bert_D[i]\n    brt_i = bert_I[i]\n\n    brt_cnt = brt_cnt_all[i]\n    brt_avg = brt_avg_all[i]\n    brt_std = brt_std_all[i]\n\n    brt_set = set(titles[i])\n    bth_d = bth_D[i]\n    bth_i = bth_I[i]\n\n    bth_cnt = bth_cnt_all[i]\n    bth_avg = bth_avg_all[i]\n    bth_std = bth_std_all[i]\n\n    bth_set = set(titles[i])\n    mut_d = mut_D[i]\n    mut_i = mut_I[i]\n\n    mut_cnt = mut_cnt_all[i]\n    mut_avg = mut_avg_all[i]\n    mut_std = mut_std_all[i]\n\n    mut_set = set(titles[i])\n\n    map_feat = {}\n    for j in range(K):\n        _w, _h = width_hight[img_i[j]]\n        _img_cnt = img_cnt_all[img_i[j]]\n        _img_avg = img_avg_all[img_i[j]]\n        _img_std = img_std_all[img_i[j]]\n\n        diff_width = abs(img_width - _w)\n        diff_hight = abs(img_hight - _h)\n        d = {\n            'img_sim': img_d[j],\n            'img_avg': img_avg, \n            'img_std': img_std,\n            'img_avg2': _img_avg, \n            'img_std2': _img_std,\n\n            'img_avg_raw': img_avg_raw_all[i],\n            'img_avg2_raw': img_avg_raw_all[img_i[j]],\n\n            'diff_width': diff_width,\n            'diff_hight': diff_hight,\n            'img_width': img_width,\n            'img_hight': img_hight,\n            'img_width2': _w,\n            'img_hight2': _h,\n\n            'st_size': st_size[i],\n            'st_size2': st_size[img_i[j]],\n            'wh_st_size': wh_st_size[i],\n            'wh_st_size2': wh_st_size[img_i[j]]\n        }\n        d.update({f'img_cnt_{ii}': img_cnt[ii] for ii in range(img_cnt.shape[0])})\n        d.update({f'img_cnt2_{ii}': _img_cnt[ii] for ii in range(_img_cnt.shape[0])})\n        map_feat[img_i[j]] = d\n        \n    for j in range(K):\n        _txt_set = titles_set[txt_i[j]]\n        _txt_cnt = txt_cnt_all[txt_i[j]]\n        _txt_avg = txt_avg_all[txt_i[j]]\n        _txt_std = txt_std_all[txt_i[j]]\n        diff_txt_set = set(titles[txt_i[j]]) & txt_set\n        diff_txt_set = len(numset & diff_txt_set) / (len(diff_txt_set) + 1)\n        xor_txt_set = set(titles[txt_i[j]]) ^ txt_set\n        xor_txt_set = len(numset & xor_txt_set) / (len(xor_txt_set) + 1)\n        jac_txt = len(txt_set & _txt_set) / (len(txt_set | _txt_set) + 1)\n        lev_dist = Levenshtein.distance(titles[i], titles[txt_i[j]])\n        d = {\n            'txt_sim': txt_d[j],\n            'txt_avg': txt_avg, \n            'txt_std': txt_std,\n            'txt_avg2': _txt_avg,\n            'txt_std2': _txt_std,\n\n            'txt_avg_raw': txt_avg_raw_all[i],\n            'txt_avg2_raw': txt_avg_raw_all[txt_i[j]],\n\n            'jac_txt': jac_txt,\n            'diff_txt_set': diff_txt_set, \n            'xor_txt_set': xor_txt_set,\n            'lev_dist': lev_dist,\n            'len_txt': len(titles[i]), \n            'len_txt2': len(titles[txt_i[j]]),\n            'lang_en': int(langs[i] == 'en'),\n            'lang_en2': int(langs[txt_i[j]] == 'en'),\n        }\n        d.update({f'txt_cnt_{ii}': txt_cnt[ii] for ii in range(txt_cnt.shape[0])})\n        d.update({f'txt_cnt2_{ii}': _txt_cnt[ii] for ii in range(_txt_cnt.shape[0])})\n        if txt_i[j] in map_feat:\n            map_feat[txt_i[j]].update(d)\n        else:\n            map_feat[txt_i[j]] = d\n            \n    for j in range(K):\n        _bth_cnt = bth_cnt_all[bth_i[j]]\n        _bth_avg = bth_avg_all[bth_i[j]]\n        _bth_std = bth_std_all[bth_i[j]]\n        if bth_i[j] in map_feat:\n            d = map_feat[bth_i[j]]\n        else:\n            d = {}\n        d.update({\n            'bth_sim': bth_d[j],\n            'bth_avg': bth_avg, \n            'bth_std': bth_std,\n            'bth_avg2': _bth_avg,\n            'bth_std2': _bth_std,\n\n            'bth_avg_raw': bth_avg_raw_all[i],\n            'bth_avg2_raw': bth_avg_raw_all[bth_i[j]],\n        })\n        d.update({f'bth_cnt_{ii}': bth_cnt[ii] for ii in range(bth_cnt.shape[0])})\n        d.update({f'bth_cnt2_{ii}': _bth_cnt[ii] for ii in range(_bth_cnt.shape[0])})\n        if 'lev_dist' not in d:\n            _bth_set = titles_set[bth_i[j]] #set(titles[bth_i[j]])\n            diff_bth_set = set(titles[bth_i[j]]) & bth_set\n            diff_bth_set = len(numset & diff_bth_set) / (len(diff_bth_set) + 1)\n            xor_bth_set = set(titles[bth_i[j]]) ^ bth_set\n            xor_bth_set = len(numset & xor_bth_set) / (len(xor_bth_set) + 1)\n            jac_bth = len(bth_set & _bth_set) / (len(bth_set | _bth_set) + 1)\n            lev_dist = Levenshtein.distance(titles[i], titles[bth_i[j]])\n            d.update({\n                'jac_txt': jac_bth,\n                'diff_txt_set': diff_bth_set, \n                'xor_txt_set': xor_bth_set,\n                'lev_dist': lev_dist,\n                'len_txt': len(titles[i]), \n                'len_txt2': len(titles[bth_i[j]]),\n                'lang_en': int(langs[i] == 'en'),\n                'lang_en2': int(langs[bth_i[j]] == 'en'),\n            })\n        if 'img_width' not in d:    \n            _w, _h = width_hight[bth_i[j]]\n            diff_width = abs(img_width - _w)\n            diff_hight = abs(img_hight - _h)\n            d.update({\n                'diff_width': diff_width,\n                'diff_hight': diff_hight,\n                 'img_width': img_width,\n                 'img_hight': img_hight,\n                 'img_width2': _w,\n                 'img_hight2': _h,\n                \n                     'st_size': st_size[i],\n                     'st_size2': st_size[bth_i[j]],\n                     'wh_st_size': wh_st_size[i],\n                     'wh_st_size2': wh_st_size[bth_i[j]]\n                     })\n        map_feat[bth_i[j]] = d\n            \n    for j in range(K):\n        _mut_cnt = mut_cnt_all[mut_i[j]]\n        _mut_avg = mut_avg_all[mut_i[j]]\n        _mut_std = mut_std_all[mut_i[j]]\n        if mut_i[j] in map_feat:\n            d = map_feat[mut_i[j]]\n        else:\n            d = {}\n        d.update({\n            'mut_sim': mut_d[j],\n            'mut_avg': mut_avg, \n            'mut_std': mut_std,\n            'mut_avg2': _mut_avg,\n            'mut_std2': _mut_std,\n            'mut_avg_raw': mut_avg_raw_all[i],\n            'mut_avg2_raw': mut_avg_raw_all[mut_i[j]],\n        })\n        d.update({f'mut_cnt_{ii}': mut_cnt[ii] for ii in range(mut_cnt.shape[0])})\n        d.update({f'mut_cnt2_{ii}': _mut_cnt[ii] for ii in range(_mut_cnt.shape[0])})\n        if 'lev_dist' not in d:\n            _mut_set = titles_set[mut_i[j]]#set(titles[mut_i[j]])\n            diff_mut_set = set(titles[mut_i[j]]) & mut_set\n            diff_mut_set = len(numset & diff_mut_set) / (len(diff_mut_set) + 1)\n            xor_mut_set = set(titles[mut_i[j]]) ^ mut_set\n            xor_mut_set = len(numset & xor_mut_set) / (len(xor_mut_set) + 1)\n            jac_mut = len(mut_set & _mut_set) / (len(mut_set | _mut_set) + 1)\n            lev_dist = Levenshtein.distance(titles[i], titles[mut_i[j]])\n            d.update({\n                'jac_txt': jac_mut,\n                'diff_txt_set': diff_mut_set, \n                'xor_txt_set': xor_mut_set,\n                'lev_dist': lev_dist,\n                'len_txt': len(titles[i]), \n                'len_txt2': len(titles[mut_i[j]]),\n                'lang_en': int(langs[i] == 'en'),\n                'lang_en2': int(langs[mut_i[j]] == 'en'),\n            })\n        if 'img_width' not in d:    \n            _w, _h = width_hight[mut_i[j]]\n            diff_width = abs(img_width - _w)\n            diff_hight = abs(img_hight - _h)\n            d.update({\n                'diff_width': diff_width,\n                'diff_hight': diff_hight,\n                'img_width': img_width,\n                'img_hight': img_hight,\n                'img_width2': _w,\n                'img_hight2': _h,\n                'st_size': st_size[i],\n                'st_size2': st_size[mut_i[j]],\n                'wh_st_size': wh_st_size[i],\n                'wh_st_size2': wh_st_size[mut_i[j]]\n            })\n        map_feat[mut_i[j]] = d\n\n    for j in range(K):\n        _brt_cnt = brt_cnt_all[brt_i[j]]\n        _brt_avg = brt_avg_all[brt_i[j]]\n        _brt_std = brt_std_all[brt_i[j]]\n        if brt_i[j] in map_feat:\n            d = map_feat[brt_i[j]]\n        else:\n            d = {}\n        d.update({\n            'brt_sim': brt_d[j],\n            'brt_avg': brt_avg, \n            'brt_std': brt_std,\n            'brt_avg2': _brt_avg,\n            'brt_std2': _brt_std,\n            'brt_avg_raw': brt_avg_raw_all[i],\n            'brt_avg2_raw': brt_avg_raw_all[brt_i[j]],\n        })\n        d.update({f'brt_cnt_{ii}': brt_cnt[ii] for ii in range(brt_cnt.shape[0])})\n        d.update({f'brt_cnt2_{ii}': _brt_cnt[ii] for ii in range(_brt_cnt.shape[0])})\n        if 'lev_dist' not in d:\n            _brt_set = titles_set[brt_i[j]]\n            diff_brt_set = set(titles[brt_i[j]]) & brt_set\n            diff_brt_set = len(numset & diff_brt_set) / (len(diff_brt_set) + 1)\n            xor_brt_set = set(titles[brt_i[j]]) ^ brt_set\n            xor_brt_set = len(numset & xor_brt_set) / (len(xor_brt_set) + 1)\n            jac_brt = len(brt_set & _brt_set) / (len(brt_set | _brt_set) + 1)\n            lev_dist = Levenshtein.distance(titles[i], titles[brt_i[j]])\n            d.update({\n                'jac_txt': jac_brt,\n                'diff_txt_set': diff_brt_set, \n                'xor_txt_set': xor_brt_set,\n                'lev_dist': lev_dist,\n                'len_txt': len(titles[i]), \n                'len_txt2': len(titles[brt_i[j]]),\n                'lang_en': int(langs[i] == 'en'),\n                'lang_en2': int(langs[brt_i[j]] == 'en'),\n            })\n        map_feat[brt_i[j]] = d\n\n    feat[:] = 0 \n    for ii, (k, map_val) in enumerate(map_feat.items()):\n        list_idx.append(i)\n        list_idx2.append(k)\n        for c, v in map_val.items():\n            if c not in map_col2id:\n                map_col2id[c] = len(map_col2id)\n            feat[ii, map_col2id[c]] = v\n\n    all_feat[ptr:ptr + len(map_feat)] = feat[:len(map_feat)]\n    ptr += len(map_feat)\n    \ndel img_D, img_I, text_D, text_I, bert_D, bert_I, bth_D, bth_I, mut_D, mut_I\ngc.collect()\n\ndel list_feats\ngc.collect()\n\nmap_weights = {sim: all_feat[:ptr, map_col2id[f'{sim}_sim']] for sim in ['img', 'bth', 'mut', 'txt', 'brt']}\n\ndel feat\ngc.collect()\n\nimport networkx as nx\n\n\nlist_idx = np.array(list_idx)\nlist_idx2 = np.array(list_idx2)\n\nfrom igraph import Graph\nmap_sim = {}\nfor sim in tqdm(['img', 'bth', 'mut', 'txt', 'brt'], desc='graph'):\n    weights = map_weights[sim]\n    idx = weights > 0\n    with timer('add edges'):\n        g = Graph()\n        g.add_vertices(len(df_test))\n        g.add_edges(list(zip(list_idx[idx], list_idx2[idx])), {'weight': weights[idx]})\n    with timer('pagerank'):\n        map_pr = np.array(g.pagerank(damping=0.85, weights='weight', niter=100, eps=1e-06, directed=False))\n    with timer('pagerank get'):\n        data1 = map_pr[list_idx]\n        data2 = map_pr[list_idx2]\n        data1[weights <= 0] = 0\n        data2[weights <= 0] = 0\n        map_sim[f'{sim}_pagerank'] = data1\n        map_sim[f'{sim}_pagerank2'] = data2\n    del map_pr, g\n    gc.collect()\n\nfor c, v in tqdm(map_sim.items()):\n    map_col2id[c] = len(map_col2id)\n    all_feat[:ptr, map_col2id[c]] = v\n\nimport treelite_runtime\nfrom cuml import ForestInference\nimport treelite\nimport pickle\nimport lightgbm as lgb\n\nall_weights = {\n    '../input/shopee-metric-resnet50d512-0328-newfold/0508_qe_best_0.345/': 1,\n}\n\ns = sum(all_weights.values())\nall_weights = {k: v / s for k, v in all_weights.items()}\n    \nlist_clf = []\nweights = []\nthresholds = [] #[0.358, 0.361, 0.350, 0.336, 0.348, 0.346]\nfor path in [\n    '../input/shopee-metric-resnet50d512-0328-newfold/0508_qe_best_0.345/',\n    ]:\n    name = os.path.dirname(path).split('/')[-1]\n    th = float(name.split('_')[-1])\n    if all_weights.get(path, 0) == 0:\n        continue\n        \n    fi = ForestInference()\n    fi.load_from_treelite_model(treelite.Model.load(f'{path}/all_data_clf_norm.lgb', model_format='lightgbm'))\n    list_clf += [fi]\n    thresholds += [th]\n    weights += [all_weights[path]]\n    \nprint(weights)\nprint(thresholds)\n\ncol = lgb.Booster(model_file=f'{path}/all_data_clf_norm.lgb').feature_name()\n\nfor sf in ['img', 'txt', 'mut', 'bth', 'brt']:\n    all_feat[:ptr, map_col2id[f'{sf}_avg']] = all_feat[:ptr, map_col2id[f'{sf}_avg_raw']]\n    all_feat[:ptr, map_col2id[f'{sf}_avg2']] = all_feat[:ptr, map_col2id[f'{sf}_avg2_raw']]\n\nCHUNK = 1000000\npreds = []\ncol_idx = [map_col2id[c] for c in col]\n\nfor ch in tqdm(range(0, ptr, CHUNK), desc='pred chunk'):\n    feat = cp.asarray(all_feat[ch:ch+CHUNK, col_idx]).astype('float32')\n    probs = np.vstack([(c.predict(feat).get() - thresholds[ii]) * weights[ii] for ii, c in enumerate(list_clf)])\n    preds += probs.sum(axis=0).tolist()\n    del feat\n    gc.collect()\n\ndf_pred = pd.DataFrame(\n    dict(\n        posting_id=list_idx,\n        posting_id_target=list_idx2,\n        pred=preds[:ptr]\n    )\n)\n\nidx = df_test.posting_id.values\ndf_pred['posting_id'] = [idx[i] for i in df_pred['posting_id'].values]\ndf_pred['posting_id_target'] = [idx[i] for i in df_pred['posting_id_target'].values]\n\ndf_pred.to_pickle('submission_tkm.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Postprocess","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\ndf_lyk = pd.read_pickle('submission_lyak.pkl')\ndf_lyk_gcn = pd.read_pickle('submission_lyak_gcn.pkl')\ndf_tkm = pd.read_pickle('submission_tkm.pkl')\n\ndf_lyk['pred'] *= 1\ndf_lyk_gcn['pred'] *= 3\ndf_tkm['pred'] *= 2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred = pd.concat([df_lyk, df_lyk_gcn, df_tkm], axis=0, ignore_index=True).groupby(['posting_id', 'posting_id_target'])[['pred']].sum() / 6\n\ndf_pred.reset_index(inplace=True)\ndf_pred.loc[df_pred['posting_id'] == df_pred['posting_id_target'], 'pred'] = 0.5\ndf_pred.set_index(['posting_id', 'posting_id_target'], inplace=True)\n\ndf_pred = df_pred.query('pred > 0')\ndf_pred = df_pred[df_pred.apply(lambda row: (row.name[1], row.name[0]) in df_pred.index, axis=1)].reset_index()\n\ndf_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import networkx as nx\nfrom tqdm import tqdm\nfrom networkx import edge_betweenness_centrality#cugraph.centrality.betweenness_centrality import edge_betweenness_centrality\n\nG = nx.Graph()\nfor i, j, w in df_pred[['posting_id', 'posting_id_target', 'pred']].values:\n    G.add_edge(i, j, weight=w)\n\nlist_remove_edges = []\nlist_add_edges = []\ndef split_graph(G):\n    list_comp = list(nx.connected_components(G))\n    n = len(G.nodes)\n    if len(list_comp) == 1:\n        map_bet = edge_betweenness_centrality(G, normalized=True)\n        map_bet = {(i, j): w  for (i, j), w in map_bet.items() \n                   if G[i][j]['weight'] < 0.15780210284453428}\n        if len(map_bet) == 0:\n            return\n        edge, val = max(map_bet.items(), key=lambda x: x[1])\n        if val > 0.11766651703447985:\n            G.remove_edge(*edge)\n            list_remove_edges.append(edge)\n            return split_graph(G)\n    else:\n        iters = list_comp\n        for comp in iters:\n            if len(comp) > 6:\n                split_graph(nx.Graph(G.subgraph(comp)))\n                \nsplit_graph(G)\nfor edge in list_remove_edges:\n    G.remove_edge(*edge)\n\ndef get_score(i, j):\n    try:\n        return G[i][j]['weight']\n    except KeyError:\n        return -1\n\nposting_ids = df_pred['posting_id'].unique()\nmatches = []\n\nfor i in posting_ids:\n    if i in G:\n        m = list(set([i] + list(G.neighbors(i))))\n    else:\n        m = [i]\n    if len(m) > 51:\n        m = sorted(m, key=lambda x: get_score(i, x), reverse=True)[:51]\n    matches.append(' '.join(m))\nmatched = pd.DataFrame(dict(posting_id=posting_ids, matches=matches))\n\nmatched.to_csv('submission.csv', index=False)\nmatched","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end_time = time.time()\nprint(\"Execution time: \", end_time - start_time,\"secs\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}