{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -lrth ../input/b7ns-final-672-300w-f0-load13-load1-14ep/b7ns_final_672_300w_f0_load13_load1_14ep_fold0_ep4.pth\n!ls -lrth ../input/b6ns-final-768-300w-f1-load28-5ep-1e-5/b6ns_final_768_300w_f1_load28_5ep_1e-5_fold1_ep5.pth\n!ls -lrth ../input/b5ns-final-768-300w-f2-load16-20ep/b5ns_final_768_300w_f2_load16_20ep_fold2_ep1.pth\n!ls -lrth ../input/b4ns-final-768-300w-f0-load16-20ep-load1-20ep/b4ns_final_768_300w_f0_load16_20ep_load1_20ep_fold0_ep4.pth\n!ls -lrth ../input/b3ns-final-768-300w-f1-load29-5ep5ep/b3ns_final_768_300w_f1_load29_5ep5ep_fold1_ep5.pth\n!ls -lrth ../input/nest101-final-768-300w-f4-load16-19ep-load1-16ep/nest101_final_768_300w_f4_load16_19ep_load1_16ep_fold4_ep5.pth\n!ls -lrth ../input/rex20-ddp-final-768-300w-f4-35ep-load20resume/rex20_DDP_final_768_300w_f4_35ep_load20resume_fold4_ep31.pth\n!ls -lrth ../input/b6ns-ddp-final-512-300w-f1-40ep/b6ns_DDP_final_512_300w_f1_40ep_fold1_ep36.pth\n!ls -lrth ../input/b5ns-final-768-300w-f2-load33-5ep-3e-5-32g/b5ns_final_768_300w_f2_load33_5ep_3e-5_32G_fold2_ep4.pth","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SKIP_COMMIT = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path = [\n    '../input/geffnet-20200820',\n    '../input/rexnetv1',\n    '../input/resnest/ResNeSt-master'    \n] + sys.path","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport glob\nimport math\nimport pickle\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport albumentations\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom tqdm.notebook import tqdm as tqdm\n\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport geffnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = '../input/landmark-recognition-2020/'\nmodel_dir = '../input/landmarkmodels/'\n\ndf = pd.read_csv(os.path.join(data_dir, 'train.csv'))\ndf['filepath'] = df['id'].apply(lambda x: os.path.join(data_dir, 'train', x[0], x[1], x[2], f'{x}.jpg'))\ndf_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))\n\ndf_test = df_sub[['id']].copy()\ndf_test['filepath'] = df_test['id'].apply(lambda x: os.path.join(data_dir, 'test', x[0], x[1], x[2], f'{x}.jpg'))\n\nuse_metric = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda')\nbatch_size = 4\nnum_workers = 4\nout_dim = 81313 \n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"transforms_672 = albumentations.Compose([\n    albumentations.Resize(672, 672),\n    albumentations.Normalize()\n])\n\ntransforms_768 = albumentations.Compose([\n    albumentations.Resize(768, 768),\n    albumentations.Normalize()\n])\ntransforms_512 = albumentations.Compose([\n    albumentations.Resize(512, 512),\n    albumentations.Normalize()\n])\n\n\nclass LandmarkDataset(Dataset):\n    def __init__(self, csv, split, mode, transforms=[transforms_672, transforms_768,transforms_512]):\n\n        self.csv = csv.reset_index()\n        self.split = split\n        self.mode = mode\n        self.transform672 = transforms[0]\n        self.transform768 = transforms[1]\n        self.transform512 = transforms[2]\n\n    def __len__(self):\n        return self.csv.shape[0]\n\n    def __getitem__(self, index):\n        row = self.csv.iloc[index]\n        \n        image = cv2.imread(row.filepath)\n        image = image[:, :, ::-1]\n        \n        res0 = self.transform672(image=image)\n        image0 = res0['image'].astype(np.float32)\n        image0 = image0.transpose(2, 0, 1)        \n\n        res1 = self.transform768(image=image)\n        image1 = res1['image'].astype(np.float32)\n        image1 = image1.transpose(2, 0, 1)    \n        \n        res3 = self.transform512(image=image)\n        image3 = res3['image'].astype(np.float32)        \n        image3 = image3.transpose(2, 0, 1)   \n               \n        \n        if self.mode == 'test':\n            return torch.tensor(image0), torch.tensor(image1) , torch.tensor(image3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if df.shape[0] > 100001: # commit\n    df = df[df.index % 10 == 0].iloc[500:1000].reset_index(drop=True)\n    df_test = df_test.head(101).copy()\n\ndataset_query = LandmarkDataset(df, 'test', 'test')\nquery_loader = torch.utils.data.DataLoader(dataset_query, batch_size=batch_size, num_workers=num_workers)\n\ndataset_test = LandmarkDataset(df_test, 'test', 'test')\ntest_loader = torch.utils.data.DataLoader(dataset_test, batch_size=batch_size, num_workers=num_workers)\n\nprint(len(dataset_query), len(dataset_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_query[0][0].shape, dataset_query[0][1].shape, dataset_query[0][2].shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ArcMarginProduct_subcenter(nn.Module):\n    def __init__(self, in_features, out_features, k=3):\n        super().__init__()\n        self.weight = nn.Parameter(torch.FloatTensor(out_features*k, in_features))\n        self.reset_parameters()\n        self.k = k\n        self.out_features = out_features\n        \n    def reset_parameters(self):\n        stdv = 1. / math.sqrt(self.weight.size(1))\n        self.weight.data.uniform_(-stdv, stdv)\n        \n    def forward(self, features):\n        cosine_all = F.linear(F.normalize(features), F.normalize(self.weight))\n        cosine_all = cosine_all.view(-1, self.out_features, self.k)\n        cosine, _ = torch.max(cosine_all, dim=2)\n        return cosine ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sigmoid = torch.nn.Sigmoid()\nclass Swish(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, i):\n        result = i * sigmoid(i)\n        ctx.save_for_backward(i)\n        return result\n    @staticmethod\n    def backward(ctx, grad_output):\n        i = ctx.saved_variables[0]\n        sigmoid_i = sigmoid(i)\n        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))\n\nclass Swish_module(nn.Module):\n    def forward(self, x):\n        return Swish.apply(x)\n\n    \n \n    \nclass enet_arcface_FINAL(nn.Module):\n\n    def __init__(self, enet_type, out_dim):\n        super(enet_arcface_FINAL, self).__init__()\n        self.enet = geffnet.create_model(enet_type.replace('-', '_'), pretrained=None)\n        self.feat = nn.Linear(self.enet.classifier.in_features, 512)\n        self.swish = Swish_module()\n        self.metric_classify = ArcMarginProduct_subcenter(512, out_dim)\n        self.enet.classifier = nn.Identity()\n \n    def forward(self, x):\n        x = self.enet(x)\n        x = self.swish(self.feat(x))\n        return F.normalize(x), self.metric_classify(x)\n    \n    \nfrom rexnetv1 import ReXNetV1\nfrom resnest.torch import resnest101    \nclass rex20_arcface(nn.Module):\n\n    def __init__(self, enet_type, out_dim, load_pretrained=False):\n        super(rex20_arcface, self).__init__()\n        self.enet = ReXNetV1(width_mult=2.0)\n        if load_pretrained:\n            pretrain_wts = \"/workspace/rexnetv1_2.0x.pth\"            \n            sd = torch.load(pretrain_wts)\n            self.enet.load_state_dict(sd, strict=True)        \n        \n        self.feat = nn.Linear(self.enet.output[1].in_channels, 512)\n        self.swish = Swish_module()\n        self.metric_classify = ArcMarginProduct_subcenter(512, out_dim)\n        self.enet.output = nn.Identity()\n    \n    def forward(self, x):\n        x = self.enet(x)\n        if x.ndim==1: \n            x = x.unsqueeze(0)          \n        x = self.swish(self.feat(x))\n        return F.normalize(x), self.metric_classify(x)    \n    \nclass nest101_arcface(nn.Module):\n\n    def __init__(self, enet_type, out_dim):\n        super(nest101_arcface, self).__init__()\n        self.enet = resnest101(pretrained=False)\n        self.feat = nn.Linear(self.enet.fc.in_features, 512)\n        self.swish = Swish_module()\n        self.metric_classify = ArcMarginProduct_subcenter(512, out_dim)\n        self.enet.fc = nn.Identity()    \n    def forward(self, x):\n        x = self.enet(x)\n        x = self.swish(self.feat(x))\n        return F.normalize(x), self.metric_classify(x)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_model(model, model_file):\n    state_dict = torch.load(model_file)\n    if \"model_state_dict\" in state_dict.keys():\n        state_dict = state_dict[\"model_state_dict\"]\n    state_dict = {k[7:] if k.startswith('module.') else k: state_dict[k] for k in state_dict.keys()}\n#     del state_dict['metric_classify.weight']\n    model.load_state_dict(state_dict, strict=True)\n    print(f\"loaded {model_file}\")\n    model.eval()    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -lrth ../input/b7ns-final-672-300w-f0-load13-load1-14ep/b7ns_final_672_300w_f0_load13_load1_14ep_fold0_ep4.pth\n!ls -lrth ../input/b6ns-final-768-300w-f1-load28-5ep-1e-5/b6ns_final_768_300w_f1_load28_5ep_1e-5_fold1_ep5.pth\n!ls -lrth ../input/b5ns-final-768-300w-f2-load16-20ep/b5ns_final_768_300w_f2_load16_20ep_fold2_ep1.pth\n!ls -lrth ../input/b4ns-final-768-300w-f0-load16-20ep-load1-20ep/b4ns_final_768_300w_f0_load16_20ep_load1_20ep_fold0_ep4.pth\n!ls -lrth ../input/b3ns-final-768-300w-f1-load29-5ep5ep/b3ns_final_768_300w_f1_load29_5ep5ep_fold1_ep5.pth\n!ls -lrth ../input/nest101-final-768-300w-f4-load16-19ep-load1-16ep/nest101_final_768_300w_f4_load16_19ep_load1_16ep_fold4_ep5.pth\n!ls -lrth ../input/rex20-ddp-final-768-300w-f4-35ep-load20resume/rex20_DDP_final_768_300w_f4_35ep_load20resume_fold4_ep31.pth\n!ls -lrth ../input/b6ns-ddp-final-512-300w-f1-40ep/b6ns_DDP_final_512_300w_f1_40ep_fold1_ep36.pth\n!ls -lrth ../input/b5ns-final-768-300w-f2-load33-5ep-3e-5-32g/b5ns_final_768_300w_f2_load33_5ep_3e-5_32G_fold2_ep4.pth","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_b7 = enet_arcface_FINAL('tf_efficientnet_b7_ns', out_dim=out_dim).to(device)\nmodel_b7 = load_model(model_b7, '../input/b7ns-final-672-300w-f0-load13-load1-14ep/b7ns_final_672_300w_f0_load13_load1_14ep_fold0_ep4.pth')\n\nmodel_b6 = enet_arcface_FINAL('tf_efficientnet_b6_ns', out_dim=out_dim).to(device)\nmodel_b6 = load_model(model_b6, '../input/b6ns-final-768-300w-f1-load28-5ep-1e-5/b6ns_final_768_300w_f1_load28_5ep_1e-5_fold1_ep5.pth')\n\nmodel_b5 = enet_arcface_FINAL('tf_efficientnet_b5_ns', out_dim=out_dim).to(device)\nmodel_b5 = load_model(model_b5, '../input/b5ns-final-768-300w-f2-load16-20ep/b5ns_final_768_300w_f2_load16_20ep_fold2_ep1.pth')\n\nmodel_b4 = enet_arcface_FINAL('tf_efficientnet_b4_ns', out_dim=out_dim).to(device)\nmodel_b4 = load_model(model_b4, '../input/b4ns-final-768-300w-f0-load16-20ep-load1-20ep/b4ns_final_768_300w_f0_load16_20ep_load1_20ep_fold0_ep4.pth')\n\nmodel_b3 = enet_arcface_FINAL('tf_efficientnet_b3_ns', out_dim=out_dim).to(device)\nmodel_b3 = load_model(model_b3, '../input/b3ns-final-768-300w-f1-load29-5ep5ep/b3ns_final_768_300w_f1_load29_5ep5ep_fold1_ep5.pth')\n\nmodel_nest101 = nest101_arcface('nest101', out_dim=out_dim).to(device)\nmodel_nest101 = load_model(model_nest101, '../input/nest101-final-768-300w-f4-load16-19ep-load1-16ep/nest101_final_768_300w_f4_load16_19ep_load1_16ep_fold4_ep5.pth')\n\nmodel_rex2 = rex20_arcface('rex2.0', out_dim=out_dim).to(device)\nmodel_rex2 = load_model(model_rex2, '../input/rex20-ddp-final-768-300w-f4-35ep-load20resume/rex20_DDP_final_768_300w_f4_35ep_load20resume_fold4_ep31.pth')\n\nmodel_b6b = enet_arcface_FINAL('tf_efficientnet_b6_ns', out_dim=out_dim).to(device)\nmodel_b6b = load_model(model_b6b, '../input/b6ns-ddp-final-512-300w-f1-40ep/b6ns_DDP_final_512_300w_f1_40ep_fold1_ep36.pth')\n\nmodel_b5b = enet_arcface_FINAL('tf_efficientnet_b5_ns', out_dim=out_dim).to(device)\nmodel_b5b = load_model(model_b5b, '../input/b5ns-final-768-300w-f2-load33-5ep-3e-5-32g/b5ns_final_768_300w_f2_load33_5ep_3e-5_32G_fold2_ep4.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(os.path.join(model_dir, 'idx2landmark_id.pkl'), 'rb') as fp:\n    idx2landmark_id = pickle.load(fp)\n    landmark_id2idx = {idx2landmark_id[idx]: idx for idx in idx2landmark_id.keys()}\n    \npred_mask = pd.Series(df.landmark_id.unique()).map(landmark_id2idx).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TOP_K = 5\nCLS_TOP_K = 5\nif True:\n    with torch.no_grad():\n        feats = []\n        for img0, img1,img3 in tqdm(query_loader): # 672, 768, 512\n            img0 = img0.cuda()\n            img1 = img1.cuda()\n            img3 = img3.cuda()\n            \n            feat_b7,_      = model_b7(img0)\n            feat_b6,_      = model_b6(img1)\n            feat_b5,_      = model_b5(img1)\n            feat_b4,_      = model_b4(img1)\n            feat_b3,_      = model_b3(img1)            \n            feat_nest101,_ = model_nest101(img1)\n            feat_rex2,_    = model_rex2(img1)\n            feat_b6b,_     = model_b6b(img3)\n            feat_b5b,_     = model_b5b(img1)            \n            feat = torch.cat([feat_b7,feat_b6,feat_b5,feat_b4,feat_b3,feat_nest101,feat_rex2,feat_b6b,feat_b5b],dim=1)            \n#             print(feat.shape)\n            feats.append(feat.detach().cpu())\n        feats = torch.cat(feats)\n        feats = feats.cuda()\n        feat = F.normalize(feat)        \n\n        PRODS = []\n        PREDS = []\n        PRODS_M = []\n        PREDS_M = []        \n        for img0, img1,img3 in tqdm(test_loader):\n            img0 = img0.cuda()\n            img1 = img1.cuda()\n            img3 = img3.cuda()\n            \n            probs_m = torch.zeros([4, 81313],device=device)\n            feat_b7,logits_m      = model_b7(img0); probs_m += logits_m\n            feat_b6,logits_m      = model_b6(img1); probs_m += logits_m\n            feat_b5,logits_m      = model_b5(img1); probs_m += logits_m\n            feat_b4,logits_m      = model_b4(img1); probs_m += logits_m\n            feat_b3,logits_m      = model_b3(img1) ; probs_m += logits_m\n            feat_nest101,logits_m = model_nest101(img1); probs_m += logits_m\n            feat_rex2,logits_m    = model_rex2(img1); probs_m += logits_m\n            feat_b6b,logits_m     = model_b6b(img3); probs_m += logits_m\n            feat_b5b,logits_m     = model_b5b(img1) ; probs_m += logits_m\n            feat = torch.cat([feat_b7,feat_b6,feat_b5,feat_b4,feat_b3,feat_nest101,feat_rex2,feat_b6b,feat_b5b],dim=1)\n            feat = F.normalize(feat)\n\n            probs_m = probs_m/9\n            probs_m[:, pred_mask] += 1.0\n            probs_m -= 1.0              \n\n            (values, indices) = torch.topk(probs_m, CLS_TOP_K, dim=1)\n            probs_m = values\n            preds_m = indices              \n            PRODS_M.append(probs_m.detach().cpu())\n            PREDS_M.append(preds_m.detach().cpu())            \n            \n            distance = feat.mm(feats.t())\n            (values, indices) = torch.topk(distance, TOP_K, dim=1)\n            probs = values\n            preds = indices    \n            PRODS.append(probs.detach().cpu())\n            PREDS.append(preds.detach().cpu())\n\n        PRODS = torch.cat(PRODS).numpy()\n        PREDS = torch.cat(PREDS).numpy()\n        PRODS_M = torch.cat(PRODS_M).numpy()\n        PREDS_M = torch.cat(PREDS_M).numpy()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# map both to landmark_id\ngallery_landmark = df['landmark_id'].values\nPREDS = gallery_landmark[PREDS]\nPREDS_M = np.vectorize(idx2landmark_id.get)(PREDS_M)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PREDS.min(), PREDS.max(), PREDS_M.min(), PREDS_M.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PREDS[:3,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PREDS_M[:3,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PRODS[:3,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PRODS_M[:3,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PRODS_F = []\nPREDS_F = []\nfor i in tqdm(range(PREDS.shape[0])):\n    tmp = {}\n    classify_dict = {PREDS_M[i,j] : PRODS_M[i,j] for j in range(CLS_TOP_K)}\n    for k in range(TOP_K):\n        lid = PREDS[i, k]\n        tmp[lid] = tmp.get(lid, 0.) + float(PRODS[i, k]) ** 9 * classify_dict.get(lid,1e-8)**10\n    pred, conf = max(tmp.items(), key=lambda x: x[1])\n    PREDS_F.append(pred)\n    PRODS_F.append(conf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PREDS_F[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PRODS_F[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test['pred_id'] = PREDS_F\ndf_test['pred_conf'] = PRODS_F","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub['landmarks'] = df_test.apply(lambda row: f'{row[\"pred_id\"]} {row[\"pred_conf\"]}', axis=1)\ndf_sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}