{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### What is this code?\n\nThis code estimates landmark IDs and confidences based on feature similarity. The features are extracted from train, validation and test images by using a inference model that has a ResNet-34 as the backbone CNN. The train images includes only landmark images, but validation images includes many non-landmark images. A confidence of a test image is calculated from the similarity with landmark images, and a penalty is derived from the similarity with non-landmark images.\n\nThe estimation algorithm is the same as in the following paper:\n\nSupporting large-scale image recognition with out-of-domain samples  \nChristof Henkel, Philipp Singer  \nhttps://arxiv.org/abs/2010.01650","metadata":{"papermill":{"duration":0.010429,"end_time":"2021-09-24T08:09:32.755344","exception":false,"start_time":"2021-09-24T08:09:32.744915","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pathlib\n\nimport torch\nimport torch.utils.data\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\nimport pandas as pd\n\nimport PIL.Image\nimport albumentations.pytorch\nimport cv2\nimport matplotlib.pyplot as plt\n\nfrom tqdm.notebook import tqdm\nfrom typing import List, Tuple\n\nIMAGE_SIZE = 320  # (2021/09/24 5:00AM) Updated.\nBATCH_SIZE = 512\n\nMODEL_FILE = pathlib.Path('../input/google-landmark-2021-validation/model.pth')\nTRAIN_LABEL_FILE = pathlib.Path('train.csv')\nTRAIN_IMAGE_DIR = pathlib.Path('../input/landmark-recognition-2021/train')\nVALID_LABEL_FILE = pathlib.Path('valid.csv')\nVALID_IMAGE_DIR = pathlib.Path('../input/google-landmark-2021-validation/valid')\nTEST_LABEL_FILE = pathlib.Path('../input/landmark-recognition-2021/sample_submission.csv')\nTEST_IMAGE_DIR = pathlib.Path('../input/landmark-recognition-2021/test')","metadata":{"papermill":{"duration":6.260797,"end_time":"2021-09-24T08:09:39.027083","exception":false,"start_time":"2021-09-24T08:09:32.766286","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-07T09:09:44.428347Z","iopub.execute_input":"2022-06-07T09:09:44.42912Z","iopub.status.idle":"2022-06-07T09:09:49.160147Z","shell.execute_reply.started":"2022-06-07T09:09:44.428996Z","shell.execute_reply":"2022-06-07T09:09:49.158866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Elimination of public training images\n(This code is updated at 2021/09/24 5:00AM GMT)\n\nIn order to reduce the processing time, only a subset of public training images are used for the feature extraction at saving the code.\nAt the submission, all private trainig images are used.","metadata":{"papermill":{"duration":0.008619,"end_time":"2021-09-24T08:09:39.044784","exception":false,"start_time":"2021-09-24T08:09:39.036165","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/landmark-recognition-2021/train.csv')\n\nif len(train_df) == 1580470:\n    records = {}\n\n    for image_id, landmark_id in train_df.values:\n        if landmark_id in records:\n            records[landmark_id].append(image_id)\n        else:\n            records[landmark_id] = [image_id]\n        \n    image_ids = []\n    landmark_ids = []\n\n    for landmark_id, img_ids in records.items():\n        num = min(len(img_ids), 2)\n        image_ids.extend(records[landmark_id][:num])\n        landmark_ids.extend([landmark_id] * num)\n\n    train_df = pd.DataFrame({'id': image_ids, 'landmark_id': landmark_ids})\n\ntrain_df.to_csv(TRAIN_LABEL_FILE, index=False)\ntrain_df","metadata":{"papermill":{"duration":4.352159,"end_time":"2021-09-24T08:09:43.40567","exception":false,"start_time":"2021-09-24T08:09:39.053511","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-07T09:09:49.16188Z","iopub.execute_input":"2022-06-07T09:09:49.162975Z","iopub.status.idle":"2022-06-07T09:09:54.420271Z","shell.execute_reply.started":"2022-06-07T09:09:49.162922Z","shell.execute_reply":"2022-06-07T09:09:54.419389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### List of non-landmark images\n(This code is added at 2021/09/20 1:00AM GMT)","metadata":{"papermill":{"duration":0.009144,"end_time":"2021-09-24T08:09:43.424616","exception":false,"start_time":"2021-09-24T08:09:43.415472","status":"completed"},"tags":[]}},{"cell_type":"code","source":"valid_df = pd.read_csv('../input/google-landmark-2021-validation/valid.csv')\nvalid_df = valid_df[valid_df['landmark_id'] == -1]\nvalid_df.to_csv(VALID_LABEL_FILE, index=False)\nvalid_df","metadata":{"papermill":{"duration":0.343822,"end_time":"2021-09-24T08:09:43.777801","exception":false,"start_time":"2021-09-24T08:09:43.433979","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-07T09:09:54.422002Z","iopub.execute_input":"2022-06-07T09:09:54.422515Z","iopub.status.idle":"2022-06-07T09:09:54.635349Z","shell.execute_reply.started":"2022-06-07T09:09:54.42246Z","shell.execute_reply":"2022-06-07T09:09:54.633468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Class and Functions for feature extraction","metadata":{"papermill":{"duration":0.009463,"end_time":"2021-09-24T08:09:43.797384","exception":false,"start_time":"2021-09-24T08:09:43.787921","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class Dataset(torch.utils.data.Dataset):\n    def __init__(self, label_file: pathlib.Path, image_dir: pathlib.Path) -> None:\n        super().__init__()\n        self.files = [\n            image_dir / n[0] / n[1] / n[2] / f'{n}.jpg'\n            for n in pd.read_csv(label_file)['id'].values]\n        \n        self.transformer = albumentations.Compose([\n            albumentations.SmallestMaxSize(IMAGE_SIZE, interpolation=cv2.INTER_CUBIC),\n            albumentations.CenterCrop(IMAGE_SIZE, IMAGE_SIZE),\n            albumentations.Normalize(),\n            albumentations.pytorch.ToTensorV2(),\n        ])\n\n    def __len__(self) -> int:\n        return len(self.files)\n\n    def __getitem__(self, index: int) -> Tuple[str, torch.Tensor]:\n        path = self.files[index]\n        image = PIL.Image.open(self.files[index])\n        image = self.transformer(image=np.array(image))['image']\n\n        return path.name[:-4], image","metadata":{"papermill":{"duration":0.020349,"end_time":"2021-09-24T08:09:43.827305","exception":false,"start_time":"2021-09-24T08:09:43.806956","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-07T09:09:54.636353Z","iopub.status.idle":"2022-06-07T09:09:54.636782Z","shell.execute_reply.started":"2022-06-07T09:09:54.636587Z","shell.execute_reply":"2022-06-07T09:09:54.636607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.no_grad()\ndef get_features(\n    model: nn.Module,\n    label_file: pathlib.Path,\n    image_dir: pathlib.Path,\n) -> Tuple[List[str], torch.Tensor]:\n    loader = torch.utils.data.DataLoader(\n        Dataset(label_file, image_dir),\n        batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\n    model = model.cuda()\n    model.eval()\n    \n    all_names = []\n    all_features = []\n\n    for names, images in tqdm(loader, desc=image_dir.name):\n        images = images.cuda()\n        features = model(images)\n        all_features.append(features)\n        all_names.extend(names)\n\n    return all_names, F.normalize(torch.cat(all_features, dim=0))","metadata":{"papermill":{"duration":0.01827,"end_time":"2021-09-24T08:09:43.855174","exception":false,"start_time":"2021-09-24T08:09:43.836904","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-07T09:09:54.638726Z","iopub.status.idle":"2022-06-07T09:09:54.639111Z","shell.execute_reply.started":"2022-06-07T09:09:54.638933Z","shell.execute_reply":"2022-06-07T09:09:54.63895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_similarity(model: nn.Module)-> Tuple[List[str], List[str]]:\n    # features\n    train_names, train_features = get_features(\n        model, TRAIN_LABEL_FILE, TRAIN_IMAGE_DIR)    \n    _, valid_features = get_features(\n        model, VALID_LABEL_FILE, VALID_IMAGE_DIR)\n    test_names, test_features = get_features(\n        model, TEST_LABEL_FILE, TEST_IMAGE_DIR)\n\n    # penalties\n    train_penalties_list = []\n    for i in range(0, train_features.shape[0], 128):\n        x = torch.mm(train_features[i:i + 128], valid_features.T)\n        x = torch.topk(x, k=5)[0].mean(dim=1)\n        train_penalties_list.append(x)\n    train_penalties = torch.cat(train_penalties_list, dim=0)\n\n    test_penalties_list = []\n    for i in range(0, test_features.shape[0], 128):\n        x = torch.mm(test_features[i:i + 128], valid_features.T)\n        x = torch.topk(x, k=10)[0].mean(dim=1)\n        test_penalties_list.append(x)\n    test_penalties = torch.cat(test_penalties_list, dim=0)\n\n    # neighbors\n    submit_ids = []\n    submit_landmark_ids = []\n    submit_confidences = []\n    \n    train_df = pd.read_csv(TRAIN_LABEL_FILE)\n    idmap = {n: v for n, v in train_df.values}\n\n    for i in range(0, test_features.shape[0], 128):\n        x = torch.mm(test_features[i:i + 128], train_features.T)\n        x -= train_penalties[None, :]\n        values, indexes = torch.topk(x, k=3)\n        \n        submit_ids.extend(test_names[i:i + 128])\n\n        for idxs, vals, penalty in zip(indexes, values, test_penalties[i:i + 128]):\n            scores = {}\n            for idx, val in zip(idxs, vals):\n                landmark_id = idmap[train_names[idx]]\n                if landmark_id in scores:\n                    scores[landmark_id] += float(val)\n                else:\n                    scores[landmark_id] = float(val)\n                    \n            landmark_id, confidence = max(\n                [(k, v) for k, v in scores.items()], key=lambda x: x[1])\n            submit_landmark_ids.append(landmark_id)\n            submit_confidences.append(confidence - penalty)\n\n    # standardize confidence values\n    max_conf = max(submit_confidences)\n    min_conf = min(submit_confidences)\n    submit_confidences = [\n        (v - min_conf) / (max_conf - min_conf) for v in submit_confidences]\n    \n    # make values for 'landmark' column\n    submit_landmarks = [\n        f'{i} {c:.8f}' for i, c in zip(submit_landmark_ids, submit_confidences)]\n    \n    return submit_ids, submit_landmarks","metadata":{"papermill":{"duration":0.037846,"end_time":"2021-09-24T08:09:43.903891","exception":false,"start_time":"2021-09-24T08:09:43.866045","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-07T09:09:54.640487Z","iopub.status.idle":"2022-06-07T09:09:54.641087Z","shell.execute_reply.started":"2022-06-07T09:09:54.640889Z","shell.execute_reply":"2022-06-07T09:09:54.640911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inference and Submission","metadata":{"papermill":{"duration":0.009333,"end_time":"2021-09-24T08:09:43.922764","exception":false,"start_time":"2021-09-24T08:09:43.913431","status":"completed"},"tags":[]}},{"cell_type":"code","source":"model = torch.jit.load(str(MODEL_FILE))\nprint(model)\nsubmit_ids, submit_landmarks = get_similarity(model)\nsubmit_df = pd.DataFrame({'id': submit_ids, 'landmarks': submit_landmarks})\nsubmit_df.to_csv('submission.csv', index=False)","metadata":{"papermill":{"duration":3278.675443,"end_time":"2021-09-24T09:04:22.607713","exception":false,"start_time":"2021-09-24T08:09:43.93227","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-07T09:09:54.64228Z","iopub.status.idle":"2022-06-07T09:09:54.643072Z","shell.execute_reply.started":"2022-06-07T09:09:54.642854Z","shell.execute_reply":"2022-06-07T09:09:54.642874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Check the submission\n\nFollowing code shows the inference results. Each figure shows a test image (LEFT), the estimated landmark image (RIGHT), landmark ID and confidence (TITLE).","metadata":{"papermill":{"duration":0.011204,"end_time":"2021-09-24T09:04:22.630541","exception":false,"start_time":"2021-09-24T09:04:22.619337","status":"completed"},"tags":[]}},{"cell_type":"code","source":"submit_df = pd.read_csv('submission.csv')\nsubmit_df['landmark_id'] = submit_df['landmarks'].apply(lambda x: int(x.split()[0]))\nsubmit_df['confidence'] = submit_df['landmarks'].apply(lambda x: float(x.split()[1]))\ntrain_df = pd.read_csv(TRAIN_LABEL_FILE)\n\ndef get_image(path, name):\n    img = PIL.Image.open(path / name[0] / name[1] / name[2] / f'{name}.jpg')\n    if img.width > img.height:\n        img = img.resize((256, round(img.height / img.width * 256)))\n        new_img = PIL.Image.new(img.mode, (256, 256), (0, 0, 0))\n        new_img.paste(img, (0, (256 - img.height) // 2))\n    else:\n        img = img.resize((round(img.width / img.height * 256), 256))\n        new_img = PIL.Image.new(img.mode, (256, 256), (0, 0, 0))\n        new_img.paste(img, ((256 - img.width) // 2, 2))\n    return np.array(new_img)\n\nrows = 10\nfig = plt.figure(figsize=(15, 4 * rows))\nfor r in range(rows):\n    for c in range(3):\n        i = r * 3 + c\n        test_name, _, label, conf = submit_df.iloc[i].values\n        test_image = get_image(TEST_IMAGE_DIR, test_name)\n        train_name = train_df.query(f'landmark_id == {label}').iloc[0]['id']\n        train_image = get_image(TRAIN_IMAGE_DIR, train_name)\n        image = np.concatenate([test_image, train_image], axis=1)\n    \n        ax = fig.add_subplot(rows, 3, i + 1)        \n        ax.set_title(f'Label={label}, Confidence={conf:.2f}')\n        ax.axis('off')\n        ax.imshow(image)\nfig.tight_layout()","metadata":{"papermill":{"duration":4.658558,"end_time":"2021-09-24T09:04:27.300387","exception":false,"start_time":"2021-09-24T09:04:22.641829","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-07T09:09:54.644499Z","iopub.status.idle":"2022-06-07T09:09:54.644851Z","shell.execute_reply.started":"2022-06-07T09:09:54.64469Z","shell.execute_reply":"2022-06-07T09:09:54.644706Z"},"trusted":true},"execution_count":null,"outputs":[]}]}