{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Google Landmark Recognition Challenge 2020\nSimplified image similarity ranking and re-ranking implementation with:\n* EfficientNetB0 backbone for global feature similarity search\n* DELF module for local feature reranking\n\nReference papers:\n* 2020 Recognition challenge winner: https://arxiv.org/abs/2010.01650\n* 2019 Recognition challend 2nd place: https://arxiv.org/abs/1906.03990","metadata":{}},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:29:48.871825Z","iopub.execute_input":"2022-07-26T06:29:48.872103Z","iopub.status.idle":"2022-07-26T06:29:49.631141Z","shell.execute_reply.started":"2022-07-26T06:29:48.872021Z","shell.execute_reply":"2022-07-26T06:29:49.630394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing libraries\nimport os\nimport cv2\nimport shutil\nimport numpy as np\nimport pandas as pd\nfrom scipy import spatial\nfrom tqdm import tqdm\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:29:49.632935Z","iopub.execute_input":"2022-07-26T06:29:49.633135Z","iopub.status.idle":"2022-07-26T06:29:56.008582Z","shell.execute_reply.started":"2022-07-26T06:29:49.633111Z","shell.execute_reply":"2022-07-26T06:29:56.007767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Directories and file paths\nTRAIN_DIR = '../input/landmark-recognition-2020/train'\nTRAIN_CSV = '../input/landmark-recognition-2020/train.csv'\ntrain_df = pd.read_csv(TRAIN_CSV)\n\nTRAIN_PATHS = [os.path.join(TRAIN_DIR, f'{img[0]}/{img[1]}/{img[2]}/{img}.jpg') for img in train_df['id']]\ntrain_df['path'] = TRAIN_PATHS\n\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:29:56.009832Z","iopub.execute_input":"2022-07-26T06:29:56.010083Z","iopub.status.idle":"2022-07-26T06:30:00.968663Z","shell.execute_reply.started":"2022-07-26T06:29:56.010050Z","shell.execute_reply":"2022-07-26T06:30:00.967839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Subsetting\ntrain_df_grouped = pd.DataFrame(train_df.landmark_id.value_counts())\ntrain_df_grouped.reset_index(inplace=True)\ntrain_df_grouped.columns = ['landmark_id','count']\n\n# Selected landmarks based on inclass frequency\nselected_landmarks = train_df_grouped[(train_df_grouped['count'] <= 155) & (train_df_grouped['count'] >= 150)]\n\ntrain_df_sub = train_df[train_df['landmark_id'].isin(selected_landmarks['landmark_id'])]\nnew_id = []\ncurrent_id = 0\nprevious_id = int(train_df_sub.head(1)['landmark_id'])\nfor landmark_id in train_df_sub['landmark_id']:\n    if landmark_id == previous_id:\n        new_id.append(current_id)\n    else:\n        current_id += 1\n        new_id.append(current_id)\n        previous_id = landmark_id\n\ntrain_df_sub['new_id'] = new_id\n\nNUM_CLASSES = train_df_sub['landmark_id'].nunique()\n\nprint(f\"Unique classes found: {NUM_CLASSES}\")\ntrain_df_sub","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:30:00.970938Z","iopub.execute_input":"2022-07-26T06:30:00.972552Z","iopub.status.idle":"2022-07-26T06:30:01.190159Z","shell.execute_reply.started":"2022-07-26T06:30:00.972519Z","shell.execute_reply":"2022-07-26T06:30:01.189420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training and validation splits\n# 90/10 stratified split for training and validation\nX_train, X_val, y_train, y_val = train_test_split(train_df_sub[['id', 'path']], train_df_sub['new_id'],\n                                                  train_size = 0.9,\n                                                  random_state = 123,\n                                                  shuffle = True,\n                                                  stratify = train_df_sub['new_id'])\n\n# Held-out test set for inference\n# Further 95/5 split -> 5% of original training set left for test set\nX_train, X_test, y_train, y_test = train_test_split(X_train, y_train,\n                                                   train_size = 0.95,\n                                                   random_state = 123,\n                                                   shuffle = True,\n                                                   stratify = y_train)\n\nassert X_train.shape[0] + X_val.shape[0] + X_test.shape[0] == train_df_sub.shape[0]\n\nprint(f\"Training data shape: {X_train.shape}\")\nprint(f\"Training label shape: {y_train.shape}\")\nprint(f\"Validation data shape: {X_val.shape}\")\nprint(f\"Validation label shape: {y_val.shape}\")\nprint(f\"Test data shape: {X_test.shape}\")\nprint(f\"Test label shape: {y_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:30:01.191571Z","iopub.execute_input":"2022-07-26T06:30:01.191798Z","iopub.status.idle":"2022-07-26T06:30:01.219223Z","shell.execute_reply.started":"2022-07-26T06:30:01.191767Z","shell.execute_reply":"2022-07-26T06:30:01.218564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Unique classes on y_train: {y_train.nunique()}\")\nprint(f\"Unique classes on y_val: {y_val.nunique()}\")\nprint(f\"Unique classes on y_test: {y_test.nunique()}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:30:01.220584Z","iopub.execute_input":"2022-07-26T06:30:01.221044Z","iopub.status.idle":"2022-07-26T06:30:01.226632Z","shell.execute_reply.started":"2022-07-26T06:30:01.221010Z","shell.execute_reply":"2022-07-26T06:30:01.225839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Classes distribution on training, validation and test sets\nplt.figure(figsize = (10, 3))\nax = sns.histplot(y_train, bins=75, kde = True)\nax.set_title('Distribution of Landmarks on training set')\nplt.tight_layout()\n\nplt.figure(figsize = (10, 3))\nax = sns.histplot(y_val, bins=75, kde = True)\nax.set_title('Distribution of Landmarks on validation set')\nplt.tight_layout()\n\nplt.figure(figsize = (10, 3))\nax = sns.histplot(y_test, bins=75, kde = True)\nax.set_title('Distribution of Landmarks on test set')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:30:01.228575Z","iopub.execute_input":"2022-07-26T06:30:01.229262Z","iopub.status.idle":"2022-07-26T06:30:02.405443Z","shell.execute_reply.started":"2022-07-26T06:30:01.229223Z","shell.execute_reply":"2022-07-26T06:30:02.404066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r train_sub, test_sub, val_sub && ls","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:30:02.406794Z","iopub.execute_input":"2022-07-26T06:30:02.407072Z","iopub.status.idle":"2022-07-26T06:30:03.093037Z","shell.execute_reply.started":"2022-07-26T06:30:02.407038Z","shell.execute_reply":"2022-07-26T06:30:03.091845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating image directories for classes subset\nNEW_BASE_DIR = \"/kaggle/working\"\n\n# Training set directory\nfor file, path, landmark in tqdm(zip(X_train['id'], X_train['path'], y_train)):\n    dir = f\"{NEW_BASE_DIR}/train_sub/{str(landmark)}\"\n    os.makedirs(dir, exist_ok = True)\n    fname = f\"{file}.jpg\"\n    shutil.copyfile(src = path, dst = f\"{dir}/{fname}\")\n\n# Validation set directory    \nfor file, path, landmark in tqdm(zip(X_val['id'], X_val['path'], y_val)):\n    dir = f\"{NEW_BASE_DIR}/val_sub/{str(landmark)}\"\n    os.makedirs(dir, exist_ok = True)\n    fname = f\"{file}.jpg\"\n    shutil.copyfile(src = path, dst = f\"{dir}/{fname}\")\n\n# Training set directory\nfor file, path, landmark in tqdm(zip(X_test['id'], X_test['path'], y_test)):\n    dir = f\"{NEW_BASE_DIR}/test_sub/{str(landmark)}\"\n    os.makedirs(dir, exist_ok = True)\n    fname = f\"{file}.jpg\"\n    shutil.copyfile(src = path, dst = f\"{dir}/{fname}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:30:03.096650Z","iopub.execute_input":"2022-07-26T06:30:03.096945Z","iopub.status.idle":"2022-07-26T06:32:45.152522Z","shell.execute_reply.started":"2022-07-26T06:30:03.096906Z","shell.execute_reply":"2022-07-26T06:32:45.151627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:32:45.155799Z","iopub.execute_input":"2022-07-26T06:32:45.156200Z","iopub.status.idle":"2022-07-26T06:32:45.840760Z","shell.execute_reply.started":"2022-07-26T06:32:45.156161Z","shell.execute_reply":"2022-07-26T06:32:45.839767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cd train_sub && ls","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:32:45.843547Z","iopub.execute_input":"2022-07-26T06:32:45.843800Z","iopub.status.idle":"2022-07-26T06:32:46.517334Z","shell.execute_reply.started":"2022-07-26T06:32:45.843774Z","shell.execute_reply":"2022-07-26T06:32:46.516340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating tensorflow tf.data.Dataset\nfrom tensorflow.keras.utils import image_dataset_from_directory\n\nIMG_SIZE = 224\nBATCH_SIZE = 16\n\nprint(\"Building training dataset...\")\n# Training tf.data.Dataset\ntrain_ds = image_dataset_from_directory(f\"{NEW_BASE_DIR}/train_sub\",\n                                        label_mode = 'int',\n                                        shuffle = True,\n                                        image_size = (IMG_SIZE, IMG_SIZE),\n                                        batch_size = BATCH_SIZE)\n\nprint(\"Building validation dataset...\")\n# Validation tf.data.Dataset\nval_ds = image_dataset_from_directory(f\"{NEW_BASE_DIR}/val_sub\",\n                                        label_mode = 'int',\n                                        shuffle = True,\n                                        image_size = (IMG_SIZE, IMG_SIZE),\n                                        batch_size = BATCH_SIZE)\n\nprint(\"Building test dataset...\")\n# Test tf.data.Dataset\ntest_ds = image_dataset_from_directory(f\"{NEW_BASE_DIR}/test_sub\",\n                                        label_mode = 'int',\n                                        shuffle = True,\n                                        image_size = (IMG_SIZE, IMG_SIZE),\n                                        batch_size = BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:32:46.519367Z","iopub.execute_input":"2022-07-26T06:32:46.519627Z","iopub.status.idle":"2022-07-26T06:32:49.802638Z","shell.execute_reply.started":"2022-07-26T06:32:46.519599Z","shell.execute_reply":"2022-07-26T06:32:49.801957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualizing a random batch from training dataset\nfor data_batch, labels_batch in train_ds.take(1):\n    ncols = 4\n    nrows = int(data_batch.shape[0]/ncols)\n    fig, ax = plt.subplots(nrows = nrows, ncols = ncols, figsize=(10, 11),\n                           sharex = True, sharey = True)\n    img_counter = 0\n    for image, label in zip(data_batch, labels_batch):\n        axi = ax.flat[img_counter]\n        axi.imshow(image/255.)\n        label = label.numpy()\n#         axi.set_title(np.where(label == 1)[0])\n        axi.set_title(label)\n        img_counter += 1\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:32:49.803704Z","iopub.execute_input":"2022-07-26T06:32:49.803924Z","iopub.status.idle":"2022-07-26T06:32:52.182790Z","shell.execute_reply.started":"2022-07-26T06:32:49.803892Z","shell.execute_reply":"2022-07-26T06:32:52.181252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####### ALTERNATIVE CODE FOR UNBATCHED DATASET #######\n# ncols = 4\n# nrows = 4\n# fig, ax = plt.subplots(nrows = nrows, ncols = ncols, figsize=(10, 11),\n#                        sharex = True, sharey = True)\n# img_counter = 0\n# for image, label in train_ds.take(16):\n#     axi = ax.flat[img_counter]\n#     axi.imshow(image[0]/255.)\n#     label = label.numpy()\n# #         axi.set_title(np.where(label == 1)[0])\n#     axi.set_title(label)\n#     img_counter += 1\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:32:52.183810Z","iopub.execute_input":"2022-07-26T06:32:52.184094Z","iopub.status.idle":"2022-07-26T06:32:52.188382Z","shell.execute_reply.started":"2022-07-26T06:32:52.184052Z","shell.execute_reply":"2022-07-26T06:32:52.187637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining a data augmentation stage\nimg_augmentation = tf.keras.Sequential(\n    # [layers.RandomFlip(\"horizontal\"),\n    [layers.RandomTranslation(height_factor = 0.1, width_factor = 0.1),\n     layers.RandomRotation(0.02),\n     layers.RandomZoom(0.2)],\n     name = \"img_augmentation\",\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:32:52.189836Z","iopub.execute_input":"2022-07-26T06:32:52.190263Z","iopub.status.idle":"2022-07-26T06:32:52.232333Z","shell.execute_reply.started":"2022-07-26T06:32:52.190227Z","shell.execute_reply":"2022-07-26T06:32:52.231712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Displaying variations of a randomly augmented training image\nplt.figure(figsize=(9, 9))\nfor image, label in train_ds.take(1):\n    for i in range(9):\n        ax = plt.subplot(3, 3, i + 1)\n        augmented_image = img_augmentation(image, training = True)\n        plt.imshow(augmented_image[15].numpy().astype(\"uint8\"))\n        plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:32:52.233571Z","iopub.execute_input":"2022-07-26T06:32:52.233823Z","iopub.status.idle":"2022-07-26T06:32:53.423814Z","shell.execute_reply.started":"2022-07-26T06:32:52.233778Z","shell.execute_reply":"2022-07-26T06:32:53.420768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####### ALTERNATIVE CODE FOR UNBATCHED DATASET #######\n# # Displaying variations of a randomly augmented training image\n# plt.figure(figsize=(9, 9))\n# for image, label in train_ds.take(16):\n#     for i in range(9):\n#         ax = plt.subplot(3, 3, i + 1)\n#         augmented_image = img_augmentation(image[0], training = True)\n#         plt.imshow(augmented_image.numpy().astype(\"uint8\"))\n#         plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:32:53.425273Z","iopub.execute_input":"2022-07-26T06:32:53.425760Z","iopub.status.idle":"2022-07-26T06:32:53.429772Z","shell.execute_reply.started":"2022-07-26T06:32:53.425722Z","shell.execute_reply":"2022-07-26T06:32:53.429050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.applications import EfficientNetB0\n\nMODELS_DIR = f\"{NEW_BASE_DIR}/models\"\n\nos.makedirs(MODELS_DIR, exist_ok = True)\n\n# Model instantiator\ndef build_model(num_classes = None):\n    inputs = keras.Input(shape = (IMG_SIZE, IMG_SIZE, 3))\n    x = img_augmentation(inputs)\n    # EfficientNetB0 backbone\n    model = EfficientNetB0(input_tensor = x,\n                           weights = 'imagenet',\n                           include_top = False,\n                           drop_connect_rate = DROP_CONNECT_RATE)\n    \n    # Freeze pretrained weights\n    model.trainable = False\n    \n    # Rebuild top\n    x = layers.GlobalAveragePooling2D(name = \"avg_pool\")(model.output)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(TOP_DROPOUT_RATE, name = \"top_dropout\")(x)\n    \n    # Embedding\n    embedding = layers.Dense(512, name = \"embedding_512\")(x)\n    outputs = layers.Dense(num_classes, activation = \"softmax\", name = \"softmax\")(embedding)\n    \n    # Compile\n    model = tf.keras.Model(inputs, outputs, name = \"EfficientNetB0\")\n    optimizer = tf.keras.optimizers.Adam(learning_rate = ADAM_LR)\n    model.compile(optimizer = optimizer,\n                 loss = \"sparse_categorical_crossentropy\",\n                 metrics = [\"accuracy\"])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:32:53.431038Z","iopub.execute_input":"2022-07-26T06:32:53.431822Z","iopub.status.idle":"2022-07-26T06:32:53.444745Z","shell.execute_reply.started":"2022-07-26T06:32:53.431776Z","shell.execute_reply":"2022-07-26T06:32:53.443934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training history visualization\ndef plot_hist(hist):\n    plt.plot(hist.history[\"accuracy\"])\n    plt.plot(hist.history[\"val_accuracy\"])\n    plt.title(\"model accuracy\")\n    plt.ylabel(\"accuracy\")\n    plt.xlabel(\"epoch\")\n    plt.legend([\"train\", \"validation\"], loc=\"upper left\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:32:53.456609Z","iopub.execute_input":"2022-07-26T06:32:53.456868Z","iopub.status.idle":"2022-07-26T06:32:53.464201Z","shell.execute_reply.started":"2022-07-26T06:32:53.456834Z","shell.execute_reply":"2022-07-26T06:32:53.463544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Instantiating model\n# Hyperparameters\nDROP_CONNECT_RATE = 0.2 # Dropout rate for stochastic depth on EfficientNet\nTOP_DROPOUT_RATE = 0.2  # Top dropout\nINIT_LR = 5e-3          # Initial learning rate\nEPOCHS = 20\n# Adam optimizer learning rate schedule\nADAM_LR = tf.keras.optimizers.schedules.ExponentialDecay(\n    INIT_LR,\n    decay_steps=100,\n    decay_rate=0.96,\n    staircase=True)\n\nmodel = build_model(num_classes = NUM_CLASSES)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:32:53.465558Z","iopub.execute_input":"2022-07-26T06:32:53.465856Z","iopub.status.idle":"2022-07-26T06:32:55.689121Z","shell.execute_reply.started":"2022-07-26T06:32:53.465819Z","shell.execute_reply":"2022-07-26T06:32:55.688439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:32:55.690513Z","iopub.execute_input":"2022-07-26T06:32:55.690754Z","iopub.status.idle":"2022-07-26T06:32:56.392495Z","shell.execute_reply.started":"2022-07-26T06:32:55.690721Z","shell.execute_reply":"2022-07-26T06:32:56.391685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training embedding layer\nmodel_file_path = os.path.join(MODELS_DIR, \"EfficientNetB0_softmax.keras\")\ncallbacks = [\n    keras.callbacks.ModelCheckpoint(model_file_path,\n                                    save_best_only=True,\n                                    monitor = \"val_accuracy\"),\n    keras.callbacks.EarlyStopping(patience = 2,\n                                  monitor = \"val_accuracy\")]\n\nhist = model.fit(train_ds,\n                 epochs = EPOCHS,\n                 validation_data = val_ds,\n                 shuffle = 'batch',\n                 callbacks = callbacks)\n\nplot_hist(hist)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:32:56.396048Z","iopub.execute_input":"2022-07-26T06:32:56.396272Z","iopub.status.idle":"2022-07-26T06:43:31.978128Z","shell.execute_reply.started":"2022-07-26T06:32:56.396247Z","shell.execute_reply":"2022-07-26T06:43:31.977327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluating best model\nmodel = keras.models.load_model(model_file_path)\nprint(\"Predictions on validation set...\")\nprint(f\"Validation accuracy: {model.evaluate(val_ds)[1]*100:.2f} %\")\nprint(\"Predictions on test set...\")\nprint(f\"Test accuracy: {model.evaluate(test_ds)[1]*100:.2f} %\")","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:43:31.979429Z","iopub.execute_input":"2022-07-26T06:43:31.979861Z","iopub.status.idle":"2022-07-26T06:43:41.958695Z","shell.execute_reply.started":"2022-07-26T06:43:31.979823Z","shell.execute_reply":"2022-07-26T06:43:41.957458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cosine Similarity\nPairwise query: key search for similarity candidates. In the following example:\n* Query images: validation set\n* Key images: training set","metadata":{}},{"cell_type":"code","source":"# Auxiliar functions\n# Load image\ndef get_image(path, resize = False, reshape = False, target_size = None):\n    img = cv2.imread(path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    if resize:\n        img = cv2.resize(img, dsize = (target_size, target_size))\n    if reshape:\n        img = tf.reshape(img, [1, target_size, target_size, 3])\n    return img\n\n# Get landmark samples\ndef get_landmark(landmark_id, samples = 16):\n    nrows = samples // 4\n    random_imgs = np.random.choice(train_df_sub[train_df_sub['new_id'] == landmark_id].index, samples, replace = False)\n    plt.figure(figsize = (12, 10))\n    for i, img in enumerate(train_df_sub.loc[random_imgs, :].values):\n        ax = plt.subplot(nrows, 4, i + 1)\n        plt.imshow(get_image(img[2]))\n        plt.title(f\"{img[0]}\")\n        plt.suptitle(f\"Samples of landmark {landmark_id}\", fontsize = 14, y = 0.94, weight = \"bold\")\n        plt.axis(\"off\")\n\n# Get image embeddings\ndef get_embeddings(model, image_paths, input_size, as_df = True):\n    embeddings = {}\n    embeddings['images_paths'] = []\n    embeddings['embedded_images'] = []\n    \n    target_dir = os.path.split(os.path.split(image_paths[0])[0])[0]\n    \n    print(f\"Retrieving embeddings for {target_dir} with {model.name}...\")\n    for image_path in tqdm(image_paths):\n        embeddings['images_paths'].append(image_path)\n        embedded_image = model.predict(get_image(image_path,\n                                                 resize = True,\n                                                 reshape = True,\n                                                 target_size = input_size))\n        embeddings['embedded_images'].append(embedded_image)\n    \n    if as_df:\n        embeddings = pd.DataFrame(embeddings)\n    \n    return embeddings\n\n# Get similarities between query key pair\ndef get_similarities(query, key):\n    '''\n    Get cosine similarity matrix between query and key pairs\n    Arguments:\n    query, key: embedded images\n    '''\n    query_array = np.stack(query.tolist()).reshape(query.shape[0],\n                                                   query[0].shape[1])\n    key_array = np.stack(key.tolist()).reshape(key.shape[0],\n                                               key[0].shape[1])\n    \n    # Initializing similarity matrix\n    similarity = np.zeros((query_array.shape[0], key_array.shape[0]))\n    \n    # Getting pairwise similarities\n    print(f\"Getting pairwise {query_array.shape[0]} query: {key_array.shape[0]} key similarities...\")\n    for query_index in tqdm(range(query_array.shape[0])):\n        similarity[query_index] = 1 - spatial.distance.cdist(query_array[np.newaxis, query_index, :],\n                                                             key_array,\n                                                             'cosine')[0]\n    return similarity\n\n# Plot top ranked images\ndef plot_similar(similar_imgs, img_paths):\n    '''\n    Plot top N similar samples from similarity index\n    '''\n    plt.figure(figsize = (18, 6))\n    nrows = similar_imgs.shape[0]//5\n    for i, img in enumerate(similar_imgs):\n        ax = plt.subplot(nrows, 5, i + 1)\n        plt.imshow(get_image(img_paths[img]))\n        plt.title(f\"Landmark id: {os.path.split(os.path.split(img_paths[img])[0])[1]}\")\n        plt.axis(\"off\")\n\n\n# Aggregate query and top similar plots\ndef query_top(image_index, top_n = 5, figsize = (6, 6), reranked = None):\n    '''\n    Plot top N similar samples against queried image\n    If reranked, provide reranked dataframe with ['top_similar'] index reordered by reranked confidence\n    '''\n    image_id = os.path.split(val_embeddings['images_paths'][image_index])[1]\n    query_landmark_id = os.path.split(os.path.split(val_embeddings['images_paths'][image_index])[0])[1]\n    \n    if type(reranked) == pd.core.frame.DataFrame:\n        similar_n = reranked['top_similar'][:top_n]\n    else:\n        similar_n = np.argsort(val_train_similarity[image_index])[::-1][:top_n]\n                \n    print(f\"Queried image: {image_id}\")\n    plt.figure(figsize = figsize)\n    plt.imshow(get_image(val_embeddings['images_paths'][image_index]))\n    plt.title(f\"Landmark id: {query_landmark_id}\")\n    plt.axis(\"off\")\n    plot_similar(similar_n, train_embeddings['images_paths'])","metadata":{"execution":{"iopub.status.busy":"2022-07-26T08:00:13.125218Z","iopub.execute_input":"2022-07-26T08:00:13.125503Z","iopub.status.idle":"2022-07-26T08:00:13.158750Z","shell.execute_reply.started":"2022-07-26T08:00:13.125473Z","shell.execute_reply":"2022-07-26T08:00:13.157457Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Embedding models\nembedding_layer = 'embedding_512'\nembedding_model = tf.keras.Model(inputs = model.input,\n                                 outputs = model.get_layer(embedding_layer).output,\n                                 name = \"EfficientNetB0_embed512\")","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:43:42.011495Z","iopub.execute_input":"2022-07-26T06:43:42.011891Z","iopub.status.idle":"2022-07-26T06:43:42.037836Z","shell.execute_reply.started":"2022-07-26T06:43:42.011856Z","shell.execute_reply":"2022-07-26T06:43:42.037128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Retrieving embeddings\ntrain_img_paths = train_ds.file_paths\nval_img_paths = val_ds.file_paths\n\ntrain_embeddings = get_embeddings(model = embedding_model,\n                                 image_paths = train_img_paths,\n                                 input_size = IMG_SIZE)\n\nval_embeddings = get_embeddings(model = embedding_model,\n                                 image_paths = val_img_paths,\n                                 input_size = IMG_SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:43:42.038991Z","iopub.execute_input":"2022-07-26T06:43:42.039210Z","iopub.status.idle":"2022-07-26T06:52:59.787793Z","shell.execute_reply.started":"2022-07-26T06:43:42.039178Z","shell.execute_reply":"2022-07-26T06:52:59.787098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_embeddings.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:52:59.788954Z","iopub.execute_input":"2022-07-26T06:52:59.789361Z","iopub.status.idle":"2022-07-26T06:52:59.806093Z","shell.execute_reply.started":"2022-07-26T06:52:59.789321Z","shell.execute_reply":"2022-07-26T06:52:59.805215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_embeddings.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:52:59.807985Z","iopub.execute_input":"2022-07-26T06:52:59.808307Z","iopub.status.idle":"2022-07-26T06:52:59.826893Z","shell.execute_reply.started":"2022-07-26T06:52:59.808255Z","shell.execute_reply":"2022-07-26T06:52:59.826252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_train_similarity = get_similarities(val_embeddings['embedded_images'],\n                                        train_embeddings['embedded_images'])\nval_train_similarity.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:52:59.828353Z","iopub.execute_input":"2022-07-26T06:52:59.828795Z","iopub.status.idle":"2022-07-26T06:53:18.407508Z","shell.execute_reply.started":"2022-07-26T06:52:59.828760Z","shell.execute_reply":"2022-07-26T06:53:18.406691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculating confidence score per submission\ndef confidence_top(query = None, key = None, similarity = None, query_image_index = None, top = 5):\n    '''\n    Arguments:\n    query_image_index = index of query image on similarity matrix query axis\n    Return confidence scores for top N predictions\n    '''\n    query_paths = query['images_paths']\n    key_paths = key['images_paths']\n    \n    similar_n = np.argsort(similarity[query_image_index])[::-1][:top]\n    \n    confidence_df = {}    \n    confidence_df['top_similar'] = []\n    for similar in similar_n:\n        confidence_df['top_similar'].append(similar)\n\n    confidence_df['image_paths'] = []\n    for similar in similar_n:\n        similar_image_path = key_paths[similar]\n        confidence_df['image_paths'].append(similar_image_path)    \n        \n    confidence_df['prediction'] = []\n    for similar in similar_n:\n        similar_image_path = key_paths[similar]\n        y = int(os.path.split(os.path.split(similar_image_path)[0])[1])\n        confidence_df['prediction'].append(y)  \n    \n    confidence_df['cos_similarity'] = []\n    for similar in similar_n:\n        confidence_df['cos_similarity'].append(similarity[query_image_index][similar]) \n    \n    return pd.DataFrame(confidence_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:53:18.409025Z","iopub.execute_input":"2022-07-26T06:53:18.409299Z","iopub.status.idle":"2022-07-26T06:53:18.419276Z","shell.execute_reply.started":"2022-07-26T06:53:18.409264Z","shell.execute_reply":"2022-07-26T06:53:18.418551Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 0\n\nquery_top(query_image_index)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T08:00:17.941302Z","iopub.execute_input":"2022-07-26T08:00:17.941875Z","iopub.status.idle":"2022-07-26T08:00:18.996997Z","shell.execute_reply.started":"2022-07-26T08:00:17.941840Z","shell.execute_reply":"2022-07-26T08:00:18.996348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:53:19.242201Z","iopub.execute_input":"2022-07-26T06:53:19.242622Z","iopub.status.idle":"2022-07-26T06:53:19.255855Z","shell.execute_reply.started":"2022-07-26T06:53:19.242582Z","shell.execute_reply":"2022-07-26T06:53:19.255138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 4\n\nquery_top(query_image_index)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T08:00:24.479497Z","iopub.execute_input":"2022-07-26T08:00:24.480120Z","iopub.status.idle":"2022-07-26T08:00:25.362765Z","shell.execute_reply.started":"2022-07-26T08:00:24.480079Z","shell.execute_reply":"2022-07-26T08:00:25.361450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:53:20.130458Z","iopub.execute_input":"2022-07-26T06:53:20.130904Z","iopub.status.idle":"2022-07-26T06:53:20.145718Z","shell.execute_reply.started":"2022-07-26T06:53:20.130868Z","shell.execute_reply":"2022-07-26T06:53:20.145070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 65\n\nquery_top(query_image_index)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T08:01:37.858945Z","iopub.execute_input":"2022-07-26T08:01:37.859210Z","iopub.status.idle":"2022-07-26T08:01:38.746785Z","shell.execute_reply.started":"2022-07-26T08:01:37.859180Z","shell.execute_reply":"2022-07-26T08:01:38.746070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:53:21.104752Z","iopub.execute_input":"2022-07-26T06:53:21.105191Z","iopub.status.idle":"2022-07-26T06:53:21.121645Z","shell.execute_reply.started":"2022-07-26T06:53:21.105150Z","shell.execute_reply":"2022-07-26T06:53:21.120798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 13\n\nquery_top(query_image_index)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T08:00:33.910213Z","iopub.execute_input":"2022-07-26T08:00:33.910781Z","iopub.status.idle":"2022-07-26T08:00:34.754754Z","shell.execute_reply.started":"2022-07-26T08:00:33.910740Z","shell.execute_reply":"2022-07-26T08:00:34.753908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:53:22.024223Z","iopub.execute_input":"2022-07-26T06:53:22.024678Z","iopub.status.idle":"2022-07-26T06:53:22.040495Z","shell.execute_reply.started":"2022-07-26T06:53:22.024641Z","shell.execute_reply":"2022-07-26T06:53:22.039654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Object oclusion example\nObject oclusion is only one of the examples of how a local feature reranking method improves query performance","metadata":{}},{"cell_type":"code","source":"query_image_index = 887\ntop_n = 10\n\nquery_top(query_image_index, top_n)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:52:10.885753Z","iopub.execute_input":"2022-07-26T07:52:10.886006Z","iopub.status.idle":"2022-07-26T07:52:12.187071Z","shell.execute_reply.started":"2022-07-26T07:52:10.885978Z","shell.execute_reply":"2022-07-26T07:52:12.186274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:52:15.688022Z","iopub.execute_input":"2022-07-26T07:52:15.688272Z","iopub.status.idle":"2022-07-26T07:52:15.703120Z","shell.execute_reply.started":"2022-07-26T07:52:15.688242Z","shell.execute_reply":"2022-07-26T07:52:15.702320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DELF module\nLocal features search\n\nReferences:\n* Large-Scale Image Retrieval with Attentive Deep Local Features: https://arxiv.org/abs/1612.06321\n* DELF on Tensorflow Hub: https://github.com/tensorflow/models/tree/master/research/delf\n","metadata":{}},{"cell_type":"code","source":"DELF_IMG_SIZE = 600","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:15:40.696838Z","iopub.execute_input":"2022-07-26T07:15:40.697682Z","iopub.status.idle":"2022-07-26T07:15:40.701538Z","shell.execute_reply.started":"2022-07-26T07:15:40.697640Z","shell.execute_reply":"2022-07-26T07:15:40.700633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_1 = get_image(val_embeddings['images_paths'][887],\n                    resize = True,\n                    target_size = DELF_IMG_SIZE)\n\nplt.figure(figsize = (6, 6))\nplt.imshow(image_1)\nplt.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:15:52.456294Z","iopub.execute_input":"2022-07-26T07:15:52.456991Z","iopub.status.idle":"2022-07-26T07:15:52.661984Z","shell.execute_reply.started":"2022-07-26T07:15:52.456954Z","shell.execute_reply":"2022-07-26T07:15:52.661384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_2 = get_image(train_embeddings['images_paths'][similar_n[5]],\n                     resize = True,\n                     target_size = DELF_IMG_SIZE)\n\nplt.figure(figsize = (6, 6))\nplt.imshow(image_2)\nplt.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:16:06.720688Z","iopub.execute_input":"2022-07-26T07:16:06.720947Z","iopub.status.idle":"2022-07-26T07:16:06.924822Z","shell.execute_reply.started":"2022-07-26T07:16:06.720919Z","shell.execute_reply":"2022-07-26T07:16:06.923104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from absl import logging\nfrom PIL import Image, ImageOps\nfrom scipy.spatial import cKDTree\nfrom skimage.feature import plot_matches\nfrom skimage.measure import ransac\nfrom skimage.transform import AffineTransform\nfrom six import BytesIO\n\nimport tensorflow_hub as hub\nfrom six.moves.urllib.request import urlopen","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:53:25.925514Z","iopub.execute_input":"2022-07-26T06:53:25.925852Z","iopub.status.idle":"2022-07-26T06:53:26.592158Z","shell.execute_reply.started":"2022-07-26T06:53:25.925821Z","shell.execute_reply":"2022-07-26T06:53:26.591389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"delf = hub.load('https://tfhub.dev/google/delf/1').signatures['default']","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:53:26.593347Z","iopub.execute_input":"2022-07-26T06:53:26.593610Z","iopub.status.idle":"2022-07-26T06:53:33.869926Z","shell.execute_reply.started":"2022-07-26T06:53:26.593577Z","shell.execute_reply":"2022-07-26T06:53:33.869173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DELF module\ndef run_delf(image):\n    '''\n    Apply DELF module to the input image\n    Arguments:\n    image: np.array resized image\n    '''\n    float_image = tf.image.convert_image_dtype(image, tf.float32)\n\n    return delf(\n      image = float_image,\n      score_threshold = tf.constant(100.0),\n      image_scales = tf.constant([0.25, 0.3536, 0.5, 0.7071, 1.0, 1.4142, 2.0]),\n      max_feature_num = tf.constant(1000))\n\ndef match_images(image1, image2, result1, result2, verbose = True):\n    distance_threshold = 0.8\n\n    # Read features.\n    num_features_1 = result1['locations'].shape[0]\n    num_features_2 = result2['locations'].shape[0]\n    \n    if verbose:\n        print(\"Loaded image 1's %d features\" % num_features_1)\n        print(\"Loaded image 2's %d features\" % num_features_2)\n\n    # Find nearest-neighbor matches using a KD tree.\n    d1_tree = cKDTree(result1['descriptors'])\n    _, indices = d1_tree.query(\n      result2['descriptors'],\n      distance_upper_bound=distance_threshold)\n\n    # Select feature locations for putative matches.\n    locations_2_to_use = np.array([\n      result2['locations'][i,]\n      for i in range(num_features_2)\n      if indices[i] != num_features_1\n    ])\n    locations_1_to_use = np.array([\n      result1['locations'][indices[i],]\n      for i in range(num_features_2)\n      if indices[i] != num_features_1\n    ])\n\n    # Perform geometric verification using RANSAC.\n    _, inliers = ransac(\n      (locations_1_to_use, locations_2_to_use),\n      AffineTransform,\n      min_samples=3,\n      residual_threshold=20,\n      max_trials=1000)\n    \n    if verbose:\n        print('Found %d inliers' % sum(inliers))\n\n    # Visualize correspondences.\n    _, ax = plt.subplots(figsize = (9, 9))\n    inlier_idxs = np.nonzero(inliers)[0]\n    plot_matches(\n      ax,\n      image1,\n      image2,\n      locations_1_to_use,\n      locations_2_to_use,\n      np.column_stack((inlier_idxs, inlier_idxs)),\n      matches_color='b')\n    ax.axis('off')\n    ax.set_title(f'DELF correspondences: Found {sum(inliers)} inliers')","metadata":{"execution":{"iopub.status.busy":"2022-07-26T06:53:33.875397Z","iopub.execute_input":"2022-07-26T06:53:33.875808Z","iopub.status.idle":"2022-07-26T06:53:33.887804Z","shell.execute_reply.started":"2022-07-26T06:53:33.875768Z","shell.execute_reply":"2022-07-26T06:53:33.886984Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"delf_result1 = run_delf(image_1)\ndelf_result2 = run_delf(image_2)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:16:13.323463Z","iopub.execute_input":"2022-07-26T07:16:13.323949Z","iopub.status.idle":"2022-07-26T07:16:13.527050Z","shell.execute_reply.started":"2022-07-26T07:16:13.323912Z","shell.execute_reply":"2022-07-26T07:16:13.526282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"match_images(image_1, image_2, delf_result1, delf_result2)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:16:15.352934Z","iopub.execute_input":"2022-07-26T07:16:15.353189Z","iopub.status.idle":"2022-07-26T07:16:16.712106Z","shell.execute_reply.started":"2022-07-26T07:16:15.353160Z","shell.execute_reply":"2022-07-26T07:16:16.711343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image_index in similar_n[:6]:\n    key_image = get_image(train_embeddings['images_paths'][image_index],\n                          resize = True,\n                          target_size = DELF_IMG_SIZE)\n    try:\n        delf_key_image_result = run_delf(key_image)\n        match_images(image_1, key_image, delf_result1, delf_key_image_result, verbose = False)\n    except:\n        print(\"No inliers found\")","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:16:24.582751Z","iopub.execute_input":"2022-07-26T07:16:24.583007Z","iopub.status.idle":"2022-07-26T07:16:32.614546Z","shell.execute_reply.started":"2022-07-26T07:16:24.582977Z","shell.execute_reply":"2022-07-26T07:16:32.613768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reranking\nReranking using DELF local features descriptor","metadata":{}},{"cell_type":"code","source":"def delf_rerank(query = None, key = None, query_image_index = None, confidence_df = None, re_sort = True):\n    distance_threshold = 0.8\n    query_paths = query['images_paths']\n    key_paths = key['images_paths']\n    \n    query_image = get_image(query_paths[query_image_index],\n                            resize = True,\n                            target_size = DELF_IMG_SIZE)\n    \n    delf_result_query = run_delf(query_image)\n    \n    # Read query features\n    num_features_query = delf_result_query['locations'].shape[0]\n    \n    inliers_list = []\n    print(f\"Retrieving local features for top {len(confidence_df['image_paths'])} key images...\")\n    for image_path in tqdm(confidence_df['image_paths']):\n        key_image = get_image(image_path,\n                          resize = True,\n                          target_size = DELF_IMG_SIZE)\n        \n        delf_result_key = run_delf(key_image)\n    \n        # Read key features\n        num_features_key = delf_result_key['locations'].shape[0]\n\n        # Find nearest-neighbor matches using a KD tree.\n        d1_tree = cKDTree(delf_result_query['descriptors'])\n        _, indices = d1_tree.query(\n          delf_result_key['descriptors'],\n          distance_upper_bound=distance_threshold)\n\n        # Select feature locations for putative matches.\n        locations_k_to_use = np.array([\n          delf_result_key['locations'][i,]\n          for i in range(num_features_key)\n          if indices[i] != num_features_query\n        ])\n        locations_q_to_use = np.array([\n          delf_result_query['locations'][indices[i],]\n          for i in range(num_features_key)\n          if indices[i] != num_features_query\n        ])\n\n        # Perform geometric verification using RANSAC.\n        try:\n            _, inliers = ransac(\n              (locations_q_to_use, locations_k_to_use),\n              AffineTransform,\n              min_samples=3,\n              residual_threshold=20,\n              max_trials=1000)\n        except:\n            inliers = [0]\n        \n        # Handling 0 inliers\n        try:\n            total_inliers = sum(inliers)\n            inliers_list.append(total_inliers)\n        except:\n            inliers_list.append(1) # Appending inlier = 1 to avoid null confidence\n    \n    confidence_df['inliers'] = inliers_list\n    \n    original_confidence = confidence_df['inliers']\n    reranked_confidence = np.sqrt(original_confidence) * confidence_df['cos_similarity']\n    confidence_df['reranked_conf'] = reranked_confidence\n    \n    if re_sort:\n        confidence_df.sort_values('reranked_conf', ascending = False, inplace = True)\n    \n    return confidence_df","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:05:56.707260Z","iopub.execute_input":"2022-07-26T07:05:56.707532Z","iopub.status.idle":"2022-07-26T07:05:56.719903Z","shell.execute_reply.started":"2022-07-26T07:05:56.707502Z","shell.execute_reply":"2022-07-26T07:05:56.718903Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reranked_df = delf_rerank(query = val_embeddings,\n                          key = train_embeddings,\n                          query_image_index = query_image_index,\n                          confidence_df = confidence_df,\n                          re_sort = True)\nreranked_df","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:52:31.447272Z","iopub.execute_input":"2022-07-26T07:52:31.447977Z","iopub.status.idle":"2022-07-26T07:52:41.126345Z","shell.execute_reply.started":"2022-07-26T07:52:31.447937Z","shell.execute_reply":"2022-07-26T07:52:41.125454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 887\ntop_n = 10\n\nquery_top(query_image_index, top_n, reranked = reranked_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:52:43.617699Z","iopub.execute_input":"2022-07-26T07:52:43.618313Z","iopub.status.idle":"2022-07-26T07:52:44.927039Z","shell.execute_reply.started":"2022-07-26T07:52:43.618275Z","shell.execute_reply":"2022-07-26T07:52:44.926308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reranking examples","metadata":{}},{"cell_type":"code","source":"query_image_index = 11\ntop_n = 10\n\nquery_top(query_image_index, top_n)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T08:07:58.817355Z","iopub.execute_input":"2022-07-26T08:07:58.818073Z","iopub.status.idle":"2022-07-26T08:08:00.267664Z","shell.execute_reply.started":"2022-07-26T08:07:58.818035Z","shell.execute_reply":"2022-07-26T08:08:00.266489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2022-07-26T08:08:07.053592Z","iopub.execute_input":"2022-07-26T08:08:07.054377Z","iopub.status.idle":"2022-07-26T08:08:07.069780Z","shell.execute_reply.started":"2022-07-26T08:08:07.054322Z","shell.execute_reply":"2022-07-26T08:08:07.069116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reranked_df = delf_rerank(query = val_embeddings,\n                          key = train_embeddings,\n                          query_image_index = query_image_index,\n                          confidence_df = confidence_df,\n                          re_sort = True)\nreranked_df","metadata":{"execution":{"iopub.status.busy":"2022-07-26T08:08:10.870630Z","iopub.execute_input":"2022-07-26T08:08:10.871055Z","iopub.status.idle":"2022-07-26T08:08:21.529868Z","shell.execute_reply.started":"2022-07-26T08:08:10.871020Z","shell.execute_reply":"2022-07-26T08:08:21.529076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 11\ntop_n = 10\n\nquery_top(query_image_index, top_n, reranked = reranked_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T08:08:30.488551Z","iopub.execute_input":"2022-07-26T08:08:30.488811Z","iopub.status.idle":"2022-07-26T08:08:31.822879Z","shell.execute_reply.started":"2022-07-26T08:08:30.488781Z","shell.execute_reply":"2022-07-26T08:08:31.822257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 395\ntop_n = 10\n\nquery_top(query_image_index, top_n)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:54:57.370885Z","iopub.execute_input":"2022-07-26T07:54:57.371141Z","iopub.status.idle":"2022-07-26T07:54:58.733946Z","shell.execute_reply.started":"2022-07-26T07:54:57.371112Z","shell.execute_reply":"2022-07-26T07:54:58.733345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:55:06.432313Z","iopub.execute_input":"2022-07-26T07:55:06.433014Z","iopub.status.idle":"2022-07-26T07:55:06.446898Z","shell.execute_reply.started":"2022-07-26T07:55:06.432977Z","shell.execute_reply":"2022-07-26T07:55:06.446097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reranked_df = delf_rerank(query = val_embeddings,\n                          key = train_embeddings,\n                          query_image_index = query_image_index,\n                          confidence_df = confidence_df,\n                          re_sort = True)\nreranked_df","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:55:09.388623Z","iopub.execute_input":"2022-07-26T07:55:09.389294Z","iopub.status.idle":"2022-07-26T07:55:21.347077Z","shell.execute_reply.started":"2022-07-26T07:55:09.389242Z","shell.execute_reply":"2022-07-26T07:55:21.346155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 395\ntop_n = 10\n\nquery_top(query_image_index, top_n, reranked = reranked_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:55:26.168922Z","iopub.execute_input":"2022-07-26T07:55:26.169183Z","iopub.status.idle":"2022-07-26T07:55:27.535916Z","shell.execute_reply.started":"2022-07-26T07:55:26.169152Z","shell.execute_reply":"2022-07-26T07:55:27.535289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 40\ntop_n = 10\n\nquery_top(query_image_index, top_n)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:55:39.733179Z","iopub.execute_input":"2022-07-26T07:55:39.733843Z","iopub.status.idle":"2022-07-26T07:55:41.009953Z","shell.execute_reply.started":"2022-07-26T07:55:39.733804Z","shell.execute_reply":"2022-07-26T07:55:41.009265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:55:47.753709Z","iopub.execute_input":"2022-07-26T07:55:47.753975Z","iopub.status.idle":"2022-07-26T07:55:47.768420Z","shell.execute_reply.started":"2022-07-26T07:55:47.753946Z","shell.execute_reply":"2022-07-26T07:55:47.767707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reranked_df = delf_rerank(query = val_embeddings,\n                          key = train_embeddings,\n                          query_image_index = query_image_index,\n                          confidence_df = confidence_df,\n                          re_sort = True)\nreranked_df","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:55:52.433548Z","iopub.execute_input":"2022-07-26T07:55:52.434016Z","iopub.status.idle":"2022-07-26T07:56:02.401582Z","shell.execute_reply.started":"2022-07-26T07:55:52.433981Z","shell.execute_reply":"2022-07-26T07:56:02.400732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 40\ntop_n = 10\n\nquery_top(query_image_index, top_n, reranked = reranked_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:56:02.403137Z","iopub.execute_input":"2022-07-26T07:56:02.403560Z","iopub.status.idle":"2022-07-26T07:56:03.756750Z","shell.execute_reply.started":"2022-07-26T07:56:02.403522Z","shell.execute_reply":"2022-07-26T07:56:03.756094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Under-represented query image\nEffect of querying images not well represented on the key set","metadata":{}},{"cell_type":"code","source":"query_image_index = 822\ntop_n = 10\n\nquery_top(query_image_index, top_n)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:56:17.006674Z","iopub.execute_input":"2022-07-26T07:56:17.006929Z","iopub.status.idle":"2022-07-26T07:56:18.370760Z","shell.execute_reply.started":"2022-07-26T07:56:17.006901Z","shell.execute_reply":"2022-07-26T07:56:18.369402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's investigate what the model has seen for landmark 36 during training...","metadata":{}},{"cell_type":"code","source":"get_landmark(36)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:56:22.119749Z","iopub.execute_input":"2022-07-26T07:56:22.120619Z","iopub.status.idle":"2022-07-26T07:56:24.134868Z","shell.execute_reply.started":"2022-07-26T07:56:22.120566Z","shell.execute_reply":"2022-07-26T07:56:24.134250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Query image has a representativeness issue when considering how the landmark appears on key set and how it was seen during training.","metadata":{}}]}