{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Google Landmark Recognition Challenge 2020\nSimplified image similarity ranking and re-ranking implementation with:\n* EfficientNetB0 backbone for global feature similarity search\n* DELF module for local feature reranking\n\nReference papers:\n* 2020 Recognition challenge winner: https://arxiv.org/abs/2010.01650\n* 2019 Recognition challend 2nd place: https://arxiv.org/abs/1906.03990","metadata":{}},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:22:42.071327Z","iopub.execute_input":"2023-04-10T22:22:42.071705Z","iopub.status.idle":"2023-04-10T22:22:43.140156Z","shell.execute_reply.started":"2023-04-10T22:22:42.07166Z","shell.execute_reply":"2023-04-10T22:22:43.139214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing libraries\nimport os\nimport cv2\nimport shutil\nimport numpy as np\nimport pandas as pd\nfrom scipy import spatial\nfrom tqdm import tqdm\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-04-12T00:13:28.648188Z","iopub.execute_input":"2023-04-12T00:13:28.648475Z","iopub.status.idle":"2023-04-12T00:13:34.810445Z","shell.execute_reply.started":"2023-04-12T00:13:28.648441Z","shell.execute_reply":"2023-04-12T00:13:34.809717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Directories and file paths\nTRAIN_DIR = '../input/landmark-recognition-2020/train'\nTRAIN_CSV = '../input/landmark-recognition-2020/train.csv'\ntrain_df = pd.read_csv(TRAIN_CSV)\n\nTRAIN_PATHS = [os.path.join(TRAIN_DIR, f'{img[0]}/{img[1]}/{img[2]}/{img}.jpg') for img in train_df['id']]\ntrain_df['path'] = TRAIN_PATHS\n\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2023-04-12T01:03:15.532228Z","iopub.execute_input":"2023-04-12T01:03:15.532517Z","iopub.status.idle":"2023-04-12T01:03:19.264049Z","shell.execute_reply.started":"2023-04-12T01:03:15.532483Z","shell.execute_reply":"2023-04-12T01:03:19.263243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Subsetting\ntrain_df_grouped = pd.DataFrame(train_df.landmark_id.value_counts())\ntrain_df_grouped.reset_index(inplace=True)\ntrain_df_grouped.columns = ['landmark_id','count']\n\n# Selected landmarks based on inclass frequency\nselected_landmarks = train_df_grouped[(train_df_grouped['count'] <= 155) & (train_df_grouped['count'] >= 150)]\n\ntrain_df_sub = train_df[train_df['landmark_id'].isin(selected_landmarks['landmark_id'])]\nnew_id = []\ncurrent_id = 0\nprevious_id = int(train_df_sub.head(1)['landmark_id'])\nfor landmark_id in train_df_sub['landmark_id']:\n    if landmark_id == previous_id:\n        new_id.append(current_id)\n    else:\n        current_id += 1\n        new_id.append(current_id)\n        previous_id = landmark_id\n\ntrain_df_sub['new_id'] = new_id\n\nNUM_CLASSES = train_df_sub['landmark_id'].nunique()\n\nprint(f\"Unique classes found: {NUM_CLASSES}\")\ntrain_df_sub","metadata":{"execution":{"iopub.status.busy":"2023-04-13T14:35:41.887528Z","iopub.execute_input":"2023-04-13T14:35:41.887811Z","iopub.status.idle":"2023-04-13T14:35:41.950798Z","shell.execute_reply.started":"2023-04-13T14:35:41.887779Z","shell.execute_reply":"2023-04-13T14:35:41.950053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training and validation splits\n# 90/10 stratified split for training and validation\nX_train, X_val, y_train, y_val = train_test_split(train_df_sub[['id','path']], train_df_sub['landmark_id'],\n                                                  train_size = 0.9,\n                                                  random_state = 123,\n                                                  shuffle = True,\n                                                 )\n\n# Held-out test set for inference\n# Further 95/5 split -> 5% of original training set left for test set\nX_train, X_test, y_train, y_test = train_test_split(X_train, y_train,\n                                                   train_size = 0.95,\n                                                   random_state = 123,\n                                                   shuffle = True)\n\nassert X_train.shape[0] + X_val.shape[0] + X_test.shape[0] == train_df_sub.shape[0]\n\nprint(f\"Training data shape: {X_train.shape}\")\nprint(f\"Training label shape: {y_train.shape}\")\nprint(f\"Validation data shape: {X_val.shape}\")\nprint(f\"Validation label shape: {y_val.shape}\")\nprint(f\"Test data shape: {X_test.shape}\")\nprint(f\"Test label shape: {y_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-13T11:23:24.834652Z","iopub.execute_input":"2023-04-13T11:23:24.834919Z","iopub.status.idle":"2023-04-13T11:23:24.942009Z","shell.execute_reply.started":"2023-04-13T11:23:24.834888Z","shell.execute_reply":"2023-04-13T11:23:24.941087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Unique classes on y_train: {y_train.nunique()}\")\nprint(f\"Unique classes on y_val: {y_val.nunique()}\")\nprint(f\"Unique classes on y_test: {y_test.nunique()}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-13T11:23:28.136195Z","iopub.execute_input":"2023-04-13T11:23:28.136594Z","iopub.status.idle":"2023-04-13T11:23:28.147561Z","shell.execute_reply.started":"2023-04-13T11:23:28.136559Z","shell.execute_reply":"2023-04-13T11:23:28.146524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Classes distribution on training, validation and test sets\nplt.figure(figsize = (10, 3))\nax = sns.histplot(y_train, bins=75, kde = True)\nax.set_title('Distribution of Landmarks on training set')\nplt.tight_layout()\n\nplt.figure(figsize = (10, 3))\nax = sns.histplot(y_val, bins=75, kde = True)\nax.set_title('Distribution of Landmarks on validation set')\nplt.tight_layout()\n\nplt.figure(figsize = (10, 3))\nax = sns.histplot(y_test, bins=75, kde = True)\nax.set_title('Distribution of Landmarks on test set')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-13T11:41:02.806663Z","iopub.execute_input":"2023-04-13T11:41:02.80708Z","iopub.status.idle":"2023-04-13T11:41:04.941449Z","shell.execute_reply.started":"2023-04-13T11:41:02.807042Z","shell.execute_reply":"2023-04-13T11:41:04.940761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r train_sub","metadata":{"execution":{"iopub.status.busy":"2023-04-13T11:23:50.587771Z","iopub.execute_input":"2023-04-13T11:23:50.588053Z","iopub.status.idle":"2023-04-13T11:23:53.299853Z","shell.execute_reply.started":"2023-04-13T11:23:50.588023Z","shell.execute_reply":"2023-04-13T11:23:53.298591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating image directories for classes subset\nNEW_BASE_DIR = \"/kaggle/working\"\n\n# Training set directory\nfor file, path, landmark in tqdm(zip(X_train['id'], X_train['path'], y_train)):\n    dir = f\"{NEW_BASE_DIR}/train_sub/{str(landmark)}\"\n    os.makedirs(dir, exist_ok = True)\n    fname = f\"{file}.jpg\"\n    shutil.copyfile(src = path, dst = f\"{dir}/{fname}\")\n\n# Validation set directory    \nfor file, path, landmark in tqdm(zip(X_val['id'], X_val['path'], y_val)):\n    dir = f\"{NEW_BASE_DIR}/val_sub/{str(landmark)}\"\n    os.makedirs(dir, exist_ok = True)\n    fname = f\"{file}.jpg\"\n    shutil.copyfile(src = path, dst = f\"{dir}/{fname}\")\n\n# Training set directory\nfor file, path, landmark in tqdm(zip(X_test['id'], X_test['path'], y_test)):\n    dir = f\"{NEW_BASE_DIR}/test_sub/{str(landmark)}\"\n    os.makedirs(dir, exist_ok = True)\n    fname = f\"{file}.jpg\"\n    shutil.copyfile(src = path, dst = f\"{dir}/{fname}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-13T11:23:57.065741Z","iopub.execute_input":"2023-04-13T11:23:57.066021Z","iopub.status.idle":"2023-04-13T11:41:02.805021Z","shell.execute_reply.started":"2023-04-13T11:23:57.065993Z","shell.execute_reply":"2023-04-13T11:41:02.803174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:23:18.26583Z","iopub.execute_input":"2023-04-10T22:23:18.266055Z","iopub.status.idle":"2023-04-10T22:23:19.276354Z","shell.execute_reply.started":"2023-04-10T22:23:18.266012Z","shell.execute_reply":"2023-04-10T22:23:19.275407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cd train_sub && ls","metadata":{"execution":{"iopub.status.busy":"2023-04-13T11:41:14.026686Z","iopub.execute_input":"2023-04-13T11:41:14.02724Z","iopub.status.idle":"2023-04-13T11:41:15.121958Z","shell.execute_reply.started":"2023-04-13T11:41:14.027203Z","shell.execute_reply":"2023-04-13T11:41:15.121082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating tensorflow tf.data.Dataset\nfrom tensorflow.keras.utils import image_dataset_from_directory\n\nIMG_SIZE = 224\nBATCH_SIZE = 16\n\nprint(\"Building training dataset...\")\n# Training tf.data.Dataset\ntrain_ds = image_dataset_from_directory(f\"{NEW_BASE_DIR}/train_sub\",\n                                        label_mode = 'int',\n                                        shuffle = True,\n                                        image_size = (IMG_SIZE, IMG_SIZE),\n                                        batch_size = BATCH_SIZE)\n\nprint(\"Building validation dataset...\")\n# Validation tf.data.Dataset\nval_ds = image_dataset_from_directory(f\"{NEW_BASE_DIR}/val_sub\",\n                                        label_mode = 'int',\n                                        shuffle = True,\n                                        image_size = (IMG_SIZE, IMG_SIZE),\n                                        batch_size = BATCH_SIZE)\n\nprint(\"Building test dataset...\")\n# Test tf.data.Dataset\ntest_ds = image_dataset_from_directory(f\"{NEW_BASE_DIR}/test_sub\",\n                                        label_mode = 'int',\n                                        shuffle = True,\n                                        image_size = (IMG_SIZE, IMG_SIZE),\n                                        batch_size = BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T11:41:33.377243Z","iopub.execute_input":"2023-04-13T11:41:33.377551Z","iopub.status.idle":"2023-04-13T11:41:48.487888Z","shell.execute_reply.started":"2023-04-13T11:41:33.377519Z","shell.execute_reply":"2023-04-13T11:41:48.487106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualizing a random batch from training dataset\nfor data_batch, labels_batch in train_ds.take(1):\n    ncols = 4\n    nrows = int(data_batch.shape[0]/ncols)\n    fig, ax = plt.subplots(nrows = nrows, ncols = ncols, figsize=(10, 11),\n                           sharex = True, sharey = True)\n    img_counter = 0\n    for image, label in zip(data_batch, labels_batch):\n        axi = ax.flat[img_counter]\n        axi.imshow(image/255.)\n        label = label.numpy()\n#         axi.set_title(np.where(label == 1)[0])\n        axi.set_title(label)\n        img_counter += 1\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-13T11:41:51.571579Z","iopub.execute_input":"2023-04-13T11:41:51.572503Z","iopub.status.idle":"2023-04-13T11:41:53.916421Z","shell.execute_reply.started":"2023-04-13T11:41:51.572436Z","shell.execute_reply":"2023-04-13T11:41:53.915557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####### ALTERNATIVE CODE FOR UNBATCHED DATASET #######\n# ncols = 4\n# nrows = 4\n# fig, ax = plt.subplots(nrows = nrows, ncols = ncols, figsize=(10, 11),\n#                        sharex = True, sharey = True)\n# img_counter = 0\n# for image, label in train_ds.take(16):\n#     axi = ax.flat[img_counter]\n#     axi.imshow(image[0]/255.)\n#     label = label.numpy()\n# #         axi.set_title(np.where(label == 1)[0])\n#     axi.set_title(label)\n#     img_counter += 1\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:23:22.6892Z","iopub.execute_input":"2023-04-10T22:23:22.689834Z","iopub.status.idle":"2023-04-10T22:23:22.694375Z","shell.execute_reply.started":"2023-04-10T22:23:22.689798Z","shell.execute_reply":"2023-04-10T22:23:22.69336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining a data augmentation stage\nimg_augmentation = tf.keras.Sequential(\n    # [layers.RandomFlip(\"horizontal\"),\n    [layers.RandomTranslation(height_factor = 0.1, width_factor = 0.1),\n     layers.RandomRotation(0.02),\n     layers.RandomZoom(0.2)],\n     name = \"img_augmentation\",\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T11:41:59.616486Z","iopub.execute_input":"2023-04-13T11:41:59.61677Z","iopub.status.idle":"2023-04-13T11:41:59.65308Z","shell.execute_reply.started":"2023-04-13T11:41:59.616741Z","shell.execute_reply":"2023-04-13T11:41:59.652346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Displaying variations of a randomly augmented training image\nplt.figure(figsize=(9, 9))\nfor image, label in train_ds.take(1):\n    for i in range(9):\n        ax = plt.subplot(3, 3, i + 1)\n        augmented_image = img_augmentation(image, training = True)\n        plt.imshow(augmented_image[15].numpy().astype(\"uint8\"))\n        plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-04-13T11:42:02.532324Z","iopub.execute_input":"2023-04-13T11:42:02.5327Z","iopub.status.idle":"2023-04-13T11:42:04.176893Z","shell.execute_reply.started":"2023-04-13T11:42:02.532641Z","shell.execute_reply":"2023-04-13T11:42:04.175412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####### ALTERNATIVE CODE FOR UNBATCHED DATASET #######\n# # Displaying variations of a randomly augmented training image\n# plt.figure(figsize=(9, 9))\n# for image, label in train_ds.take(16):\n#     for i in range(9):\n#         ax = plt.subplot(3, 3, i + 1)\n#         augmented_image = img_augmentation(image[0], training = True)\n#         plt.imshow(augmented_image.numpy().astype(\"uint8\"))\n#         plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:23:23.913152Z","iopub.execute_input":"2023-04-10T22:23:23.913607Z","iopub.status.idle":"2023-04-10T22:23:23.917353Z","shell.execute_reply.started":"2023-04-10T22:23:23.913573Z","shell.execute_reply":"2023-04-10T22:23:23.916669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODELS_DIR = f\"{NEW_BASE_DIR}/models\"\n\nos.makedirs(MODELS_DIR, exist_ok = True)\n\n# Model instantiator\ndef build_model(num_classes = None):\n    inputs = keras.Input(shape = (IMG_SIZE, IMG_SIZE, 3))\n    x = img_augmentation(inputs)\n    # EfficientNetB0 backbone\n    model = EfficientNetB0(input_tensor = x,\n                           weights = 'imagenet',\n                           include_top = False,\n                           drop_connect_rate = DROP_CONNECT_RATE)\n    \n    # Freeze pretrained weights\n    model.trainable = False\n    \n    # Rebuild top\n    x = layers.GlobalAveragePooling2D(name = \"avg_pool\")(model.output)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(TOP_DROPOUT_RATE, name = \"top_dropout\")(x)\n    \n    # Embedding\n    embedding = layers.Dense(512, name = \"embedding_512\")(x)\n    outputs = layers.Dense(num_classes, activation = \"softmax\", name = \"softmax\")(embedding)\n    \n    # Compile\n    model = tf.keras.Model(inputs, outputs, name = \"EfficientNetB0\")\n    optimizer = tf.keras.optimizers.Adam(learning_rate = ADAM_LR)\n    model.compile(optimizer = optimizer,\n                 loss = \"sparse_categorical_crossentropy\",\n                 metrics = [\"accuracy\"])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-13T13:50:12.414261Z","iopub.execute_input":"2023-04-13T13:50:12.414566Z","iopub.status.idle":"2023-04-13T13:50:12.426366Z","shell.execute_reply.started":"2023-04-13T13:50:12.414532Z","shell.execute_reply":"2023-04-13T13:50:12.424549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training history visualization\ndef plot_hist(hist):\n    plt.plot(hist.history[\"accuracy\"])\n    plt.plot(hist.history[\"val_accuracy\"])\n    plt.title(\"model accuracy\")\n    plt.ylabel(\"accuracy\")\n    plt.xlabel(\"epoch\")\n    plt.legend([\"train\", \"validation\"], loc=\"upper left\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-13T13:10:35.293263Z","iopub.execute_input":"2023-04-13T13:10:35.293718Z","iopub.status.idle":"2023-04-13T13:10:35.307662Z","shell.execute_reply.started":"2023-04-13T13:10:35.293666Z","shell.execute_reply":"2023-04-13T13:10:35.305887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-04-13T13:10:38.209013Z","iopub.execute_input":"2023-04-13T13:10:38.209299Z","iopub.status.idle":"2023-04-13T13:10:42.615992Z","shell.execute_reply.started":"2023-04-13T13:10:38.209267Z","shell.execute_reply":"2023-04-13T13:10:42.615238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:23:25.812492Z","iopub.execute_input":"2023-04-10T22:23:25.812723Z","iopub.status.idle":"2023-04-10T22:23:26.863913Z","shell.execute_reply.started":"2023-04-10T22:23:25.812691Z","shell.execute_reply":"2023-04-10T22:23:26.862951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Instantiating model\n# Hyperparameters\nDROP_CONNECT_RATE = 0.2 # Dropout rate for stochastic depth on EfficientNet\nTOP_DROPOUT_RATE = 0.2  # Top dropout\nINIT_LR = 5e-3          # Initial learning rate\nEPOCHS = 20\n\n# Adam optimizer learning rate schedule\nADAM_LR = tf.keras.optimizers.schedules.ExponentialDecay(\n    INIT_LR,\n    decay_steps=100,\n    decay_rate=0.96,\n    staircase=True)\n\nmodel = build_model(num_classes = NUM_CLASSES)\n\n# Training embedding layer\nmodel_file_path = os.path.join(MODELS_DIR, \"EfficientNetB0_softmax.keras\")\ncallbacks = [\n    keras.callbacks.ModelCheckpoint(model_file_path,\n                                    save_best_only=True,\n                                    monitor = \"val_accuracy\"),\n    keras.callbacks.EarlyStopping(patience = 2,\n                                  monitor = \"val_accuracy\")]\n\n# Training history visualization\ndef plot_hist(hist):\n    plt.plot(hist.history[\"accuracy\"])\n    plt.plot(hist.history[\"val_accuracy\"])\n    plt.title(\"model accuracy\")\n    plt.ylabel(\"accuracy\")\n    plt.xlabel(\"epoch\")\n    plt.legend([\"train\", \"validation\"], loc=\"upper left\")\n    plt.show()\n\nhist = model.fit(train_ds,\n                 epochs = EPOCHS,\n                 validation_data = val_ds,\n                 shuffle = 'batch',\n                 callbacks = callbacks)\n\nplot_hist(hist)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T13:50:16.900917Z","iopub.execute_input":"2023-04-13T13:50:16.901408Z","iopub.status.idle":"2023-04-13T14:23:42.261959Z","shell.execute_reply.started":"2023-04-13T13:50:16.901364Z","shell.execute_reply":"2023-04-13T14:23:42.260349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluating best model\nmodel = keras.models.load_model(model_file_path)\nprint(\"Predictions on validation set...\")\nprint(f\"Validation accuracy: {model.evaluate(val_ds)[1]*100:.2f} %\")\nprint(\"Predictions on test set...\")\nprint(f\"Test accuracy: {model.evaluate(test_ds)[1]*100:.2f} %\")","metadata":{"execution":{"iopub.status.busy":"2023-04-12T02:15:38.282456Z","iopub.execute_input":"2023-04-12T02:15:38.282741Z","iopub.status.idle":"2023-04-12T02:15:56.992132Z","shell.execute_reply.started":"2023-04-12T02:15:38.282712Z","shell.execute_reply":"2023-04-12T02:15:56.990323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cosine Similarity\nPairwise query: key search for similarity candidates. In the following example:\n* Query images: validation set\n* Key images: training set","metadata":{}},{"cell_type":"code","source":"# Auxiliar functions\n# Load image\ndef get_image(path, resize = False, reshape = False, target_size = None):\n    img = cv2.imread(path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    if resize:\n        img = cv2.resize(img, dsize = (target_size, target_size))\n    if reshape:\n        img = tf.reshape(img, [1, target_size, target_size, 3])\n    return img\n\n# Get landmark samples\ndef get_landmark(landmark_id, samples = 16):\n    nrows = samples // 4\n    random_imgs = np.random.choice(train_df_sub[train_df_sub['new_id'] == landmark_id].index, samples, replace = False)\n    plt.figure(figsize = (12, 10))\n    for i, img in enumerate(train_df_sub.loc[random_imgs, :].values):\n        ax = plt.subplot(nrows, 4, i + 1)\n        plt.imshow(get_image(img[2]))\n        plt.title(f\"{img[0]}\")\n        plt.suptitle(f\"Samples of landmark {landmark_id}\", fontsize = 14, y = 0.94, weight = \"bold\")\n        plt.axis(\"off\")\n\n# Get image embeddings\ndef get_embeddings(model, image_paths, input_size, as_df = True):\n    embeddings = {}\n    embeddings['images_paths'] = []\n    embeddings['embedded_images'] = []\n    \n    target_dir = os.path.split(os.path.split(image_paths[0])[0])[0]\n    \n    print(f\"Retrieving embeddings for {target_dir} with {model.name}...\")\n    for image_path in tqdm(image_paths):\n        embeddings['images_paths'].append(image_path)\n        embedded_image = model.predict(get_image(image_path,\n                                                 resize = True,\n                                                 reshape = True,\n                                                 target_size = input_size))\n        embeddings['embedded_images'].append(embedded_image)\n    \n    if as_df:\n        embeddings = pd.DataFrame(embeddings)\n    \n    return embeddings\n\n# Get similarities between query key pair\ndef get_similarities(query, key):\n    '''\n    Get cosine similarity matrix between query and key pairs\n    Arguments:\n    query, key: embedded images\n    '''\n    query_array = np.stack(query.tolist()).reshape(query.shape[0],\n                                                   query[0].shape[1])\n    key_array = np.stack(key.tolist()).reshape(key.shape[0],\n                                               key[0].shape[1])\n    \n    # Initializing similarity matrix\n    similarity = np.zeros((query_array.shape[0], key_array.shape[0]))\n    \n    # Getting pairwise similarities\n    print(f\"Getting pairwise {query_array.shape[0]} query: {key_array.shape[0]} key similarities...\")\n    for query_index in tqdm(range(query_array.shape[0])):\n        similarity[query_index] = 1 - spatial.distance.cdist(query_array[np.newaxis, query_index, :],\n                                                             key_array,\n                                                             'cosine')[0]\n    return similarity\n\n# Plot top ranked images\ndef plot_similar(similar_imgs, img_paths):\n    '''\n    Plot top N similar samples from similarity index\n    '''\n    plt.figure(figsize = (18, 6))\n    nrows = similar_imgs.shape[0]//5\n    for i, img in enumerate(similar_imgs):\n        ax = plt.subplot(nrows, 5, i + 1)\n        plt.imshow(get_image(img_paths[img]))\n        plt.title(f\"Landmark id: {os.path.split(os.path.split(img_paths[img])[0])[1]}\")\n        plt.axis(\"off\")\n\n\n# Aggregate query and top similar plots\ndef query_top(image_index, top_n = 5, figsize = (6, 6), reranked = None):\n    '''\n    Plot top N similar samples against queried image\n    If reranked, provide reranked dataframe with ['top_similar'] index reordered by reranked confidence\n    '''\n    image_id = os.path.split(val_embeddings['images_paths'][image_index])[1]\n    query_landmark_id = os.path.split(os.path.split(val_embeddings['images_paths'][image_index])[0])[1]\n    \n    if type(reranked) == pd.core.frame.DataFrame:\n        similar_n = reranked['top_similar'][:top_n]\n    else:\n        similar_n = np.argsort(val_train_similarity[image_index])[::-1][:top_n]\n                \n    print(f\"Queried image: {image_id}\")\n    plt.figure(figsize = figsize)\n    plt.imshow(get_image(val_embeddings['images_paths'][image_index]))\n    plt.title(f\"Landmark id: {image_id}\")\n    plt.axis(\"off\")\n    plot_similar(similar_n, train_embeddings['images_paths'])\n    return similar_n","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:53:28.215548Z","iopub.execute_input":"2023-04-10T22:53:28.215819Z","iopub.status.idle":"2023-04-10T22:53:28.234877Z","shell.execute_reply.started":"2023-04-10T22:53:28.215791Z","shell.execute_reply":"2023-04-10T22:53:28.233996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Embedding models\nembedding_layer = 'embedding_512'\nembedding_model = tf.keras.Model(inputs = model.input,\n                                 outputs = model.get_layer(embedding_layer).output,\n                                 name = \"EfficientNetB5_embed512\")","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:53:32.331773Z","iopub.execute_input":"2023-04-10T22:53:32.332047Z","iopub.status.idle":"2023-04-10T22:53:32.353321Z","shell.execute_reply.started":"2023-04-10T22:53:32.332003Z","shell.execute_reply":"2023-04-10T22:53:32.352607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:53:41.132554Z","iopub.execute_input":"2023-04-10T22:53:41.132826Z","iopub.status.idle":"2023-04-10T22:53:41.152015Z","shell.execute_reply.started":"2023-04-10T22:53:41.132797Z","shell.execute_reply":"2023-04-10T22:53:41.151016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Retrieving embeddings\ntrain_img_paths = train_ds.file_paths\nval_img_paths = val_ds.file_paths\n\ntrain_embeddings = get_embeddings(model = embedding_model,\n                                 image_paths = train_img_paths,\n                                 input_size = IMG_SIZE)\n\nval_embeddings = get_embeddings(model = embedding_model,\n                                 image_paths = val_img_paths,\n                                 input_size = IMG_SIZE)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:33:48.023081Z","iopub.execute_input":"2023-04-10T22:33:48.023874Z","iopub.status.idle":"2023-04-10T22:43:32.804897Z","shell.execute_reply.started":"2023-04-10T22:33:48.023839Z","shell.execute_reply":"2023-04-10T22:43:32.803463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_embeddings.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:32.806289Z","iopub.execute_input":"2023-04-10T22:43:32.806625Z","iopub.status.idle":"2023-04-10T22:43:32.822296Z","shell.execute_reply.started":"2023-04-10T22:43:32.806588Z","shell.execute_reply":"2023-04-10T22:43:32.821535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_embeddings.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:32.82932Z","iopub.execute_input":"2023-04-10T22:43:32.829619Z","iopub.status.idle":"2023-04-10T22:43:32.844203Z","shell.execute_reply.started":"2023-04-10T22:43:32.829593Z","shell.execute_reply":"2023-04-10T22:43:32.843296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_train_similarity = get_similarities(val_embeddings['embedded_images'],\n                                        train_embeddings['embedded_images'])\nval_train_similarity.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:32.845756Z","iopub.execute_input":"2023-04-10T22:43:32.846433Z","iopub.status.idle":"2023-04-10T22:43:53.382698Z","shell.execute_reply.started":"2023-04-10T22:43:32.846396Z","shell.execute_reply":"2023-04-10T22:43:53.381797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculating confidence score per submission\ndef confidence_top(query = None, key = None, similarity = None, query_image_index = None, top = 5):\n    '''\n    Arguments:\n    query_image_index = index of query image on similarity matrix query axis\n    Return confidence scores for top N predictions\n    '''\n    query_paths = query['images_paths']\n    key_paths = key['images_paths']\n    \n    similar_n = np.argsort(similarity[query_image_index])[::-1][:top]\n    \n    confidence_df = {}    \n    confidence_df['top_similar'] = []\n    for similar in similar_n:\n        confidence_df['top_similar'].append(similar)\n\n    confidence_df['image_paths'] = []\n    for similar in similar_n:\n        similar_image_path = key_paths[similar]\n        confidence_df['image_paths'].append(similar_image_path)    \n        \n    confidence_df['prediction'] = []\n    for similar in similar_n:\n        similar_image_path = key_paths[similar]\n        y = int(os.path.split(os.path.split(similar_image_path)[0])[1])\n        confidence_df['prediction'].append(y)  \n    \n    confidence_df['cos_similarity'] = []\n    for similar in similar_n:\n        confidence_df['cos_similarity'].append(similarity[query_image_index][similar]) \n    \n    return pd.DataFrame(confidence_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:53.384127Z","iopub.execute_input":"2023-04-10T22:43:53.384457Z","iopub.status.idle":"2023-04-10T22:43:53.395125Z","shell.execute_reply.started":"2023-04-10T22:43:53.384422Z","shell.execute_reply":"2023-04-10T22:43:53.394266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 0\n\nquery_top(query_image_index)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:53.396516Z","iopub.execute_input":"2023-04-10T22:43:53.396792Z","iopub.status.idle":"2023-04-10T22:43:54.306716Z","shell.execute_reply.started":"2023-04-10T22:43:53.396757Z","shell.execute_reply":"2023-04-10T22:43:54.305903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:54.307929Z","iopub.execute_input":"2023-04-10T22:43:54.308672Z","iopub.status.idle":"2023-04-10T22:43:54.324657Z","shell.execute_reply.started":"2023-04-10T22:43:54.308633Z","shell.execute_reply":"2023-04-10T22:43:54.323967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 4\n\nquery_top(query_image_index)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:54.32593Z","iopub.execute_input":"2023-04-10T22:43:54.326531Z","iopub.status.idle":"2023-04-10T22:43:55.749558Z","shell.execute_reply.started":"2023-04-10T22:43:54.326495Z","shell.execute_reply":"2023-04-10T22:43:55.748926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = 5)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:55.750748Z","iopub.execute_input":"2023-04-10T22:43:55.751376Z","iopub.status.idle":"2023-04-10T22:43:55.765338Z","shell.execute_reply.started":"2023-04-10T22:43:55.751338Z","shell.execute_reply":"2023-04-10T22:43:55.764514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 65\n\nquery_top(query_image_index)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:55.766658Z","iopub.execute_input":"2023-04-10T22:43:55.767009Z","iopub.status.idle":"2023-04-10T22:43:56.679452Z","shell.execute_reply.started":"2023-04-10T22:43:55.766972Z","shell.execute_reply":"2023-04-10T22:43:56.677838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = 5)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:56.680885Z","iopub.execute_input":"2023-04-10T22:43:56.681803Z","iopub.status.idle":"2023-04-10T22:43:56.695995Z","shell.execute_reply.started":"2023-04-10T22:43:56.681764Z","shell.execute_reply":"2023-04-10T22:43:56.695316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 13\n\nquery_top(query_image_index)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:56.697299Z","iopub.execute_input":"2023-04-10T22:43:56.697898Z","iopub.status.idle":"2023-04-10T22:43:57.575477Z","shell.execute_reply.started":"2023-04-10T22:43:56.697862Z","shell.execute_reply":"2023-04-10T22:43:57.574874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:57.576787Z","iopub.execute_input":"2023-04-10T22:43:57.577268Z","iopub.status.idle":"2023-04-10T22:43:57.592904Z","shell.execute_reply.started":"2023-04-10T22:43:57.577226Z","shell.execute_reply":"2023-04-10T22:43:57.592098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Object oclusion example\nObject oclusion is only one of the examples of how a local feature reranking method improves query performance","metadata":{}},{"cell_type":"code","source":"query_image_index = 887\ntop_n = 10\n\nquery_top(query_image_index, top_n)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:04:47.211747Z","iopub.execute_input":"2023-04-10T23:04:47.212061Z","iopub.status.idle":"2023-04-10T23:04:49.072548Z","shell.execute_reply.started":"2023-04-10T23:04:47.212012Z","shell.execute_reply":"2023-04-10T23:04:49.071904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:58.920732Z","iopub.execute_input":"2023-04-10T22:43:58.921437Z","iopub.status.idle":"2023-04-10T22:43:58.936995Z","shell.execute_reply.started":"2023-04-10T22:43:58.921398Z","shell.execute_reply":"2023-04-10T22:43:58.936312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DELF module\nLocal features search\n\nReferences:\n* Large-Scale Image Retrieval with Attentive Deep Local Features: https://arxiv.org/abs/1612.06321\n* DELF on Tensorflow Hub: https://github.com/tensorflow/models/tree/master/research/delf\n","metadata":{}},{"cell_type":"code","source":"DELF_IMG_SIZE = 600","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:58.938175Z","iopub.execute_input":"2023-04-10T22:43:58.939056Z","iopub.status.idle":"2023-04-10T22:43:58.942964Z","shell.execute_reply.started":"2023-04-10T22:43:58.939002Z","shell.execute_reply":"2023-04-10T22:43:58.942067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_1 = get_image(val_embeddings['images_paths'][887],\n                    resize = True,\n                    target_size = DELF_IMG_SIZE)\n\nplt.figure(figsize = (6, 6))\nplt.imshow(image_1)\nplt.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:58.944538Z","iopub.execute_input":"2023-04-10T22:43:58.944815Z","iopub.status.idle":"2023-04-10T22:43:59.169774Z","shell.execute_reply.started":"2023-04-10T22:43:58.944779Z","shell.execute_reply":"2023-04-10T22:43:59.168927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 888\ntop_n = 5\n\n\nsimilar_n = query_top(query_image_index, top_n)","metadata":{"execution":{"iopub.status.busy":"2023-04-11T01:27:43.787205Z","iopub.execute_input":"2023-04-11T01:27:43.787866Z","iopub.status.idle":"2023-04-11T01:27:43.869228Z","shell.execute_reply.started":"2023-04-11T01:27:43.787758Z","shell.execute_reply":"2023-04-11T01:27:43.868258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_landmark(20)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:55:57.025494Z","iopub.execute_input":"2023-04-10T23:55:57.025811Z","iopub.status.idle":"2023-04-10T23:55:58.594504Z","shell.execute_reply.started":"2023-04-10T23:55:57.025781Z","shell.execute_reply":"2023-04-10T23:55:58.593876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DELF_IMG_SIZE = 600\n\n# Looping through similar candidates\nfor image_index in similar_n[:6]:\n    key_image = get_image(train_embeddings['images_paths'][image_index],\n                          resize = True,\n                          target_size = DELF_IMG_SIZE)\n    try:\n        delf_key_image_result = run_delf(key_image)\n        match_images(image_1, key_image, delf_result1, delf_key_image_result, verbose = False)\n    except:\n        print(\"No inliers found\")","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:12:19.034345Z","iopub.execute_input":"2023-04-10T23:12:19.034616Z","iopub.status.idle":"2023-04-10T23:12:28.079726Z","shell.execute_reply.started":"2023-04-10T23:12:19.034583Z","shell.execute_reply":"2023-04-10T23:12:28.078936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from absl import logging\nfrom PIL import Image, ImageOps\nfrom scipy.spatial import cKDTree\nfrom skimage.feature import plot_matches\nfrom skimage.measure import ransac\nfrom skimage.transform import AffineTransform\nfrom six import BytesIO\n\nimport tensorflow_hub as hub\nfrom six.moves.urllib.request import urlopen","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:12:37.149931Z","iopub.execute_input":"2023-04-10T23:12:37.150587Z","iopub.status.idle":"2023-04-10T23:12:37.156019Z","shell.execute_reply.started":"2023-04-10T23:12:37.150543Z","shell.execute_reply":"2023-04-10T23:12:37.154918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"delf = hub.load('https://tfhub.dev/google/delf/1').signatures['default']","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:12:39.948427Z","iopub.execute_input":"2023-04-10T23:12:39.94903Z","iopub.status.idle":"2023-04-10T23:12:44.553268Z","shell.execute_reply.started":"2023-04-10T23:12:39.948992Z","shell.execute_reply":"2023-04-10T23:12:44.552435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DELF module\ndef run_delf(image):\n    '''\n    Apply DELF module to the input image\n    Arguments:\n    image: np.array resized image\n    '''\n    float_image = tf.image.convert_image_dtype(image, tf.float32)\n\n    return delf(\n      image = float_image,\n      score_threshold = tf.constant(100.0),\n      image_scales = tf.constant([0.25, 0.3536, 0.5, 0.7071, 1.0, 1.4142, 2.0]),\n      max_feature_num = tf.constant(1000))\n\ndef match_images(image1, image2, result1, result2, verbose = True):\n    distance_threshold = 0.8\n\n    # Read features.\n    num_features_1 = result1['locations'].shape[0]\n    num_features_2 = result2['locations'].shape[0]\n    \n    if verbose:\n        print(\"Loaded image 1's %d features\" % num_features_1)\n        print(\"Loaded image 2's %d features\" % num_features_2)\n\n    # Find nearest-neighbor matches using a KD tree.\n    d1_tree = cKDTree(result1['descriptors'])\n    _, indices = d1_tree.query(\n      result2['descriptors'],\n      distance_upper_bound=distance_threshold)\n\n    # Select feature locations for putative matches.\n    locations_2_to_use = np.array([\n      result2['locations'][i,]\n      for i in range(num_features_2)\n      if indices[i] != num_features_1\n    ])\n    locations_1_to_use = np.array([\n      result1['locations'][indices[i],]\n      for i in range(num_features_2)\n      if indices[i] != num_features_1\n    ])\n\n    # Perform geometric verification using RANSAC.\n    _, inliers = ransac(\n      (locations_1_to_use, locations_2_to_use),\n      AffineTransform,\n      min_samples=3,\n      residual_threshold=20,\n      max_trials=1000)\n    \n    if verbose:\n        print('Found %d inliers' % sum(inliers))\n\n    # Visualize correspondences.\n    _, ax = plt.subplots(figsize = (9, 9))\n    inlier_idxs = np.nonzero(inliers)[0]\n    plot_matches(\n      ax,\n      image1,\n      image2,\n      locations_1_to_use,\n      locations_2_to_use,\n      np.column_stack((inlier_idxs, inlier_idxs)),\n      matches_color='b')\n    ax.axis('off')\n    ax.set_title(f'DELF correspondences: Found {sum(inliers)} inliers')","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:13:25.312287Z","iopub.execute_input":"2023-04-10T23:13:25.312578Z","iopub.status.idle":"2023-04-10T23:13:25.328467Z","shell.execute_reply.started":"2023-04-10T23:13:25.312547Z","shell.execute_reply":"2023-04-10T23:13:25.326103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:55:34.38447Z","iopub.execute_input":"2023-04-10T22:55:34.384754Z","iopub.status.idle":"2023-04-10T22:55:34.49308Z","shell.execute_reply.started":"2023-04-10T22:55:34.384724Z","shell.execute_reply":"2023-04-10T22:55:34.492211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"match_images(image_1, image_2, delf_result1, delf_result2)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:59.209882Z","iopub.status.idle":"2023-04-10T22:43:59.210678Z","shell.execute_reply.started":"2023-04-10T22:43:59.210443Z","shell.execute_reply":"2023-04-10T22:43:59.210467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image_index in similar_n[:6]:\n    key_image = get_image(train_embeddings['images_paths'][image_index],\n                          resize = True,\n                          target_size = DELF_IMG_SIZE)\n    try:\n        delf_key_image_result = run_delf(key_image)\n        match_images(image_1, key_image, delf_result1, delf_key_image_result, verbose = False)\n    except:\n        print(\"No inliers found\")","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:59.211895Z","iopub.status.idle":"2023-04-10T22:43:59.21268Z","shell.execute_reply.started":"2023-04-10T22:43:59.212448Z","shell.execute_reply":"2023-04-10T22:43:59.212472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reranking\nReranking using DELF local features descriptor","metadata":{}},{"cell_type":"code","source":"def delf_rerank(query = None, key = None, query_image_index = None, confidence_df = None, re_sort = True):\n    distance_threshold = 0.8\n    query_paths = query['images_paths']\n    key_paths = key['images_paths']\n    \n    query_image = get_image(query_paths[query_image_index],\n                            resize = True,\n                            target_size = DELF_IMG_SIZE)\n    \n    delf_result_query = run_delf(query_image)\n    \n    # Read query features\n    num_features_query = delf_result_query['locations'].shape[0]\n    \n    inliers_list = []\n    print(f\"Retrieving local features for top {len(confidence_df['image_paths'])} key images...\")\n    for image_path in tqdm(confidence_df['image_paths']):\n        key_image = get_image(image_path,\n                          resize = True,\n                          target_size = DELF_IMG_SIZE)\n        \n        delf_result_key = run_delf(key_image)\n    \n        # Read key features\n        num_features_key = delf_result_key['locations'].shape[0]\n\n        # Find nearest-neighbor matches using a KD tree.\n        d1_tree = cKDTree(delf_result_query['descriptors'])\n        _, indices = d1_tree.query(\n          delf_result_key['descriptors'],\n          distance_upper_bound=distance_threshold)\n\n        # Select feature locations for putative matches.\n        locations_k_to_use = np.array([\n          delf_result_key['locations'][i,]\n          for i in range(num_features_key)\n          if indices[i] != num_features_query\n        ])\n        locations_q_to_use = np.array([\n          delf_result_query['locations'][indices[i],]\n          for i in range(num_features_key)\n          if indices[i] != num_features_query\n        ])\n\n        # Perform geometric verification using RANSAC.\n        try:\n            _, inliers = ransac(\n              (locations_q_to_use, locations_k_to_use),\n              AffineTransform,\n              min_samples=3,\n              residual_threshold=20,\n              max_trials=1000)\n        except:\n            inliers = [0]\n        \n        # Handling 0 inliers\n        try:\n            total_inliers = sum(inliers)\n            inliers_list.append(total_inliers)\n        except:\n            inliers_list.append(1) # Appending inlier = 1 to avoid null confidence\n    \n    confidence_df['inliers'] = inliers_list\n    \n    original_confidence = confidence_df['inliers']\n    reranked_confidence = np.sqrt(original_confidence) * confidence_df['cos_similarity']\n    confidence_df['reranked_conf'] = reranked_confidence\n    \n    if re_sort:\n        confidence_df.sort_values('reranked_conf', ascending = False, inplace = True)\n    \n    return confidence_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:17:14.798527Z","iopub.execute_input":"2023-04-10T23:17:14.798809Z","iopub.status.idle":"2023-04-10T23:17:14.810481Z","shell.execute_reply.started":"2023-04-10T23:17:14.79878Z","shell.execute_reply":"2023-04-10T23:17:14.809702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reranked_df = delf_rerank(query = val_embeddings,\n                          key = train_embeddings,\n                          query_image_index = query_image_index,\n                          confidence_df = confidence_df,\n                          re_sort = True)\nreranked_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:17:20.489698Z","iopub.execute_input":"2023-04-10T23:17:20.489967Z","iopub.status.idle":"2023-04-10T23:17:33.649572Z","shell.execute_reply.started":"2023-04-10T23:17:20.489938Z","shell.execute_reply":"2023-04-10T23:17:33.64863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 887\ntop_n = 10\n\nquery_top(query_image_index, top_n, reranked = reranked_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:17:55.119201Z","iopub.execute_input":"2023-04-10T23:17:55.119469Z","iopub.status.idle":"2023-04-10T23:17:56.450363Z","shell.execute_reply.started":"2023-04-10T23:17:55.119441Z","shell.execute_reply":"2023-04-10T23:17:56.449562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reranking examples","metadata":{}},{"cell_type":"code","source":"query_image_index = 11\ntop_n = 10\n\nquery_top(query_image_index, top_n)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:18:17.160713Z","iopub.execute_input":"2023-04-10T23:18:17.161518Z","iopub.status.idle":"2023-04-10T23:18:18.463596Z","shell.execute_reply.started":"2023-04-10T23:18:17.161471Z","shell.execute_reply":"2023-04-10T23:18:18.462898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:18:24.4059Z","iopub.execute_input":"2023-04-10T23:18:24.406474Z","iopub.status.idle":"2023-04-10T23:18:24.420287Z","shell.execute_reply.started":"2023-04-10T23:18:24.406436Z","shell.execute_reply":"2023-04-10T23:18:24.41945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reranked_df = delf_rerank(query = val_embeddings,\n                          key = train_embeddings,\n                          query_image_index = query_image_index,\n                          confidence_df = confidence_df,\n                          re_sort = True)\nreranked_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:18:32.792662Z","iopub.execute_input":"2023-04-10T23:18:32.792937Z","iopub.status.idle":"2023-04-10T23:18:45.433551Z","shell.execute_reply.started":"2023-04-10T23:18:32.792903Z","shell.execute_reply":"2023-04-10T23:18:45.432653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 11\ntop_n = 10\n\nquery_top(query_image_index, top_n, reranked = reranked_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:18:57.43346Z","iopub.execute_input":"2023-04-10T23:18:57.433745Z","iopub.status.idle":"2023-04-10T23:18:58.79787Z","shell.execute_reply.started":"2023-04-10T23:18:57.433715Z","shell.execute_reply":"2023-04-10T23:18:58.797202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 395\ntop_n = 10\n\nquery_top(query_image_index, top_n)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:19:02.15095Z","iopub.execute_input":"2023-04-10T23:19:02.151536Z","iopub.status.idle":"2023-04-10T23:19:03.440441Z","shell.execute_reply.started":"2023-04-10T23:19:02.1515Z","shell.execute_reply":"2023-04-10T23:19:03.438722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:19:07.917506Z","iopub.execute_input":"2023-04-10T23:19:07.91824Z","iopub.status.idle":"2023-04-10T23:19:07.932664Z","shell.execute_reply.started":"2023-04-10T23:19:07.918202Z","shell.execute_reply":"2023-04-10T23:19:07.931947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reranked_df = delf_rerank(query = val_embeddings,\n                          key = train_embeddings,\n                          query_image_index = query_image_index,\n                          confidence_df = confidence_df,\n                          re_sort = True)\nreranked_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:19:13.064117Z","iopub.execute_input":"2023-04-10T23:19:13.064837Z","iopub.status.idle":"2023-04-10T23:19:25.636068Z","shell.execute_reply.started":"2023-04-10T23:19:13.064802Z","shell.execute_reply":"2023-04-10T23:19:25.635093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 395\ntop_n = 10\n\nquery_top(query_image_index, top_n, reranked = reranked_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:19:30.060898Z","iopub.execute_input":"2023-04-10T23:19:30.061381Z","iopub.status.idle":"2023-04-10T23:19:31.373603Z","shell.execute_reply.started":"2023-04-10T23:19:30.061345Z","shell.execute_reply":"2023-04-10T23:19:31.372969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 40\ntop_n = 10\n\nquery_top(query_image_index, top_n)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:59.235848Z","iopub.status.idle":"2023-04-10T22:43:59.236674Z","shell.execute_reply.started":"2023-04-10T22:43:59.236433Z","shell.execute_reply":"2023-04-10T22:43:59.236462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:59.237807Z","iopub.status.idle":"2023-04-10T22:43:59.238602Z","shell.execute_reply.started":"2023-04-10T22:43:59.238366Z","shell.execute_reply":"2023-04-10T22:43:59.238395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reranked_df = delf_rerank(query = val_embeddings,\n                          key = train_embeddings,\n                          query_image_index = query_image_index,\n                          confidence_df = confidence_df,\n                          re_sort = True)\nreranked_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:59.239742Z","iopub.status.idle":"2023-04-10T22:43:59.240405Z","shell.execute_reply.started":"2023-04-10T22:43:59.240174Z","shell.execute_reply":"2023-04-10T22:43:59.240197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_image_index = 40\ntop_n = 10\n\nquery_top(query_image_index, top_n, reranked = reranked_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T22:43:59.241693Z","iopub.status.idle":"2023-04-10T22:43:59.2425Z","shell.execute_reply.started":"2023-04-10T22:43:59.242263Z","shell.execute_reply":"2023-04-10T22:43:59.242293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Under-represented query image\nEffect of querying images not well represented on the key set","metadata":{}},{"cell_type":"code","source":"query_image_index = 36\ntop_n = 10\n\nquery_top(query_image_index, top_n)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:20:01.984945Z","iopub.execute_input":"2023-04-10T23:20:01.985228Z","iopub.status.idle":"2023-04-10T23:20:03.403423Z","shell.execute_reply.started":"2023-04-10T23:20:01.9852Z","shell.execute_reply":"2023-04-10T23:20:03.399169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confidence_df = confidence_top(query = val_embeddings,\n                               key = train_embeddings,\n                               similarity = val_train_similarity,\n                               query_image_index = query_image_index,\n                               top = top_n)\n\nconfidence_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:20:34.458564Z","iopub.execute_input":"2023-04-10T23:20:34.458836Z","iopub.status.idle":"2023-04-10T23:20:34.471697Z","shell.execute_reply.started":"2023-04-10T23:20:34.458807Z","shell.execute_reply":"2023-04-10T23:20:34.47086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reranked_df = delf_rerank(query = val_embeddings,\n                          key = train_embeddings,\n                          query_image_index = query_image_index,\n                          confidence_df = confidence_df,\n                          re_sort = True)\nreranked_df","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:20:58.553431Z","iopub.execute_input":"2023-04-10T23:20:58.553736Z","iopub.status.idle":"2023-04-10T23:21:10.074799Z","shell.execute_reply.started":"2023-04-10T23:20:58.553705Z","shell.execute_reply":"2023-04-10T23:21:10.073691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's investigate what the model has seen for landmark 36 during training...","metadata":{}},{"cell_type":"code","source":"get_landmark(25)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:22:00.664457Z","iopub.execute_input":"2023-04-10T23:22:00.66474Z","iopub.status.idle":"2023-04-10T23:22:02.520635Z","shell.execute_reply.started":"2023-04-10T23:22:00.66471Z","shell.execute_reply":"2023-04-10T23:22:02.519914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Query image has a representativeness issue when considering how the landmark appears on key set and how it was seen during training.","metadata":{}},{"cell_type":"code","source":"! cd test_sub","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:28:32.777341Z","iopub.execute_input":"2023-04-10T23:28:32.77769Z","iopub.status.idle":"2023-04-10T23:28:34.008924Z","shell.execute_reply.started":"2023-04-10T23:28:32.777653Z","shell.execute_reply":"2023-04-10T23:28:34.007675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls test_sub","metadata":{"execution":{"iopub.status.busy":"2023-04-10T23:29:01.883236Z","iopub.execute_input":"2023-04-10T23:29:01.88356Z","iopub.status.idle":"2023-04-10T23:29:02.91292Z","shell.execute_reply.started":"2023-04-10T23:29:01.883528Z","shell.execute_reply":"2023-04-10T23:29:02.911996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! python --version","metadata":{"execution":{"iopub.status.busy":"2023-04-11T17:21:28.705282Z","iopub.execute_input":"2023-04-11T17:21:28.70561Z","iopub.status.idle":"2023-04-11T17:21:29.641584Z","shell.execute_reply.started":"2023-04-11T17:21:28.705579Z","shell.execute_reply":"2023-04-11T17:21:29.640723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip freeze","metadata":{"execution":{"iopub.status.busy":"2023-04-11T16:56:14.677374Z","iopub.execute_input":"2023-04-11T16:56:14.678084Z","iopub.status.idle":"2023-04-11T16:56:18.805398Z","shell.execute_reply.started":"2023-04-11T16:56:14.677992Z","shell.execute_reply":"2023-04-11T16:56:18.804524Z"},"trusted":true},"execution_count":null,"outputs":[]}]}