{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Kaggle kernel for \"Google Landmarks Recognition Challenge 2020\".**\n\nMany Kagglers are familiar with image classification challenges like the ImageNet Large \nScale Visual Recognition Challenge (ILSVRC), which aims to recognize 1K general object categories. \nLandmark recognition is a little different from that: it contains a much larger number of classes (there are more than 81K classes in this challenge),\nand the number of training examples per class may not be very large. \nLandmark recognition is challenging in its own way.\n\nIn the previous editions of this challenge (2018 and 2019), \nsubmissions were handled by uploading prediction files to the system. \nThis year's competition is structured in a synchronous rerun format, \nwhere participants need to submit their Kaggle notebooks for scoring.\n\nThis challenge is organized in conjunction with the Landmark Retrieval Challenge 2020, \nwhich was launched June 30, 2020. \nBoth challenges are affiliated with the Instance-Level Recognition workshop in ECCV’20.\n"},{"metadata":{"_uuid":"c8c52dac-dad9-4043-92fa-3809ce6eb20f","_cell_guid":"8f0ba39f-da65-4c71-8e10-9bb5959e9b61","trusted":true},"cell_type":"code","source":"# import required libraries\nimport copy\nimport csv\nimport gc\nimport operator\nimport os\nimport pathlib\nimport shutil\n\nimport numpy as np\nimport PIL\nimport pydegensac\nfrom scipy import spatial\nimport tensorflow as tf\nprint(\"Successfully imported\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Input data\n# Dataset parameters\n\nINPUT_DIR = os.path.join('..', 'input')\n\nDATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')\nTEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')\nTRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')\nTRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DEBUGGING PARAMS\nNUM_PUBLIC_TRAIN_IMAGES = 1580470  # Used to detect if in session or re-run.\nMAX_NUM_EMBEDDINGS = -1            # Set to > 1 to subsample dataset while debugging.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Retrieval & re-ranking parameters\nNUM_TO_RERANK = 3\nTOP_K = 3           #Number of retrieved images used to make prediction for a test image.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# RANSAC parameters\nMAX_INLIER_SCORE = 35\nMAX_REPROJECTION_ERROR = 6.0\nMAX_RANSAC_ITERATIONS = 10_000_000\nHOMOGRAPHY_CONFIDENCE = 0.99","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DELG model:\nSAVED_MODEL_DIR = '../input/delg-saved-models/local_and_global'\nDELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)\nDELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])\nDELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)\nDELG_INPUT_TENSOR_NAMES = [\n    'input_image:0', 'input_scales:0', 'input_abs_thres:0']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Global feature extraction:\nNUM_EMBEDDING_DIMENSIONS = 2048\nGLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,\n                                                ['global_descriptors:0'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Local feature extraction:\nLOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)\nLOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(\n    DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],\n    ['boxes:0', 'features:0'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to convert to Hexdecimal\ndef to_hex(image_id) -> str:\n  return '{0:0{1}x}'.format(image_id, 16)\n\n# Function to get image path\ndef get_image_path(subset, image_id):\n  name = to_hex(image_id)\n  return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],\n                      '{}.jpg'.format(name))\n\n\n# Function to load images into tensor\ndef load_image_tensor(image_path):\n  return tf.convert_to_tensor(\n      np.array(PIL.Image.open(image_path).convert('RGB')))\n\n\n# Function for extracting global features\ndef extract_global_features(image_root_dir):\n  \"\"\"Extracts embeddings for all the images in given `image_root_dir`.\"\"\"\n\n  image_paths = [x for x in pathlib.Path(image_root_dir).rglob('*.jpg')]\n\n  num_embeddings = len(image_paths)\n  if MAX_NUM_EMBEDDINGS > 0:\n    num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)\n\n  ids = num_embeddings * [None]\n  embeddings = np.empty((num_embeddings, NUM_EMBEDDING_DIMENSIONS))\n\n  for i, image_path in enumerate(image_paths):\n    if i >= num_embeddings:\n      break\n\n    ids[i] = int(image_path.name.split('.')[0], 16)\n    image_tensor = load_image_tensor(image_path)\n    features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,\n                                            DELG_IMAGE_SCALES_TENSOR,\n                                            DELG_SCORE_THRESHOLD_TENSOR)\n    embeddings[i, :] = tf.nn.l2_normalize(\n        tf.reduce_sum(features[0], axis=0, name='sum_pooling'),\n        axis=0,\n        name='final_l2_normalization').numpy()\n\n  return ids, embeddings\n\n\n# Function for extracting local features\ndef extract_local_features(image_path):\n  \"\"\"Extracts local features for the given `image_path`.\"\"\"\n\n  image_tensor = load_image_tensor(image_path)\n\n  features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,\n                                         DELG_SCORE_THRESHOLD_TENSOR,\n                                         LOCAL_FEATURE_NUM_TENSOR)\n\n  # Shape: (N, 2)\n  keypoints = tf.divide(\n      tf.add(\n          tf.gather(features[0], [0, 1], axis=1),\n          tf.gather(features[0], [2, 3], axis=1)), 2.0).numpy()\n\n  # Shape: (N, 128)\n  descriptors = tf.nn.l2_normalize(\n      features[1], axis=1, name='l2_normalization').numpy()\n\n  return keypoints, descriptors\n\n\n# Function to get Putative matches of the feature points in both images which are computed \n# by using a correlation measure for points in one image with a features in the other image.\ndef get_putative_matching_keypoints(test_keypoints,\n                                    test_descriptors,\n                                    train_keypoints,\n                                    train_descriptors,\n                                    max_distance=0.9):\n  \"\"\"Finds matches from `test_descriptors` to KD-tree of `train_descriptors`.\"\"\"\n\n  train_descriptor_tree = spatial.cKDTree(train_descriptors)\n  _, matches = train_descriptor_tree.query(\n      test_descriptors, distance_upper_bound=max_distance)\n\n  test_kp_count = test_keypoints.shape[0]\n  train_kp_count = train_keypoints.shape[0]\n\n  test_matching_keypoints = np.array([\n      test_keypoints[i,]\n      for i in range(test_kp_count)\n      if matches[i] != train_kp_count\n  ])\n  train_matching_keypoints = np.array([\n      train_keypoints[matches[i],]\n      for i in range(test_kp_count)\n      if matches[i] != train_kp_count\n  ])\n\n  return test_matching_keypoints, train_matching_keypoints\n\n\n# Function to get no of RANSAC inliers.\ndef get_num_inliers(test_keypoints, test_descriptors, train_keypoints,\n                    train_descriptors):\n  \"\"\"Returns the number of RANSAC inliers.\"\"\"\n\n  test_match_kp, train_match_kp = get_putative_matching_keypoints(\n      test_keypoints, test_descriptors, train_keypoints, train_descriptors)\n\n  if test_match_kp.shape[\n      0] <= 4:  # Min keypoints supported by `pydegensac.findHomography()`\n    return 0\n\n  try:\n    _, mask = pydegensac.findHomography(test_match_kp, train_match_kp,\n                                        MAX_REPROJECTION_ERROR,\n                                        HOMOGRAPHY_CONFIDENCE,\n                                        MAX_RANSAC_ITERATIONS)\n  except np.linalg.LinAlgError:  # When det(H)=0, can't invert matrix.\n    return 0\n\n  return int(copy.deepcopy(mask).astype(np.float32).sum())\n\n\n# Function to get total score\ndef get_total_score(num_inliers, global_score):\n  local_score = min(num_inliers, MAX_INLIER_SCORE) / MAX_INLIER_SCORE\n  return local_score + global_score\n\n# Function to get rescored and sorted training images by local feature extraction\ndef rescore_and_rerank_by_num_inliers(test_image_id,\n                                      train_ids_labels_and_scores):\n  \"\"\"Returns rescored and sorted training images by local feature extraction.\"\"\"\n\n  test_image_path = get_image_path('test', test_image_id)\n  test_keypoints, test_descriptors = extract_local_features(test_image_path)\n\n  for i in range(len(train_ids_labels_and_scores)):\n    train_image_id, label, global_score = train_ids_labels_and_scores[i]\n\n    train_image_path = get_image_path('train', train_image_id)\n    train_keypoints, train_descriptors = extract_local_features(\n        train_image_path)\n\n    num_inliers = get_num_inliers(test_keypoints, test_descriptors,\n                                  train_keypoints, train_descriptors)\n    total_score = get_total_score(num_inliers, global_score)\n    train_ids_labels_and_scores[i] = (train_image_id, label, total_score)\n\n  train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)\n\n  return train_ids_labels_and_scores\n\n# function to load labelmap\ndef load_labelmap():\n  with open(TRAIN_LABELMAP_PATH, mode='r') as csv_file:\n    csv_reader = csv.DictReader(csv_file)\n    labelmap = {row['id']: row['landmark_id'] for row in csv_reader}\n\n  return labelmap\n\n# function for prediction map\ndef get_prediction_map(test_ids, train_ids_labels_and_scores):\n  \"\"\"Makes dict from test ids and ranked training ids, labels, scores.\"\"\"\n\n  prediction_map = dict()\n\n  for test_index, test_id in enumerate(test_ids):\n    hex_test_id = to_hex(test_id)\n\n    aggregate_scores = {}\n    for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:\n      if label not in aggregate_scores:\n        aggregate_scores[label] = 0\n      aggregate_scores[label] += score\n\n    label, score = max(aggregate_scores.items(), key=operator.itemgetter(1))\n\n    prediction_map[hex_test_id] = {'score': score, 'class': label}\n\n  return prediction_map\n\n# function to get predictions\ndef get_predictions(labelmap):\n  \"\"\"Gets predictions using embedding similarity and local feature reranking.\"\"\"\n\n  test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)\n\n  train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)\n\n  train_ids_labels_and_scores = [None] * test_embeddings.shape[0]\n\n  # Using (slow) for-loop, as distance matrix doesn't fit in memory.\n  for test_index in range(test_embeddings.shape[0]):\n    distances = spatial.distance.cdist(\n        test_embeddings[np.newaxis, test_index, :], train_embeddings,\n        'cosine')[0]\n    partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]\n\n    nearest = sorted([(train_ids[p], distances[p]) for p in partition],\n                     key=lambda x: x[1])\n\n    train_ids_labels_and_scores[test_index] = [\n        (train_id, labelmap[to_hex(train_id)], 1. - cosine_distance)\n        for train_id, cosine_distance in nearest\n    ]\n\n  del test_embeddings\n  del train_embeddings\n  del labelmap\n  gc.collect()\n\n  pre_verification_predictions = get_prediction_map(\n      test_ids, train_ids_labels_and_scores)\n\n#  return None, pre_verification_predictions\n\n  for test_index, test_id in enumerate(test_ids):\n    train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(\n        test_id, train_ids_labels_and_scores[test_index])\n\n  post_verification_predictions = get_prediction_map(\n      test_ids, train_ids_labels_and_scores)\n\n  return pre_verification_predictions, post_verification_predictions\n\n# function to save submission csv file.\ndef save_submission_csv(predictions=None):\n  \"\"\"Saves optional `predictions` as submission.csv.\n\n  The csv has columns {id, landmarks}. The landmarks column is a string\n  containing the label and score for the id, separated by a ws delimeter.\n\n  If `predictions` is `None` (default), submission.csv is copied from\n  sample_submission.csv in `IMAGE_DIR`.\n\n  Args:\n    predictions: Optional dict of image ids to dicts with keys {class, score}.\n  \"\"\"\n\n  if predictions is None:\n    # Dummy submission!\n    shutil.copyfile(\n        os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')\n    return\n\n  with open('submission.csv', 'w') as submission_csv:\n    csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])\n    csv_writer.writeheader()\n    for image_id, prediction in predictions.items():\n      label = prediction['class']\n      score = prediction['score']\n      csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})\n\n\n# Main Function\ndef main():\n  labelmap = load_labelmap()\n  num_training_images = len(labelmap.keys())\n  print(f'Found {num_training_images} training images.')\n\n  if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:\n    print('Copying sample submission.')\n    save_submission_csv()\n    return\n\n  _, post_verification_predictions = get_predictions(labelmap)\n  save_submission_csv(post_verification_predictions)\n\n\nif __name__ == '__main__':\n  main()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}