{"cells":[{"metadata":{},"cell_type":"markdown","source":"As I don't have enough resourses, just experimenting with RANSAC hyperparameters. \nHere's what I observed so far...\n\nIncressing MAX_INLIER_SCORE,MAX_REPROJECTION_ERROR decreases score\nDecreasing HOMOGRAPHY_CONFIDENCE decreases score\nIncressing MAX_RANSAC_ITERATIONS a little over 8.5 million increases score, however increasing it more decreases score\n\nBy incressing MAX_RANSAC_ITERATIONS by 500k million, the score improved by 0.0003"},{"metadata":{},"cell_type":"markdown","source":"Any recommendations to better tune these parametes are welcomed."},{"metadata":{},"cell_type":"markdown","source":"**At the set hyperparameters as of now, the score is 0.4856\nAlso, I didn't find any much learning with this approach. Any new approach suggestions would be highly appreciated.** which work with limited resources and constrained time limits."},{"metadata":{"trusted":true},"cell_type":"code","source":"import copy\nimport csv\nimport gc\nimport operator\nimport os\nimport pathlib\nimport shutil\n\nimport numpy as np\nimport PIL\nimport pydegensac\nfrom scipy import spatial\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dataset parameters:\nINPUT_DIR = os.path.join('..', 'input')\n\nDATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')\nTEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')\nTRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')\nTRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')\n\n# DEBUGGING PARAMS:\nNUM_PUBLIC_TRAIN_IMAGES = 1580470 # Used to detect if in session or re-run.\nMAX_NUM_EMBEDDINGS = -1  # Set to > 1 to subsample dataset while debugging.\n\n# Retrieval & re-ranking parameters:\nNUM_TO_RERANK = 3\nTOP_K = 3 # Number of retrieved images used to make prediction for a test image.\n\n# RANSAC parameters:\nMAX_INLIER_SCORE = 35\nMAX_REPROJECTION_ERROR = 7.0\nMAX_RANSAC_ITERATIONS = 8500000\nHOMOGRAPHY_CONFIDENCE = 0.99\n\n# DELG model:\nSAVED_MODEL_DIR = '../input/delg-saved-models/local_and_global'\nDELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)\nDELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])\nDELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)\nDELG_INPUT_TENSOR_NAMES = [\n    'input_image:0', 'input_scales:0', 'input_abs_thres:0'\n]\n\n# Global feature extraction:\nNUM_EMBEDDING_DIMENSIONS = 2048\nGLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,\n                                                ['global_descriptors:0'])\n\n# Local feature extraction:\nLOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)\nLOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(\n    DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],\n    ['boxes:0', 'features:0'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"def to_hex(image_id) -> str:\n  return '{0:0{1}x}'.format(image_id, 16)\n\n\ndef get_image_path(subset, image_id):\n  name = to_hex(image_id)\n  return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],\n                      '{}.jpg'.format(name))\n\n\ndef load_image_tensor(image_path):\n  return tf.convert_to_tensor(\n      np.array(PIL.Image.open(image_path).convert('RGB')))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_global_features(image_root_dir):\n  \"\"\"Extracts embeddings for all the images in given `image_root_dir`.\"\"\"\n\n  image_paths = [x for x in pathlib.Path(image_root_dir).rglob('*.jpg')]\n\n  num_embeddings = len(image_paths)\n  if MAX_NUM_EMBEDDINGS > 0:\n    num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)\n\n  ids = num_embeddings * [None]\n  embeddings = np.empty((num_embeddings, NUM_EMBEDDING_DIMENSIONS))\n\n  for i, image_path in enumerate(image_paths):\n    if i >= num_embeddings:\n      break\n\n    ids[i] = int(image_path.name.split('.')[0], 16)\n    image_tensor = load_image_tensor(image_path)\n    features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,\n                                            DELG_IMAGE_SCALES_TENSOR,\n                                            DELG_SCORE_THRESHOLD_TENSOR)\n    embeddings[i, :] = tf.nn.l2_normalize(\n        tf.reduce_sum(features[0], axis=0, name='sum_pooling'),\n        axis=0,\n        name='final_l2_normalization').numpy()\n\n  return ids, embeddings","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_local_features(image_path):\n  \"\"\"Extracts local features for the given `image_path`.\"\"\"\n\n  image_tensor = load_image_tensor(image_path)\n\n  features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,\n                                         DELG_SCORE_THRESHOLD_TENSOR,\n                                         LOCAL_FEATURE_NUM_TENSOR)\n\n  # Shape: (N, 2)\n  keypoints = tf.divide(\n      tf.add(\n          tf.gather(features[0], [0, 1], axis=1),\n          tf.gather(features[0], [2, 3], axis=1)), 2.0).numpy()\n\n  # Shape: (N, 128)\n  descriptors = tf.nn.l2_normalize(\n      features[1], axis=1, name='l2_normalization').numpy()\n\n  return keypoints, descriptors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_putative_matching_keypoints(test_keypoints,\n                                    test_descriptors,\n                                    train_keypoints,\n                                    train_descriptors,\n                                    max_distance=0.9):\n  \"\"\"Finds matches from `test_descriptors` to KD-tree of `train_descriptors`.\"\"\"\n\n  train_descriptor_tree = spatial.cKDTree(train_descriptors)\n  _, matches = train_descriptor_tree.query(\n      test_descriptors, distance_upper_bound=max_distance)\n\n  test_kp_count = test_keypoints.shape[0]\n  train_kp_count = train_keypoints.shape[0]\n\n  test_matching_keypoints = np.array([\n      test_keypoints[i,]\n      for i in range(test_kp_count)\n      if matches[i] != train_kp_count\n  ])\n  train_matching_keypoints = np.array([\n      train_keypoints[matches[i],]\n      for i in range(test_kp_count)\n      if matches[i] != train_kp_count\n  ])\n\n  return test_matching_keypoints, train_matching_keypoints","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,\n                    train_descriptors):\n  \"\"\"Returns the number of RANSAC inliers.\"\"\"\n\n  test_match_kp, train_match_kp = get_putative_matching_keypoints(\n      test_keypoints, test_descriptors, train_keypoints, train_descriptors)\n\n  if test_match_kp.shape[\n      0] <= 4:  # Min keypoints supported by `pydegensac.findHomography()`\n    return 0\n\n  try:\n    _, mask = pydegensac.findHomography(test_match_kp, train_match_kp,\n                                        MAX_REPROJECTION_ERROR,\n                                        HOMOGRAPHY_CONFIDENCE,\n                                        MAX_RANSAC_ITERATIONS)\n  except np.linalg.LinAlgError:  # When det(H)=0, can't invert matrix.\n    return 0\n\n  return int(copy.deepcopy(mask).astype(np.float32).sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_total_score(num_inliers, global_score):\n  local_score = min(num_inliers, MAX_INLIER_SCORE) / MAX_INLIER_SCORE\n  return local_score + global_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rescore_and_rerank_by_num_inliers(test_image_id,\n                                      train_ids_labels_and_scores):\n  \"\"\"Returns rescored and sorted training images by local feature extraction.\"\"\"\n\n  test_image_path = get_image_path('test', test_image_id)\n  test_keypoints, test_descriptors = extract_local_features(test_image_path)\n\n  for i in range(len(train_ids_labels_and_scores)):\n    train_image_id, label, global_score = train_ids_labels_and_scores[i]\n\n    train_image_path = get_image_path('train', train_image_id)\n    train_keypoints, train_descriptors = extract_local_features(\n        train_image_path)\n\n    num_inliers = get_num_inliers(test_keypoints, test_descriptors,\n                                  train_keypoints, train_descriptors)\n    total_score = get_total_score(num_inliers, global_score)\n    train_ids_labels_and_scores[i] = (train_image_id, label, total_score)\n\n  train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)\n\n  return train_ids_labels_and_scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_labelmap():\n  with open(TRAIN_LABELMAP_PATH, mode='r') as csv_file:\n    csv_reader = csv.DictReader(csv_file)\n    labelmap = {row['id']: row['landmark_id'] for row in csv_reader}\n\n  return labelmap","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_prediction_map(test_ids, train_ids_labels_and_scores):\n  \"\"\"Makes dict from test ids and ranked training ids, labels, scores.\"\"\"\n\n  prediction_map = dict()\n\n  for test_index, test_id in enumerate(test_ids):\n    hex_test_id = to_hex(test_id)\n\n    aggregate_scores = {}\n    for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:\n      if label not in aggregate_scores:\n        aggregate_scores[label] = 0\n      aggregate_scores[label] += score\n\n    label, score = max(aggregate_scores.items(), key=operator.itemgetter(1))\n\n    prediction_map[hex_test_id] = {'score': score, 'class': label}\n\n  return prediction_map","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_predictions(labelmap):\n  \"\"\"Gets predictions using embedding similarity and local feature reranking.\"\"\"\n\n  test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)\n\n  train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)\n\n  train_ids_labels_and_scores = [None] * test_embeddings.shape[0]\n\n  # Using (slow) for-loop, as distance matrix doesn't fit in memory.\n  for test_index in range(test_embeddings.shape[0]):\n    distances = spatial.distance.cdist(\n        test_embeddings[np.newaxis, test_index, :], train_embeddings,\n        'cosine')[0]\n    partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]\n\n    nearest = sorted([(train_ids[p], distances[p]) for p in partition],\n                     key=lambda x: x[1])\n\n    train_ids_labels_and_scores[test_index] = [\n        (train_id, labelmap[to_hex(train_id)], 1. - cosine_distance)\n        for train_id, cosine_distance in nearest\n    ]\n\n  del test_embeddings\n  del train_embeddings\n  del labelmap\n  gc.collect()\n\n  pre_verification_predictions = get_prediction_map(\n      test_ids, train_ids_labels_and_scores)\n\n#  return None, pre_verification_predictions\n\n  for test_index, test_id in enumerate(test_ids):\n    train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(\n        test_id, train_ids_labels_and_scores[test_index])\n\n  post_verification_predictions = get_prediction_map(\n      test_ids, train_ids_labels_and_scores)\n\n  return pre_verification_predictions, post_verification_predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_submission_csv(predictions=None):\n  \"\"\"Saves optional `predictions` as submission.csv.\n\n  The csv has columns {id, landmarks}. The landmarks column is a string\n  containing the label and score for the id, separated by a ws delimeter.\n\n  If `predictions` is `None` (default), submission.csv is copied from\n  sample_submission.csv in `IMAGE_DIR`.\n\n  Args:\n    predictions: Optional dict of image ids to dicts with keys {class, score}.\n  \"\"\"\n\n  if predictions is None:\n    # Dummy submission!\n    shutil.copyfile(\n        os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')\n    return\n\n  with open('submission.csv', 'w') as submission_csv:\n    csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])\n    csv_writer.writeheader()\n    for image_id, prediction in predictions.items():\n      label = prediction['class']\n      score = prediction['score']\n      csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def main():\n  labelmap = load_labelmap()\n  num_training_images = len(labelmap.keys())\n  print(f'Found {num_training_images} training images.')\n\n  if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:\n    print(\n        f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images. Copying sample submission.'\n    )\n    save_submission_csv()\n    return\n\n  _, post_verification_predictions = get_predictions(labelmap)\n  save_submission_csv(post_verification_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"main()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}