{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaK80","dataSources":[{"sourceId":19231,"databundleVersionId":1413778,"sourceType":"competition"},{"sourceId":1383617,"sourceType":"datasetVersion","datasetId":807160}],"dockerImageVersionId":29985,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"Baseline Notebook for \"Google Landmarks Recognition Challenge 2020\".\n\nGenerates `submission.csv` in Kaggle format. When the number of Training images\nindicates that the kernel is being run against the public dataset,\nsimply copies `sample_submission.csv` to allow for quickly starting reruns\non the private dataset. When in a rerun against the private dataset,\nmakes predictions via retrieval, using DELG TensorFlow SavedModels for global\nand local feature extraction.\n\nFirst, ranks all training images by embedding similarity to each test image.\nThen, performs geometric-verification and re-ranking on the `NUM_TO_RERANK`\nmost similar training images. For a given test image, each class' score is\nthe sum of the scores of re-ranked training images, and the predicted\nclass is the one with the highest aggregate score.\n\nNOTE: For speed, this uses `pydegensac` as its RANSAC implementation.\nSince the module has no interface for setting random seeds, RANSAC results\nand submission scores will vary slightly between reruns.\n\"\"\"\n\nimport copy\nimport csv\nimport gc\nimport operator\nimport os\nimport pathlib\nimport shutil\n\nimport numpy as np\nimport PIL\nimport pydegensac\nfrom scipy import spatial\nimport tensorflow as tf\n\n# Dataset parameters:\nINPUT_DIR = os.path.join('..', 'input')\n\nDATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')\nTEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')\nTRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')\nTRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')\n\n# DEBUGGING PARAMS:\nNUM_PUBLIC_TRAIN_IMAGES = 1580470 # Used to detect if in session or re-run.\nMAX_NUM_EMBEDDINGS = -1  # Set to > 1 to subsample dataset while debugging.\n\n# Retrieval & re-ranking parameters:\nNUM_TO_RERANK = 3\nTOP_K = 3 # Number of retrieved images used to make prediction for a test image.\n\n# RANSAC parameters:\nMAX_INLIER_SCORE = 35\nMAX_REPROJECTION_ERROR = 7.0\nMAX_RANSAC_ITERATIONS = 8000000\nHOMOGRAPHY_CONFIDENCE = 0.99\n\n# DELG model:\nSAVED_MODEL_DIR = '../input/delg-saved-models/local_and_global'\nDELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)\nDELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])\nDELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)\nDELG_INPUT_TENSOR_NAMES = [\n    'input_image:0', 'input_scales:0', 'input_abs_thres:0'\n]\n\n# Global feature extraction:\nNUM_EMBEDDING_DIMENSIONS = 2048\nGLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,\n                                                ['global_descriptors:0'])\n\n# Local feature extraction:\nLOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)\nLOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(\n    DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],\n    ['boxes:0', 'features:0'])\n\n\ndef to_hex(image_id) -> str:\n  return '{0:0{1}x}'.format(image_id, 16)\n\n\ndef get_image_path(subset, image_id):\n  name = to_hex(image_id)\n  return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],\n                      '{}.jpg'.format(name))\n\n\ndef load_image_tensor(image_path):\n  return tf.convert_to_tensor(\n      np.array(PIL.Image.open(image_path).convert('RGB')))\n\n\ndef extract_global_features(image_root_dir):\n  \"\"\"Extracts embeddings for all the images in given `image_root_dir`.\"\"\"\n\n  image_paths = [x for x in pathlib.Path(image_root_dir).rglob('*.jpg')]\n\n  num_embeddings = len(image_paths)\n  if MAX_NUM_EMBEDDINGS > 0:\n    num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)\n\n  ids = num_embeddings * [None]\n  embeddings = np.empty((num_embeddings, NUM_EMBEDDING_DIMENSIONS))\n\n  for i, image_path in enumerate(image_paths):\n    if i >= num_embeddings:\n      break\n\n    ids[i] = int(image_path.name.split('.')[0], 16)\n    image_tensor = load_image_tensor(image_path)\n    features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,\n                                            DELG_IMAGE_SCALES_TENSOR,\n                                            DELG_SCORE_THRESHOLD_TENSOR)\n    embeddings[i, :] = tf.nn.l2_normalize(\n        tf.reduce_sum(features[0], axis=0, name='sum_pooling'),\n        axis=0,\n        name='final_l2_normalization').numpy()\n\n  return ids, embeddings\n\n\ndef extract_local_features(image_path):\n  \"\"\"Extracts local features for the given `image_path`.\"\"\"\n\n  image_tensor = load_image_tensor(image_path)\n\n  features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,\n                                         DELG_SCORE_THRESHOLD_TENSOR,\n                                         LOCAL_FEATURE_NUM_TENSOR)\n\n  # Shape: (N, 2)\n  keypoints = tf.divide(\n      tf.add(\n          tf.gather(features[0], [0, 1], axis=1),\n          tf.gather(features[0], [2, 3], axis=1)), 2.0).numpy()\n\n  # Shape: (N, 128)\n  descriptors = tf.nn.l2_normalize(\n      features[1], axis=1, name='l2_normalization').numpy()\n\n  return keypoints, descriptors\n\n\ndef get_putative_matching_keypoints(test_keypoints,\n                                    test_descriptors,\n                                    train_keypoints,\n                                    train_descriptors,\n                                    max_distance=0.9):\n  \"\"\"Finds matches from `test_descriptors` to KD-tree of `train_descriptors`.\"\"\"\n\n  train_descriptor_tree = spatial.cKDTree(train_descriptors)\n  _, matches = train_descriptor_tree.query(\n      test_descriptors, distance_upper_bound=max_distance)\n\n  test_kp_count = test_keypoints.shape[0]\n  train_kp_count = train_keypoints.shape[0]\n\n  test_matching_keypoints = np.array([\n      test_keypoints[i,]\n      for i in range(test_kp_count)\n      if matches[i] != train_kp_count\n  ])\n  train_matching_keypoints = np.array([\n      train_keypoints[matches[i],]\n      for i in range(test_kp_count)\n      if matches[i] != train_kp_count\n  ])\n\n  return test_matching_keypoints, train_matching_keypoints\n\n\ndef get_num_inliers(test_keypoints, test_descriptors, train_keypoints,\n                    train_descriptors):\n  \"\"\"Returns the number of RANSAC inliers.\"\"\"\n\n  test_match_kp, train_match_kp = get_putative_matching_keypoints(\n      test_keypoints, test_descriptors, train_keypoints, train_descriptors)\n\n  if test_match_kp.shape[\n      0] <= 4:  # Min keypoints supported by `pydegensac.findHomography()`\n    return 0\n\n  try:\n    _, mask = pydegensac.findHomography(test_match_kp, train_match_kp,\n                                        MAX_REPROJECTION_ERROR,\n                                        HOMOGRAPHY_CONFIDENCE,\n                                        MAX_RANSAC_ITERATIONS)\n  except np.linalg.LinAlgError:  # When det(H)=0, can't invert matrix.\n    return 0\n\n  return int(copy.deepcopy(mask).astype(np.float32).sum())\n\n\ndef get_total_score(num_inliers, global_score):\n  local_score = min(num_inliers, MAX_INLIER_SCORE) / MAX_INLIER_SCORE\n  return local_score + global_score\n\n\ndef rescore_and_rerank_by_num_inliers(test_image_id,\n                                      train_ids_labels_and_scores):\n  \"\"\"Returns rescored and sorted training images by local feature extraction.\"\"\"\n\n  test_image_path = get_image_path('test', test_image_id)\n  test_keypoints, test_descriptors = extract_local_features(test_image_path)\n\n  for i in range(len(train_ids_labels_and_scores)):\n    train_image_id, label, global_score = train_ids_labels_and_scores[i]\n\n    train_image_path = get_image_path('train', train_image_id)\n    train_keypoints, train_descriptors = extract_local_features(\n        train_image_path)\n\n    num_inliers = get_num_inliers(test_keypoints, test_descriptors,\n                                  train_keypoints, train_descriptors)\n    total_score = get_total_score(num_inliers, global_score)\n    train_ids_labels_and_scores[i] = (train_image_id, label, total_score)\n\n  train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)\n\n  return train_ids_labels_and_scores\n\n\ndef load_labelmap():\n  with open(TRAIN_LABELMAP_PATH, mode='r') as csv_file:\n    csv_reader = csv.DictReader(csv_file)\n    labelmap = {row['id']: row['landmark_id'] for row in csv_reader}\n\n  return labelmap\n\n\ndef get_prediction_map(test_ids, train_ids_labels_and_scores):\n  \"\"\"Makes dict from test ids and ranked training ids, labels, scores.\"\"\"\n\n  prediction_map = dict()\n\n  for test_index, test_id in enumerate(test_ids):\n    hex_test_id = to_hex(test_id)\n\n    aggregate_scores = {}\n    for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:\n      if label not in aggregate_scores:\n        aggregate_scores[label] = 0\n      aggregate_scores[label] += score\n\n    label, score = max(aggregate_scores.items(), key=operator.itemgetter(1))\n\n    prediction_map[hex_test_id] = {'score': score, 'class': label}\n\n  return prediction_map\n\n\ndef get_predictions(labelmap):\n  \"\"\"Gets predictions using embedding similarity and local feature reranking.\"\"\"\n\n  test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)\n\n  train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)\n\n  train_ids_labels_and_scores = [None] * test_embeddings.shape[0]\n\n  # Using (slow) for-loop, as distance matrix doesn't fit in memory.\n  for test_index in range(test_embeddings.shape[0]):\n    distances = spatial.distance.cdist(\n        test_embeddings[np.newaxis, test_index, :], train_embeddings,\n        'cosine')[0]\n    partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]\n\n    nearest = sorted([(train_ids[p], distances[p]) for p in partition],\n                     key=lambda x: x[1])\n\n    train_ids_labels_and_scores[test_index] = [\n        (train_id, labelmap[to_hex(train_id)], 1. - cosine_distance)\n        for train_id, cosine_distance in nearest\n    ]\n\n  del test_embeddings\n  del train_embeddings\n  del labelmap\n  gc.collect()\n\n  pre_verification_predictions = get_prediction_map(\n      test_ids, train_ids_labels_and_scores)\n\n#  return None, pre_verification_predictions\n\n  for test_index, test_id in enumerate(test_ids):\n    train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(\n        test_id, train_ids_labels_and_scores[test_index])\n\n  post_verification_predictions = get_prediction_map(\n      test_ids, train_ids_labels_and_scores)\n\n  return pre_verification_predictions, post_verification_predictions\n\n\ndef save_submission_csv(predictions=None):\n  \"\"\"Saves optional `predictions` as submission.csv.\n\n  The csv has columns {id, landmarks}. The landmarks column is a string\n  containing the label and score for the id, separated by a ws delimeter.\n\n  If `predictions` is `None` (default), submission.csv is copied from\n  sample_submission.csv in `IMAGE_DIR`.\n\n  Args:\n    predictions: Optional dict of image ids to dicts with keys {class, score}.\n  \"\"\"\n\n  if predictions is None:\n    # Dummy submission!\n    shutil.copyfile(\n        os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')\n    return\n\n  with open('submission.csv', 'w') as submission_csv:\n    csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])\n    csv_writer.writeheader()\n    for image_id, prediction in predictions.items():\n      label = prediction['class']\n      score = prediction['score']\n      csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})\n\n\ndef main():\n  labelmap = load_labelmap()\n  num_training_images = len(labelmap.keys())\n  print(f'Found {num_training_images} training images.')\n\n  if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:\n    print(\n        f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images. Copying sample submission.'\n    )\n    save_submission_csv()\n    return\n\n  _, post_verification_predictions = get_predictions(labelmap)\n  save_submission_csv(post_verification_predictions)\n\n\n\nmain()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-09T21:35:17.280496Z","iopub.execute_input":"2024-02-09T21:35:17.281264Z","iopub.status.idle":"2024-02-09T21:35:37.341619Z","shell.execute_reply.started":"2024-02-09T21:35:17.281208Z","shell.execute_reply":"2024-02-09T21:35:37.340258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]}]}