{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import copy\nimport csv\nimport gc\nimport operator\nimport os\nimport pathlib\nimport shutil\nimport pandas as pd\nimport numpy as np\nimport PIL\nimport pydegensac\nfrom scipy import spatial\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset parameters:"},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_DIR = os.path.join('..', 'input')\n\nDATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')\nTEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')\nTRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')\nTRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv  =pd.read_csv('../input/landmark-recognition-2020/train.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# debugging params\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"num_public_train_images = len(train_csv) # Used to detect if in session or re-run.\nmax_num_emdeddings= -1 # Set to > 1 to subsample dataset while debugging.\n\n# Retrieval & re-ranking parameters:\nnum_to_rerank = 3\ntop_k = 3 #Number of retrieved images used to make prediction for a test image.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# RANSAC parameters:"},{"metadata":{"trusted":true},"cell_type":"code","source":"max_inlier_score = 35\nmax_preprojection_error = 6.0\nmax_rensac_iterations = 10_000_000\nhomography_confidence = 0.99","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DELG model"},{"metadata":{"trusted":true},"cell_type":"code","source":"save_model_dir = '../input/delg-saved-models/local_and_global'\ndelg_model = tf.saved_model.load(save_model_dir)\n\ndelg_image_scales_tensor = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])\ndelg_score_threshold_tensor = tf.constant(175.)\ndelg_input_tensor_names = [\n    'input_image:0', 'input_scales:0', 'input_abs_thres:0'\n]\n\n# Global feature extraction:\n\nnum_embedding_dimensions = 2048\n\nglobal_feature_extraction_fn = delg_model.prune(delg_input_tensor_names,\n                                                ['global_descriptors:0'])\n\n# Local feature extraction:\n\nlocal_reature_num_tensor = tf.constant(1000)\nlocal_feature_extraction_fn = delg_model.prune(\n    delg_input_tensor_names + ['input_max_feature_num:0'],\n    ['boxes:0', 'features:0'])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# data_loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_hex(image_id) -> str:\n  return '{0:0{1}x}'.format(image_id, 16)\n\n\ndef get_image_path(subset, image_id):\n  name = to_hex(image_id)\n  return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],\n                      '{}.jpg'.format(name))\n\n\ndef load_image_tensor(image_path):\n  return tf.convert_to_tensor(\n      np.array(PIL.Image.open(image_path).convert('RGB')))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"directory = \"../input/landmark-recognition-2020\"\ntest = pd.read_csv(os.path.join(directory,'sample_submission.csv'))\ntest['image_']=test.id.str[0]+\"/\"+test.id.str[1]+\"/\"+test.id.str[2]+\"/\"+test.id+\".jpg\"\ntest.head()\ntrain = pd.read_csv(os.path.join(directory,'train.csv'))\ntrain[\"image_\"] = train.id.str[0]+\"/\"+train.id.str[1]+\"/\"+train.id.str[2]+\"/\"+train.id+\".jpg\"\ntrain[\"target_\"] = train.landmark_id.astype(str)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 5656\ntrain_data = pd.read_csv(\"../input/landmark-recognition-2020/train.csv\")\ntemp = train_data[\"landmark_id\"].value_counts().to_frame()\ntemp.reset_index(inplace=True)\ntemp.columns = ['landmark_id', 'n_images']\ntemp.head(10).style.background_gradient(subset='n_images')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_ten = [138982,126637,20409,83144,113209,177870,194914,149980,139894,1924]\nx = [i  for i in range(0,len(train_csv)) if train_csv[\"landmark_id\"][i] in  top_ten]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"train_top_csv = train.iloc[x,] ##需要儲存\ntrain_top_csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train\ntemp = X_train[\"landmark_id\"].value_counts().to_frame()\ntemp.reset_index(inplace=True)\ntemp.columns = ['landmark_id', 'n_images']\ntemp.head(10).style.background_gradient(subset='n_images')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_top_csv.to_csv(\"train.csv\",index=False,sep=',')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import csv \ncsv_file=csv.reader(open('train.csv','r'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_dir =\"../input/landmark-recognition-2020/train/\"\ntmp  = list(train_top_csv[\"image_\"])\npath = train_dir+tmp[0]\npath","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#! mkdir train\n!rm -rf image1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\norg_path = '../input/landmark-recognition-2020/train/'\nfor i in range(0,len(tmp)):\n     image = cv2.imread(org_path + tmp[i])\n     cv2.imwrite('./train/'+train_top_csv.iloc[i,0]+'.jpg',image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.getcwd()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a = os.listdir('./image1/')\nprint(a)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# extract_global_features"},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_global_features(image_root_dir):\n  \"\"\"Extracts embeddings for all the images in given `image_root_dir`.\"\"\"\n\n  image_paths = [x for x in pathlib.Path(image_root_dir).rglob('*.jpg')]\n\n  num_embeddings = len(image_paths)\n  if max_num_emdeddings > 0:\n    num_embeddings = min(max_num_emdeddings, num_embeddings)\n\n  ids = num_embeddings * [None]\n  embeddings = np.empty((num_embeddings, num_embedding_dimensions))\n\n  for i, image_path in enumerate(image_paths):\n    if i >= num_embeddings:\n      break\n\n    ids[i] = int(image_path.name.split('.')[0], 16)\n    image_tensor = load_image_tensor(image_path)\n    features = global_feature_extraction_fn(image_tensor,\n                                            delg_image_scales_tensor,\n                                            delg_score_threshold_tensor)\n    embeddings[i, :] = tf.nn.l2_normalize(\n        tf.reduce_sum(features[0], axis=0, name='sum_pooling'),\n        axis=0,\n        name='final_l2_normalization').numpy()\n\n  return ids, embeddings\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# extract_local_features"},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_local_features(image_path):\n  \"\"\"Extracts local features for the given `image_path`.\"\"\"\n\n  image_tensor = load_image_tensor(image_path)\n\n  features = local_feature_extraction_fn(image_tensor, delg_image_scales_tensor,\n                                         delg_score_threshold_tensor,\n                                         local_reature_num_tensor)\n\n  # Shape: (N, 2)\n  keypoints = tf.divide(\n      tf.add(\n          tf.gather(features[0], [0, 1], axis=1),\n          tf.gather(features[0], [2, 3], axis=1)), 2.0).numpy()\n\n  # Shape: (N, 128)\n  descriptors = tf.nn.l2_normalize(\n      features[1], axis=1, name='l2_normalization').numpy()\n\n  return keypoints, descriptors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"extract_local_features(train_dir+tmp[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# matching_keypoints"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_putative_matching_keypoints(test_keypoints,\n                                    test_descriptors,\n                                    train_keypoints,\n                                    train_descriptors,\n                                    max_distance=0.9):\n  \"\"\"Finds matches from `test_descriptors` to KD-tree of `train_descriptors`.\"\"\"\n\n  train_descriptor_tree = spatial.cKDTree(train_descriptors)\n  _, matches = train_descriptor_tree.query(\n      test_descriptors, distance_upper_bound=max_distance)\n\n  test_kp_count = test_keypoints.shape[0]\n  train_kp_count = train_keypoints.shape[0]\n\n  test_matching_keypoints = np.array([\n      test_keypoints[i,]\n      for i in range(test_kp_count)\n      if matches[i] != train_kp_count\n  ])\n  train_matching_keypoints = np.array([\n      train_keypoints[matches[i],]\n      for i in range(test_kp_count)\n      if matches[i] != train_kp_count\n  ])\n\n  return test_matching_keypoints, train_matching_keypoints\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,\n                    train_descriptors):\n  \"\"\"Returns the number of RANSAC inliers.\"\"\"\n\n  test_match_kp, train_match_kp = get_putative_matching_keypoints(\n      test_keypoints, test_descriptors, train_keypoints, train_descriptors)\n\n  if test_match_kp.shape[0] <= 4:  # Min keypoints supported by `pydegensac.findHomography()`\n    return 0\n\n  try:\n    _, mask = pydegensac.findHomography(test_match_kp, train_match_kp,\n                                        max_preprojection_error,\n                                        homography_confidence,\n                                        max_rensac_iterations)\n  except np.linalg.LinAlgError:  # When det(H)=0, can't invert matrix.\n    return 0\n\n  return int(copy.deepcopy(mask).astype(np.float32).sum())\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# get train score"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_total_score(num_inliers, global_score):\n  local_score = min(num_inliers, max_inlier_score) / max_inlier_score\n  return local_score + global_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rescore_and_rerank_by_num_inliers(test_image_id,train_ids_labels_and_scores):\n  \"\"\"Returns rescored and sorted training images by local feature extraction.\"\"\"\n\n  test_image_path = get_image_path('test', test_image_id)\n  test_keypoints, test_descriptors = extract_local_features(test_image_path)\n\n  for i in range(len(train_ids_labels_and_scores)):\n    train_image_id, label, global_score = train_ids_labels_and_scores[i]\n\n    train_image_path = get_image_path('train', train_image_id)\n    train_keypoints, train_descriptors = extract_local_features(\n        train_image_path)\n\n    num_inliers = get_num_inliers(test_keypoints, test_descriptors,\n                                  train_keypoints, train_descriptors)\n    total_score = get_total_score(num_inliers, global_score)\n    train_ids_labels_and_scores[i] = (train_image_id, label, total_score)\n\n  train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)\n\n  return train_ids_labels_and_scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_labelmap():\n  with open(TRAIN_LABELMAP_PATH, mode='r') as csv_file:\n    csv_reader = csv.DictReader(csv_file)\n    labelmap = {row['id']: row['landmark_id'] for row in csv_reader}\n\n  return labelmap","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"load_labelmap()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"extract_global_features(TEST_IMAGE_DIR)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_prediction_map(test_ids, train_ids_labels_and_scores):\n  \"\"\"Makes dict from test ids and ranked training ids, labels, scores.\"\"\"\n\n  prediction_map = dict()\n\n  for test_index, test_id in enumerate(test_ids):\n    hex_test_id = to_hex(test_id)\n\n    aggregate_scores = {}\n    for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:\n      if label not in aggregate_scores:\n        aggregate_scores[label] = 0\n      aggregate_scores[label] += score\n\n    label, score = max(aggregate_scores.items(), key=operator.itemgetter(1))\n\n    prediction_map[hex_test_id] = {'score': score, 'class': label}\n\n  return prediction_map","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_predictions(labelmap):\n  \"\"\"Gets predictions using embedding similarity and local feature reranking.\"\"\"\n\n  test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)\n\n  train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)\n\n  train_ids_labels_and_scores = [None] * test_embeddings.shape[0]\n\n  # Using (slow) for-loop, as distance matrix doesn't fit in memory.\n  for test_index in range(test_embeddings.shape[0]):\n    distances = spatial.distance.cdist(\n        test_embeddings[np.newaxis, test_index, :], train_embeddings,\n        'cosine')[0]\n    partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]\n\n    nearest = sorted([(train_ids[p], distances[p]) for p in partition],\n                     key=lambda x: x[1])\n\n    train_ids_labels_and_scores[test_index] = [\n        (train_id, labelmap[to_hex(train_id)], 1. - cosine_distance)\n        for train_id, cosine_distance in nearest\n    ]\n\n  del test_embeddings\n  del train_embeddings\n  del labelmap\n  gc.collect()\n\n  pre_verification_predictions = get_prediction_map(\n      test_ids, train_ids_labels_and_scores)\n\n#  return None, pre_verification_predictions\n\n  for test_index, test_id in enumerate(test_ids):\n    train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(\n        test_id, train_ids_labels_and_scores[test_index])\n\n  post_verification_predictions = get_prediction_map(\n      test_ids, train_ids_labels_and_scores)\n\n  return pre_verification_predictions, post_verification_predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_submission_csv(predictions=None):\n\n  if predictions is None:\n    # Dummy submission!\n    shutil.copyfile(\n        os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')\n    return\n\n  with open('submission.csv', 'w') as submission_csv:\n    csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])\n    csv_writer.writeheader()\n    for image_id, prediction in predictions.items():\n      label = prediction['class']\n      score = prediction['score']\n      csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})\n\n\ndef main():\n  labelmap = load_labelmap()\n  num_training_images = len(labelmap.keys())\n  print(f'Found {num_training_images} training images.')\n\n  if num_training_images == num_public_train_images:\n    print('Copying sample submission.')\n    save_submission_csv()\n    return\n\n    _, post_verification_predictions = get_predictions(labelmap)\n    save_submission_csv(post_verification_predictions)\n\n\nif __name__ == '__main__':\n  main()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_csv(\"./submission.csv\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}