{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Private 0.51+ and Public 0.54+ kernel with some straightforward methodologies\n\n\n**How we approach:**\n\nWhen this competition first launch, we just fiddled and played around with the parameters of the original baseline. \n\nAfter several weeks, thanks to [Ragnar's](http://www.kaggle.com/ragnar123) awesome kernel of training Effnet, we find some practical ways to train our own models. After reading Keetar's fantastic writeup of his GLD retrieval, we trained our Effnet B6 and B7 first with 384 sized images. CV is 0.84 and 0.85 respectively. Then we use the increasing 512 sized images to further tuned our B6 and B7. Training environment is Colab Pro. Then we simply put the two model ensembling predictions to the golbal features extraction. "},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/glrec2020/Keras_Applications-1.0.8-py3-none-any.whl\n!pip install ../input/glrec2020/efficientnet-1.1.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import operator\nimport gc\nimport pathlib\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom scipy import spatial\nimport cv2\nimport efficientnet.tfkeras as efn\nimport math\nimport copy\nimport csv\nimport os\nimport numpy as np\nimport PIL\nimport pydegensac","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUMBER_OF_CLASSES = 81313\nIMAGE_SIZE = [384, 384]\nLR = 0.0001","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ArcMarginProduct(tf.keras.layers.Layer):\n    '''\n    Implements large margin arc distance.\n\n    Reference:\n        https://arxiv.org/pdf/1801.07698.pdf\n        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n            blob/master/src/modeling/metric_learning.py\n    '''\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n\n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n\n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n        one_hot = tf.cast(\n            tf.one_hot(y, depth=self.n_classes),\n            dtype=cosine.dtype\n        )\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output\n\n\n# Function to build our model using fine tunning (efficientnet)\ndef get_model_B6():\n\n    margin = ArcMarginProduct(\n        n_classes = NUMBER_OF_CLASSES, \n        s = 64, \n        m = 0.15, \n        name='head/arc_margin', \n        dtype='float32'\n        )\n\n    inp = tf.keras.layers.Input(shape = (384, 384, 3), name = 'inp1')\n    label = tf.keras.layers.Input(shape = (), name = 'inp2')\n    x4 = efn.EfficientNetB6(weights = None, include_top = False)(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x4)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    x = tf.keras.layers.Dense(512)(x)\n    x = margin([x, label])\n\n    output = tf.keras.layers.Softmax(dtype='float32')(x)\n\n    model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])\n\n    opt = tf.keras.optimizers.Adam(learning_rate = LR)\n\n    model.compile(\n        optimizer = opt,\n        loss = [tf.keras.losses.SparseCategoricalCrossentropy()],\n        metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]\n        ) \n\n    return model\n\ndef get_model_B7():\n\n    margin = ArcMarginProduct(\n        n_classes = NUMBER_OF_CLASSES, \n        s = 64, \n        m = 0.15, \n        name='head/arc_margin', \n        dtype='float32'\n        )\n\n    inp = tf.keras.layers.Input(shape = (384, 384, 3), name = 'inp1')\n    label = tf.keras.layers.Input(shape = (), name = 'inp2')\n    x4 = efn.EfficientNetB7(weights = None, include_top = False)(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x4)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    x = tf.keras.layers.Dense(512)(x)\n    x = margin([x, label])\n\n    output = tf.keras.layers.Softmax(dtype='float32')(x)\n\n    model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])\n\n    opt = tf.keras.optimizers.Adam(learning_rate = LR)\n\n    model.compile(\n        optimizer = opt,\n        loss = [tf.keras.losses.SparseCategoricalCrossentropy()],\n        metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]\n        ) \n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL1 = get_model_B6()\nMODEL2 = get_model_B7()\n\nMODEL1.load_weights('../input/effb6-512-ep18/effb6model512-18.h5')\nMODEL1 = tf.keras.models.Model(inputs = MODEL1.input[0], outputs = MODEL1.layers[-4].output)\n\nMODEL2.load_weights('../input/effb7-512-ep12/effb7model512-12.h5')\nMODEL2 = tf.keras.models.Model(inputs = MODEL2.input[0], outputs = MODEL2.layers[-4].output)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TO_RERANK = 3 #originally 5\nNUM_PUBLIC_TEST_IMAGES = 10345 # Used to detect if in session or re-run.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_image(image_path, size = (384, 384)):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, size)\n    img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 100))[1].tostring()\n    img = tf.image.decode_jpeg(img, channels = 3)\n    img = tf.image.resize(img, (512, 512))\n    img = tf.cast(img, tf.float32) / 255.0\n    img = tf.reshape(img, [1, 512, 512, 3])\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_DIR = os.path.join('..', 'input')\n\nDATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')\nTEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')\nTRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')\nTRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_PUBLIC_TRAIN_IMAGES = 1580470 # Used to detect if in session or re-run.\nMAX_NUM_EMBEDDINGS = -1  # Set to > 1 to subsample dataset while debugging.\n\n# Retrieval & re-ranking parameters:\nNUM_TO_RERANK = 3\nTOP_K = 3 #Number of retrieved images used to make prediction for a test image.\n\n# RANSAC parameters:\nMAX_INLIER_SCORE = 25\nMAX_REPROJECTION_ERROR = 7.0\n# MAX_RANSAC_ITERATIONS = 1000000\nMAX_RANSAC_ITERATIONS = 100000\nHOMOGRAPHY_CONFIDENCE = 0.99","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SAVED_MODEL_DIR = '../input/delg-saved-models/local_and_global'\nDELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)\nDELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 2])\nDELG_SCORE_THRESHOLD_TENSOR = tf.constant(375.)\nDELG_INPUT_TENSOR_NAMES = [\n    'input_image:0', 'input_scales:0', 'input_abs_thres:0'\n]\n# NUM_EMBEDDING_DIMENSIONS = 1024\n# NUM_EMBEDDING_DIMENSIONS = 2048\nNUM_EMBEDDING_DIMENSIONS = 3072\n\nGLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,\n                                                ['global_descriptors:0'])\n\nLOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)\nLOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(\n    DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],\n    ['boxes:0', 'features:0'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_hex(image_id) -> str:\n  return '{0:0{1}x}'.format(image_id, 16)\n\n\ndef get_image_path(subset, image_id):\n  name = to_hex(image_id)\n  return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],\n                      '{}.jpg'.format(name))\n\n\ndef load_image_tensor(image_path):\n  return tf.convert_to_tensor(\n      np.array(PIL.Image.open(image_path).convert('RGB')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def extract_global_features(filepaths):\n#     image_paths = [x for x in pathlib.Path(filepaths).rglob('*.jpg')]\n#     num_images = len(image_paths)\n#     ids = num_images * [None]\n#     # Generate an empty matrix where we can store the embeddings of each image\n#     embeddings = np.empty((num_images, NUM_EMBEDDING_DIMENSIONS))\n#     for i, image_path in enumerate(image_paths):\n#         ids[i] = int(image_path.name.split('.')[0], 16)\n#         image_tensor = read_image(str(image_path), (384, 384)) #384\n#         prediction1 = MODEL1.predict(image_tensor)\n#         prediction2 = MODEL2.predict(image_tensor)\n#         prediction = tf.concat([prediction1, prediction2], 1)\n# #         prediction = tf.concat([prediction1, prediction2 , prediction1, prediction2], 1)\n        \n#         embeddings[i, :] = prediction\n#     return ids, embeddings\n\ndef extract_global_features(filepaths):\n    image_paths = [x for x in pathlib.Path(filepaths).rglob('*.jpg')]\n    num_images = len(image_paths)\n    ids = num_images * [None]\n    # Generate an empty matrix where we can store the embeddings of each image\n    embeddings = np.empty((num_images, NUM_EMBEDDING_DIMENSIONS))\n    for i, image_path in enumerate(image_paths):\n        ids[i] = int(image_path.name.split('.')[0], 16)\n        image_tensor = read_image(str(image_path), (384, 384)) #384\n        image_tensor_delg = load_image_tensor(image_path)\n        features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor_delg,\n                                            DELG_IMAGE_SCALES_TENSOR,\n                                            DELG_SCORE_THRESHOLD_TENSOR)\n        prediction3 = tf.nn.l2_normalize( tf.reduce_sum(features[0], axis=0, name='sum_pooling'), axis=0, name='final_l2_normalization').numpy().reshape(1, 2048)\n\n        prediction1 = MODEL1.predict(image_tensor)\n        prediction2 = MODEL2.predict(image_tensor)\n        prediction = tf.concat([prediction3 ,prediction1, prediction2 ], 1)\n        embeddings[i, :] = prediction\n    return ids, embeddings\n\n\ndef extract_local_features(image_path):\n  \"\"\"Extracts local features for the given `image_path`.\"\"\"\n\n  input_image = load_image_tensor(image_path)\n  width = tf.cast(tf.shape(input_image)[0], dtype=tf.float32)\n  height = tf.cast(tf.shape(input_image)[1], dtype=tf.float32)\n  image_tensor = tf.cast(tf.image.resize(input_image, [tf.cast(pow(2,0.6) * width, dtype=tf.int32), tf.cast(pow(2,0.6) * height, dtype = tf.int32)]), dtype = input_image.dtype)\n\n  features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,\n                                         DELG_SCORE_THRESHOLD_TENSOR,\n                                         LOCAL_FEATURE_NUM_TENSOR)\n\n  # Shape: (N, 2)\n  keypoints = tf.divide(\n      tf.add(\n          tf.gather(features[0], [0, 1], axis=1),\n          tf.gather(features[0], [2, 3], axis=1)), 2.0).numpy()\n\n  # Shape: (N, 128)\n  descriptors = tf.nn.l2_normalize(\n      features[1], axis=1, name='l2_normalization').numpy()\n\n  return keypoints, descriptors\n\n# def extract_local_features(image_path):\n#   \"\"\"Extracts local features for the given `image_path`.\"\"\"\n\n# #   image_tensor = load_image_tensor(image_path)\n#   input_image = load_image_tensor(image_path)\n#   width = tf.cast(tf.shape(input_image)[0], dtype=tf.float32)\n#   height = tf.cast(tf.shape(input_image)[1], dtype=tf.float32)\n    \n#   image_tensor = tf.cast(tf.image.resize(input_image, [tf.cast(pow(2,0.6) * width, dtype=tf.int32), tf.cast(pow(2,0.6) * height, dtype = tf.int32)]), dtype = input_image.dtype)\n\n#   features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,\n#                                          DELG_SCORE_THRESHOLD_TENSOR,\n#                                          LOCAL_FEATURE_NUM_TENSOR)\n\n#   # Shape: (N, 2)\n#   keypoints = tf.divide(\n#       tf.add(\n#           tf.gather(features[0], [0, 1], axis=1),\n#           tf.gather(features[0], [2, 3], axis=1)), 2.0).numpy()\n\n#   # Shape: (N, 128)\n#   descriptors = tf.nn.l2_normalize(\n#       features[1], axis=1, name='l2_normalization').numpy()\n\n#   return keypoints, descriptors\n\n\ndef get_putative_matching_keypoints(test_keypoints,\n                                    test_descriptors,\n                                    train_keypoints,\n                                    train_descriptors,\n                                    max_distance=0.75):\n  \"\"\"Finds matches from `test_descriptors` to KD-tree of `train_descriptors`.\"\"\"\n\n  train_descriptor_tree = spatial.cKDTree(train_descriptors)\n  _, matches = train_descriptor_tree.query(\n      test_descriptors, distance_upper_bound=max_distance)\n\n  test_kp_count = test_keypoints.shape[0]\n  train_kp_count = train_keypoints.shape[0]\n\n  test_matching_keypoints = np.array([\n      test_keypoints[i,]\n      for i in range(test_kp_count)\n      if matches[i] != train_kp_count\n  ])\n  train_matching_keypoints = np.array([\n      train_keypoints[matches[i],]\n      for i in range(test_kp_count)\n      if matches[i] != train_kp_count\n  ])\n\n  return test_matching_keypoints, train_matching_keypoints\n\n\ndef get_num_inliers(test_keypoints, test_descriptors, train_keypoints,\n                    train_descriptors):\n  \"\"\"Returns the number of RANSAC inliers.\"\"\"\n\n  test_match_kp, train_match_kp = get_putative_matching_keypoints(\n      test_keypoints, test_descriptors, train_keypoints, train_descriptors)\n\n  if test_match_kp.shape[\n      0] <= 4:  # Min keypoints supported by `pydegensac.findHomography()`\n    return 0\n\n  try:\n    _, mask = pydegensac.findHomography(test_match_kp, train_match_kp,\n                                        MAX_REPROJECTION_ERROR,\n                                        HOMOGRAPHY_CONFIDENCE,\n                                        MAX_RANSAC_ITERATIONS)\n  except np.linalg.LinAlgError:  # When det(H)=0, can't invert matrix.\n    return 0\n\n  return int(copy.deepcopy(mask).astype(np.float32).sum())\n\n\ndef get_total_score(num_inliers, global_score):\n  local_score = min(num_inliers, MAX_INLIER_SCORE) / MAX_INLIER_SCORE\n  return local_score + global_score\n\n\ndef rescore_and_rerank_by_num_inliers(test_image_id,\n                                      train_ids_labels_and_scores):\n  \"\"\"Returns rescored and sorted training images by local feature extraction.\"\"\"\n\n  test_image_path = get_image_path('test', test_image_id)\n  test_keypoints, test_descriptors = extract_local_features(test_image_path)\n\n  for i in range(len(train_ids_labels_and_scores)):\n    train_image_id, label, global_score = train_ids_labels_and_scores[i]\n\n    train_image_path = get_image_path('train', train_image_id)\n    train_keypoints, train_descriptors = extract_local_features(\n        train_image_path)\n\n    num_inliers = get_num_inliers(test_keypoints, test_descriptors,\n                                  train_keypoints, train_descriptors)\n    total_score = get_total_score(num_inliers, global_score)\n    train_ids_labels_and_scores[i] = (train_image_id, label, total_score)\n\n  train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)\n\n  return train_ids_labels_and_scores\n\n\ndef load_labelmap():\n  with open(TRAIN_LABELMAP_PATH, mode='r') as csv_file:\n    csv_reader = csv.DictReader(csv_file)\n    labelmap = {row['id']: row['landmark_id'] for row in csv_reader}\n\n  return labelmap\n\n\ndef get_prediction_map(test_ids, train_ids_labels_and_scores):\n  \"\"\"Makes dict from test ids and ranked training ids, labels, scores.\"\"\"\n\n  prediction_map = dict()\n\n  for test_index, test_id in enumerate(test_ids):\n    hex_test_id = to_hex(test_id)\n\n    aggregate_scores = {}\n    for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:\n      if label not in aggregate_scores:\n        aggregate_scores[label] = 0\n      aggregate_scores[label] += score\n\n    label, score = max(aggregate_scores.items(), key=operator.itemgetter(1))\n\n    prediction_map[hex_test_id] = {'score': score, 'class': label}\n\n  return prediction_map\n\n\ndef get_predictions(labelmap):\n  \"\"\"Gets predictions using embedding similarity and local feature reranking.\"\"\"\n\n  test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)\n\n  train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)\n\n  train_ids_labels_and_scores = [None] * test_embeddings.shape[0]\n\n  # Using (slow) for-loop, as distance matrix doesn't fit in memory.\n  for test_index in range(test_embeddings.shape[0]):\n    distances = spatial.distance.cdist(\n        test_embeddings[np.newaxis, test_index, :], train_embeddings,\n        'cosine')[0]\n    partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]\n\n    nearest = sorted([(train_ids[p], distances[p]) for p in partition],\n                     key=lambda x: x[1])\n\n    train_ids_labels_and_scores[test_index] = [\n        (train_id, labelmap[to_hex(train_id)], 1. - cosine_distance)\n        for train_id, cosine_distance in nearest\n    ]\n\n  del test_embeddings\n  del train_embeddings\n  del labelmap\n  gc.collect()\n\n  pre_verification_predictions = get_prediction_map(\n      test_ids, train_ids_labels_and_scores)\n\n#  return None, pre_verification_predictions\n\n  for test_index, test_id in enumerate(test_ids):\n    train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(\n        test_id, train_ids_labels_and_scores[test_index])\n\n  post_verification_predictions = get_prediction_map(\n      test_ids, train_ids_labels_and_scores)\n\n  return pre_verification_predictions, post_verification_predictions\n\n\ndef save_submission_csv(predictions=None):\n\n  if predictions is None:\n    # Dummy submission!\n    shutil.copyfile(\n        os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')\n    return\n\n  with open('submission.csv', 'w') as submission_csv:\n    csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])\n    csv_writer.writeheader()\n    for image_id, prediction in predictions.items():\n      label = prediction['class']\n      score = prediction['score']\n      csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})\n\n\ndef main():\n  labelmap = load_labelmap()\n  num_training_images = len(labelmap.keys())\n  print(f'Found {num_training_images} training images.')\n\n  if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:\n    print('Copying sample submission.')\n    save_submission_csv()\n    return\n\n  _, post_verification_predictions = get_predictions(labelmap)\n  save_submission_csv(post_verification_predictions)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n  main()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}