{"cells":[{"metadata":{},"cell_type":"markdown","source":"# SMALL WRITEUP\n\n* Global features from EfnetB5 model - (Public-0.5160, Private-0.4952)\n* Combined with DELG baseline local features - (Public-0.5432, Private-0.5184)\n\n### Global features\n\n* EfnetB5 - embedding dimension -->2048\n* Images resized to 384x384. See tfrecords --> [tfrecords1](https://www.kaggle.com/ragnar123/landmark-tfrecords-384), [tfrecords2](https://www.kaggle.com/ragnar123/landmark-tfrecords-384-2)\n* Only left-right augmentation\n* Arcface head with scale 64 and margin 0.1\n* Weighted cross-entropy loss.\n* You can see this notebook for more details --> [training notebook](https://www.kaggle.com/ragnar123/efficientnetb3-data-pipeline-and-model)\n* Inference on 512x512 resized images.\n\n### Local features\n\n* We took the local features from a high scoring public notebook and used the same parameters."},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = False\n#if true check parameters","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport re, csv, copy, gc, operator\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\nfrom sklearn import metrics\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tqdm.notebook import tqdm as tqdm\nimport shutil\nimport pathlib\nimport pydegensac\nimport cv2\nimport PIL\nfrom scipy import spatial\n!pip install -q ../input/whl-files/Keras_Applications-1.0.8-py3-none-any.whl\n!pip install -q ../input/whl-files/efficientnet-1.1.0-py3-none-any.whl\nimport efficientnet.tfkeras as efn\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = [512, 512]\nNUMBER_OF_CLASSES = 81313\nLR = 1e-6\n\nDIM = IMAGE_SIZE[0]\nEFNET = 5\nQUALITY = 100\nMARGIN = 0.3\nEMBEDDING_DIMENSION = 2048   #1280 b0 2304 for b6 1536 b3 2048 b5\nNUM_TO_RERANK = 3\nTOP_K = 3\n\nif DEBUG:\n    NUM_PUBLIC_TRAIN_IMAGES = -1\nelse:\n    NUM_PUBLIC_TRAIN_IMAGES = 1580470\n\nDATASET_DIR = '../input/landmark-recognition-2020'\nTRAIN_IMAGE_DIR = '../input/landmark-recognition-2020/train'\nTEST_IMAGE_DIR = '../input/landmark-recognition-2020/test'\nTRAIN_LABELMAP_PATH = '../input/landmark-recognition-2020/train.csv'\nMODEL_PATH = '../input/tpu-train-final-dataset/fold-0_epoch-3_valloss-1.3284_loss-0.0056_margin-0.10_scale-64_logweight.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ArcMarginProduct(tf.keras.layers.Layer):\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n\n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n\n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n        one_hot = tf.cast(\n            tf.one_hot(y, depth=self.n_classes),\n            dtype=cosine.dtype\n        )\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output\n\nEFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\ndef get_model(ef):\n    margin = ArcMarginProduct(\n        n_classes = NUMBER_OF_CLASSES, \n        s = 64, \n        m = 0.1, \n        name='head/arc_margin', \n        dtype='float32'\n        )\n    inp = tf.keras.layers.Input(shape = (*IMAGE_SIZE, 3), name = 'inp1')\n    label = tf.keras.layers.Input(shape = (), name = 'inp2')\n    x = EFNS[ef](weights = None, include_top = False)(inp)   \n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = margin([x, label])\n\n    output = tf.keras.layers.Softmax(dtype='float32')(x)\n\n    model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])\n\n    opt = tf.keras.optimizers.Adam(learning_rate = LR)\n\n    model.compile(\n        optimizer = opt,\n        loss = [tf.keras.losses.SparseCategoricalCrossentropy()],\n        metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]\n        ) \n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL = get_model(ef = EFNET)\nMODEL.load_weights(MODEL_PATH)\nembedding_model = tf.keras.models.Model(inputs = MODEL.input[0], outputs = MODEL.layers[-4].output)\nprint(embedding_model.layers[2].output.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_INLIER_SCORE = 35\nMAX_REPROJECTION_ERROR = 7.0\nMAX_RANSAC_ITERATIONS = 8500000\nHOMOGRAPHY_CONFIDENCE = 0.99\n\nSAVED_MODEL_DIR = '../input/delg-saved-models/local_and_global'\nDELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)\nDELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])\nDELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)\nDELG_INPUT_TENSOR_NAMES = ['input_image:0', 'input_scales:0', 'input_abs_thres:0']\n\nLOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)\nLOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],\n                                               ['boxes:0', 'features:0'])\n\ndef rescore_and_rerank_by_num_inliers(test_image_id,train_ids_labels_and_scores):\n    test_image_path = get_image_path('test', test_image_id)\n    test_keypoints, test_descriptors = extract_local_features(test_image_path)\n    for i in range(len(train_ids_labels_and_scores)):\n        train_image_id, label, global_score = train_ids_labels_and_scores[i]\n        train_image_path = get_image_path('train', train_image_id)\n        train_keypoints, train_descriptors = extract_local_features(\n            train_image_path)\n        num_inliers = get_num_inliers(test_keypoints, test_descriptors,\n                                      train_keypoints, train_descriptors)\n        total_score = get_total_score(num_inliers, global_score)\n        train_ids_labels_and_scores[i] = (train_image_id, label, total_score)\n    train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)\n    return train_ids_labels_and_scores\n\ndef get_image_path(subset, image_id):\n    name = to_hex(image_id)\n    return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],\n                        '{}.jpg'.format(name))\n\ndef extract_local_features(image_path):\n    image_tensor = load_image_tensor(image_path)\n    features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,\n                                           DELG_SCORE_THRESHOLD_TENSOR,\n                                           LOCAL_FEATURE_NUM_TENSOR)\n    keypoints = tf.divide(\n        tf.add(\n            tf.gather(features[0], [0, 1], axis=1),\n            tf.gather(features[0], [2, 3], axis=1)), 2.0).numpy()\n    descriptors = tf.nn.l2_normalize(\n        features[1], axis=1, name='l2_normalization').numpy()\n    return keypoints, descriptors\n\ndef get_num_inliers(test_keypoints, test_descriptors, train_keypoints,train_descriptors):\n    test_match_kp, train_match_kp = get_putative_matching_keypoints(test_keypoints, test_descriptors, \n                                                                    train_keypoints, train_descriptors)\n    if test_match_kp.shape[0] <= 4:\n        return 0\n    try:\n        _, mask = pydegensac.findHomography(test_match_kp, train_match_kp,\n                                            MAX_REPROJECTION_ERROR,\n                                            HOMOGRAPHY_CONFIDENCE,\n                                            MAX_RANSAC_ITERATIONS)\n    except np.linalg.LinAlgError:\n        return 0\n    return int(copy.deepcopy(mask).astype(np.float32).sum())\n\ndef get_putative_matching_keypoints(test_keypoints,\n                                    test_descriptors,\n                                    train_keypoints,\n                                    train_descriptors,\n                                    max_distance=0.9):\n    train_descriptor_tree = spatial.cKDTree(train_descriptors)\n    _, matches = train_descriptor_tree.query(test_descriptors, distance_upper_bound=max_distance)\n    test_kp_count = test_keypoints.shape[0]\n    train_kp_count = train_keypoints.shape[0]\n    test_matching_keypoints = np.array([\n        test_keypoints[i,] for i in range(test_kp_count) if matches[i] != train_kp_count])\n    train_matching_keypoints = np.array([\n        train_keypoints[matches[i],] for i in range(test_kp_count) if matches[i] != train_kp_count])\n    return test_matching_keypoints, train_matching_keypoints\n\ndef get_total_score(num_inliers, global_score):\n    local_score = min(num_inliers, MAX_INLIER_SCORE) / MAX_INLIER_SCORE\n    return local_score + global_score\n\ndef load_image_tensor(image_path):\n    return tf.convert_to_tensor(np.array(PIL.Image.open(image_path).convert('RGB')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_image(image_path, size = (DIM, DIM)):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, size)\n    img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, QUALITY))[1].tostring()\n    img = tf.image.decode_jpeg(img, channels = 3)\n    img = tf.cast(img, tf.float32) / 255.0\n    img = tf.reshape(img, [1, DIM, DIM, 3])\n    return img\n\ndef get_predictions(labelmap):\n    if DEBUG:\n        test_image_paths = [x for x in pathlib.Path(TEST_IMAGE_DIR).rglob('*.jpg')][:100]\n        train_image_paths = [x for x in pathlib.Path(TRAIN_IMAGE_DIR).rglob('*.jpg')][:100]        \n    else:\n        test_image_paths = [x for x in pathlib.Path(TEST_IMAGE_DIR).rglob('*.jpg')]\n        train_image_paths = [x for x in pathlib.Path(TRAIN_IMAGE_DIR).rglob('*.jpg')]\n    print('Extracting global features of test images')\n    test_ids, test_embeddings = extract_global_features(test_image_paths)\n    print('Extracting global features of train images')\n    train_ids, train_embeddings = extract_global_features(train_image_paths)\n    train_ids_labels_and_scores = [None] * test_embeddings.shape[0]\n    for test_index in range(test_embeddings.shape[0]):\n        distances = spatial.distance.cdist(test_embeddings[np.newaxis, test_index, : ],\n                                         train_embeddings, 'cosine')[0]\n        partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]\n        nearest = sorted([(train_ids[p], distances[p]) for p in partition],\n                         key = lambda x: x[1])\n        train_ids_labels_and_scores[test_index] = [(train_id, labelmap[to_hex(train_id)],\n                                                    1.0 - cosine_distance)\n                                                   for train_id, cosine_distance in nearest]\n    del test_embeddings\n    del train_embeddings\n    del labelmap\n    gc.collect()\n    pre_verification_predictions = get_prediction_map(test_ids, train_ids_labels_and_scores)\n    \n    for test_index, test_id in tqdm(enumerate(test_ids)):\n        train_ids_labels_and_scores[test_index]=rescore_and_rerank_by_num_inliers(\n            test_id,train_ids_labels_and_scores[test_index])\n    post_verification_predictions = get_prediction_map(test_ids, train_ids_labels_and_scores)\n    \n    return pre_verification_predictions, post_verification_predictions\n\ndef extract_global_features(image_paths):\n    num_images = len(image_paths)\n    ids = num_images * [None]\n    embeddings = np.empty((num_images, EMBEDDING_DIMENSION))\n    for i, image_path in tqdm(enumerate(image_paths)):           \n        ids[i] = int(image_path.name.split('.')[0], 16)\n        image_tensor = read_image(str(image_path), size = (DIM, DIM))\n        features = embedding_model.predict(image_tensor)\n        embeddings[i, :] = tf.nn.l2_normalize(tf.reduce_sum(features, axis=0),\n                                              axis=0).numpy()\n    return ids, embeddings\n\ndef get_prediction_map(test_ids, train_ids_labels_and_scores):\n    prediction_map = dict()\n    for test_index, test_id in enumerate(test_ids):\n        hex_test_id = to_hex(test_id)\n        aggregate_scores = {}\n        for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:\n            if label not in aggregate_scores:\n                aggregate_scores[label] = 0\n            aggregate_scores[label] += score\n        label, score = max(aggregate_scores.items(), key = operator.itemgetter(1))\n        prediction_map[hex_test_id] = {'score': score, 'class': label}\n    return prediction_map\n\ndef to_hex(image_id):\n    return '{0:0{1}x}'.format(image_id, 16)\n\ndef load_labelmap():\n    with open(TRAIN_LABELMAP_PATH, mode = 'r') as csv_file:\n        csv_reader = csv.DictReader(csv_file)\n        labelmap = {row['id']: row['landmark_id'] for row in csv_reader}\n    return labelmap\n\ndef save_submission_csv(predictions = None):\n    if predictions is None:\n        shutil.copyfile(os.path.join(DATASET_DIR, 'sample_submission.csv'), \n                        'submission.csv')\n        return\n    with open('submission.csv', 'w') as submission_csv:\n        csv_writer = csv.DictWriter(submission_csv, fieldnames = ['id', 'landmarks'])\n        csv_writer.writeheader()\n        for image_id, prediction in predictions.items():\n            label = prediction['class']\n            score = prediction['score']\n            csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def main():\n    labelmap = load_labelmap()\n    num_training_images = len(labelmap.keys())\n    print(f'Found {num_training_images} training images')\n    if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:\n        print(f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images. Copying sample submission')\n        save_submission_csv()\n        return\n\n    pre_verification_predictions, post_verification_predictions = get_predictions(labelmap)\n    save_submission_csv(post_verification_predictions)\n    print('Done')\n\nmain()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}