{"cells":[{"metadata":{},"cell_type":"markdown","source":"# This inference kernel is mainly taken from @[Ragnar](http://www.kaggle.com/ragnar123)\n\nI would sincerely appreciate his awesome kernel and training scripts. This is just a testing kernel from my team as we want to see the effnets' performance. We used both effb7 and effb6 first trained with the 384 sized images, then we used these two models to further trained with 512 sized images. (Even though due to time limit, we could not have finished the entire training process and the b7 is not fully converged) Thus, we will just share it as a reference. Feel free to ask if you have any questions.\n\nAgain, thanks Ragnar."},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/glrec2020/Keras_Applications-1.0.8-py3-none-any.whl\n!pip install ../input/glrec2020/efficientnet-1.1.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import operator\nimport gc\nimport pathlib\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom scipy import spatial\nimport cv2\nimport efficientnet.tfkeras as efn\nimport math","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUMBER_OF_CLASSES = 81313\nIMAGE_SIZE = [384, 384]\nLR = 0.0001","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ArcMarginProduct(tf.keras.layers.Layer):\n    '''\n    Implements large margin arc distance.\n\n    Reference:\n        https://arxiv.org/pdf/1801.07698.pdf\n        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n            blob/master/src/modeling/metric_learning.py\n    '''\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n\n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n\n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n        one_hot = tf.cast(\n            tf.one_hot(y, depth=self.n_classes),\n            dtype=cosine.dtype\n        )\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output\n\n\n# Function to build our model using fine tunning (efficientnet)\ndef get_model_B6():\n\n    margin = ArcMarginProduct(\n        n_classes = NUMBER_OF_CLASSES, \n        s = 64, \n        m = 0.15, \n        name='head/arc_margin', \n        dtype='float32'\n        )\n\n    inp = tf.keras.layers.Input(shape = (384, 384, 3), name = 'inp1')\n    label = tf.keras.layers.Input(shape = (), name = 'inp2')\n    x4 = efn.EfficientNetB6(weights = None, include_top = False)(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x4)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    x = tf.keras.layers.Dense(512)(x)\n    x = margin([x, label])\n\n    output = tf.keras.layers.Softmax(dtype='float32')(x)\n\n    model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])\n\n    opt = tf.keras.optimizers.Adam(learning_rate = LR)\n\n    model.compile(\n        optimizer = opt,\n        loss = [tf.keras.losses.SparseCategoricalCrossentropy()],\n        metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]\n        ) \n\n    return model\n\ndef get_model_B7():\n\n    margin = ArcMarginProduct(\n        n_classes = NUMBER_OF_CLASSES, \n        s = 64, \n        m = 0.15, \n        name='head/arc_margin', \n        dtype='float32'\n        )\n\n    inp = tf.keras.layers.Input(shape = (384, 384, 3), name = 'inp1')\n    label = tf.keras.layers.Input(shape = (), name = 'inp2')\n    x4 = efn.EfficientNetB7(weights = None, include_top = False)(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x4)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    x = tf.keras.layers.Dense(512)(x)\n    x = margin([x, label])\n\n    output = tf.keras.layers.Softmax(dtype='float32')(x)\n\n    model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])\n\n    opt = tf.keras.optimizers.Adam(learning_rate = LR)\n\n    model.compile(\n        optimizer = opt,\n        loss = [tf.keras.losses.SparseCategoricalCrossentropy()],\n        metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]\n        ) \n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_EMBEDDING_DIMENSIONS = 1024\nDATASET_DIR = '../input/landmark-image-train/train_encoded.csv'\nTEST_IMAGE_DIR = '../input/landmark-recognition-2020/test'\nTRAIN_IMAGE_DIR = '../input/landmark-recognition-2020/train'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL1 = get_model_B6()\nMODEL2 = get_model_B7()\n\nMODEL1.load_weights('../input/effb6-512-ep18/effb6model512-18.h5')\nMODEL1 = tf.keras.models.Model(inputs = MODEL1.input[0], outputs = MODEL1.layers[-4].output)\n\nMODEL2.load_weights('../input/effb7-512-ep12/effb7model512-12.h5')\nMODEL2 = tf.keras.models.Model(inputs = MODEL2.input[0], outputs = MODEL2.layers[-4].output)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TO_RERANK = 3 #originally 5\nNUM_PUBLIC_TEST_IMAGES = 10345 # Used to detect if in session or re-run.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_image(image_path, size = (384, 384)):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, size)\n    img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 100))[1].tostring()\n    img = tf.image.decode_jpeg(img, channels = 3)\n    img = tf.image.resize(img, (512, 512))\n    img = tf.cast(img, tf.float32) / 255.0\n    img = tf.reshape(img, [1, 512, 512, 3])\n    return img\n\n# def read_image(image_path, size = (384, 384)):\n#     img = cv2.imread(image_path) #384\n#     img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n#     img = cv2.resize(img, size)\n#     img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 100))[1].tostring()\n#     img = tf.image.decode_jpeg(img, channels = 3)\n#     img = tf.cast(img, tf.float32) / 255.0\n#     img = tf.reshape(img, [1, 384, 384, 3])\n#     return img\n\n# Function to get training and test embeddings\ndef generate_embeddings(filepaths):\n    image_paths = [x for x in pathlib.Path(filepaths).rglob('*.jpg')]\n    num_images = len(image_paths)\n    ids = num_images * [None]\n    # Generate an empty matrix where we can store the embeddings of each image\n    embeddings = np.empty((num_images, NUM_EMBEDDING_DIMENSIONS))\n    for i, image_path in enumerate(image_paths):\n        ids[i] = image_path.name.split('.')[0]\n        image_tensor = read_image(str(image_path), (384, 384)) #384\n        prediction1 = MODEL1.predict(image_tensor)\n        prediction2 = MODEL2.predict(image_tensor)\n        prediction = tf.concat([prediction1, prediction2], 1)\n        \n        embeddings[i, :] = prediction\n    return ids, embeddings\n\n# def generate_embeddings(filepaths):\n#     image_paths = [x for x in pathlib.Path(filepaths).rglob('*.jpg')]\n#     num_images = len(image_paths)\n#     ids = num_images * [None]\n#     # Generate an empty matrix where we can store the embeddings of each image\n#     embeddings = np.empty((num_images, NUM_EMBEDDING_DIMENSIONS))\n#     for i, image_path in enumerate(image_paths):\n#         ids[i] = image_path.name.split('.')[0]\n#         image_tensor = read_image(str(image_path), (384, 384))\n#         prediction1 = MODEL1.predict(image_tensor)\n#         prediction2 = MODEL2.predict(image_tensor)\n#         prediction = np.average([prediction1, prediction2], axis = 0)\n#         embeddings[i, :] = prediction\n#     return ids, embeddings\n\n# This function get the most similar train images for each test image based on cosine similarity\ndef get_similarities(train_csv, test_directory, train_directory):\n    # Get target dictionary\n    df = pd.read_csv(train_csv)\n    df = df[['id', 'landmark_id']]\n    df.set_index('id', inplace = True)\n    df = df.to_dict()['landmark_id']\n    # Extract the test ids and global feature for the test images\n    test_ids, test_embeddings = generate_embeddings(test_directory)\n    # Extract the train ids and global features for the train images\n    train_ids, train_embeddings = generate_embeddings(train_directory)\n    # Initiate a list were we will store the similar training images for each test image (also score)\n    train_ids_labels_and_scores = [None] * test_embeddings.shape[0]\n    # Using (slow) for-loop, as distance matrix doesn't fit in memory\n    for test_index in range(test_embeddings.shape[0]):\n        distances = spatial.distance.cdist(\n            test_embeddings[np.newaxis, test_index, : ], train_embeddings, 'cosine')[0]\n        # Get the indices of the closest images\n        top_k = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]\n        # Get the nearest ids and distances using the previous indices\n        nearest = sorted([(train_ids[p], distances[p]) for p in top_k], key = lambda x: x[1])\n        # Get the labels and score results\n        train_ids_labels_and_scores[test_index] = [(df[train_id], 1.0 - cosine_distance) for \\\n                                                   train_id, cosine_distance in nearest]\n        \n    del test_embeddings\n    del train_embeddings\n    gc.collect()\n    return test_ids, train_ids_labels_and_scores\n\n# This function aggregate top simlarities and make predictions\ndef generate_predictions(test_ids, train_ids_labels_and_scores):\n    targets = []\n    scores = []\n    \n    # Iterate through each test id\n    for test_index, test_id in enumerate(test_ids):\n        aggregate_scores = {}\n        # Iterate through the similar images with their corresponing score for the given test image\n        for target, score in train_ids_labels_and_scores[test_index]:\n            if target not in aggregate_scores:\n                aggregate_scores[target] = 0\n            aggregate_scores[target] += score\n        # Get the best score\n        target, score = max(aggregate_scores.items(), key = operator.itemgetter(1))\n        targets.append(target)\n        scores.append(score)\n        \n    final = pd.DataFrame({'id': test_ids, 'target': targets, 'scores': scores})\n    final['landmarks'] = final['target'].astype(str) + ' ' + final['scores'].astype(str)\n    final[['id', 'landmarks']].to_csv('submission.csv', index = False)\n    return final\n\ndef inference_and_save_submission_csv(train_csv, test_directory, train_directory):\n    image_paths = [x for x in pathlib.Path(test_directory).rglob('*.jpg')]\n    test_len = len(image_paths)\n    if test_len == NUM_PUBLIC_TEST_IMAGES:\n        # Dummy submission\n        shutil.copyfile('../input/landmark-recognition-2020/sample_submission.csv', 'submission.csv')\n        return 'Job Done'\n    else:\n        test_ids, train_ids_labels_and_scores = get_similarities(train_csv, test_directory, train_directory)\n        final = generate_predictions(test_ids, train_ids_labels_and_scores)\n        return final\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final = inference_and_save_submission_csv(DATASET_DIR, TEST_IMAGE_DIR, TRAIN_IMAGE_DIR)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}