{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import operator\nimport gc\nimport pathlib\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom scipy import spatial\nimport cv2\n!pip install ../input/landmark-lib/Keras_Applications-1.0.8-py3-none-any.whl\n!pip install ../input/landmark-lib/efficientnet-1.1.0-py3-none-any.whl\nimport efficientnet.tfkeras as efn\nimport math\n\nNUMBER_OF_CLASSES = 81313\nLR = 0.0001\n\nIMAGE_SIZE = [600, 600]\nIMG_H = IMAGE_SIZE[0]\nIMG_W = IMAGE_SIZE[1]\nS = 64\nM = 0.15\nEFF = 7\nWEIGHTS_PATH = \"../input/b7res600drop5ep14/b7res600drop5.h5\"\nNUM_TO_RERANK = 3\nDROPOUT = 0.5\n\nclass ArcMarginProduct(tf.keras.layers.Layer):\n    '''\n    Implements large margin arc distance.\n    '''\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n\n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n\n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n        one_hot = tf.cast(\n            tf.one_hot(y, depth=self.n_classes),\n            dtype=cosine.dtype\n        )\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output\n\n\n# Function to build our model using fine tunning (efficientnet)\ndef get_model(eff = 1):\n    \n\n    margin = ArcMarginProduct(\n        n_classes = NUMBER_OF_CLASSES, \n        s = S, \n        m = M, \n        name='head/arc_margin', \n        dtype='float32'\n        )\n\n    inp = tf.keras.layers.Input(shape = (*IMAGE_SIZE, 3), name = 'inp1')\n    label = tf.keras.layers.Input(shape = (), name = 'inp2')\n    if eff == 0:\n        x = efn.EfficientNetB0(weights = None, include_top = False)(inp)\n    elif eff == 1:\n        x = efn.EfficientNetB1(weights = None, include_top = False)(inp)\n    elif eff == 2:\n        x = efn.EfficientNetB2(weights = None, include_top = False)(inp)\n    elif eff == 3:\n        x = efn.EfficientNetB3(weights = None, include_top = False)(inp)\n    elif eff == 4:\n        x = efn.EfficientNetB4(weights = None, include_top = False)(inp)\n    elif eff == 5:\n        x = efn.EfficientNetB5(weights = None, include_top = False)(inp)\n    elif eff == 6:\n        x = efn.EfficientNetB6(weights = None, include_top = False)(inp)\n    elif eff == 7:\n        x = efn.EfficientNetB7(weights = None, include_top = False)(inp)\n        \n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dropout(DROPOUT)(x)\n    x = tf.keras.layers.Dense(512)(x)\n    x = margin([x, label])\n\n    output = tf.keras.layers.Softmax(dtype='float32')(x)\n\n    model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])\n\n    opt = tf.keras.optimizers.Adam(learning_rate = LR)\n\n    model.compile(\n        optimizer = opt,\n        loss = [tf.keras.losses.SparseCategoricalCrossentropy()],\n        metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]\n        ) \n\n    return model\n\n\nNUM_EMBEDDING_DIMENSIONS = 512\nDATASET_DIR = '../input/landmark-image-train/train_encoded.csv'\nTEST_IMAGE_DIR = '../input/landmark-recognition-2020/test'\nTRAIN_IMAGE_DIR = '../input/landmark-recognition-2020/train'\nMODEL = get_model(eff = EFF)\nMODEL.load_weights(WEIGHTS_PATH)\nMODEL = tf.keras.models.Model(inputs = MODEL.input[0], outputs = MODEL.layers[-4].output)\n\n\n\nNUM_PUBLIC_TEST_IMAGES = 10345 # Used to detect if in session or re-run.\n\n# Read image and resize it\ndef read_image(image_path, size = (384, 384)):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, size)\n    img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 100))[1].tostring()\n    img = tf.image.decode_jpeg(img, channels = 3)\n    img = tf.cast(img, tf.float32) / 255.0\n    img = tf.reshape(img, [1, IMG_H, IMG_W, 3])\n    return img\n\n# Function to get training and test embeddings\ndef generate_embeddings(filepaths):\n    image_paths = [x for x in pathlib.Path(filepaths).rglob('*.jpg')]\n    num_images = len(image_paths)\n    ids = num_images * [None]\n    # Generate an empty matrix where we can store the embeddings of each image\n    embeddings = np.empty((num_images, NUM_EMBEDDING_DIMENSIONS))\n    for i, image_path in enumerate(image_paths):\n        ids[i] = image_path.name.split('.')[0]\n        image_tensor = read_image(str(image_path), (IMG_H, IMG_W))\n        prediction = MODEL.predict(image_tensor)\n#         prediction2 = MODEL2.predict(image_tensor)\n#         prediction = np.average([prediction1, prediction2], axis = 0)\n        embeddings[i, :] = prediction\n    return ids, embeddings\n\n# This function get the most similar train images for each test image based on cosine similarity\ndef get_similarities(train_csv, test_directory, train_directory):\n    # Get target dictionary\n    df = pd.read_csv(train_csv)\n    df = df[['id', 'landmark_id']]\n    df.set_index('id', inplace = True)\n    df = df.to_dict()['landmark_id']\n    # Extract the test ids and global feature for the test images\n    test_ids, test_embeddings = generate_embeddings(test_directory)\n    # Extract the train ids and global features for the train images\n    train_ids, train_embeddings = generate_embeddings(train_directory)\n    # Initiate a list were we will store the similar training images for each test image (also score)\n    train_ids_labels_and_scores = [None] * test_embeddings.shape[0]\n    # Using (slow) for-loop, as distance matrix doesn't fit in memory\n    for test_index in range(test_embeddings.shape[0]):\n        distances = spatial.distance.cdist(\n            test_embeddings[np.newaxis, test_index, : ], train_embeddings, 'cosine')[0]\n        # Get the indices of the closest images\n        top_k = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]\n        # Get the nearest ids and distances using the previous indices\n        nearest = sorted([(train_ids[p], distances[p]) for p in top_k], key = lambda x: x[1])\n        # Get the labels and score results\n        train_ids_labels_and_scores[test_index] = [(df[train_id], 1.0 - cosine_distance) for \\\n                                                   train_id, cosine_distance in nearest]\n        \n    del test_embeddings\n    del train_embeddings\n    gc.collect()\n    return test_ids, train_ids_labels_and_scores\n\n# This function aggregate top simlarities and make predictions\ndef generate_predictions(test_ids, train_ids_labels_and_scores):\n    targets = []\n    scores = []\n    \n    # Iterate through each test id\n    for test_index, test_id in enumerate(test_ids):\n        aggregate_scores = {}\n        # Iterate through the similar images with their corresponing score for the given test image\n        for target, score in train_ids_labels_and_scores[test_index]:\n            if target not in aggregate_scores:\n                aggregate_scores[target] = 0\n            aggregate_scores[target] += score\n        # Get the best score\n        target, score = max(aggregate_scores.items(), key = operator.itemgetter(1))\n        targets.append(target)\n        scores.append(score)\n        \n    final = pd.DataFrame({'id': test_ids, 'target': targets, 'scores': scores})\n    final['landmarks'] = final['target'].astype(str) + ' ' + final['scores'].astype(str)\n    final[['id', 'landmarks']].to_csv('submission.csv', index = False)\n    return final\n\ndef inference_and_save_submission_csv(train_csv, test_directory, train_directory):\n    image_paths = [x for x in pathlib.Path(test_directory).rglob('*.jpg')]\n    test_len = len(image_paths)\n    if test_len == NUM_PUBLIC_TEST_IMAGES:\n        # Dummy submission\n        shutil.copyfile('../input/landmark-recognition-2020/sample_submission.csv', 'submission.csv')\n        return 'Job Done'\n    else:\n        test_ids, train_ids_labels_and_scores = get_similarities(train_csv, test_directory, train_directory)\n        final = generate_predictions(test_ids, train_ids_labels_and_scores)\n        return final\n    \nfinal = inference_and_save_submission_csv(DATASET_DIR, TEST_IMAGE_DIR, TRAIN_IMAGE_DIR)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}