{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## About\n\nIn this notebook, I'll create a submission with the models of [GLRet21: EfficientNetB0 Baseline Training](https://www.kaggle.com/hidehisaarai1213/glret21-efficientnetb0-baseline-training).\n\nThis notebook is based on [GLRet21: EfficientNetB0 Baseline Inference](https://www.kaggle.com/hidehisaarai1213/glret21-efficientnetb0-baseline-inference) and\n\n[Recognition Kernel](https://www.kaggle.com/camaskew/host-baseline-example?scriptVersionId=40191321)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# If you copy the notbook please upvote it","metadata":{}},{"cell_type":"code","source":"EFF_SZ = 6","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ../input/kerasapplications/ > /dev/null\n!pip install ../input/efficientnet-keras-source-code/ > /dev/null","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport math\nimport random\nimport re\nimport warnings\nfrom pathlib import Path\nfrom PIL import Image\nfrom typing import Optional, Tuple\nimport csv\nimport operator\n\nimport efficientnet.tfkeras as efn\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom scipy import spatial\nfrom sklearn.preprocessing import normalize\nfrom tqdm import tqdm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Settings","metadata":{}},{"cell_type":"code","source":"DATADIR = Path(\"../input/landmark-recognition-2021\")\nTEST_IMAGE_DIR = DATADIR / \"test\"\nTRAIN_IMAGE_DIR = DATADIR / \"train\"\nTRAIN_LABELMAP_PATH = '../input/landmark-recognition-2021/train.csv'\n\nNUM_TO_RERANK = 6\n#TOP_K = 5  \n\nN_CLASSES = 81313\nTHRESHOLD = 0.5\n\nTEST = False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/landmark-recognition-2021/train.csv')\n\ndf_test = pd.read_csv('../input/landmark-recognition-2021/sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utilities","metadata":{}},{"cell_type":"code","source":"import time\n\nfrom contextlib import contextmanager\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    print(f\"[{name}]\")\n    yield\n    print(f'[{name}] done in {time.time() - t0:.0f} s')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\n\nset_seed(1213)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    return strategy","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy = auto_select_accelerator()\nREPLICAS = strategy.num_replicas_in_sync\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"class GeM(tf.keras.layers.Layer):\n    def __init__(self, pool_size, init_norm=3.0, normalize=False, **kwargs):\n        self.pool_size = pool_size\n        self.init_norm = init_norm\n        self.normalize = normalize\n\n        super(GeM, self).__init__(**kwargs)\n\n    def get_config(self):\n        config = super().get_config().copy()\n        config.update({\n            'pool_size': self.pool_size,\n            'init_norm': self.init_norm,\n            'normalize': self.normalize,\n        })\n        return config\n\n    def build(self, input_shape):\n        feature_size = input_shape[-1]\n        self.p = self.add_weight(name='norms', shape=(feature_size,),\n                                 initializer=tf.keras.initializers.constant(self.init_norm),\n                                 trainable=True)\n        super(GeM, self).build(input_shape)\n\n    def call(self, inputs):\n        x = inputs\n        x = tf.math.maximum(x, 1e-6)\n        x = tf.pow(x, self.p)\n\n        x = tf.nn.avg_pool(x, self.pool_size, self.pool_size, 'VALID')\n        x = tf.pow(x, 1.0 / self.p)\n\n        if self.normalize:\n            x = tf.nn.l2_normalize(x, 1)\n        return x\n\n    def compute_output_shape(self, input_shape):\n        return tuple([None, input_shape[-1]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ArcMarginProduct(tf.keras.layers.Layer):\n    '''\n    Implements large margin arc distance.\n\n    Reference:\n        https://arxiv.org/pdf/1801.07698.pdf\n        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n            blob/master/src/modeling/metric_learning.py\n    '''\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n\n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n\n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n        one_hot = tf.cast(\n            tf.one_hot(y, depth=self.n_classes),\n            dtype=cosine.dtype\n        )\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def build_model(size=256, efficientnet_size=0, weights=\"imagenet\", count=0):\ndef build_model(size=256, efficientnet_size=EFF_SZ, weights=\"imagenet\", count=0):   \n    inp = tf.keras.layers.Input(shape=(size, size, 3), name=\"inp1\")\n    label = tf.keras.layers.Input(shape=(), name=\"inp2\")\n    x = getattr(efn, f\"EfficientNetB{efficientnet_size}\")(\n        weights=weights, include_top=False, input_shape=(size, size, 3))(inp)\n    x = GeM(8)(x)\n    x = tf.keras.layers.Flatten()(x)\n    x = tf.keras.layers.Dense(512, name=\"dense_before_arcface\", kernel_initializer=\"he_normal\")(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = ArcMarginProduct(\n        n_classes=N_CLASSES,\n        s=30,\n        m=0.5,\n        name=\"head/arc_margin\",\n        dtype=\"float32\"\n    )([x, label])\n    output = tf.keras.layers.Softmax(dtype=\"float32\")(x)\n    model = tf.keras.Model(inputs=[inp, label], outputs=[output])\n    opt = tf.optimizers.Adam(learning_rate=1e-4)\n    model.compile(\n        optimizer=opt,\n        loss=[tf.keras.losses.SparseCategoricalCrossentropy()],\n        metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]\n    )\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model_for_inference(weights_path: str, EFF_SZ):\n    with strategy.scope():\n        base_model = build_model(\n            size=256,\n#             efficientnet_size=0,\n            efficientnet_size=EFF_SZ,\n            weights=None,\n            count=0) \n        \n# def create_model_for_inference(weights_path: str):\n#     with strategy.scope():\n#         base_model = build_model(\n#             size=256,\n#             efficientnet_size=0,\n#             weights=None,\n#             count=0)\n        base_model.load_weights(weights_path)\n        model = tf.keras.Model(inputs=base_model.get_layer(\"inp1\").input,\n                               outputs=base_model.get_layer(\"dense_before_arcface\").output)\n        return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Extraction","metadata":{}},{"cell_type":"code","source":"def to_hex(image_id) -> str:\n    return '{0:0{1}x}'.format(image_id, 16)\n\n\ndef get_image_path(subset, image_id):\n    name = to_hex(image_id)\n    return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2], '{}.jpg'.format(name))\n\n\ndef load_image_tensor(image_path):\n    tensor = tf.convert_to_tensor(np.array(Image.open(image_path).convert(\"RGB\")))\n    tensor = tf.image.resize(tensor, size=(256, 256))\n    tensor = tf.expand_dims(tensor, axis=0)\n    return tf.cast(tensor, tf.float32) / 255.0\n\n\ndef create_batch(files):\n    images = []\n    for f in files:\n        images.append(load_image_tensor(f))\n    return tf.concat(images, axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_global_features(image_root_dir, n_models=4,DEBUG='TEST'):\n    image_paths = []\n    for root, dirs, files in os.walk(image_root_dir):\n        for file in files:\n            if file.endswith('.jpg'):\n                 image_paths.append(os.path.join(root, file))\n                    \n    if DEBUG == 'TRAIN':\n        \n        #image_paths = random.shuffle(image_paths)\n        \n        if TEST:\n            \n            image_paths = image_paths[:100000]\n        \n        else:\n        \n            image_paths = image_paths\n    \n    elif DEBUG == 'TEST':\n        \n        #image_paths = random.shuffle(image_paths)\n        \n        if TEST:\n            \n            image_paths = image_paths#[:1000]\n        else:\n        \n            image_paths = image_paths\n        \n    \n    \n    \n    num_embeddings = len(image_paths)\n\n    ids = num_embeddings * [None]\n    ids = []\n    for path in image_paths:\n        ids.append(path.split('/')[-1][:-4])\n    \n    embeddings = np.zeros((num_embeddings, 512))\n    image_paths = np.array(image_paths)\n    chunk_size = 512\n    \n    n_chunks = len(image_paths) // chunk_size\n    if len(image_paths) % chunk_size != 0:\n        n_chunks += 1\n\n    for n in range(n_models):\n        print(f\"Getting Embedding for fold{n} model.\")\n        model = create_model_for_inference(f\"../input/glret21-efficientnetb0-baseline-training/fold{n}.h5\")\n        for i in tqdm(range(n_chunks)):\n            files = image_paths[i * chunk_size:(i + 1) * chunk_size]\n            batch = create_batch(files)\n            embedding_tensor = model.predict(batch)\n            embeddings[i * chunk_size:(i + 1) * chunk_size] += embedding_tensor / n_models\n        del model\n        gc.collect()\n        tf.keras.backend.clear_session()\n\n    embeddings = normalize(embeddings, axis=1)\n\n    return ids, embeddings\n\ndef load_labelmap():\n    \n    \n    with open(TRAIN_LABELMAP_PATH, mode='r') as csv_file:\n        \n        csv_reader = csv.DictReader(csv_file)\n        labelmap = {row['id']: row['landmark_id'] for row in csv_reader}\n\n    return labelmap\n\ndef get_aggregate_score(dict_map):\n    \n    aggregate_scores = {}\n\n    for ids, label, score in dict_map:\n\n\n        if label not in aggregate_scores:\n\n            aggregate_scores[label] = score\n\n            #aggregate_scores[label] = label_map[ids]\n\n        else:\n\n            aggregate_scores[label] += score\n            \n    return aggregate_scores\n    \n    \ndef fill_prediction(ID):\n    \n    if ID in prediction_dict:\n        \n        if prediction_dict[ID][1] <= THRESHOLD:\n            \n            return ''\n        \n        else:\n        \n            return str(prediction_dict[ID][0]) + ' ' + str(prediction_dict[ID][1])\n    \n    else: \n        return ''\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Main","metadata":{}},{"cell_type":"code","source":"def get_predictions():\n    with timer(\"Getting Test Embeddings\"):\n        test_ids, test_embeddings = extract_global_features(str(TEST_IMAGE_DIR))\n\n    with timer(\"Getting Train Embeddings\"):\n        train_ids, train_embeddings = extract_global_features(str(TRAIN_IMAGE_DIR),DEBUG='TRAIN')\n\n    PredictionString_list = []\n    labelmap = load_labelmap()\n    train_ids_labels_and_scores = {}\n    test_ids_labels_and_scores = {}\n    with timer(\"Matching...\"):\n        for test_index in range(test_embeddings.shape[0]):\n            distances = spatial.distance.cdist(test_embeddings[np.newaxis, test_index, :], train_embeddings, 'cosine')[0]\n            partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]\n            nearest = sorted([(train_ids[p], distances[p]) for p in partition], key=lambda x: x[1])\n            df = pd.DataFrame(columns=['ids', 'distance', 'label'])\n            with timer(test_index):\n                \n                train_ids_labels_and_scores[test_ids[test_index]] = [\n                    (train_id, labelmap[train_id],1 - cosine_distance)\n                                    for train_id, cosine_distance in nearest\n                                ]\n    \n        for ids in train_ids_labels_and_scores:\n            \n            a = get_aggregate_score(train_ids_labels_and_scores[ids])\n            \n            test_ids_labels_and_scores[ids] =  max(a.items(), key=operator.itemgetter(1))\n            #test_ids_labels_and_scores[ids] = a\n            \n            \n\n    return test_ids_labels_and_scores #test_ids, PredictionString_list\n\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#prediction_dict['777f9efff0fc6b81']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#prediction_dict = get_predictions()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TEST:\n    num_pict = 1\nelse:\n    num_pict = 10345\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(df_test) == num_pict:\n    df_test[['id', 'landmarks']].to_csv('submission.csv', index = False)\n\nelse:\n    prediction_dict = get_predictions()\n    df_test['landmarks'] = df_test['id'].apply(fill_prediction)\n    df_test[['id', 'landmarks']].to_csv('submission.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}