{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Set your own project id here\nPROJECT_ID = 'daring-runway-287322'\nfrom google.cloud import storage\nstorage_client = storage.Client(project=PROJECT_ID)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Downloading and installing the repository"},{"metadata":{"trusted":true},"cell_type":"code","source":"!git clone https://hav4ik:3bbdb6bd02341ef0ebdf165ff819fc4f45d64ef9@github.com/hav4ik/google-landmarks-2020.git\n!cd google-landmarks-2020; git checkout delg_train_loop\n!pip install google-landmarks-2020/python/\n!cd google-landmarks-2020; git branch -a; git log --oneline -n 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"json_file = \"\"\"{\n  \"type\": \"service_account\",\n  \"project_id\": \"daring-runway-287322\",\n  \"private_key_id\": \"170e6a965549c385476a35db9eca944cd93821df\",\n  \"private_key\": \"-----BEGIN PRIVATE KEY-----\\\\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC3DKtJPhaafJOK\\\\nBWw8kBUR23SZDl6EUOPOrtP5rT18W+SCJch5KSqSFrgBEBehAkj+p/yqpJvlCqwX\\\\nd3Gt0p6obqVeIU6h8fh6xEc3VDs8wX6lokxr4TpNoUZUQ9U2fJo0bL3ALlp6w23k\\\\nkaYjvUWI6QmUVyrgiocF/hoDWumWevgT16a6GxHQJ37iPalZnHvOBNfhsQUehJz/\\\\nJkKRF+ZmhsNsXQkhK9lpdxus10INBRwmpDbgaq2r5LplVfp/r5E4sH2XjRVA2HxS\\\\n7RCTMXx587OjvIuMPSIC6yhqN6qNP3oKYIu7YBYkZcaiH6CIr528aU9DFWEnS+Yg\\\\n8dd0zkP/AgMBAAECggEAG8lwDPfAXaYwjWCiXBUxJZtPOVa9kU9oLX1Lj1i1fzXB\\\\n2+HZa08M9808FGk5dQcQRpwAlnTTd3LlydAncRAKZp4ZO09KDPpa0mQOPX9rWJ80\\\\n1vbqq+uuOo+TSgJcEN/cVr31FgHTWVRmT66ubjlyn0Rk4GBVEW3l7kpy73RzBhUV\\\\nuQGCNeiMFgy/yTg5SyvlXvnuYglELE1j8gO09hHD3RJ5IHAUoeG1RY1HDqKqVAz7\\\\nbYPJEeG1QCZ/yciP31+7mdxbKugJh8C/w+5JL+QzJYQSycP2IFiJAUtXn69i9rPV\\\\nXYoX7vwPrTVjKZ/sAw36oTvpAcitsACbl5LJkWWsOQKBgQDqo5jIUiCAq79NN5wi\\\\nfNHLKvCZJI7AgxJ9TTjXwofi3OGL62b/DqkKa9t+LLWj+m5YrMeexnz4/u6dc00g\\\\nHqRbTXEqG3UGI9C3tjlOdEhCfxMg+epeKFCptSWpv1FRBIvAbsVsmZTWFtfRI8AY\\\\nOL0qfUImqfO50l2pqx58NtEb5wKBgQDHtr8qYjH4HwvHpjr+x1oIK2OsEHOxdDpR\\\\nA4wLxQKNDnSDTRrPLzTza9T1FVk2qTGBtCqF3ZDVbIVNoxC5ImHkKg/CQam7g4qh\\\\nWptcXqqRGtGJi5rHKPQXMZKSfmhzFYdLzC1Y7Ny+giw0VQgdllavvQXG/V4jf9hI\\\\nrbT2cW9UKQKBgQDT7fP/a36D+ZyOaB4UYF01fpFWIVj2tOysVGV1K/WEiTEHKhYb\\\\nZeh0yzqzWjqt43JrkZOz61/RnqFzgUM2MbcN6ILAH5CxfQP2Cxbzr7/cn95tOI3h\\\\ngdRplH5yNaWC4fJAW0zibE2smXFRK1NEDWt+xyhKt9K2EjYbG74rP2/cHQKBgAry\\\\nMTgZebOnv/WJXuJn+r9H4YakwDKRtECMeTiL67/fcvNfXoDMjZJp7pogOWuNinEZ\\\\ndtsvcajA8e13aos7HzJqO8Lh5nOomgiN9sXxSlf5qFJpnGoeDILY7Leqxf7Ix0mY\\\\nP0QId3DZoaKcpn04qDepnI9zg51efHQ/URPQ09FBAoGALtLTvCk9T8iCIUYP09k2\\\\naay2DZgS/Y/g1Qfb5ps5BllKNPTZK8eVn/2myalpDG5n11Y4VSmnzDoLLwb/8Wac\\\\nxEb7ILuIbsTK9iOCpF+bCKqWaOvYocetRY7+5vwR4a50Kk0/pFEfKPyAbUtImX1l\\\\nDJ2ZiXpo8i1vp+08J7f8oao=\\\\n-----END PRIVATE KEY-----\\\\n\",\n  \"client_email\": \"975295620615-compute@developer.gserviceaccount.com\",\n  \"client_id\": \"102626917878770955141\",\n  \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n  \"token_uri\": \"https://oauth2.googleapis.com/token\",\n  \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n  \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/975295620615-compute%40developer.gserviceaccount.com\"\n}\"\"\"\n\nwith open('credentials.json', 'w') as f:\n    f.write(json_file)\n    \nimport os\nos.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'credentials.json'\n\n!export GOOGLE_APPLICATION_CREDENTIALS=credentials.json\n!export GCLOUD_PROJECT=daring-runway-287322\n!cat credentials.json","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading the trained DELG model from GCS; perform surgery"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport sys\nimport yaml\nimport math\nimport tensorflow as tf\nfrom matplotlib import pyplot as plt\nimport numpy as np\n\nsys.path.append('google-landmarks-2020/training')\nfrom train import train_delg\nfrom glrec.train import utils as train_utils\nfrom delg_model import DelgModel","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Local features model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_gcs_path = 'gs://khch-glc2020-models/local_only/RN152V2/RN152V2-512x512-512-GeM+ArcFace_002-from-003-Faster/checkpoints/018_val_loss=0.28348.hdf5'\n\nmodel_file = train_utils.resolve_file_path(model_gcs_path)\n!find /tmp/glrec | sed -e \"s/[^-][^\\/]*\\// |/g\" -e \"s/|\\([^ ]\\)/|-\\1/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_config_yaml=\"\"\"\n# Model configurations\nmodel_config:\n\n  # Backbone CNN configuration\n  backbone_config:\n    architecture: ResNet152V2\n    weights: null\n    trainable: false\n\n  # Global branch: [backbone]->[pooling]->[dense]->[head]\n  global_branch_config:\n    # Currently, the following pooling methods are supported:\n    #   - GAP()\n    #   - GeM(p=3, train_p=False)\n    pooling_config:\n      method: GeM\n      kwargs:\n        p: 3.0\n        train_p: false\n\n    # Embedding after pooling (output units of a Dense layer\n    # without activation function because the backbone already\n    # have it, and without bias.\n    embedding_dim: 512\n\n    # Currently, the following heads are supported:\n    #   - ArcFace(s=30, m=0.5)\n    #   - ArcMarginProduct(s=30, m=0.5, easy_margin=False)\n    #   - AdaCos(m=0.5, is_dynamic=True)\n    #   - CosFace(s=30, m=0.35)\n    head_config:\n      layer: ArcFace\n      kwargs:\n        s: 45.45 # = sqrt(2) * log(C-1), where C=81313\n        m: 0.3 # Was 0.1 initially\n\n    # Trainability of global branch\n    trainable: false\n\n  # Shallow feature map to be fed to local branch\n  shallow_layer_name: conv5_block1_preact_relu\n\n  # Local features extractor configurations.\n  local_branch_config:\n    attention_config: {} # use default settings\n    autoencoder_config: {} # use default settings\n    trainable: true\n\n  # Places classifier\n  places_branch_config: null\n\n  # Supported training modes right now:\n  #   - global_only\n  #   - local_only\n  #   - local_and_global\n  training_mode: local_only\n  inference_mode: local_only\n\"\"\"\n\nlocal_model = DelgModel(**yaml.safe_load(model_config_yaml)['model_config'])\nlocal_model.build(input_shape=[\n    [1, 512, 512, 3],\n    [1, ]\n])\nlocal_model.load_weights(model_file)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Global Features Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"global_model_gcs_path = 'gs://khch-glc2020-models/global/RN152V2/RN152V2-512x512-512-GeM+ArcFace_003-Faster/checkpoints/034_val_loss=0.05133.hdf5'\nglobal_model_file = train_utils.resolve_file_path(global_model_gcs_path)\n!find /tmp/glrec | sed -e \"s/[^-][^\\/]*\\// |/g\" -e \"s/|\\([^ ]\\)/|-\\1/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"global_cfg = \"\"\"\n# Model configurations\nmodel_config:\n\n  # Backbone CNN configuration\n  backbone_config:\n    architecture: ResNet152V2\n    weights: null\n    trainable: true\n\n  # Global branch: [backbone]->[pooling]->[dense]->[head]\n  global_branch_config:\n    # Currently, the following pooling methods are supported:\n    #   - GAP()\n    #   - GeM(p=3, train_p=False)\n    pooling_config:\n      method: GeM\n      kwargs:\n        p: 3.0\n        train_p: false\n\n    # Embedding after pooling (output units of a Dense layer\n    # without activation function because the backbone already\n    # have it, and without bias.\n    embedding_dim: 512\n\n    # Currently, the following heads are supported:\n    #   - ArcFace(s=30, m=0.5)\n    #   - ArcMarginProduct(s=30, m=0.5, easy_margin=False)\n    #   - AdaCos(m=0.5, is_dynamic=True)\n    #   - CosFace(s=30, m=0.35)\n    head_config:\n      layer: ArcFace\n      kwargs:\n        s: 45.45 # = sqrt(2) * log(C-1), where C=81313\n        m: 0.3 # Was 0.1 initially\n\n  # Shallow feature map to be fed to local branch\n  shallow_layer_name: conv5_block1_preact_relu\n\n  # Local features extractor configurations.\n  local_branch_config:\n    attention_config: {} # use default settings\n    autoencoder_config: {} # use default settings\n\n  # Places classifier\n  places_branch_config: null\n\n  # Supported training modes right now:\n  #   - global_only\n  #   - local_only\n  #   - local_and_global\n  training_mode: global_only\n  inference_mode: global_only\n\"\"\"\n\nglobal_model = DelgModel(**yaml.safe_load(global_cfg)['model_config'])\nglobal_model.build(input_shape=[\n  [1, 512, 512, 3],\n  [1, ]\n])\nglobal_model.load_weights(global_model_file)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Global and Local model"},{"metadata":{"trusted":true},"cell_type":"code","source":"loc_n_glob_cfg = \"\"\"\n# Model configurations\nmodel_config:\n\n  # Backbone CNN configuration\n  backbone_config:\n    architecture: ResNet152V2\n    weights: null\n    trainable: true\n\n  # Global branch: [backbone]->[pooling]->[dense]->[head]\n  global_branch_config:\n    # Currently, the following pooling methods are supported:\n    #   - GAP()\n    #   - GeM(p=3, train_p=False)\n    pooling_config:\n      method: GeM\n      kwargs:\n        p: 3.0\n        train_p: false\n\n    # Embedding after pooling (output units of a Dense layer\n    # without activation function because the backbone already\n    # have it, and without bias.\n    embedding_dim: 512\n\n    # Currently, the following heads are supported:\n    #   - ArcFace(s=30, m=0.5)\n    #   - ArcMarginProduct(s=30, m=0.5, easy_margin=False)\n    #   - AdaCos(m=0.5, is_dynamic=True)\n    #   - CosFace(s=30, m=0.35)\n    head_config:\n      layer: ArcFace\n      kwargs:\n        s: 45.45 # = sqrt(2) * log(C-1), where C=81313\n        m: 0.35 # Was 0.1 initially\n\n  # Shallow feature map to be fed to local branch\n  shallow_layer_name: conv5_block1_preact_relu\n\n  # Local features extractor configurations.\n  local_branch_config:\n    attention_config: {} # use default settings\n    autoencoder_config: {} # use default settings\n\n  # Places classifier\n  places_branch_config: null\n\n  # Supported training modes right now:\n  #   - global_only\n  #   - local_only\n  #   - local_and_global\n  training_mode: local_and_global\n  inference_mode: local_and_global\n\"\"\"\n\nloc_n_gloc_model = DelgModel(**yaml.safe_load(loc_n_glob_cfg)['model_config'])\nloc_n_gloc_model.build(input_shape=[\n  [1, 512, 512, 3],\n  [1, ]\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Surgery itself"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Random tensors for testing\ninput_image = tf.convert_to_tensor(np.random.rand(4, 512, 512, 3),\n                                  dtype=tf.float32)\ninput_label = tf.convert_to_tensor(np.random.randint(0, 100, 4),\n                                  dtype=tf.int32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# BACKBONE WEIGHTS\nloc_n_gloc_model.backbone.set_weights(global_model.backbone.get_weights())\n\n# Another representation of backbone, but not sure that we can leave it as-is,\n# so let's just load weights for it as well.\nloc_n_gloc_model.backbone_infer.set_weights(\n        global_model.backbone_infer.get_weights())\n\n# Check that all 3 backbones are equal\nbb_local_output = local_model.backbone(input_image)\nbb_global_output = global_model.backbone(input_image)\nbb_locngl_output = loc_n_gloc_model.backbone(input_image)\n\nprint('||local - global|| =',\n      np.mean(np.abs(bb_local_output.numpy() - bb_global_output.numpy())))\nprint('||local - locngloc|| =',\n      np.mean(np.abs(bb_local_output.numpy() - bb_locngl_output.numpy())))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# GLOBAL HEAD WEIGHTS\nloc_n_gloc_model.global_branch.set_weights(\n    global_model.global_branch.get_weights())\n\nglob_output = global_model([input_image, input_label]).numpy()\nloc_n_glob_output = loc_n_gloc_model([input_image, input_label])[0].numpy()\nprint('Output differences:', np.mean(np.abs(loc_n_glob_output - glob_output)))\n\nx = global_model.delg_inference(input_image).numpy()\ny = loc_n_gloc_model.delg_inference(input_image)[0].numpy()\nprint('Inference differences:', np.mean(np.abs(x - y)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Local head warmup\ndummy_output = loc_n_gloc_model([input_image, input_label])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# LOCAL HEAD WEIGHTS\n# we need to re-order local model's weights in order to match our loc_n_gloc\n\nlng_names = [w.name for w in loc_n_gloc_model.local_branch.weights]\nloc_wdict = dict([(w.name, w) for w in local_model.local_branch.weights])\nloc_wlist = [loc_wdict[name].numpy() for name in lng_names]\nloc_n_gloc_model.local_branch.set_weights(loc_wlist)\n\nloc_desc, loc_probs, loc_scores = local_model.delg_inference(input_image)\n_, lng_desc, lng_probs, lng_scores = loc_n_gloc_model.delg_inference(input_image)\n\nprint('desc differences:', np.mean(np.abs(loc_desc.numpy() - lng_desc.numpy())))\nprint('probs differences:', np.mean(np.abs(loc_probs.numpy() - lng_probs.numpy())))\nprint('scores differences:', np.mean(np.abs(loc_scores.numpy() - lng_scores.numpy())))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# NUM_CLASSES = 81313\n# \n# \n# class WrapperModel(tf.keras.models.Model):\n#     def __init__(self):\n#         super().__init__()\n#         self.delg_model = model\n#     \n#     # The model only have 1 goal: to output \n#     @tf.function(input_signature=[\n#         tf.TensorSpec(shape=[None, None, 3], dtype=tf.uint8, name='input_image')\n#     ])\n#     def single_scale_original(self, input_image):\n#         float_image = tf.cast(input_image, tf.float32)\n#         float_image = (float_image - 128.) / 128.\n#         height, width = tf.shape(float_image)[0], tf.shape(float_image)[1]\n#         float_image = tf.reshape(float_image, [1, height, width, 3])\n#         \n#     \n# wrapper_model = WrapperModel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# img = cv2.imread('../input/landmark-retrieval-2020/train/0/0/0/000054af45275b62.jpg')\n# plt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# emb = wrapped_model(tf.convert_to_tensor(np.random.randint(0, 256, size=(600, 600, 3), dtype=np.uint8)))\n# emb = wrapper_model.multi_scale_original(tf.convert_to_tensor(img))\n# emb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# np.sum(emb['global_descriptor'].numpy() ** 2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Form submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"# tf.saved_model.save(\n#     wrapper_model,\n#     export_dir='./submission',\n#     signatures={\n#         'single_scale_512x512': wrapper_model.single_scale_512x512,\n#         'single_scale_original': wrapper_model.single_scale_original,\n#         'multi_scale_512x512': wrapper_model.multi_scale_512x512,\n#         'multi_scale_original': wrapper_model.multi_scale_original,\n#         'classifier': wrapper_model.global_classifier,\n#     })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !saved_model_cli show --dir ./submission/ --all","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Testing model load"},{"metadata":{"trusted":true},"cell_type":"code","source":"# NUM_EMBEDDING_DIMENSIONS = 512\n# GLOBAL_MODEL_DIR = './submission'\n# GLOBAL_MODEL = tf.saved_model.load(GLOBAL_MODEL_DIR)\n# GLOBAL_FEATURE_EXTRACTION_FN = GLOBAL_MODEL.signatures['serving_default']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# emb = wrapper_model(tf.convert_to_tensor(img))\n# emb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# emb_loaded = GLOBAL_FEATURE_EXTRACTION_FN(tf.convert_to_tensor(img))['global_descriptor']\n# emb_loaded","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# object_detection -> shape_utils (for get_dim_as_int)\n\nIn `shape_utils.py`, `get_dim_as_int` is an alias for the same function from `static_shape.py`"},{"metadata":{"trusted":true},"cell_type":"code","source":"# just a static namespace\nclass shape_utils:\n    \n    @staticmethod\n    def get_dim_as_int(dim):\n      \"\"\"Utility to get v1 or v2 TensorShape dim as an int.\n    \n      Args:\n        dim: The TensorShape dimension to get as an int\n    \n      Returns:\n        None or an int.\n      \"\"\"\n      try:\n        return dim.value\n      except AttributeError:\n        return dim","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# object_detection.core -> box_utils\n\nThis is mainly for `class BoxList` definition."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n\n\"\"\"Bounding Box List definition.\n\nBoxList represents a list of bounding boxes as tensorflow\ntensors, where each bounding box is represented as a row of 4 numbers,\n[y_min, x_min, y_max, x_max].  It is assumed that all bounding boxes\nwithin a given list correspond to a single image.  See also\nbox_list_ops.py for common box related operations (such as area, iou, etc).\n\nOptionally, users can add additional related fields (such as weights).\nWe assume the following things to be true about fields:\n* they correspond to boxes in the box_list along the 0th dimension\n* they have inferrable rank at graph construction time\n* all dimensions except for possibly the 0th can be inferred\n  (i.e., not None) at graph construction time.\n\nSome other notes:\n  * Following tensorflow conventions, we use height, width ordering,\n  and correspondingly, y,x (or ymin, xmin, ymax, xmax) ordering\n  * Tensors are always provided as (flat) [N, 4] tensors.\n\"\"\"\n\nimport tensorflow.compat.v1 as tf\n\nclass box_list:\n    # static namespace\n\n    class BoxList(object):\n      \"\"\"Box collection.\"\"\"\n\n      def __init__(self, boxes):\n        \"\"\"Constructs box collection.\n\n        Args:\n          boxes: a tensor of shape [N, 4] representing box corners\n\n        Raises:\n          ValueError: if invalid dimensions for bbox data or if bbox data is not in\n              float32 format.\n        \"\"\"\n        if len(boxes.get_shape()) != 2 or boxes.get_shape()[-1] != 4:\n          raise ValueError('Invalid dimensions for box data: {}'.format(\n              boxes.shape))\n        if boxes.dtype != tf.float32:\n          raise ValueError('Invalid tensor type: should be tf.float32')\n        self.data = {'boxes': boxes}\n\n      def num_boxes(self):\n        \"\"\"Returns number of boxes held in collection.\n\n        Returns:\n          a tensor representing the number of boxes held in the collection.\n        \"\"\"\n        return tf.shape(self.data['boxes'])[0]\n\n      def num_boxes_static(self):\n        \"\"\"Returns number of boxes held in collection.\n\n        This number is inferred at graph construction time rather than run-time.\n\n        Returns:\n          Number of boxes held in collection (integer) or None if this is not\n            inferrable at graph construction time.\n        \"\"\"\n        return shape_utils.get_dim_as_int(self.data['boxes'].get_shape()[0])\n\n      def get_all_fields(self):\n        \"\"\"Returns all fields.\"\"\"\n        return self.data.keys()\n\n      def get_extra_fields(self):\n        \"\"\"Returns all non-box fields (i.e., everything not named 'boxes').\"\"\"\n        return [k for k in self.data.keys() if k != 'boxes']\n\n      def add_field(self, field, field_data):\n        \"\"\"Add field to box list.\n\n        This method can be used to add related box data such as\n        weights/labels, etc.\n\n        Args:\n          field: a string key to access the data via `get`\n          field_data: a tensor containing the data to store in the BoxList\n        \"\"\"\n        self.data[field] = field_data\n\n      def has_field(self, field):\n        return field in self.data\n\n      def get(self):\n        \"\"\"Convenience function for accessing box coordinates.\n\n        Returns:\n          a tensor with shape [N, 4] representing box coordinates.\n        \"\"\"\n        return self.get_field('boxes')\n\n      def set(self, boxes):\n        \"\"\"Convenience function for setting box coordinates.\n\n        Args:\n          boxes: a tensor of shape [N, 4] representing box corners\n\n        Raises:\n          ValueError: if invalid dimensions for bbox data\n        \"\"\"\n        if len(boxes.get_shape()) != 2 or boxes.get_shape()[-1] != 4:\n          raise ValueError('Invalid dimensions for box data.')\n        self.data['boxes'] = boxes\n\n      def get_field(self, field):\n        \"\"\"Accesses a box collection and associated fields.\n\n        This function returns specified field with object; if no field is specified,\n        it returns the box coordinates.\n\n        Args:\n          field: this optional string parameter can be used to specify\n            a related field to be accessed.\n\n        Returns:\n          a tensor representing the box collection or an associated field.\n\n        Raises:\n          ValueError: if invalid field\n        \"\"\"\n        if not self.has_field(field):\n          raise ValueError('field ' + str(field) + ' does not exist')\n        return self.data[field]\n\n      def set_field(self, field, value):\n        \"\"\"Sets the value of a field.\n\n        Updates the field of a box_list with a given value.\n\n        Args:\n          field: (string) name of the field to set value.\n          value: the value to assign to the field.\n\n        Raises:\n          ValueError: if the box_list does not have specified field.\n        \"\"\"\n        if not self.has_field(field):\n          raise ValueError('field %s does not exist' % field)\n        self.data[field] = value\n\n      def get_center_coordinates_and_sizes(self, scope=None):\n        \"\"\"Computes the center coordinates, height and width of the boxes.\n\n        Args:\n          scope: name scope of the function.\n\n        Returns:\n          a list of 4 1-D tensors [ycenter, xcenter, height, width].\n        \"\"\"\n        with tf.name_scope(scope, 'get_center_coordinates_and_sizes'):\n          box_corners = self.get()\n          ymin, xmin, ymax, xmax = tf.unstack(tf.transpose(box_corners))\n          width = xmax - xmin\n          height = ymax - ymin\n          ycenter = ymin + height / 2.\n          xcenter = xmin + width / 2.\n          return [ycenter, xcenter, height, width]\n\n      def transpose_coordinates(self, scope=None):\n        \"\"\"Transpose the coordinate representation in a boxlist.\n\n        Args:\n          scope: name scope of the function.\n        \"\"\"\n        with tf.name_scope(scope, 'transpose_coordinates'):\n          y_min, x_min, y_max, x_max = tf.split(\n              value=self.get(), num_or_size_splits=4, axis=1)\n          self.set(tf.concat([x_min, y_min, x_max, y_max], 1))\n\n      def as_tensor_dict(self, fields=None):\n        \"\"\"Retrieves specified fields as a dictionary of tensors.\n\n        Args:\n          fields: (optional) list of fields to return in the dictionary.\n            If None (default), all fields are returned.\n\n        Returns:\n          tensor_dict: A dictionary of tensors specified by fields.\n\n        Raises:\n          ValueError: if specified field is not contained in boxlist.\n        \"\"\"\n        tensor_dict = {}\n        if fields is None:\n          fields = self.get_all_fields()\n        for field in fields:\n          if not self.has_field(field):\n            raise ValueError('boxlist must contain all specified fields')\n          tensor_dict[field] = self.get_field(field)\n        return tensor_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# object_detection.core -> box_utils_ops\n\nThis is for `non_max_supression` method"},{"metadata":{"trusted":true},"cell_type":"code","source":"def gather(boxlist, indices, fields=None, scope=None, use_static_shapes=False):\n  \"\"\"Gather boxes from BoxList according to indices and return new BoxList.\n  By default, `gather` returns boxes corresponding to the input index list, as\n  well as all additional fields stored in the boxlist (indexing into the\n  first dimension).  However one can optionally only gather from a\n  subset of fields.\n  Args:\n    boxlist: BoxList holding N boxes\n    indices: a rank-1 tensor of type int32 / int64\n    fields: (optional) list of fields to also gather from.  If None (default),\n      all fields are gathered from.  Pass an empty fields list to only gather\n      the box coordinates.\n    scope: name scope.\n    use_static_shapes: Whether to use an implementation with static shape\n      gurantees.\n  Returns:\n    subboxlist: a BoxList corresponding to the subset of the input BoxList\n    specified by indices\n  Raises:\n    ValueError: if specified field is not contained in boxlist or if the\n      indices are not of type int32\n  \"\"\"\n  with tf.name_scope(scope, 'Gather'):\n    if len(indices.shape.as_list()) != 1:\n      raise ValueError('indices should have rank 1')\n    if indices.dtype != tf.int32 and indices.dtype != tf.int64:\n      raise ValueError('indices should be an int32 / int64 tensor')\n    gather_op = tf.gather\n    if use_static_shapes:\n      gather_op = ops.matmul_gather_on_zeroth_axis\n    subboxlist = box_list.BoxList(gather_op(boxlist.get(), indices))\n    if fields is None:\n      fields = boxlist.get_extra_fields()\n    fields += ['boxes']\n    for field in fields:\n      if not boxlist.has_field(field):\n        raise ValueError('boxlist must contain all specified fields')\n      subfieldlist = gather_op(boxlist.get_field(field), indices)\n      subboxlist.add_field(field, subfieldlist)\n    return subboxlist\n\n\nclass box_list_ops:\n    # dummy namespace\n    \n    @staticmethod\n    def non_max_suppression(boxlist, thresh, max_output_size, scope=None):\n      \"\"\"Non maximum suppression.\n      This op greedily selects a subset of detection bounding boxes, pruning\n      away boxes that have high IOU (intersection over union) overlap (> thresh)\n      with already selected boxes.  Note that this only works for a single class ---\n      to apply NMS to multi-class predictions, use MultiClassNonMaxSuppression.\n      Args:\n        boxlist: BoxList holding N boxes.  Must contain a 'scores' field\n          representing detection scores.\n        thresh: scalar threshold\n        max_output_size: maximum number of retained boxes\n        scope: name scope.\n      Returns:\n        a BoxList holding M boxes where M <= max_output_size\n      Raises:\n        ValueError: if thresh is not in [0, 1]\n      \"\"\"\n      with tf.name_scope(scope, 'NonMaxSuppression'):\n        if not 0 <= thresh <= 1.0:\n          raise ValueError('thresh must be between 0 and 1')\n        if not isinstance(boxlist, box_list.BoxList):\n          raise ValueError('boxlist must be a BoxList')\n        if not boxlist.has_field('scores'):\n          raise ValueError('input boxlist must have \\'scores\\' field')\n        selected_indices = tf.image.non_max_suppression(\n            boxlist.get(), boxlist.get_field('scores'),\n            max_output_size, iou_threshold=thresh)\n        return gather(boxlist, selected_indices)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# delf -> feature_extractor\n\nmainly for `CalculateReceptiveBoxes` method"},{"metadata":{"trusted":true},"cell_type":"code","source":"class feature_extractor:\n    # just a dummy namespace\n    \n    @staticmethod\n    def CalculateReceptiveBoxes(height, width, rf, stride, padding):\n      \"\"\"Calculate receptive boxes for each feature point.\n      Args:\n        height: The height of feature map.\n        width: The width of feature map.\n        rf: The receptive field size.\n        stride: The effective stride between two adjacent feature points.\n        padding: The effective padding size.\n      Returns:\n        rf_boxes: [N, 4] receptive boxes tensor. Here N equals to height x width.\n        Each box is represented by [ymin, xmin, ymax, xmax].\n      \"\"\"\n      x, y = tf.meshgrid(tf.range(width), tf.range(height))\n      coordinates = tf.reshape(tf.stack([y, x], axis=2), [-1, 2])\n      # [y,x,y,x]\n      point_boxes = tf.cast(\n          tf.concat([coordinates, coordinates], 1), dtype=tf.float32)\n      bias = [-padding, -padding, -padding + rf - 1, -padding + rf - 1]\n      rf_boxes = stride * point_boxes + bias\n      return rf_boxes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# delf.python.training.datasets -> gld"},{"metadata":{"trusted":true},"cell_type":"code","source":"class gld:\n    # just a dummy namespace\n    \n    @staticmethod\n    def NormalizeImages(images, pixel_value_scale=0.5, pixel_value_offset=0.5):\n      \"\"\"Normalize pixel values in image.\n      Output is computed as\n      normalized_images = (images - pixel_value_offset) / pixel_value_scale.\n      Args:\n        images: `Tensor`, images to normalize.\n        pixel_value_scale: float, scale.\n        pixel_value_offset: float, offset.\n      Returns:\n        normalized_images: `Tensor`, normalized images.\n      \"\"\"\n      images = tf.cast(images, tf.float32)\n      normalized_images = tf.math.divide(\n          tf.subtract(images, pixel_value_offset), pixel_value_scale)\n      return normalized_images","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# The main code for keypoints calculation"},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef ExtractLocalAndGlobalFeatures(image, image_scales, max_feature_num,\n                                  abs_thres, global_scales_ind, iou, model_fn,\n                                  stride_factor):\n  \"\"\"Extract local+global features for input image.\n  Args:\n    image: image tensor of type tf.uint8 with shape [h, w, channels].\n    image_scales: 1D float tensor which contains float scales used for image\n      pyramid construction.\n    max_feature_num: int tensor denoting the maximum selected feature points.\n    abs_thres: float tensor denoting the score threshold for feature selection.\n    global_scales_ind: Global feature extraction happens only for a subset of\n      `image_scales`, those with corresponding indices from this tensor.\n    iou: float scalar denoting the iou threshold for NMS.\n    model_fn: model function. Follows the signature:\n      * Args:\n        * `images`: Batched image tensor.\n      * Returns:\n        * `global_descriptors`: Global descriptors for input images.\n        * `attention_prob`: Attention map after the non-linearity.\n        * `feature_map`: Feature map after ResNet convolution.\n    stride_factor: integer accounting for striding after block3.\n  Returns:\n    boxes: [N, 4] float tensor which denotes the selected receptive boxes. N is\n      the number of final feature points which pass through keypoint selection\n      and NMS steps.\n    local_descriptors: [N, depth] float tensor.\n    feature_scales: [N] float tensor. It is the inverse of the input image\n      scales such that larger image scales correspond to larger image regions,\n      which is compatible with keypoints detected with other techniques, for\n      example Congas.\n    scores: [N, 1] float tensor denoting the attention score.\n    global_descriptors: [S, D] float tensor, with the global descriptors for\n      each scale; S is the number of scales, and D the global descriptor\n      dimensionality.\n  \"\"\"\n  original_image_shape_float = tf.gather(\n      tf.dtypes.cast(tf.shape(image), tf.float32), [0, 1])\n  image_tensor = gld.NormalizeImages(\n      image, pixel_value_offset=128.0, pixel_value_scale=128.0)\n  image_tensor = tf.expand_dims(image_tensor, 0, name='image/expand_dims')\n\n  # Hard code the receptive field parameters for now.\n  # We need to revisit this once we change the architecture and selected\n  # convolutional blocks to use as local features.\n  rf, stride, padding = [291.0, 32.0 * stride_factor, 145.0]\n\n  def _ResizeAndExtract(scale_index):\n    \"\"\"Helper function to resize image then extract features.\n    Args:\n      scale_index: A valid index in image_scales.\n    Returns:\n      global_descriptor: [1,D] tensor denoting the extracted global descriptor.\n      boxes: Box tensor with the shape of [K, 4].\n      local_descriptors: Local descriptor tensor with the shape of [K, depth].\n      scales: Scale tensor with the shape of [K].\n      scores: Score tensor with the shape of [K].\n    \"\"\"\n    scale = tf.gather(image_scales, scale_index)\n    new_image_size = tf.dtypes.cast(\n        tf.round(original_image_shape_float * scale), tf.int32)\n    resized_image = tf.image.resize(image_tensor, new_image_size)\n    global_descriptor, attention_prob, feature_map = model_fn(resized_image)\n\n    attention_prob = tf.squeeze(attention_prob, axis=[0])\n    feature_map = tf.squeeze(feature_map, axis=[0])\n\n    # Compute RF boxes and re-project them to the original image space.\n    rf_boxes = feature_extractor.CalculateReceptiveBoxes(\n        tf.shape(feature_map)[0],\n        tf.shape(feature_map)[1], rf, stride, padding)\n    rf_boxes = tf.divide(rf_boxes, scale)\n\n    attention_prob = tf.reshape(attention_prob, [-1])\n    feature_map = tf.reshape(feature_map, [-1, tf.shape(feature_map)[2]])\n\n    # Use attention score to select local features.\n    indices = tf.reshape(tf.where(attention_prob >= abs_thres), [-1])\n    boxes = tf.gather(rf_boxes, indices)\n    local_descriptors = tf.gather(feature_map, indices)\n    scores = tf.gather(attention_prob, indices)\n    scales = tf.ones_like(scores, tf.float32) / scale\n\n    return global_descriptor, boxes, local_descriptors, scales, scores\n\n  # TODO(andrearaujo): Currently, a global feature is extracted even for scales\n  # which are not using it. The obtained result is correct, however feature\n  # extraction is slower than expected. We should try to fix this in the future.\n\n  # Run first scale.\n  (output_global_descriptors, output_boxes, output_local_descriptors,\n   output_scales, output_scores) = _ResizeAndExtract(0)\n  if not tf.reduce_any(tf.equal(global_scales_ind, 0)):\n    # If global descriptor is not using the first scale, clear it out.\n    output_global_descriptors = tf.zeros(\n        [0, tf.shape(output_global_descriptors)[1]])\n\n  # Loop over subsequent scales.\n  num_scales = tf.shape(image_scales)[0]\n  for scale_index in tf.range(1, num_scales):\n    # Allow an undefined number of global feature scales to be extracted.\n    tf.autograph.experimental.set_loop_options(\n        shape_invariants=[(output_global_descriptors,\n                           tf.TensorShape([None, None]))])\n\n    (global_descriptor, boxes, local_descriptors, scales,\n     scores) = _ResizeAndExtract(scale_index)\n    output_boxes = tf.concat([output_boxes, boxes], 0)\n    output_local_descriptors = tf.concat(\n        [output_local_descriptors, local_descriptors], 0)\n    output_scales = tf.concat([output_scales, scales], 0)\n    output_scores = tf.concat([output_scores, scores], 0)\n    if tf.reduce_any(tf.equal(global_scales_ind, scale_index)):\n      output_global_descriptors = tf.concat(\n          [output_global_descriptors, global_descriptor], 0)\n\n  feature_boxes = box_list.BoxList(output_boxes)\n  feature_boxes.add_field('local_descriptors', output_local_descriptors)\n  feature_boxes.add_field('scales', output_scales)\n  feature_boxes.add_field('scores', output_scores)\n\n  nms_max_boxes = tf.minimum(max_feature_num, feature_boxes.num_boxes())\n  final_boxes = box_list_ops.non_max_suppression(feature_boxes, iou,\n                                                 nms_max_boxes)\n\n  return (final_boxes.get(), final_boxes.get_field('local_descriptors'),\n          final_boxes.get_field('scales'),\n          tf.expand_dims(final_boxes.get_field('scores'),\n                         1), output_global_descriptors)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class WrapperModel(tf.keras.models.Model):\n    def __init__(self):\n        super().__init__()\n        self.delg_model = loc_n_gloc_model\n        self._stride_factor = 1.\n        self._iou = 1.0\n    \n    @tf.function\n    def model_fn(self, image_tensor):\n        glob_desc, loc_desc, loc_probs, loc_scores = self.delg_model.delg_inference(image_tensor)\n        return glob_desc, loc_probs, loc_desc\n    \n    @tf.function(input_signature=[\n      tf.TensorSpec(shape=[None, None, 3], dtype=tf.uint8, name='input_image'),\n      tf.TensorSpec(shape=[None], dtype=tf.float32, name='input_scales'),\n      tf.TensorSpec(shape=(), dtype=tf.int32, name='input_max_feature_num'),\n      tf.TensorSpec(shape=(), dtype=tf.float32, name='input_abs_thres'),\n      tf.TensorSpec(\n          shape=[None], dtype=tf.int32, name='input_global_scales_ind')\n      ])\n    def ExtractFeatures(self, input_image, input_scales, input_max_feature_num,\n                        input_abs_thres, input_global_scales_ind):\n      extracted_features = ExtractLocalAndGlobalFeatures(\n          input_image, input_scales, input_max_feature_num, input_abs_thres,\n          input_global_scales_ind, self._iou, self.model_fn,\n          self._stride_factor)\n  \n      named_output_tensors = {}\n      named_output_tensors['boxes'] = tf.identity(\n          extracted_features[0], name='boxes')\n      named_output_tensors['features'] = tf.identity(\n          extracted_features[1], name='features')\n      named_output_tensors['scales'] = tf.identity(\n          extracted_features[2], name='scales')\n      named_output_tensors['scores'] = tf.identity(\n          extracted_features[3], name='scores')\n      named_output_tensors['global_descriptors'] = tf.identity(\n          extracted_features[4], name='global_descriptors')\n      return named_output_tensors\n    \nwrapper_model = WrapperModel()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test out the new shiny toy!"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\n\ntrain_df = pd.read_csv('../input/landmark-recognition-2020/train.csv')\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_indices = np.arange(len(train_df))\nimage_ids = np.random.choice(all_indices, 5)\nimage_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import PIL\n\n\nINPUT_DIR = os.path.join('..', 'input')\nDATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')\n\n\ndef get_image_path(subset, image_id):\n  # name = to_hex(image_id)\n  name = image_id\n  return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],\n                      '{}.jpg'.format(name))\n\n\ndef load_image_tensor(image_path, scale_factor=1.0):\n  img = np.array(PIL.Image.open(image_path).convert('RGB'))\n  if abs(scale_factor - 1.0) > 1e-5:\n    img = cv2.resize(img, (0, 0), fx=scale_factor, fy=scale_factor)\n  return tf.convert_to_tensor(img)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize Raw model and DELG model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n\nrows = 3\nplt.figure(figsize=(15, rows * 4))\n\n\nDELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])\nDELG_SCORE_THRESHOLD_TENSOR = tf.constant(0.0)\nLOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)\nGLOBAL_SCALES_IND = tf.convert_to_tensor([0, 1, 2], dtype=tf.int32)\n\n\nall_indices = np.arange(len(train_df))\nindices = np.random.choice(all_indices, rows)\n\nfor i, idx in enumerate(indices):\n    img_ts = load_image_tensor(get_image_path('train', train_df['id'][idx]))\n    \n    # RAW MODEL INFERENCE\n    float_image = tf.cast(img_ts, tf.float32)\n    float_image = (float_image - 128.) / 128.\n    height, width = tf.shape(float_image)[0], tf.shape(float_image)[1]\n    float_image = tf.reshape(float_image, [1, height, width, 3])\n\n    glob_desc, loc_probs, loc_desc = wrapper_model.model_fn(float_image)\n    glob_desc, loc_probs, loc_desc = glob_desc.numpy(), loc_probs.numpy(), loc_desc.numpy()\n    \n    plt.subplot(rows, 3, i*3 + 1)\n    plt.imshow(img_ts.numpy())\n    plt.axis('off')\n\n    plt.subplot(rows, 3, i*3 + 2)\n    heatmap = np.squeeze(loc_probs) / loc_probs.max()\n    plt.imshow(heatmap)\n    plt.axis('off')\n    \n    # DELG MODEL PIPELINE INFERENCE\n    named_output_tensors = wrapper_model.ExtractFeatures(\n        img_ts,\n        DELG_IMAGE_SCALES_TENSOR,\n        LOCAL_FEATURE_NUM_TENSOR,\n        DELG_SCORE_THRESHOLD_TENSOR,\n        GLOBAL_SCALES_IND\n    )\n    \n    boxes = named_output_tensors['boxes'].numpy()\n    boxes_cy = ((boxes[:, 2] + boxes[:, 0]) / 2).astype(np.int32)\n    boxes_cx = ((boxes[:, 3] + boxes[:, 1]) / 2).astype(np.int32)\n    scores = named_output_tensors['scores'].numpy()\n\n    image_vis = img_ts.numpy()\n\n    num_to_vis = 20\n    for box_idx in range(num_to_vis - 1, -1, -1):\n        cv2.circle(\n            image_vis,\n            (boxes_cx[box_idx], boxes_cy[box_idx]),\n            10 + int(40 * (scores[box_idx] / np.max(scores))),\n            (\n                int(255 * (1. - scores[box_idx] / np.max(scores))),\n                int(255 * (scores[box_idx] / np.max(scores))),\n                0\n            ),\n            5 + int(10 * (scores[box_idx] / np.max(scores)))\n        )\n\n    plt.subplot(rows, 3, i*3 + 3)\n    plt.axis('off')\n    plt.imshow(image_vis)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SAVE THIS AWESOME MODEL"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}