{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/glrec2020/Keras_Applications-1.0.8-py3-none-any.whl\n!pip install ../input/glrec2020/efficientnet-1.1.0-py3-none-any.whl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import operator\nimport gc\nimport pathlib\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom scipy import spatial\nimport cv2\nimport efficientnet.tfkeras as efn\nimport math\nimport copy\nimport csv\nimport os\nimport numpy as np\nimport PIL\nimport pydegensac\nfrom sklearn.preprocessing import normalize","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUMBER_OF_CLASSES = 81313\nIMAGE_SIZE = [512, 512]\nLR = 0.0001","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GeM(tf.keras.layers.Layer):\n    def __init__(self, pool_size, init_norm=3.0, normalize=False, **kwargs):\n        self.pool_size = pool_size\n        self.init_norm = init_norm\n        self.normalize = normalize\n\n        super(GeM, self).__init__(**kwargs)\n\n    def get_config(self):\n        config = super().get_config().copy()\n        config.update({\n            'pool_size': self.pool_size,\n            'init_norm': self.init_norm,\n            'normalize': self.normalize,\n        })\n        return config\n\n    def build(self, input_shape):\n        feature_size = input_shape[-1]\n        self.p = self.add_weight(name='norms', shape=(feature_size,),\n                                 initializer=tf.keras.initializers.constant(self.init_norm),\n                                 trainable=True)\n        super(GeM, self).build(input_shape)\n\n    def call(self, inputs):\n        x = inputs\n        x = tf.math.maximum(x, 1e-6)\n        x = tf.pow(x, self.p)\n\n        x = tf.nn.avg_pool(x, self.pool_size, self.pool_size, 'VALID')\n        x = tf.pow(x, 1.0 / self.p)\n\n        if self.normalize:\n            x = tf.nn.l2_normalize(x, 1)\n        return x\n\n    def compute_output_shape(self, input_shape):\n        return tuple([None, input_shape[-1]])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ArcMarginProduct(tf.keras.layers.Layer):\n    '''\n    Implements large margin arc distance.\n\n    Reference:\n        https://arxiv.org/pdf/1801.07698.pdf\n        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n            blob/master/src/modeling/metric_learning.py\n    '''\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n\n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n\n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n        one_hot = tf.cast(\n            tf.one_hot(y, depth=self.n_classes),\n            dtype=cosine.dtype\n        )\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output\n\n\ndef get_model_B7():\n\n    margin = ArcMarginProduct(\n        n_classes = NUMBER_OF_CLASSES, \n        s = 64, \n        m = 0.15, \n        name='head/arc_margin', \n        dtype='float32'\n        )\n\n    inp = tf.keras.layers.Input(shape = (512, 512, 3), name = 'inp1')\n    label = tf.keras.layers.Input(shape = (), name = 'inp2')\n    x4 = efn.EfficientNetB7(weights = None, include_top = False)(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x4)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    x = tf.keras.layers.Dense(512)(x)\n    x = margin([x, label])\n\n    output = tf.keras.layers.Softmax(dtype='float32')(x)\n\n    model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])\n\n    opt = tf.keras.optimizers.Adam(learning_rate = LR)\n\n    model.compile(\n        optimizer = opt,\n        loss = [tf.keras.losses.SparseCategoricalCrossentropy()],\n        metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]\n        ) \n\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(size=256, efficientnet_size=0, weights=\"imagenet\", count=0):\n    inp = tf.keras.layers.Input(shape=(size, size, 3), name=\"inp1\")\n    label = tf.keras.layers.Input(shape=(), name=\"inp2\")\n    x = getattr(efn, f\"EfficientNetB{efficientnet_size}\")(\n        weights=weights, include_top=False, input_shape=(size, size, 3))(inp)\n    x = GeM(16)(x)\n    x = tf.keras.layers.Flatten()(x)\n    x = tf.keras.layers.Dense(512, name=\"dense_before_arcface\", kernel_initializer=\"he_normal\")(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = ArcMarginProduct(\n        n_classes=NUMBER_OF_CLASSES,\n        s=30,\n        m=0.5,\n        name=\"head/arc_margin\",\n        dtype=\"float32\"\n    )([x, label])\n    output = tf.keras.layers.Softmax(dtype=\"float32\")(x)\n    model = tf.keras.Model(inputs=[inp, label], outputs=[output])\n    #lr_decayed_fn = tf.keras.experimental.CosineDecay(1e-3, count)\n    opt = tf.optimizers.Adam(learning_rate=1e-4)\n    model.compile(\n        optimizer=opt,\n        loss=[tf.keras.losses.SparseCategoricalCrossentropy()],\n        metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]\n    )\n    return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model2(size=256, efficientnet_size=0, weights=\"noisy-student\", count=0):\n    inp = tf.keras.layers.Input(shape=(size, size, 3), name=\"inp1\")\n    label = tf.keras.layers.Input(shape=(), name=\"inp2\")\n    x = getattr(efn, f\"EfficientNetB{efficientnet_size}\")(\n        weights=weights, include_top=False, input_shape=(size, size, 3))(inp)\n    #x = GeM(16)(x)\n    #x = tf.keras.layers.Flatten()(x)\n    x = tf.keras.layers.GlobalAvgPool2D()(x)\n    x = tf.keras.layers.Dense(512, name=\"dense_before_arcface\", kernel_initializer=\"he_normal\")(x)\n    #x = tf.keras.layers.BatchNormalization()(x)\n    #x = tf.keras.layers.ReLU()(x)\n    x = ArcMarginProduct(\n        n_classes=NUMBER_OF_CLASSES,\n        s=30,\n        m=0.3,\n        name=\"head/arc_margin\",\n        dtype=\"float32\"\n    )([x, label])\n    output = tf.keras.layers.Softmax(dtype=\"float32\")(x)\n    model = tf.keras.Model(inputs=[inp, label], outputs=[output])\n    opt = tf.optimizers.Adam(learning_rate=1e-4)\n    model.compile(\n        optimizer=opt,\n        loss=[tf.keras.losses.SparseCategoricalCrossentropy()],\n        metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]\n    )\n    return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model_for_inference(weights_path: str, efficientnet_size=7):\n\n        base_model = build_model(\n            size=IMAGE_SIZE[0],\n            efficientnet_size=efficientnet_size,\n            weights=None,\n            count=0)\n        base_model.load_weights(weights_path)\n        model = tf.keras.Model(inputs=base_model.get_layer(\"inp1\").input,\n                               outputs=base_model.get_layer(\"dense_before_arcface\").output)\n        return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model_for_inference2(weights_path: str, efficientnet_size=7):\n\n        base_model = build_model2(\n            size=IMAGE_SIZE[0],\n            efficientnet_size=efficientnet_size,\n            weights=None,\n            count=0)\n        base_model.load_weights(weights_path)\n        model = tf.keras.Model(inputs=base_model.get_layer(\"inp1\").input,\n                               outputs=base_model.get_layer(\"dense_before_arcface\").output)\n        return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nBATCH_SIZE = 8","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL1 = create_model_for_inference(f\"../input/glret21-efficientnetb7-training-f1-gem-m3/fold1.h5\", efficientnet_size=7)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL2 = get_model_B7()\nMODEL2.load_weights('../input/effb7-512-ep12/effb7model512-12.h5')\nMODEL2 = tf.keras.models.Model(inputs = MODEL2.input[0], outputs = MODEL2.layers[-4].output)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL3 = create_model_for_inference(f\"../input/glret21-efficientnetb6-training-f3-gem-m3/fold3.h5\", efficientnet_size=6)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#MODEL4 = create_model_for_inference2(f\"../input/glret21-efficientnetb7-training-f0-m3/fold0.h5\", efficientnet_size=7)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_TO_RERANK = 3 #originally 5\nNUM_PUBLIC_TEST_IMAGES = 10345 # Used to detect if in session or re-run.","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to decode our images\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels = 3)\n    image = tf.image.resize(image, IMAGE_SIZE)\n    image = tf.cast(image, tf.float32) /255\n    return image\n\n# Function to read our test image and return image\ndef read_image(image):\n    image = tf.io.read_file(image)\n    image = decode_image(image)\n    return image\n\n# Function to get our dataset that read images\ndef get_dataset(image):\n    dataset = tf.data.Dataset.from_tensor_slices(image)\n    dataset = dataset.map(read_image, num_parallel_calls = AUTO)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_image_embeddings(filepaths, model, return_ids=True):\n    image_paths = [x for x in pathlib.Path(filepaths).rglob('*.jpg')]\n    df = pd.DataFrame(image_paths)\n    df.columns = ['path']\n    df['path'] = df['path'].map(str)\n    df['ids'] = df['path'].map(lambda x:str(x).split('/')[-1].split('.')[0])\n    \n    embeds = []\n    chunk = 5000\n    iterator = np.arange(np.ceil(len(image_paths) / chunk))\n    for j in iterator:\n        a = int(j * chunk)\n        b = int((j + 1) * chunk)\n        image_dataset = get_dataset(df['path'].iloc[a:b])\n        image_embeddings = model.predict(image_dataset)\n        embeds.append(image_embeddings)\n    #del model\n    image_embeddings = np.concatenate(embeds)\n    image_embeddings = normalize(image_embeddings, axis=1)\n    print(f'Our image embeddings shape is {image_embeddings.shape}')\n    del embeds\n    gc.collect()\n    if return_ids:\n        return list(df['ids']), image_embeddings\n    else:\n        return image_embeddings","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR = os.path.join('..', 'input')\n\nDATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2021')\nTEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')\nTRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')\nTRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')\n\nNUM_PUBLIC_TRAIN_IMAGES = 1580470 # Used to detect if in session or re-run.\nMAX_NUM_EMBEDDINGS = -1  # Set to > 1 to subsample dataset while debugging.\n\n# Retrieval & re-ranking parameters:\nNUM_TO_RERANK = 2\nTOP_K = 2 #Number of retrieved images used to make prediction for a test image.\n\n# RANSAC parameters:\nMAX_INLIER_SCORE = 25\nMAX_REPROJECTION_ERROR = 7.0\nMAX_RANSAC_ITERATIONS = 20_000\nHOMOGRAPHY_CONFIDENCE = 0.99","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAVED_MODEL_DIR = '../input/notebook369f5200fc/delf_gld_20190411/model'\nDELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)\nDELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])\nDELG_SCORE_THRESHOLD_TENSOR = tf.constant(200.)\nDELG_INPUT_TENSOR_NAMES = [\n    'input_image:0', 'input_scales:0', 'input_abs_thres:0'\n]\nNUM_EMBEDDING_DIMENSIONS = 1024\n\nLOCAL_FEATURE_NUM_TENSOR = tf.constant(1300)\nLOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(\n    DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'], ['boxes:0', 'features:0'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_hex(image_id) -> str:\n  return '{0:0{1}x}'.format(image_id, 16)\n\n\ndef get_image_path(subset, image_id):\n  name = image_id\n  return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],\n                      '{}.jpg'.format(name))\n\n\ndef load_image_tensor(image_path):\n  return tf.convert_to_tensor(\n      np.array(PIL.Image.open(image_path).convert('RGB')))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_local_features(image_path):\n  \"\"\"Extracts local features for the given `image_path`.\"\"\"\n\n  image_tensor = load_image_tensor(image_path)\n\n  features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,\n                                         DELG_SCORE_THRESHOLD_TENSOR,\n                                         LOCAL_FEATURE_NUM_TENSOR)\n\n  # Shape: (N, 2)\n  keypoints = tf.divide(\n      tf.add(\n          tf.gather(features[0], [0, 1], axis=1),\n          tf.gather(features[0], [2, 3], axis=1)), 2.0).numpy()\n\n  # Shape: (N, 128)\n  descriptors = tf.nn.l2_normalize(\n      features[1], axis=1, name='l2_normalization').numpy()\n\n  return keypoints, descriptors\n\n\ndef get_putative_matching_keypoints(test_keypoints,\n                                    test_descriptors,\n                                    train_keypoints,\n                                    train_descriptors,\n                                    max_distance=0.9):\n  \"\"\"Finds matches from `test_descriptors` to KD-tree of `train_descriptors`.\"\"\"\n\n  train_descriptor_tree = spatial.cKDTree(train_descriptors)\n  _, matches = train_descriptor_tree.query(\n      test_descriptors, distance_upper_bound=max_distance)\n\n  test_kp_count = test_keypoints.shape[0]\n  train_kp_count = train_keypoints.shape[0]\n\n  test_matching_keypoints = np.array([\n      test_keypoints[i,]\n      for i in range(test_kp_count)\n      if matches[i] != train_kp_count\n  ])\n  train_matching_keypoints = np.array([\n      train_keypoints[matches[i],]\n      for i in range(test_kp_count)\n      if matches[i] != train_kp_count\n  ])\n\n  return test_matching_keypoints, train_matching_keypoints\n\n\ndef get_num_inliers(test_keypoints, test_descriptors, train_keypoints,\n                    train_descriptors):\n  \"\"\"Returns the number of RANSAC inliers.\"\"\"\n\n  test_match_kp, train_match_kp = get_putative_matching_keypoints(\n      test_keypoints, test_descriptors, train_keypoints, train_descriptors)\n\n  if test_match_kp.shape[\n      0] <= 4:  # Min keypoints supported by `pydegensac.findHomography()`\n    return 0\n\n  try:\n    _, mask = pydegensac.findHomography(test_match_kp, train_match_kp,\n                                        MAX_REPROJECTION_ERROR,\n                                        HOMOGRAPHY_CONFIDENCE,\n                                        MAX_RANSAC_ITERATIONS)\n  except np.linalg.LinAlgError:  # When det(H)=0, can't invert matrix.\n    return 0\n\n  return int(copy.deepcopy(mask).astype(np.float32).sum())\n\n\ndef get_total_score(num_inliers, global_score):\n  local_score = min(num_inliers, MAX_INLIER_SCORE) / MAX_INLIER_SCORE\n  return local_score + global_score\n\n\ndef rescore_and_rerank_by_num_inliers(test_image_id,\n                                      train_ids_labels_and_scores):\n  \"\"\"Returns rescored and sorted training images by local feature extraction.\"\"\"\n\n  test_image_path = get_image_path('test', test_image_id)\n  test_keypoints, test_descriptors = extract_local_features(test_image_path)\n\n  for i in range(len(train_ids_labels_and_scores)):\n    train_image_id, label, global_score = train_ids_labels_and_scores[i]\n\n    train_image_path = get_image_path('train', train_image_id)\n    train_keypoints, train_descriptors = extract_local_features(\n        train_image_path)\n\n    num_inliers = get_num_inliers(test_keypoints, test_descriptors,\n                                  train_keypoints, train_descriptors)\n    total_score = get_total_score(num_inliers, global_score)\n    train_ids_labels_and_scores[i] = (train_image_id, label, total_score)\n\n  train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)\n\n  return train_ids_labels_and_scores\n\n\ndef load_labelmap():\n  with open(TRAIN_LABELMAP_PATH, mode='r') as csv_file:\n    csv_reader = csv.DictReader(csv_file)\n    labelmap = {row['id']: row['landmark_id'] for row in csv_reader}\n\n  return labelmap\n\n\ndef get_prediction_map(test_ids, train_ids_labels_and_scores):\n  \"\"\"Makes dict from test ids and ranked training ids, labels, scores.\"\"\"\n\n  prediction_map = dict()\n\n  for test_index, test_id in enumerate(test_ids):\n    hex_test_id = test_id\n\n    aggregate_scores = {}\n    for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:\n      if label not in aggregate_scores:\n        aggregate_scores[label] = 0\n      aggregate_scores[label] += score\n\n    label, score = max(aggregate_scores.items(), key=operator.itemgetter(1))\n\n    prediction_map[hex_test_id] = {'score': score, 'class': label}\n\n  return prediction_map\n\n\ndef get_predictions(labelmap):\n  \"\"\"Gets predictions using embedding similarity and local feature reranking.\"\"\"\n  test_ids, test_embeddings = get_image_embeddings(TEST_IMAGE_DIR, MODEL1, return_ids=True)\n  train_ids, train_embeddings = get_image_embeddings(TRAIN_IMAGE_DIR, MODEL1, return_ids=True)\n\n  test_embeddings2 = get_image_embeddings(TEST_IMAGE_DIR, MODEL2, return_ids=False)\n  train_embeddings2 = get_image_embeddings(TRAIN_IMAGE_DIR, MODEL2, return_ids=False)\n\n  test_embeddings3 = get_image_embeddings(TEST_IMAGE_DIR, MODEL3, return_ids=False)\n  train_embeddings3 = get_image_embeddings(TRAIN_IMAGE_DIR, MODEL3, return_ids=False)\n  #test_embeddings4 = get_image_embeddings(TEST_IMAGE_DIR, MODEL4, return_ids=False)\n  #train_embeddings4 = get_image_embeddings(TRAIN_IMAGE_DIR, MODEL4, return_ids=False)\n  test_embeddings = np.concatenate([test_embeddings, test_embeddings2, test_embeddings3], axis=1)\n  train_embeddings = np.concatenate([train_embeddings, train_embeddings2, train_embeddings3], axis=1)\n  del test_embeddings2,train_embeddings2,test_embeddings3,train_embeddings3\n  gc.collect()\n  train_ids_labels_and_scores = [None] * test_embeddings.shape[0]\n\n  # Using (slow) for-loop, as distance matrix doesn't fit in memory.\n  for test_index in range(test_embeddings.shape[0]):\n    distances = spatial.distance.cdist(\n        test_embeddings[np.newaxis, test_index, :], train_embeddings,\n        'cosine')[0]\n    partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]\n\n    nearest = sorted([(train_ids[p], distances[p]) for p in partition],\n                     key=lambda x: x[1])\n\n    train_ids_labels_and_scores[test_index] = [\n        (train_id, labelmap[train_id], 1. - cosine_distance)\n        for train_id, cosine_distance in nearest\n    ]\n\n  del test_embeddings\n  del train_embeddings\n  del labelmap\n  gc.collect()\n\n  pre_verification_predictions = get_prediction_map(\n      test_ids, train_ids_labels_and_scores)\n\n#  return None, pre_verification_predictions\n\n  for test_index, test_id in enumerate(test_ids):\n    train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(\n        test_id, train_ids_labels_and_scores[test_index])\n\n  post_verification_predictions = get_prediction_map(\n      test_ids, train_ids_labels_and_scores)\n\n  return pre_verification_predictions, post_verification_predictions\n\n\ndef save_submission_csv(predictions=None):\n\n  if predictions is None:\n    # Dummy submission!\n    shutil.copyfile(\n        os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')\n    return\n\n  with open('submission.csv', 'w') as submission_csv:\n    csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])\n    csv_writer.writeheader()\n    for image_id, prediction in predictions.items():\n      label = prediction['class']\n      score = prediction['score']\n      csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})\n\n\ndef main():\n  labelmap = load_labelmap()\n  num_training_images = len(labelmap.keys())\n  print(f'Found {num_training_images} training images.')\n\n  if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:\n    print('Copying sample submission.')\n    save_submission_csv()\n    return\n\n  _, post_verification_predictions = get_predictions(labelmap)\n  save_submission_csv(post_verification_predictions)\n\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n  main()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}