{"cells":[{"metadata":{"execution":{"iopub.execute_input":"2020-09-26T03:48:57.42971Z","iopub.status.busy":"2020-09-26T03:48:57.428912Z","iopub.status.idle":"2020-09-26T03:49:03.285733Z","shell.execute_reply":"2020-09-26T03:49:03.285184Z"},"papermill":{"duration":5.880613,"end_time":"2020-09-26T03:49:03.285853","exception":false,"start_time":"2020-09-26T03:48:57.40524","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"\"\"\"\nBased on Baseline kernel for \"Google Landmarks Recognition Challenge 2020\".\n\nFirst, ranks all training images by embedding similarity to each test image.\nThen, performs geometric-verification and re-ranking on the `NUM_TO_RERANK`\nmost similar training images. For a given test image, each class' score is\nthe sum of the scores of re-ranked training images, and the predicted\nclass is the one with the highest aggregate score.\n\nTODO:\n* Resize image if needed\n* kNN by cuML\n* Faster RANSAC (better with GPU)\n\"\"\"\n\nimport copy\nimport csv\nimport gc\nimport operator\nimport os\nimport pathlib\nimport shutil\nimport time\n\nimport numpy as np\nimport pandas as pd\nimport PIL\nimport pydegensac\nfrom scipy import spatial\nimport random\n\n# import multiprocessing as mp\n# from multiprocessing import set_start_method\n# set_start_method(\"spawn\")\n# from multiprocessing import get_context\n\nimport gc\ngc.enable()\n\n# os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"  # see issue #152\n# # os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n\nimport tensorflow as tf\ntf.compat.v1.enable_resource_variables()\ntf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\n\nprint(f\"Tensorflow Version: {tf.__version__}\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-26T03:49:03.322162Z","iopub.status.busy":"2020-09-26T03:49:03.321402Z","iopub.status.idle":"2020-09-26T03:49:03.325143Z","shell.execute_reply":"2020-09-26T03:49:03.324614Z"},"papermill":{"duration":0.025054,"end_time":"2020-09-26T03:49:03.325235","exception":false,"start_time":"2020-09-26T03:49:03.300181","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def set_seed(seed: int = 42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    tf.random.set_seed(seed)\n\nrand_seed = 1120\nset_seed(rand_seed)\n\n# debug_mode = True\ndebug_mode = False\n\n# DEBUGGING PARAMS:\nNUM_PUBLIC_TRAIN_IMAGES = 1580470  # Used to detect if in session or re-run.\nif debug_mode:\n    MAX_NUM_EMBEDDINGS = 10\n    NUM_PUBLIC_TRAIN_IMAGES = 10\n    # MAX_NUM_EMBEDDINGS = 100\n    # NUM_PUBLIC_TRAIN_IMAGES = 100\nelse:\n    MAX_NUM_EMBEDDINGS = -1  # Set to > 1 to subsample dataset while debugging.","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-26T03:49:03.363152Z","iopub.status.busy":"2020-09-26T03:49:03.361235Z","iopub.status.idle":"2020-09-26T03:49:03.3638Z","shell.execute_reply":"2020-09-26T03:49:03.364312Z"},"papermill":{"duration":0.025084,"end_time":"2020-09-26T03:49:03.364426","exception":false,"start_time":"2020-09-26T03:49:03.339342","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Dataset parameters:\nINPUT_DIR = os.path.join('..', 'input')\nDATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')\nTEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')\nTRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')\nTRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')\n\n# INPUT_DIR = \"/workspace/Kaggle/Landmark\"\n# DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')\n# TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')\n# TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')\n# TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')\n\n# Retrieval & re-ranking parameters:\n# NUM_TO_RERANK = 4\n# TOP_K = 4  # Number of retrieved images used to make prediction for a test image (<=NUM_TO_RERANK)\n# NUM_TO_RERANK = 2\n# TOP_K = 2  # Number of retrieved images used to make prediction for a test image (<=NUM_TO_RERANK)\nNUM_TO_RERANK = 10\nTOP_K = 10  # Number of retrieved images used to make prediction for a test image (<=NUM_TO_RERANK)\n\n\n# RANSAC parameters:\nMAX_INLIER_SCORE = 70\nMAX_REPROJECTION_ERROR = 4.0\nMAX_RANSAC_ITERATIONS = 10000\nHOMOGRAPHY_CONFIDENCE = 0.995\n\n# MAX_INLIER_SCORE = 20\n# MAX_REPROJECTION_ERROR = 4.0\n# MAX_RANSAC_ITERATIONS = 60000\n# # MAX_RANSAC_ITERATIONS = 120000\n# HOMOGRAPHY_CONFIDENCE = 0.99","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.01451,"end_time":"2020-09-26T03:49:03.393249","exception":false,"start_time":"2020-09-26T03:49:03.378739","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Pretrained Model Config"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-26T03:49:05.912164Z","iopub.status.busy":"2020-09-26T03:49:04.240761Z","iopub.status.idle":"2020-09-26T03:49:06.208761Z","shell.execute_reply":"2020-09-26T03:49:06.20769Z"},"papermill":{"duration":2.80121,"end_time":"2020-09-26T03:49:06.208928","exception":false,"start_time":"2020-09-26T03:49:03.407718","status":"completed"},"scrolled":true,"tags":[],"trusted":true},"cell_type":"code","source":"# R101-DELG model pretrained on GLDv2-clean\n# SAVED_MODEL_DIR = \"/workspace/Kaggle/Landmark/pretrained_delg/r101delg_gldv2clean_20200914\"\n\nmodels = [\"r50\", \"r101\", \"rn101af\"]\n# models = [\"r50\", \"r101\", \"r50v1\", \"r101v1\", \"rn101af\"]\nmodel_config = {\n    \"r50\": {\n        \"SAVED_MODEL_DIR\":\n        \"../input/pretrained-delg/r50delg_gldv2clean_20200914\",\n        \"DELG_SCORE_THRESHOLD_TENSOR\": tf.constant(454.6),  # score_threshold,\n        \"global_weight\": 0.4,\n        \"global_only\": False,\n        \"is_tf2\": False,\n    },\n    \"r101\": {\n        \"SAVED_MODEL_DIR\":\n        \"../input/pretrained-delg/r101delg_gldv2clean_20200914\",\n        \"DELG_SCORE_THRESHOLD_TENSOR\": tf.constant(357.48),  # score_threshold\n        \"global_weight\": 0.4,\n        \"global_only\": False,\n        \"is_tf2\": False,\n    },\n#     \"r50v1\": {\n#         \"SAVED_MODEL_DIR\": \"../input/pretrained-delg/r50delg_gld_20200814\",\n#         \"DELG_SCORE_THRESHOLD_TENSOR\": tf.constant(175.0),  # score_threshold,\n#         \"global_weight\": 0.2,\n#         \"global_only\": True,\n#         \"is_tf2\": False,\n#     },\n#     \"r101v1\": {\n#         \"SAVED_MODEL_DIR\": \"../input/pretrained-delg/r101delg_gld_20200814\",\n#         \"DELG_SCORE_THRESHOLD_TENSOR\": tf.constant(166.1),  # score_threshold\n#         \"global_weight\": 0.2,\n#         \"global_only\": True,\n#         \"is_tf2\": False,\n#     },\n    \"rn101af\": {\n        \"SAVED_MODEL_DIR\":\n        \"../input/pretrained-delg/rn101_af_gldv2clean_20200814\",\n        \"global_weight\": 0.2,\n        \"global_only\": True,\n        \"is_tf2\": True,\n    }\n}\n\n# models = [\"r50\", \"r101\", \"r50v1\", \"r101v1\", \"rn101af\"]\n# model_config = {\n#     \"r50\": {\n#         \"SAVED_MODEL_DIR\":\n#         \"/workspace/Kaggle/Landmark/pretrained_delg/r50delg_gldv2clean_20200914\",\n#         \"DELG_SCORE_THRESHOLD_TENSOR\": tf.constant(454.6),  # score_threshold,\n#         \"global_weight\": 0.2,\n#         \"global_only\": False,\n#         \"is_tf2\": False,\n#     },\n#     \"r101\": {\n#         \"SAVED_MODEL_DIR\":\n#         \"/workspace/Kaggle/Landmark/pretrained_delg/r101delg_gldv2clean_20200914\",\n#         \"DELG_SCORE_THRESHOLD_TENSOR\": tf.constant(357.48),  # score_threshold\n#         \"global_weight\": 0.2,\n#         \"global_only\": False,\n#         \"is_tf2\": False,\n#     },\n#     \"r50v1\": {\n#         \"SAVED_MODEL_DIR\":\n#         \"/workspace/Kaggle/Landmark/pretrained_delg/r50delg_gld_20200814\",\n#         \"DELG_SCORE_THRESHOLD_TENSOR\": tf.constant(175.0),  # score_threshold,\n#         \"global_weight\": 0.2,\n#         \"global_only\": True,\n#         \"is_tf2\": False,\n#     },\n#     \"r101v1\": {\n#         \"SAVED_MODEL_DIR\":\n#         \"/workspace/Kaggle/Landmark/pretrained_delg/r101delg_gld_20200814\",\n#         \"DELG_SCORE_THRESHOLD_TENSOR\": tf.constant(166.1),  # score_threshold\n#         \"global_weight\": 0.2,\n#         \"global_only\": True,\n#         \"is_tf2\": False,\n#     },\n#     \"rn101af\": {\n#         \"SAVED_MODEL_DIR\":\n#         \"/workspace/Kaggle/Landmark/pretrained_delg/rn101_af_gldv2clean_20200814\",\n#         \"global_weight\": 0.2,\n#         \"global_only\": True,\n#         \"is_tf2\": True,\n#     }\n# }\n\n# To stay below the time limit, the host only extracted local features for 3 scales\nDELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0,\n                                                 1.4142135])  # image_scales\n\n# Local feature extraction:\nLOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)  # max_feature_num\n\n# Global feature extraction:\nNUM_EMBEDDING_DIMENSIONS = 2048\nGLOBAL_SCALES_IND_TENSOR = tf.range(\n    len(DELG_IMAGE_SCALES_TENSOR))  # image_scales_ind","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.014747,"end_time":"2020-09-26T03:49:06.23997","exception":false,"start_time":"2020-09-26T03:49:06.225223","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Utility Functions"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-26T03:49:06.294217Z","iopub.status.busy":"2020-09-26T03:49:06.289597Z","iopub.status.idle":"2020-09-26T03:49:06.296873Z","shell.execute_reply":"2020-09-26T03:49:06.296408Z"},"papermill":{"duration":0.041791,"end_time":"2020-09-26T03:49:06.296967","exception":false,"start_time":"2020-09-26T03:49:06.255176","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def to_hex(image_id) -> str:\n    return '{0:0{1}x}'.format(image_id, 16)\n\n\ndef get_image_path(subset, image_id):\n    name = to_hex(image_id)\n    return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],\n                        '{}.jpg'.format(name))\n\n\ndef load_image_tensor(image_path):\n    return tf.convert_to_tensor(\n        np.array(PIL.Image.open(image_path).convert('RGB')))\n\n\ndef extract_global_features(image_root_dir, extractor, is_tf2=False):\n    \"\"\"Extracts embeddings for all the images in given `image_root_dir`.\"\"\"\n\n    image_paths = [x for x in pathlib.Path(image_root_dir).rglob('*.jpg')]\n\n    num_embeddings = len(image_paths)\n    if MAX_NUM_EMBEDDINGS > 0:\n        num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)\n\n    print(\n        f\"Extracting global features from {num_embeddings} images (dim={NUM_EMBEDDING_DIMENSIONS}) ......\"\n    )\n\n    ids = num_embeddings * [None]\n    embeddings = np.empty((num_embeddings, NUM_EMBEDDING_DIMENSIONS))\n\n    for i, image_path in enumerate(image_paths):\n        if i >= num_embeddings:\n            break\n\n        ids[i] = int(image_path.name.split('.')[0], 16)\n\n        image_tensor = load_image_tensor(image_path)\n\n        if not is_tf2:\n            output = extractor(\n                input_image=image_tensor,\n                input_scales=DELG_IMAGE_SCALES_TENSOR,\n                input_global_scales_ind=GLOBAL_SCALES_IND_TENSOR)\n            raw_global_descriptors = output[-1]\n        else:\n            output_dict = extractor(\n                input_image=image_tensor,\n                input_scales=DELG_IMAGE_SCALES_TENSOR,\n                input_global_scales_ind=GLOBAL_SCALES_IND_TENSOR)\n            raw_global_descriptors = output_dict[\"global_descriptors\"]\n\n        embedding_tensor = tf.nn.l2_normalize(raw_global_descriptors,\n                                              axis=1,\n                                              name='l2_normalization')\n        embedding_tensor = tf.reduce_sum(embedding_tensor,\n                                         axis=0,\n                                         name='sum_pooling')\n        embeddings[i, :] = tf.nn.l2_normalize(\n            embedding_tensor, axis=0, name='final_l2_normalization').numpy()\n\n    return ids, embeddings\n\n\ndef extract_local_features(model_name, image_path, extractor):\n    \"\"\"Extracts local features for the given `image_path`.\"\"\"\n\n    image_tensor = load_image_tensor(image_path)\n\n    output = extractor(input_image=image_tensor,\n                       input_scales=DELG_IMAGE_SCALES_TENSOR,\n                       input_global_scales_ind=GLOBAL_SCALES_IND_TENSOR,\n                       input_max_feature_num=LOCAL_FEATURE_NUM_TENSOR,\n                       input_abs_thres=model_config[model_name]\n                       [\"DELG_SCORE_THRESHOLD_TENSOR\"])\n\n    boxes, features = output[0], output[1]\n\n    # Shape: (N, 2)\n    keypoints = tf.divide(\n        tf.add(tf.gather(boxes, [0, 1], axis=1),\n               tf.gather(boxes, [2, 3], axis=1)), 2.0).numpy()\n\n    # Shape: (N, 128)\n    local_descriptors = tf.nn.l2_normalize(features,\n                                           axis=1,\n                                           name='l2_normalization').numpy()\n\n    return keypoints, local_descriptors","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-26T03:49:06.463405Z","iopub.status.busy":"2020-09-26T03:49:06.448056Z","iopub.status.idle":"2020-09-26T03:49:06.466201Z","shell.execute_reply":"2020-09-26T03:49:06.465689Z"},"papermill":{"duration":0.15554,"end_time":"2020-09-26T03:49:06.466331","exception":false,"start_time":"2020-09-26T03:49:06.310791","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def compute_putative_matching_keypoints(test_keypoints,\n                                        test_descriptors,\n                                        train_keypoints,\n                                        train_descriptors,\n                                        max_distance=0.9):\n    \"\"\"Finds matches from `test_descriptors` to KD-tree of `train_descriptors`.\"\"\"\n\n    train_descriptor_tree = spatial.cKDTree(train_descriptors)\n    _, matches = train_descriptor_tree.query(test_descriptors,\n                                             distance_upper_bound=max_distance)\n\n    test_kp_count = test_keypoints.shape[0]\n    train_kp_count = train_keypoints.shape[0]\n\n    test_matching_keypoints = np.array([\n        test_keypoints[i, ] for i in range(test_kp_count)\n        if matches[i] != train_kp_count\n    ])\n    train_matching_keypoints = np.array([\n        train_keypoints[matches[i], ] for i in range(test_kp_count)\n        if matches[i] != train_kp_count\n    ])\n\n    return test_matching_keypoints, train_matching_keypoints\n\n\ndef compute_num_inliers(test_keypoints, test_descriptors, train_keypoints,\n                        train_descriptors):\n    \"\"\"Returns the number of RANSAC inliers.\"\"\"\n\n    test_match_kp, train_match_kp = compute_putative_matching_keypoints(\n        test_keypoints, test_descriptors, train_keypoints, train_descriptors)\n\n    if test_match_kp.shape[\n            0] <= 4:  # Min keypoints supported by `pydegensac.findHomography()`\n        return 0\n\n    try:\n        _, mask = pydegensac.findHomography(test_match_kp, train_match_kp,\n                                            MAX_REPROJECTION_ERROR,\n                                            HOMOGRAPHY_CONFIDENCE,\n                                            MAX_RANSAC_ITERATIONS)\n    except np.linalg.LinAlgError:  # When det(H)=0, can't invert matrix.\n        return 0\n\n    return int(copy.deepcopy(mask).astype(np.float32).sum())\n\n\ndef get_total_score(num_inliers, global_score):\n    local_score = min(num_inliers, MAX_INLIER_SCORE) / MAX_INLIER_SCORE\n    return local_score + global_score\n\ndef rescore_and_rerank_by_num_inliers(test_image_id,\n                                      train_ids_labels_and_scores,\n                                      local_models):\n    \"\"\"Returns rescored and sorted training images by local feature extraction.\"\"\"\n\n    test_image_path = get_image_path('test', test_image_id)\n    \n    model_name = \"r50\"\n    extractor = local_models[model_name]\n    test_keypoints, test_descriptors = extract_local_features(\n        model_name, test_image_path, extractor)\n\n    for i in range(len(train_ids_labels_and_scores)):\n        train_image_id, label, global_score = train_ids_labels_and_scores[i]\n\n        train_image_path = get_image_path('train', train_image_id)\n        train_keypoints, train_descriptors = extract_local_features(\n            model_name, train_image_path, extractor)\n\n        num_inliers = compute_num_inliers(test_keypoints, test_descriptors,\n                                          train_keypoints, train_descriptors)\n        total_score = get_total_score(num_inliers, global_score)\n        train_ids_labels_and_scores[i] = (train_image_id, label, total_score)\n\n    train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)\n\n    return train_ids_labels_and_scores\n\n# def rescore_and_rerank_by_num_inliers(test_image_id,\n#                                       train_ids_labels_and_scores,\n#                                       local_models):\n#     \"\"\"Returns rescored and sorted training images by local feature extraction.\"\"\"\n\n#     test_image_path = get_image_path('test', test_image_id)\n    \n#     model_test_embeds = []\n#     for i, model_name in enumerate(models):\n#         local_extractor = local_models[model_name]\n#         keypoints, descriptors = extract_local_features(\n#             model_name, test_image_path, local_extractor)\n        \n#         model_test_embeds.append((keypoints, descriptors))\n    \n#     test_keypoints = []\n#     test_descriptors = []\n#     for i in range(len(models)):\n#         test_keypoints.append(model_test_embeds[i][0])\n#         test_descriptors.append(model_test_embeds[i][1])\n#     test_keypoints = np.concatenate(test_keypoints, axis=0)\n#     test_descriptors = np.concatenate(test_descriptors, axis=0)\n#     # print(test_keypoints.shape, test_descriptors.shape)\n\n#     for i in range(len(train_ids_labels_and_scores)):\n#         train_image_id, label, global_score = train_ids_labels_and_scores[i]\n\n#         train_image_path = get_image_path('train', train_image_id)\n        \n#         model_train_embeds = []\n#         for i, model_name in enumerate(models):\n#             local_extractor = local_models[model_name]\n#             keypoints, descriptors = extract_local_features(\n#                 model_name, train_image_path, local_extractor)\n\n#             model_train_embeds.append((keypoints, descriptors))\n\n#         train_keypoints = []\n#         train_descriptors = []\n#         for i in range(len(models)):\n#             train_keypoints.append(model_train_embeds[i][0])\n#             train_descriptors.append(model_train_embeds[i][1])\n#         train_keypoints = np.concatenate(train_keypoints, axis=0)\n#         train_descriptors = np.concatenate(train_descriptors, axis=0)\n        \n#         # print(train_keypoints.shape, train_descriptors.shape)\n\n#         num_inliers = compute_num_inliers(test_keypoints, test_descriptors,\n#                                           train_keypoints, train_descriptors)\n#         total_score = get_total_score(num_inliers, global_score)\n#         train_ids_labels_and_scores[i] = (train_image_id, label, total_score)\n\n#     train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)\n\n#     return train_ids_labels_and_scores","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.01451,"end_time":"2020-09-26T03:49:06.496271","exception":false,"start_time":"2020-09-26T03:49:06.481761","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Generate Predictions"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-26T03:49:06.533817Z","iopub.status.busy":"2020-09-26T03:49:06.533011Z","iopub.status.idle":"2020-09-26T03:49:06.535589Z","shell.execute_reply":"2020-09-26T03:49:06.53616Z"},"papermill":{"duration":0.025456,"end_time":"2020-09-26T03:49:06.536289","exception":false,"start_time":"2020-09-26T03:49:06.510833","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def get_global_extractor(model, is_tf2=False):\n    if not is_tf2:\n        DELG_INPUT_TENSOR_NAMES = [\n            'input_image:0', 'input_scales:0', 'input_global_scales_ind:0'\n        ]\n\n        # Global feature extractor graph\n        GLOBAL_FEATURE_EXTRACTION_FN = model.prune(DELG_INPUT_TENSOR_NAMES,\n                                                   ['global_descriptors:0'])\n\n        return GLOBAL_FEATURE_EXTRACTION_FN\n    else:\n        return model.signatures['serving_default']\n\n\ndef get_local_extractor(model):\n    DELG_INPUT_TENSOR_NAMES = [\n        'input_image:0', 'input_scales:0', 'input_global_scales_ind:0'\n    ]\n\n    # Local feature extractor graph\n    LOCAL_FEATURE_EXTRACTION_FN = model.prune(\n        DELG_INPUT_TENSOR_NAMES +\n        ['input_max_feature_num:0', 'input_abs_thres:0'],\n        ['boxes:0', 'features:0'])\n\n    return LOCAL_FEATURE_EXTRACTION_FN","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5eef0b98-5231-47ef-9482-c30f1dcef785","_uuid":"6beda86b-a0cd-4b8f-b832-cb98d81d82ce","execution":{"iopub.execute_input":"2020-09-26T03:49:06.594566Z","iopub.status.busy":"2020-09-26T03:49:06.586981Z","iopub.status.idle":"2020-09-26T03:49:06.597345Z","shell.execute_reply":"2020-09-26T03:49:06.596828Z"},"papermill":{"duration":0.044601,"end_time":"2020-09-26T03:49:06.597448","exception":false,"start_time":"2020-09-26T03:49:06.552847","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def load_labelmap():\n    with open(TRAIN_LABELMAP_PATH, mode='r') as csv_file:\n        csv_reader = csv.DictReader(csv_file)\n        labelmap = {row['id']: row['landmark_id'] for row in csv_reader}\n\n    return labelmap\n\n\ndef get_prediction_map(test_ids, train_ids_labels_and_scores):\n    \"\"\"Makes dict from test ids and ranked training ids, labels, scores.\"\"\"\n\n    prediction_map = dict()\n\n    for test_index, test_id in enumerate(test_ids):\n        hex_test_id = to_hex(test_id)\n\n        aggregate_scores = {}\n        # Extract Top-K ranked training images with their labels\n        for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:\n            if label not in aggregate_scores:\n                aggregate_scores[label] = 0\n            aggregate_scores[label] += score\n\n        label, score = max(aggregate_scores.items(),\n                           key=operator.itemgetter(1))\n\n        prediction_map[hex_test_id] = {'score': score, 'class': label}\n\n    return prediction_map\n\n\ndef get_predictions(labelmap, global_models, local_models):\n    \"\"\"Gets predictions using embedding similarity and local feature reranking.\"\"\"\n\n    test_ids, train_ids = None, None\n    model_test_embeds, model_train_embeds = [], []\n    for model_name in models:\n        global_extractor = global_models[model_name]\n        test_ids, model_test_embeddings = extract_global_features(\n            TEST_IMAGE_DIR, global_extractor,\n            model_config[model_name][\"is_tf2\"])\n        train_ids, model_train_embeddings = extract_global_features(\n            TRAIN_IMAGE_DIR, global_extractor,\n            model_config[model_name][\"is_tf2\"])\n\n        model_test_embeds.append(model_test_embeddings)\n        model_train_embeds.append(model_train_embeddings)\n\n        del global_extractor\n        gc.collect()\n        tf.keras.backend.clear_session()\n\n    test_embeddings = np.zeros_like(model_test_embeds[0])\n    train_embeddings = np.zeros_like(model_train_embeds[0])\n    for i in range(len(models)):\n        test_embeddings += model_config[model_name][\n            \"global_weight\"] * model_test_embeds[i]\n        train_embeddings += model_config[model_name][\n            \"global_weight\"] * model_train_embeds[i]\n    # test_embeddings /= len(models)\n    # train_embeddings /= len(models)\n\n    train_ids_labels_and_scores = [None] * test_embeddings.shape[0]\n\n    # Using (slow) for-loop, as distance matrix doesn't fit in memory.\n    for test_index in range(test_embeddings.shape[0]):\n        # Extract nearest training images\n        distances = spatial.distance.cdist(\n            test_embeddings[np.newaxis, test_index, :], train_embeddings,\n            'cosine')[0]\n        partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]\n\n        nearest = sorted([(train_ids[p], distances[p]) for p in partition],\n                         key=lambda x: x[1])\n\n        train_ids_labels_and_scores[test_index] = [\n            (train_id, labelmap[to_hex(train_id)], 1. - cosine_distance)\n            for train_id, cosine_distance in nearest\n        ]\n\n    del test_embeddings\n    del train_embeddings\n    del labelmap\n    gc.collect()\n\n    pre_verification_predictions = get_prediction_map(\n        test_ids, train_ids_labels_and_scores)\n\n    for test_index, test_id in enumerate(test_ids):\n        train_ids_labels_and_scores[\n            test_index] = rescore_and_rerank_by_num_inliers(\n                test_id, train_ids_labels_and_scores[test_index], local_models)\n\n    post_verification_predictions = get_prediction_map(\n        test_ids, train_ids_labels_and_scores)\n\n    return pre_verification_predictions, post_verification_predictions","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-26T03:49:06.637562Z","iopub.status.busy":"2020-09-26T03:49:06.636812Z","iopub.status.idle":"2020-09-26T03:49:06.640461Z","shell.execute_reply":"2020-09-26T03:49:06.640961Z"},"papermill":{"duration":0.028121,"end_time":"2020-09-26T03:49:06.641078","exception":false,"start_time":"2020-09-26T03:49:06.612957","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def save_submission_csv(predictions=None):\n    \"\"\"Saves optional `predictions` as submission.csv.\n\n  The csv has columns {id, landmarks}. The landmarks column is a string\n  containing the label and score for the id, separated by a ws delimeter.\n\n  If `predictions` is \"None\" (default), submission.csv is copied from\n  sample_submission.csv in `IMAGE_DIR`.\n\n  Args:\n    predictions: Optional dict of image ids to dicts with keys {class, score}.\n  \"\"\"\n\n    if predictions is None:\n        # Dummy submission!\n        shutil.copyfile(os.path.join(DATASET_DIR, 'sample_submission.csv'),\n                        'submission.csv')\n        return\n\n    with open('submission.csv', 'w') as submission_csv:\n        csv_writer = csv.DictWriter(submission_csv,\n                                    fieldnames=['id', 'landmarks'])\n        csv_writer.writeheader()\n        for image_id, prediction in predictions.items():\n            label = prediction['class']\n            score = prediction['score']\n            csv_writer.writerow({\n                'id': image_id,\n                'landmarks': f'{label} {score}'\n            })","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-26T03:49:06.682819Z","iopub.status.busy":"2020-09-26T03:49:06.681829Z","iopub.status.idle":"2020-09-26T03:49:06.685681Z","shell.execute_reply":"2020-09-26T03:49:06.686175Z"},"papermill":{"duration":0.02972,"end_time":"2020-09-26T03:49:06.686322","exception":false,"start_time":"2020-09-26T03:49:06.656602","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def run():\n    # with get_context(\"spawn\").Pool() as pool:\n\n    # Load model topology graph and pretrained weights\n    global_models = {}\n    local_models = {}\n    for model_name in models:\n        DELG_MODEL = tf.saved_model.load(\n            model_config[model_name][\"SAVED_MODEL_DIR\"])\n        global_models[model_name] = get_global_extractor(\n            DELG_MODEL, model_config[model_name][\"is_tf2\"])\n        if not model_config[model_name][\"global_only\"]:\n            local_models[model_name] = get_local_extractor(DELG_MODEL)\n\n        del DELG_MODEL\n        gc.collect()\n        tf.keras.backend.clear_session()\n\n    labelmap = load_labelmap()\n    num_training_images = len(labelmap.keys())\n    print(f'Found {num_training_images} training images.')\n\n    if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:\n        print(\n            f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images. Copying sample submission.'\n        )\n        save_submission_csv()\n    else:\n        _, post_verification_predictions = get_predictions(\n            labelmap, global_models, local_models)\n        save_submission_csv(post_verification_predictions)\n\n    del labelmap, global_models, local_models\n    gc.collect()\n    tf.keras.backend.clear_session()\n\n    # pool.close()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-26T03:49:06.722352Z","iopub.status.busy":"2020-09-26T03:49:06.721702Z","iopub.status.idle":"2020-09-26T03:50:04.826456Z","shell.execute_reply":"2020-09-26T03:50:04.82702Z"},"papermill":{"duration":58.124889,"end_time":"2020-09-26T03:50:04.827192","exception":false,"start_time":"2020-09-26T03:49:06.702303","status":"completed"},"scrolled":true,"tags":[],"trusted":true},"cell_type":"code","source":"%%time\n\nrun()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.017901,"end_time":"2020-09-26T03:50:04.862399","exception":false,"start_time":"2020-09-26T03:50:04.844498","status":"completed"},"tags":[]},"cell_type":"markdown","source":"### Verify Submission File"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-26T03:50:04.905562Z","iopub.status.busy":"2020-09-26T03:50:04.904632Z","iopub.status.idle":"2020-09-26T03:50:04.939604Z","shell.execute_reply":"2020-09-26T03:50:04.940101Z"},"papermill":{"duration":0.060467,"end_time":"2020-09-26T03:50:04.940263","exception":false,"start_time":"2020-09-26T03:50:04.879796","status":"completed"},"scrolled":true,"tags":[],"trusted":true},"cell_type":"code","source":"submit_df = pd.read_csv(\"submission.csv\", engine=\"c\")\nprint(submit_df.shape)\nsubmit_df.head()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.017924,"end_time":"2020-09-26T03:50:04.976523","exception":false,"start_time":"2020-09-26T03:50:04.958599","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.017536,"end_time":"2020-09-26T03:50:05.012241","exception":false,"start_time":"2020-09-26T03:50:04.994705","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## EOF\n"},{"metadata":{"papermill":{"duration":0.017861,"end_time":"2020-09-26T03:50:05.048444","exception":false,"start_time":"2020-09-26T03:50:05.030583","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}