{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import copy\nimport csv\nimport gc\nimport operator\nimport os\nimport pathlib\nimport shutil\n\nimport numpy as np\nimport PIL\nimport pydegensac\nfrom scipy import spatial\nimport tensorflow as tf\n\n#parameters:\nINPUT_DIR = os.path.join('..', 'input')\n\ndf_dir = os.path.join(INPUT_DIR, 'landmark-recognition-2021')\ntestiMAGE_dir = os.path.join(df_dir, 'test')\ntrainImage_dir = os.path.join(df_dir, 'train')\ntarinLabelPath_dir = os.path.join(df_dir, 'train.csv')\n\nnum_public_trainImage = 1580470  # Used to detect if in session or re-run.\nmax_num_embeddings = -1  # Set to > 1 to subsample dataset while debugging.\n\nnum_toRerank = 6\ntop_K_folds = 3  # Number of retrieved images used to make prediction for a test image.\n\nmax_inlinearScore = 26\nmax_projectionError = 6.0\nmax_ransacIterations = 900000\nhomography_confidence = 0.95\n\n#model:\nsaveModel_dir = '../input/delg-saved-models/local_and_global'\ndelgModel = tf.saved_model.load(saveModel_dir)\ndelg_imageScalerTensor = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])\ndelg_scoreThreshooldTensor = tf.constant(175.)\ndelg_inputTensorNames = [\n    'input_image:0', 'input_scales:0', 'input_abs_thres:0'\n]\n\n#feature extraction:\nnumEmbeddingDimensions = 2048\nglobalFeatureExtraction_fn = delgModel.prune(delg_inputTensorNames,\n                                                ['global_descriptors:0'])\nlocalFeatureNumeTensor = tf.constant(1000)\nlocalFeatureExtraction_FN = delgModel.prune(\n    delg_inputTensorNames + ['input_max_feature_num:0'],\n    ['boxes:0', 'features:0'])\n\n\ndef toHEX(image_id) -> str:\n    return '{0:0{1}x}'.format(image_id, 16)\n\n\ndef getImagePath(subset, image_id):\n    name = toHEX(image_id)\n    return os.path.join(df_dir, subset, name[0], name[1], name[2],\n                        '{}.jpg'.format(name))\n\n\ndef loadImageTensor(image_path):\n    return tf.convert_to_tensor(\n        np.array(PIL.Image.open(image_path).convert('RGB')))\n\n\ndef extractGlobalFeatures(image_root_dir):\n    \"\"\"Extracts embeddings for all the images in given `image_root_dir`.\"\"\"\n\n    image_paths = [x for x in pathlib.Path(image_root_dir).rglob('*.jpg')]\n\n    num_embeddings = len(image_paths)\n    if max_num_embeddings > 0:\n        num_embeddings = min(max_num_embeddings, num_embeddings)\n\n    ids = num_embeddings * [None]\n    embeddings = np.empty((num_embeddings, numEmbeddingDimensions))\n\n    for i, image_path in enumerate(image_paths):\n        if i >= num_embeddings:\n            break\n\n        ids[i] = int(image_path.name.split('.')[0], 16)\n        image_tensor = loadImageTensor(image_path)\n        features = globalFeatureExtraction_fn(image_tensor,\n                                                delg_imageScalerTensor,\n                                                delg_scoreThreshooldTensor)\n        embeddings[i, :] = tf.nn.l2_normalize(\n            tf.reduce_sum(features[0], axis=0, name='sum_pooling'),\n            axis=0,\n            name='final_l2_normalization').numpy()\n\n    return ids, embeddings\n\n\ndef extractLocalFeatures(image_path):\n\n    image_tensor = loadImageTensor(image_path)\n\n    features = localFeatureExtraction_FN(image_tensor, delg_imageScalerTensor,\n                                           delg_scoreThreshooldTensor,\n                                           localFeatureNumeTensor)\n\n    keypoints = tf.divide(\n        tf.add(\n            tf.gather(features[0], [0, 1], axis=1),\n            tf.gather(features[0], [2, 3], axis=1)), 2.0).numpy()\n\n    descriptors = tf.nn.l2_normalize(\n        features[1], axis=1, name='l2_normalization').numpy()\n\n    return keypoints, descriptors\n\n\ndef getPutuativeMatchingPinPoints(test_keypoints,\n                                    test_descriptors,\n                                    train_keypoints,\n                                    train_descriptors,\n                                    max_distance=0.9):\n\n    train_descriptor_tree = spatial.cKDTree(train_descriptors)\n    _, matches = train_descriptor_tree.query(\n        test_descriptors, distance_upper_bound=max_distance)\n\n    test_kp_count = test_keypoints.shape[0]\n    train_kp_count = train_keypoints.shape[0]\n\n    test_matching_keypoints = np.array([\n        test_keypoints[i,]\n        for i in range(test_kp_count)\n        if matches[i] != train_kp_count\n    ])\n    train_matching_keypoints = np.array([\n        train_keypoints[matches[i],]\n        for i in range(test_kp_count)\n        if matches[i] != train_kp_count\n    ])\n\n    return test_matching_keypoints, train_matching_keypoints\n\n\ndef getNumInliners(test_keypoints, test_descriptors, train_keypoints,\n                    train_descriptors):\n\n    test_match_kp, train_match_kp = getPutuativeMatchingPinPoints(\n        test_keypoints, test_descriptors, train_keypoints, train_descriptors)\n\n    if test_match_kp.shape[\n        0] <= 4:  \n        return 0\n\n    try:\n        _, mask = pydegensac.findHomography(test_match_kp, train_match_kp,\n                                            max_projectionError,\n                                            homography_confidence,\n                                            max_ransacIterations)\n    except np.linalg.LinAlgError:  \n        return 0\n\n    return int(copy.deepcopy(mask).astype(np.float32).sum())\n\n\ndef getTotalSocre(num_inliers, global_score):\n    local_score = min(num_inliers, max_inlinearScore) / max_inlinearScore\n    return local_score + global_score\n\n\ndef rescoreAndRerank_byInliners(test_image_id,\n                                      train_ids_labels_and_scores):\n\n    test_image_path = getImagePath('test', test_image_id)\n    test_keypoints, test_descriptors = extractLocalFeatures(test_image_path)\n\n    for i in range(len(train_ids_labels_and_scores)):\n        train_image_id, label, global_score = train_ids_labels_and_scores[i]\n\n        train_image_path = getImagePath('train', train_image_id)\n        train_keypoints, train_descriptors = extractLocalFeatures(\n            train_image_path)\n\n        num_inliers = getNumInliners(test_keypoints, test_descriptors,\n                                      train_keypoints, train_descriptors)\n        total_score = getTotalSocre(num_inliers, global_score)\n        train_ids_labels_and_scores[i] = (train_image_id, label, total_score)\n\n    train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)\n\n    return train_ids_labels_and_scores\n\n\ndef loadLabelMap():\n    with open(tarinLabelPath_dir, mode='r') as csv_file:\n        csv_reader = csv.DictReader(csv_file)\n        labelmap = {row['id']: row['landmark_id'] for row in csv_reader}\n\n    return labelmap\n\n\ndef getPredictionMAp(test_ids, train_ids_labels_and_scores):\n\n    prediction_map = dict()\n\n    for test_index, test_id in enumerate(test_ids):\n        hex_test_id = toHEX(test_id)\n\n        aggregate_scores = {}\n        for _, label, score in train_ids_labels_and_scores[test_index][:top_K_folds]:\n            if label not in aggregate_scores:\n                aggregate_scores[label] = 0\n            aggregate_scores[label] += score\n\n        label, score = max(aggregate_scores.items(), key=operator.itemgetter(1))\n\n        prediction_map[hex_test_id] = {'score': score, 'class': label}\n\n    return prediction_map\n\n\ndef get_Predictions(labelmap):\n\n    test_ids, test_embeddings = extractGlobalFeatures(testiMAGE_dir)\n\n    train_ids, train_embeddings = extractGlobalFeatures(trainImage_dir)\n\n    train_ids_labels_and_scores = [None] * test_embeddings.shape[0]\n\n    for test_index in range(test_embeddings.shape[0]):\n        distances = spatial.distance.cdist(\n            test_embeddings[np.newaxis, test_index, :], train_embeddings,\n            'cosine')[0]\n        partition = np.argpartition(distances, num_toRerank)[:num_toRerank]\n\n        nearest = sorted([(train_ids[p], distances[p]) for p in partition],\n                         key=lambda x: x[1])\n\n        train_ids_labels_and_scores[test_index] = [\n            (train_id, labelmap[toHEX(train_id)], 1. - cosine_distance)\n            for train_id, cosine_distance in nearest\n        ]\n\n    del test_embeddings\n    del train_embeddings\n    del labelmap\n    gc.collect()\n\n    pre_verification_predictions = getPredictionMAp(\n        test_ids, train_ids_labels_and_scores)\n\n    for test_index, test_id in enumerate(test_ids):\n        train_ids_labels_and_scores[test_index] = rescoreAndRerank_byInliners(\n            test_id, train_ids_labels_and_scores[test_index])\n\n    post_verification_predictions = getPredictionMAp(\n        test_ids, train_ids_labels_and_scores)\n\n    return pre_verification_predictions, post_verification_predictions\n\n\ndef saveSubmission_scv(predictions=None):\n\n    if predictions is None:\n        shutil.copyfile(\n            os.path.join(df_dir, 'sample_submission.csv'), 'submission.csv')\n        return\n\n    with open('submission.csv', 'w') as submission_csv:\n        csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])\n        csv_writer.writeheader()\n        for image_id, prediction in predictions.items():\n            label = prediction['class']\n            score = prediction['score']\n            csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})\n\n\ndef main():\n    labelmap = loadLabelMap()\n    num_training_images = len(labelmap.keys())\n    print(f'Found {num_training_images} training images.')\n\n    if num_training_images == num_public_trainImage:\n        print(\n            f'Found {num_public_trainImage} training images. Copying sample submission.'\n        )\n        saveSubmission_scv()\n        return\n\n    _, post_verification_predictions = get_Predictions(labelmap)\n    saveSubmission_scv(post_verification_predictions)\n\n\nif __name__ == '__main__':\n    main()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-13T17:52:56.097777Z","iopub.execute_input":"2021-09-13T17:52:56.098099Z","iopub.status.idle":"2021-09-13T17:53:10.716509Z","shell.execute_reply.started":"2021-09-13T17:52:56.098067Z","shell.execute_reply":"2021-09-13T17:53:10.715662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}