{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Documentation - In progress","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2021-09-26T20:02:50.164985Z","iopub.status.busy":"2021-09-26T20:02:50.164636Z","iopub.status.idle":"2021-09-26T20:02:50.900568Z","shell.execute_reply":"2021-09-26T20:02:50.899508Z","shell.execute_reply.started":"2021-09-26T20:02:50.164887Z"},"papermill":{"duration":0.025056,"end_time":"2021-09-26T21:38:25.228287","exception":false,"start_time":"2021-09-26T21:38:25.203231","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!mkdir -p /tmp/concatenated-embeddings","metadata":{"papermill":{"duration":0.737553,"end_time":"2021-09-26T21:38:25.989749","exception":false,"start_time":"2021-09-26T21:38:25.252196","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-01T11:27:55.383552Z","iopub.execute_input":"2021-10-01T11:27:55.384505Z","iopub.status.idle":"2021-10-01T11:27:56.175004Z","shell.execute_reply.started":"2021-10-01T11:27:55.384458Z","shell.execute_reply":"2021-10-01T11:27:56.17402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/input/","metadata":{"execution":{"iopub.status.busy":"2021-10-01T11:27:56.176815Z","iopub.execute_input":"2021-10-01T11:27:56.177245Z","iopub.status.idle":"2021-10-01T11:27:56.982258Z","shell.execute_reply.started":"2021-10-01T11:27:56.177198Z","shell.execute_reply":"2021-10-01T11:27:56.981157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Part 1: Extracting Embeddings ","metadata":{}},{"cell_type":"code","source":"# %%python\n\n### Script to concatenate all embeddings based on given model votes\n\n\nimport numpy as np\nimport pandas as pd\nimport pickle\nimport glob\nimport os\nfrom sklearn.preprocessing import normalize\nfrom tqdm.auto import tqdm\n\nmodel_votes = {\n    'v2s-15epochs-800t800':1,\n    'v2m-15epochs-640t640':1,\n    'v2xl-15epochs-512t640':1,\n    'v2m-15epochs-732t732': 1,\n     'v2l-12epochs-720t720': 1\n}\n\nsample_submission = pd.read_csv('../input/landmark-retrieval-2021/sample_submission.csv')\nIS_PRIVATE = sample_submission.shape[0]!=1129\nprint(IS_PRIVATE)\n\nif not IS_PRIVATE:\n    model_votes = {\n        'v2xl-15n2epochs-720t720':1\n    }\n\n\nmodel_names = list(model_votes.keys())\nmodel_votes = [model_votes[x] for x in model_names]\nsum_votes = sum(model_votes)\nmodel_votes = [x/sum_votes for x in model_votes]\n\npickle.dump(model_names, open(\"model_names.pkl\", \"wb\"))\npickle.dump(model_votes, open(\"model_votes.pkl\", \"wb\"))\n\nnpy_array_files = glob.glob('../input/glr-precomputed-embed-v2l-12epochs-720t720/*.npy')\nnpy_array_files = [x.split('/')[-1] for x in npy_array_files]\n\n\n\nfor file in tqdm(npy_array_files):\n    if 'train' in file:\n        if 'name' not in file:\n            array = []\n            for i,model_name in enumerate(model_names):\n                if model_name=='v2xl-15n2epochs-720t720':\n                    model_name = 'v2xl-15n2epochs'\n                array.append(np.load(f'../input/glr-precomputed-embed-{model_name}/{file}')*model_votes[i])\n            array = normalize(np.concatenate(array,axis=1),axis=1)\n        else:\n            reference = None\n            for i,model_name in enumerate(model_names):\n                if model_name=='v2xl-15n2epochs-720t720':\n                    model_name = 'v2xl-15n2epochs'\n                array = np.load(f'../input/glr-precomputed-embed-{model_name}/{file}')\n                if i==0:\n                    reference=array\n                else:\n                    assert (reference==array).sum()==len(reference)\n        np.save('/tmp/concatenated-embeddings/'+file,array)\n#     except:\n#         print(\"Missing file\",file)","metadata":{"papermill":{"duration":480.559559,"end_time":"2021-09-26T21:46:26.574653","exception":false,"start_time":"2021-09-26T21:38:26.015094","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-01T11:28:55.455536Z","iopub.execute_input":"2021-10-01T11:28:55.455851Z","iopub.status.idle":"2021-10-01T11:32:40.530472Z","shell.execute_reply.started":"2021-10-01T11:28:55.455821Z","shell.execute_reply":"2021-10-01T11:32:40.529778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -lth /tmp/concatenated-embeddings/","metadata":{"papermill":{"duration":0.775651,"end_time":"2021-09-26T21:46:27.376347","exception":false,"start_time":"2021-09-26T21:46:26.600696","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-01T11:33:18.936468Z","iopub.execute_input":"2021-10-01T11:33:18.93704Z","iopub.status.idle":"2021-10-01T11:33:19.786269Z","shell.execute_reply.started":"2021-10-01T11:33:18.936995Z","shell.execute_reply":"2021-10-01T11:33:19.785223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!du -sh /tmp/concatenated-embeddings/","metadata":{"papermill":{"duration":0.673791,"end_time":"2021-09-26T21:46:28.074358","exception":false,"start_time":"2021-09-26T21:46:27.400567","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-01T11:33:19.788337Z","iopub.execute_input":"2021-10-01T11:33:19.789046Z","iopub.status.idle":"2021-10-01T11:33:20.58281Z","shell.execute_reply.started":"2021-10-01T11:33:19.789012Z","shell.execute_reply":"2021-10-01T11:33:20.582011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport os\nimport numpy as np\nimport pandas as pd\nimport glob\nimport pickle\nfrom tqdm.notebook import tqdm\nfrom sklearn.preprocessing import normalize","metadata":{"papermill":{"duration":0.642292,"end_time":"2021-09-26T21:46:28.741676","exception":false,"start_time":"2021-09-26T21:46:28.099384","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-01T11:33:20.822301Z","iopub.execute_input":"2021-10-01T11:33:20.822735Z","iopub.status.idle":"2021-10-01T11:33:20.82818Z","shell.execute_reply.started":"2021-10-01T11:33:20.822704Z","shell.execute_reply":"2021-10-01T11:33:20.827159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/landmark-retrieval-2021/sample_submission.csv')\nIS_PRIVATE = sample_submission.shape[0]!=1129\nprint(IS_PRIVATE)","metadata":{"papermill":{"duration":0.048027,"end_time":"2021-09-26T21:46:28.814458","exception":false,"start_time":"2021-09-26T21:46:28.766431","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-01T11:33:22.543682Z","iopub.execute_input":"2021-10-01T11:33:22.544083Z","iopub.status.idle":"2021-10-01T11:33:22.560011Z","shell.execute_reply.started":"2021-10-01T11:33:22.544055Z","shell.execute_reply":"2021-10-01T11:33:22.559388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if IS_PRIVATE:\n    test_images = glob.glob('../input/landmark-retrieval-2021/test/*/*/*/*.jpg')\n    index_images = glob.glob('../input/landmark-retrieval-2021/index/*/*/*/*.jpg')\nelse:\n    test_images = glob.glob('../input/landmark-retrieval-2021/test/0/0/*/*.jpg')\n    index_images = glob.glob('../input/landmark-retrieval-2021/index/0/0/0/*.jpg')\n\nprint(len(test_images),len(index_images))\n\npickle.dump(test_images, open(\"test_images.pkl\", \"wb\"))\npickle.dump(index_images, open(\"index_images.pkl\", \"wb\"))","metadata":{"papermill":{"duration":0.060134,"end_time":"2021-09-26T21:46:28.899781","exception":false,"start_time":"2021-09-26T21:46:28.839647","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-01T11:33:23.062948Z","iopub.execute_input":"2021-10-01T11:33:23.063379Z","iopub.status.idle":"2021-10-01T11:33:23.116878Z","shell.execute_reply.started":"2021-10-01T11:33:23.06335Z","shell.execute_reply":"2021-10-01T11:33:23.116056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile get_embeddings.py\n\nimport sys\nimport tensorflow as tf\nimport tensorflow_hub as tfhub\nfrom sklearn.preprocessing import normalize\nimport pickle\nimport numpy as np\n\nMODEL_IMAGE_SIZE_MAP = {\n    'v2m-15epochs-640t640':640,\n    'v2l-15epochs-512t640':640,\n    'v2m-15epochs-800t800':800,\n    'v2m-15epochs-732t732':732,\n    'v2xl-15n2epochs-720t720':720,\n    'v2s-15epochs-800t800':800,\n    'v2l-10n4epochs-720t720':720,\n    'v2xl-15epochs-512t640':640,\n    'b6-15epochs-800t800':800,\n    'v2l-12epochs-720t720':720\n}\n\nmodel_name = sys.argv[1]\nIMAGE_SIZE = MODEL_IMAGE_SIZE_MAP[model_name]\nIMAGE_SIZE = [IMAGE_SIZE,IMAGE_SIZE]\n\nstrategy = tf.distribute.get_strategy()  \nAUTO = tf.data.experimental.AUTOTUNE\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\nTTA = ['rotate0',  'rotate0_lr']\n\ntest_images = pickle.load(open(\"test_images.pkl\",'rb'))\nindex_images = pickle.load(open(\"index_images.pkl\",'rb'))\n\n# Function to decode our images\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels = 3)\n    image = tf.image.resize(image, IMAGE_SIZE)\n    image = tf.cast(image, tf.float32) / 255.0\n    return image\n\n# Function to read our test image and return image\ndef read_image(image):\n    image = tf.io.read_file(image)\n    image = decode_image(image)\n    return image\n\ndef test_time_augmentation(img,tta=None):\n    if tta:\n        if tta[-3:]=='_lr':\n            img = tf.image.flip_left_right(img)\n            tta = tta[:-3]\n\n        if tta[-3:]=='_ud':\n            img = tf.image.flip_up_down(img)\n            tta = tta[:-3]\n    return img\n    \n# Function to get our dataset that read images\ndef get_test_dataset(image_paths,tta=None):\n    dataset = tf.data.Dataset.from_tensor_slices((image_paths))\n    dataset = dataset.map(lambda image_path: read_image(image_path), num_parallel_calls = AUTO)\n    dataset = dataset.map(lambda image: test_time_augmentation(image,tta), num_parallel_calls = AUTO)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n\n### Loading model from zoo\nembed_model = tf.keras.models.load_model(f'../input/google-landmark-model-zoo/{model_name}.h5', \n                           custom_objects={'KerasLayer': tfhub.KerasLayer})\n\nprint(\"Predicting on image size\",IMAGE_SIZE)\n\nfor i,tta in enumerate(TTA):\n    index_ds = get_test_dataset(index_images,tta)\n    if i==0:\n        index_embeddings = normalize(embed_model.predict(index_ds,verbose=1),axis=1)\n    else:\n        index_embeddings += normalize(embed_model.predict(index_ds,verbose=1),axis=1)\n\nfor i,tta in enumerate(TTA):\n    test_ds = get_test_dataset(test_images,tta)\n    if i==0:\n        test_embeddings = normalize(embed_model.predict(test_ds,verbose=1),axis=1)\n    else:\n        test_embeddings += normalize(embed_model.predict(test_ds,verbose=1),axis=1)\n\nindex_embeddings = normalize(index_embeddings,axis=1)\ntest_embeddings = normalize(test_embeddings,axis=1)\n\nnp.save(f'/tmp/{model_name}/index_embeddings.npy',index_embeddings)\nnp.save(f'/tmp/{model_name}/test_embeddings.npy',test_embeddings)","metadata":{"papermill":{"duration":0.035308,"end_time":"2021-09-26T21:46:28.963324","exception":false,"start_time":"2021-09-26T21:46:28.928016","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-01T11:33:23.52539Z","iopub.execute_input":"2021-10-01T11:33:23.525685Z","iopub.status.idle":"2021-10-01T11:33:23.533854Z","shell.execute_reply.started":"2021-10-01T11:33:23.525656Z","shell.execute_reply":"2021-10-01T11:33:23.532725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_names = pickle.load(open(\"model_names.pkl\",'rb'))\nmodel_votes = pickle.load(open(\"model_votes.pkl\",'rb'))","metadata":{"papermill":{"duration":0.035045,"end_time":"2021-09-26T21:46:29.023927","exception":false,"start_time":"2021-09-26T21:46:28.988882","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-01T11:33:23.821651Z","iopub.execute_input":"2021-10-01T11:33:23.821968Z","iopub.status.idle":"2021-10-01T11:33:23.828881Z","shell.execute_reply.started":"2021-10-01T11:33:23.821938Z","shell.execute_reply":"2021-10-01T11:33:23.82778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for model_name in model_names:\n    print(\"Predicting for\",model_name)\n    !mkdir -p /tmp/{model_name}\n    !python get_embeddings.py {model_name}","metadata":{"papermill":{"duration":168.35975,"end_time":"2021-09-26T21:49:17.408654","exception":false,"start_time":"2021-09-26T21:46:29.048904","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-01T11:33:24.464414Z","iopub.execute_input":"2021-10-01T11:33:24.465154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nimport numpy as np\nimport pickle\nfrom sklearn.preprocessing import normalize\n\nmodel_names = pickle.load(open(\"model_names.pkl\",'rb'))\nmodel_votes = pickle.load(open(\"model_votes.pkl\",'rb'))\n\nindex_embeddings = []\nfor model_name,vote in zip(model_names,model_votes):\n    index_embeddings.append(np.load(f'/tmp/{model_name}/index_embeddings.npy')*vote)\nindex_embeddings = np.concatenate(index_embeddings,axis=1)\nindex_embeddings = normalize(index_embeddings,axis=1)\nnp.save(f'/tmp/index_embeddings.npy',index_embeddings)","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:49:17.492643Z","iopub.status.busy":"2021-09-26T21:49:17.492115Z","iopub.status.idle":"2021-09-26T21:49:18.470859Z","shell.execute_reply":"2021-09-26T21:49:18.470236Z","shell.execute_reply.started":"2021-09-26T20:57:36.137388Z"},"papermill":{"duration":1.022974,"end_time":"2021-09-26T21:49:18.471011","exception":false,"start_time":"2021-09-26T21:49:17.448037","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nimport numpy as np\nimport pickle\nfrom sklearn.preprocessing import normalize\n\nmodel_names = pickle.load(open(\"model_names.pkl\",'rb'))\nmodel_votes = pickle.load(open(\"model_votes.pkl\",'rb'))\n\ntest_embeddings = []\nfor model_name,vote in zip(model_names,model_votes):\n    test_embeddings.append(np.load(f'/tmp/{model_name}/test_embeddings.npy')*vote)\ntest_embeddings = np.concatenate(test_embeddings,axis=1)\ntest_embeddings = normalize(test_embeddings,axis=1)\nnp.save(f'/tmp/test_embeddings.npy',test_embeddings)","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:49:18.558612Z","iopub.status.busy":"2021-09-26T21:49:18.553993Z","iopub.status.idle":"2021-09-26T21:49:19.522065Z","shell.execute_reply":"2021-09-26T21:49:19.52107Z","shell.execute_reply.started":"2021-09-26T20:57:46.819126Z"},"papermill":{"duration":1.011832,"end_time":"2021-09-26T21:49:19.522215","exception":false,"start_time":"2021-09-26T21:49:18.510383","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for model_name in model_names:\n    !rm -r /tmp/{model_name}","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:49:19.609534Z","iopub.status.busy":"2021-09-26T21:49:19.608038Z","iopub.status.idle":"2021-09-26T21:49:21.484328Z","shell.execute_reply":"2021-09-26T21:49:21.485095Z","shell.execute_reply.started":"2021-09-26T20:57:52.361253Z"},"papermill":{"duration":1.923919,"end_time":"2021-09-26T21:49:21.485265","exception":false,"start_time":"2021-09-26T21:49:19.561346","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!du -sh /tmp/","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:49:21.571142Z","iopub.status.busy":"2021-09-26T21:49:21.570256Z","iopub.status.idle":"2021-09-26T21:49:22.219331Z","shell.execute_reply":"2021-09-26T21:49:22.218363Z","shell.execute_reply.started":"2021-09-26T20:57:56.137631Z"},"papermill":{"duration":0.695805,"end_time":"2021-09-26T21:49:22.219521","exception":false,"start_time":"2021-09-26T21:49:21.523716","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Part 2: Nearest Neighbour Search","metadata":{"papermill":{"duration":0.063762,"end_time":"2021-09-26T21:49:22.347566","exception":false,"start_time":"2021-09-26T21:49:22.283804","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import gc\nfrom cuml.neighbors import NearestNeighbors\nmodel_root = '/tmp/concatenated-embeddings/'","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:49:22.480272Z","iopub.status.busy":"2021-09-26T21:49:22.47796Z","iopub.status.idle":"2021-09-26T21:49:25.680798Z","shell.execute_reply":"2021-09-26T21:49:25.681198Z","shell.execute_reply.started":"2021-09-26T21:03:52.150211Z"},"papermill":{"duration":3.269978,"end_time":"2021-09-26T21:49:25.681361","exception":false,"start_time":"2021-09-26T21:49:22.411383","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index_image_ids = [x.split('/')[-1].split('.')[0] for x in index_images]\ntest_image_ids = [x.split('/')[-1].split('.')[0] for x in test_images]\nprint(len(index_image_ids),len(test_image_ids))","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:49:25.76472Z","iopub.status.busy":"2021-09-26T21:49:25.764167Z","iopub.status.idle":"2021-09-26T21:49:25.76873Z","shell.execute_reply":"2021-09-26T21:49:25.769284Z","shell.execute_reply.started":"2021-09-26T20:59:00.259471Z"},"papermill":{"duration":0.048617,"end_time":"2021-09-26T21:49:25.769454","exception":false,"start_time":"2021-09-26T21:49:25.720837","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index_embeddings = np.load('/tmp/index_embeddings.npy')\nprint(index_embeddings.shape)","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:49:25.852537Z","iopub.status.busy":"2021-09-26T21:49:25.851743Z","iopub.status.idle":"2021-09-26T21:49:25.855292Z","shell.execute_reply":"2021-09-26T21:49:25.855781Z","shell.execute_reply.started":"2021-09-26T20:58:44.759419Z"},"papermill":{"duration":0.047158,"end_time":"2021-09-26T21:49:25.855896","exception":false,"start_time":"2021-09-26T21:49:25.808738","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_embeddings = np.load('/tmp/test_embeddings.npy')\nprint(test_embeddings.shape)","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:49:25.938213Z","iopub.status.busy":"2021-09-26T21:49:25.937317Z","iopub.status.idle":"2021-09-26T21:49:25.941996Z","shell.execute_reply":"2021-09-26T21:49:25.941433Z","shell.execute_reply.started":"2021-09-26T20:59:16.854278Z"},"papermill":{"duration":0.046972,"end_time":"2021-09-26T21:49:25.942131","exception":false,"start_time":"2021-09-26T21:49:25.895159","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/glr-validation-metadata/train.csv')\nlandmark_map = train_df.set_index('id').landmark_id.to_dict()","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:49:26.024957Z","iopub.status.busy":"2021-09-26T21:49:26.024479Z","iopub.status.idle":"2021-09-26T21:49:41.416269Z","shell.execute_reply":"2021-09-26T21:49:41.415335Z","shell.execute_reply.started":"2021-09-26T21:01:43.757606Z"},"papermill":{"duration":15.434343,"end_time":"2021-09-26T21:49:41.416428","exception":false,"start_time":"2021-09-26T21:49:25.982085","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pick top 300 neighbours from train set for each image in test and index set ","metadata":{}},{"cell_type":"code","source":"## Find 15*20 neighbours for each index images in batches\nlandmark_knn=15\nindex_distances_file = '/tmp/index_distances.dat'\nindex_neighbours_file = '/tmp/index_neighbours.dat'\nindex_distances = np.memmap(index_distances_file, dtype='float32', mode='w+', \n                            shape=(len(index_image_ids),landmark_knn*20))\nindex_neighbours = np.memmap(index_neighbours_file, dtype='int32', mode='w+',\n                             shape=(len(index_image_ids),landmark_knn*20))\n\ntest_distances_file = '/tmp/test_distances.dat'\ntest_neighbours_file = '/tmp/test_neighbours.dat'\ntest_distances = np.memmap(test_distances_file, dtype='float32', mode='w+', \n                            shape=(len(test_image_ids),landmark_knn*20))\ntest_neighbours = np.memmap(test_neighbours_file, dtype='int32', mode='w+',\n                             shape=(len(test_image_ids),landmark_knn*20))\n\nfor part in tqdm(range(20)):\n    train_names = np.load(model_root+f'train-names-{part}.npy')\n    train_embed = np.load(model_root+f'train-predictions-{part}.npy')\n    non_landmark_scores = pd.read_csv(f'../input/glr-non-landmark-scores-for-train-images/non_landmark_score-{part}.csv',\n                      index_col='id').non_landmark_confidence.to_dict()\n    neighbors_model = NearestNeighbors(n_neighbors = landmark_knn, metric = 'cosine')\n    neighbors_model.fit(train_embed)\n    \n    distances, indices = neighbors_model.kneighbors(index_embeddings)\n    distances = np.abs(distances)\n    func = np.vectorize(lambda x: landmark_map[train_names[x]])\n    neighbours = func(indices)\n    func = np.vectorize(lambda x: non_landmark_scores[train_names[x]])\n    nlr_scores = func(indices)\n    distances = distances-0.4*(1-nlr_scores)\n    start_idx = part*landmark_knn\n    end_idx = (part+1)*landmark_knn\n    index_distances[:,start_idx:end_idx] = distances\n    index_neighbours[:,start_idx:end_idx] = neighbours\n    del indices,distances,neighbours,nlr_scores\n    gc.collect()\n    \n    distances, indices = neighbors_model.kneighbors(test_embeddings)\n    distances = np.abs(distances)\n    func = np.vectorize(lambda x: landmark_map[train_names[x]])\n    neighbours = func(indices)\n    func = np.vectorize(lambda x: non_landmark_scores[train_names[x]])\n    nlr_scores = func(indices)\n    distances = distances-0.4*(1-nlr_scores)\n    start_idx = part*landmark_knn\n    end_idx = (part+1)*landmark_knn\n    test_distances[:,start_idx:end_idx] = distances\n    test_neighbours[:,start_idx:end_idx] = neighbours\n    del train_names, train_embed, neighbors_model,indices,distances,neighbours,nlr_scores\n    gc.collect()","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:49:41.510828Z","iopub.status.busy":"2021-09-26T21:49:41.510131Z","iopub.status.idle":"2021-09-26T21:56:01.133152Z","shell.execute_reply":"2021-09-26T21:56:01.132713Z","shell.execute_reply.started":"2021-09-26T21:03:55.471685Z"},"papermill":{"duration":379.676761,"end_time":"2021-09-26T21:56:01.133297","exception":false,"start_time":"2021-09-26T21:49:41.456536","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /tmp/index_landmarks/\n!mkdir -p /tmp/test_landmarks/","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:56:01.2602Z","iopub.status.busy":"2021-09-26T21:56:01.219746Z","iopub.status.idle":"2021-09-26T21:56:02.68095Z","shell.execute_reply":"2021-09-26T21:56:02.680272Z","shell.execute_reply.started":"2021-09-26T21:10:21.756471Z"},"papermill":{"duration":1.50656,"end_time":"2021-09-26T21:56:02.681097","exception":false,"start_time":"2021-09-26T21:56:01.174537","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pick top 5 landmarks for each index image","metadata":{}},{"cell_type":"code","source":"BATCHSIZE = 10000 # Higher Batchsize -> Faster code, high chances of memory error\nfor start_idx in tqdm(range(0,len(index_distances),BATCHSIZE)):\n    BATCH_distances = index_distances[start_idx:start_idx+BATCHSIZE]\n    BATCH_neighbours = index_neighbours[start_idx:start_idx+BATCHSIZE]\n    names = index_image_ids[start_idx:start_idx+BATCHSIZE]\n\n    BATCH_META = pd.DataFrame(np.stack([BATCH_neighbours.reshape(-1),BATCH_distances.reshape(-1)],\n                                       axis=1),columns=['landmark','distance'])\n    BATCH_META['image_id'] = BATCH_META.index//BATCH_distances.shape[1]\n    BATCH_META['image_id'] = BATCH_META['image_id'].apply(lambda x: names[x])\n    BATCH_META = BATCH_META.sort_values('distance').groupby(['image_id','landmark']).head(2)\n    BATCH_META['confidence'] = 1-BATCH_META['distance']\n    BATCH_META = BATCH_META.groupby(['image_id','landmark']).confidence.sum().sort_values(ascending=False).reset_index()\n    BATCH_META_max = BATCH_META.groupby('image_id').head(1).sort_values(['image_id','landmark'])\n    BATCH_META_max['confidence'] = 0.5\n    BATCH_META = pd.concat([BATCH_META,BATCH_META_max])\n    BATCH_META = BATCH_META.groupby(['image_id','landmark']).confidence.sum().sort_values(ascending=False).reset_index()\n    BATCH_META = BATCH_META.groupby('image_id').head(5).sort_values(['image_id','landmark'])\n    \n    BATCH_META.to_csv(f'/tmp/index_landmarks/index_landmarks_{start_idx}.csv')\n    del BATCH_META,BATCH_META_max\n    gc.collect()","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:56:02.799298Z","iopub.status.busy":"2021-09-26T21:56:02.79825Z","iopub.status.idle":"2021-09-26T21:56:03.086349Z","shell.execute_reply":"2021-09-26T21:56:03.085918Z","shell.execute_reply.started":"2021-09-26T21:10:24.806922Z"},"papermill":{"duration":0.364106,"end_time":"2021-09-26T21:56:03.086481","exception":false,"start_time":"2021-09-26T21:56:02.722375","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pick top 5 landmarks for each test image","metadata":{}},{"cell_type":"code","source":"BATCHSIZE = 10 # Higher Batchsize -> Faster code, high chances of memory error\nfor start_idx in tqdm(range(0,len(test_distances),BATCHSIZE)):\n    BATCH_distances = test_distances[start_idx:start_idx+BATCHSIZE]\n    BATCH_neighbours = test_neighbours[start_idx:start_idx+BATCHSIZE]\n    names = test_image_ids[start_idx:start_idx+BATCHSIZE]\n\n    BATCH_META = pd.DataFrame(np.stack([BATCH_neighbours.reshape(-1),BATCH_distances.reshape(-1)],\n                                       axis=1),columns=['landmark','distance'])\n    BATCH_META['image_id'] = BATCH_META.index//BATCH_distances.shape[1]\n    BATCH_META['image_id'] = BATCH_META['image_id'].apply(lambda x: names[x])\n    BATCH_META = BATCH_META.sort_values('distance').groupby(['image_id','landmark']).head(2)\n    BATCH_META['confidence'] = 1-BATCH_META['distance']\n    BATCH_META = BATCH_META.groupby(['image_id','landmark']).confidence.sum().sort_values(ascending=False).reset_index()\n    BATCH_META_max = BATCH_META.groupby('image_id').head(1).sort_values(['image_id','landmark'])\n    BATCH_META_max['confidence'] = 0.5\n    BATCH_META = pd.concat([BATCH_META,BATCH_META_max])\n    BATCH_META = BATCH_META.groupby(['image_id','landmark']).confidence.sum().sort_values(ascending=False).reset_index()\n    BATCH_META = BATCH_META.groupby('image_id').head(5).sort_values(['image_id','landmark'])\n    BATCH_META.to_csv(f'/tmp/test_landmarks/test_landmarks_{start_idx}.csv')\n    del BATCH_META,BATCH_META_max\n    gc.collect()","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:56:03.198181Z","iopub.status.busy":"2021-09-26T21:56:03.197608Z","iopub.status.idle":"2021-09-26T21:56:03.353274Z","shell.execute_reply":"2021-09-26T21:56:03.352366Z","shell.execute_reply.started":"2021-09-26T21:10:26.343129Z"},"papermill":{"duration":0.225179,"end_time":"2021-09-26T21:56:03.353396","exception":false,"start_time":"2021-09-26T21:56:03.128217","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_landmarks_paths = os.listdir('/tmp/test_landmarks/')\nindex_landmarks_paths = os.listdir('/tmp/index_landmarks/')","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:56:03.440769Z","iopub.status.busy":"2021-09-26T21:56:03.440103Z","iopub.status.idle":"2021-09-26T21:56:03.443896Z","shell.execute_reply":"2021-09-26T21:56:03.443496Z","shell.execute_reply.started":"2021-09-26T21:10:27.618329Z"},"papermill":{"duration":0.048945,"end_time":"2021-09-26T21:56:03.444004","exception":false,"start_time":"2021-09-26T21:56:03.395059","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index_landmarks = []\nfor index_landmarks_path in tqdm(index_landmarks_paths):\n    index_landmarks.append(pd.read_csv('/tmp/index_landmarks/'+index_landmarks_path,index_col=0))\nindex_landmarks = pd.concat(index_landmarks)\nindex_landmarks.columns = [x+'_index' if x!='landmark' else x for x in index_landmarks.columns]\nprint(index_landmarks.shape)","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:56:03.535158Z","iopub.status.busy":"2021-09-26T21:56:03.5342Z","iopub.status.idle":"2021-09-26T21:56:03.569398Z","shell.execute_reply":"2021-09-26T21:56:03.568805Z","shell.execute_reply.started":"2021-09-26T21:10:29.122209Z"},"papermill":{"duration":0.08446,"end_time":"2021-09-26T21:56:03.569555","exception":false,"start_time":"2021-09-26T21:56:03.485095","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not IS_PRIVATE:\n    index_landmarks['landmark'] = (index_landmarks['landmark']%10).astype(int)","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:56:03.662142Z","iopub.status.busy":"2021-09-26T21:56:03.66164Z","iopub.status.idle":"2021-09-26T21:56:03.664745Z","shell.execute_reply":"2021-09-26T21:56:03.665144Z","shell.execute_reply.started":"2021-09-26T21:10:30.882164Z"},"papermill":{"duration":0.050489,"end_time":"2021-09-26T21:56:03.66526","exception":false,"start_time":"2021-09-26T21:56:03.614771","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Finding Direct Neighbours","metadata":{}},{"cell_type":"code","source":"def db_aug(V, n_neighbors=3):\n    \n    V = normalize(V,axis=1)\n    model = NearestNeighbors(n_neighbors=n_neighbors, metric=\"cosine\")\n    model.fit(V)\n    distances, indices = model.kneighbors(V)\n\n    w = np.power(np.clip(2.0 - distances, 0, 2.0), 0.5)\n    V_tmp = None\n    for i in range(n_neighbors):\n        if V_tmp is None:\n            V_tmp = w[:, i, None]*V[indices[:, i]]\n        else:\n            V_tmp += w[:, i, None]*V[indices[:, i]]\n    V_tmp /= w.sum(axis=1)[:, None]\n\n    return V_tmp","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:56:03.755864Z","iopub.status.busy":"2021-09-26T21:56:03.755216Z","iopub.status.idle":"2021-09-26T21:56:03.758495Z","shell.execute_reply":"2021-09-26T21:56:03.758067Z","shell.execute_reply.started":"2021-09-26T21:10:36.48895Z"},"papermill":{"duration":0.051312,"end_time":"2021-09-26T21:56:03.758597","exception":false,"start_time":"2021-09-26T21:56:03.707285","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_embeddings = np.concatenate([index_embeddings,test_embeddings])\nall_embeddings = db_aug(all_embeddings, n_neighbors=2)\n# index_embeddings = all_embeddings[:index_embeddings.shape[0]]\ntest_embeddings = all_embeddings[index_embeddings.shape[0]:]\nprint(index_embeddings.shape,test_embeddings.shape)","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:56:03.847427Z","iopub.status.busy":"2021-09-26T21:56:03.846607Z","iopub.status.idle":"2021-09-26T21:56:04.017829Z","shell.execute_reply":"2021-09-26T21:56:04.018228Z","shell.execute_reply.started":"2021-09-26T21:10:37.326049Z"},"papermill":{"duration":0.218257,"end_time":"2021-09-26T21:56:04.01838","exception":false,"start_time":"2021-09-26T21:56:03.800123","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"KNN = min(len(index_embeddings),100)\nneighbors_model = NearestNeighbors(n_neighbors = KNN, metric = 'cosine')\nneighbors_model.fit(index_embeddings)\ndistances, indices = neighbors_model.kneighbors(test_embeddings)\ndistances = np.abs(distances)","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:56:04.107794Z","iopub.status.busy":"2021-09-26T21:56:04.106971Z","iopub.status.idle":"2021-09-26T21:56:04.283174Z","shell.execute_reply":"2021-09-26T21:56:04.283775Z","shell.execute_reply.started":"2021-09-26T21:10:38.349934Z"},"papermill":{"duration":0.223638,"end_time":"2021-09-26T21:56:04.283975","exception":false,"start_time":"2021-09-26T21:56:04.060337","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"direct_neighbours = pd.DataFrame(np.stack([np.take(np.array(index_image_ids),indices).reshape(-1),\n                              distances.reshape(-1)],axis=1),columns=['image_id_index','direct_distance'])\ndirect_neighbours['image_id_test'] = np.take(test_image_ids,direct_neighbours.index.values//KNN)\ndirect_neighbours['direct_confidence'] = (1-direct_neighbours.direct_distance.astype(float)).clip(0,1)","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:56:04.3862Z","iopub.status.busy":"2021-09-26T21:56:04.385423Z","iopub.status.idle":"2021-09-26T21:56:04.387342Z","shell.execute_reply":"2021-09-26T21:56:04.388051Z","shell.execute_reply.started":"2021-09-26T21:10:39.388954Z"},"papermill":{"duration":0.053944,"end_time":"2021-09-26T21:56:04.388189","exception":false,"start_time":"2021-09-26T21:56:04.334245","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Finding Bridged Neighbours \n\n### Ensembling direct & bridged neighbours with power average ","metadata":{}},{"cell_type":"code","source":"retrieval_results = []\nfor test_landmarks_path in tqdm(test_landmarks_paths):\n    test_landmarks = pd.read_csv('/tmp/test_landmarks/'+test_landmarks_path,index_col=0)\n    test_image_ids_subset = test_landmarks.image_id.unique()\n    direct_neighbours_subset = direct_neighbours[direct_neighbours.image_id_test.isin(test_image_ids_subset)]\n    if not IS_PRIVATE:\n        test_landmarks['landmark'] = (test_landmarks['landmark']%10).astype(int)\n    test_landmarks.columns = [x+'_test' if x!='landmark' else x for x in test_landmarks.columns]\n    merged_landmarks = pd.merge(test_landmarks,index_landmarks,on='landmark')\n    merged_landmarks['confidence'] = merged_landmarks.apply(lambda row:\n                                                                min(row.confidence_test,row.confidence_index),\n                                                               axis=1)\n    merged_landmarks['bridged_confidence'] = merged_landmarks.confidence/3\n    merged_landmarks =  merged_landmarks.sort_values(['confidence','confidence_index'],ascending=False).drop_duplicates(\n        ['image_id_test','image_id_index'],keep='first')\n    merged_landmarks = pd.merge(direct_neighbours_subset[['image_id_index','image_id_test','direct_confidence']],\n             merged_landmarks[['image_id_test','image_id_index','bridged_confidence']],\n             on=['image_id_index','image_id_test'],\n            how='outer').fillna(0)\n    merged_landmarks['confidence'] = (merged_landmarks['bridged_confidence'] ** 3) + (merged_landmarks['direct_confidence'] ** 3)\n    merged_landmarks =  merged_landmarks.sort_values(['confidence','bridged_confidence'],ascending=False).drop_duplicates(\n        ['image_id_test','image_id_index'],keep='first')\n    subset = merged_landmarks.groupby('image_id_test').image_id_index.apply(\n        lambda x: \" \".join(x.values)).reset_index()\n    subset.columns = ['id','images']\n    retrieval_results.append(subset)\nretrieval_results = pd.concat(retrieval_results)","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:56:04.502278Z","iopub.status.busy":"2021-09-26T21:56:04.47788Z","iopub.status.idle":"2021-09-26T21:56:04.553374Z","shell.execute_reply":"2021-09-26T21:56:04.552967Z","shell.execute_reply.started":"2021-09-26T21:10:40.003205Z"},"papermill":{"duration":0.123949,"end_time":"2021-09-26T21:56:04.55349","exception":false,"start_time":"2021-09-26T21:56:04.429541","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"retrieval_results.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.execute_input":"2021-09-26T21:56:04.642373Z","iopub.status.busy":"2021-09-26T21:56:04.641704Z","iopub.status.idle":"2021-09-26T21:56:04.644459Z","shell.execute_reply":"2021-09-26T21:56:04.64399Z","shell.execute_reply.started":"2021-09-26T21:10:40.79157Z"},"papermill":{"duration":0.048802,"end_time":"2021-09-26T21:56:04.644569","exception":false,"start_time":"2021-09-26T21:56:04.595767","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.042,"end_time":"2021-09-26T21:56:04.728949","exception":false,"start_time":"2021-09-26T21:56:04.686949","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}