{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U ../input/kerasapplications/Keras_Applications-1.0.8-py3-none-any.whl\n!pip install -U ../input/efficientnetwhl/efficientnet-1.1.0-py3-none-any.whl\nimport efficientnet.tfkeras as efn","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nprint(tf.__version__)\nimport numpy as np\nimport math\nimport pandas as pd\nfrom sklearn import model_selection\nimport glob\nimport os\nfrom zipfile import ZipFile\nimport shutil\nimport tqdm.notebook as tqdm\nimport random, re, math, os\nfrom kaggle_datasets import KaggleDatasets\n\nimport matplotlib.pyplot as plt\nimport logging\ntf.get_logger().setLevel(logging.ERROR)\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport copy\nimport csv\nimport gc\nimport operator\nimport os\nimport pathlib\n\nimport numpy as np\nimport PIL\nimport pydegensac\nfrom scipy import spatial\n\nmixed_precision = False\n\nDEVICE = \"TPU\"\nif DEVICE == \"TPU\":\n print(\"connecting to TPU...\")\n try:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n print('Running on TPU ', tpu.master())\n except ValueError:\n print(\"Could not connect to TPU\")\n tpu = None\n\n if tpu:\n try:\n print(\"initializing TPU ...\")\n tf.config.experimental_connect_to_cluster(tpu)\n tf.tpu.experimental.initialize_tpu_system(tpu)\n strategy = tf.distribute.experimental.TPUStrategy(tpu)\n print(\"TPU initialized\")\n except _:\n print(\"failed to initialize TPU\")\n else:\n DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n print(\"Using default strategy for CPU and single GPU\")\n strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n\nstrategy_num = strategy.num_replicas_in_sync\nprint(f'strategy.num: {strategy_num}')\n \nconfig = {\n 'learning_rate': 5e-3,\n 'momentum': 0.9,\n 'scale': 30,\n 'margin': 0.3,\n 'clip_grad': 10.0,\n 'n_epochs': 3,\n 'input_size': (800,800, 3),\n 'n_classes': 81313,\n}\n\nBATCH_SIZE=32*strategy_num\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class AddMarginProduct(tf.keras.layers.Layer):\n '''\n Implements large margin cosine distance.\n\n References:\n https://arxiv.org/pdf/1801.07698.pdf\n https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n blob/master/src/modeling/metric_learning.py\n '''\n def __init__(self, n_classes, s=30, m=0.30, **kwargs):\n super(AddMarginProduct, self).__init__(**kwargs)\n self.n_classes = n_classes\n self.s = s\n self.m = m\n\n def build(self, input_shape):\n super(AddMarginProduct, self).build(input_shape[0])\n\n self.W = self.add_weight(\n name='W',\n shape=(int(input_shape[0][-1]), self.n_classes),\n initializer='glorot_uniform',\n dtype='float32',\n trainable=True,\n regularizer=None)\n\n def call(self, inputs):\n X, y = inputs\n y = tf.cast(y, dtype=tf.int32)\n cosine = tf.matmul(\n tf.math.l2_normalize(X, axis=1),\n tf.math.l2_normalize(self.W, axis=0)\n )\n phi = cosine - self.m\n one_hot = tf.cast(\n tf.one_hot(y, depth=self.n_classes),\n dtype=cosine.dtype\n )\n output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n output *= self.s\n return output\n\n\ndef create_model_b7_1024(input_shape=config['input_size'],n_classes=config['n_classes'],scale=30,margin=0.0):\n\n backbone = efn.EfficientNetB7(weights=None, include_top=False, input_shape=input_shape)\n pooling = tf.keras.layers.GlobalAveragePooling2D(name='head/pooling')\n margin = AddMarginProduct(n_classes=n_classes,s=scale,m=margin,name='head/cos_margin',dtype='float32')\n\n dense = tf.keras.layers.Dense(1024, name='head/dense')\n bn_0 = tf.keras.layers.BatchNormalization(name='head/bn_0')\n softmax = tf.keras.layers.Softmax(dtype='float32')\n\n image = tf.keras.layers.Input(input_shape, name='input/image')\n label = tf.keras.layers.Input((), name='input/label')\n\n x = backbone(image) \n x = pooling(x)\n x=dense(x)\n x=bn_0(x)\n x = margin([x, label])\n \n x = softmax(x)\n\n return tf.keras.Model(inputs=[image, label], outputs=x)\n\n\ndef create_model_b6_512(input_shape=config['input_size'],n_classes=config['n_classes'],scale=30,margin=0.0):\n backbone = efn.EfficientNetB6(weights=None, include_top=False, input_shape=input_shape)\n \n pooling = tf.keras.layers.GlobalAveragePooling2D(name='head/pooling')\n bn_0 = tf.keras.layers.BatchNormalization(name='head/bn_0')\n dense = tf.keras.layers.Dense(512, name='head/dense')\n bn_1 = tf.keras.layers.BatchNormalization(name='head/bn_1')\n margin = AddMarginProduct(n_classes=n_classes,s=scale,m=margin,name='head/cos_margin',dtype='float32')\n \n softmax = tf.keras.layers.Softmax(dtype='float32')\n\n image = tf.keras.layers.Input(input_shape, name='input/image')\n label = tf.keras.layers.Input((), name='input/label')\n\n x = backbone(image)\n x = pooling(x)\n x=bn_0(x)\n x= dense(x)\n x=bn_1(x)\n x = margin([x, label])\n \n x = softmax(x)\n return tf.keras.Model(inputs=[image, label], outputs=x)\n\n\ndef create_model_b7_512(input_shape=config['input_size'],n_classes=config['n_classes'],scale=30,margin=0.0):\n backbone = efn.EfficientNetB7(weights=None, include_top=False, input_shape=input_shape)\n \n pooling = tf.keras.layers.GlobalAveragePooling2D(name='head/pooling')\n bn_0 = tf.keras.layers.BatchNormalization(name='head/bn_0')\n dense = tf.keras.layers.Dense(512, name='head/dense')\n bn_1 = tf.keras.layers.BatchNormalization(name='head/bn_1')\n margin = AddMarginProduct(n_classes=n_classes,s=scale,m=margin,name='head/cos_margin',dtype='float32')\n \n softmax = tf.keras.layers.Softmax(dtype='float32')\n\n image = tf.keras.layers.Input(input_shape, name='input/image')\n label = tf.keras.layers.Input((), name='input/label')\n\n x = backbone(image)\n x = pooling(x)\n x=bn_0(x)\n x= dense(x)\n x=bn_1(x)\n x = margin([x, label])\n \n x = softmax(x)\n return tf.keras.Model(inputs=[image, label], outputs=x)\n\ndef create_model_b7_2560(input_shape=config['input_size'],n_classes=config['n_classes'],scale=30,margin=0.0):\n backbone =efn.EfficientNetB7(weights=None, include_top=False,input_shape=input_shape)\n pooling = tf.keras.layers.GlobalAveragePooling2D(name='head/pooling')\n margin = AddMarginProduct(n_classes=n_classes,s=scale,m=margin,name='head/cos_margin',dtype='float32')\n softmax = tf.keras.layers.Softmax(dtype='float32')\n\n image = tf.keras.layers.Input(input_shape, name='input/image')\n label = tf.keras.layers.Input((), name='input/label')\n\n x = backbone(image)\n x = pooling(x)\n x = margin([x, label])\n x = softmax(x)\n return tf.keras.Model(inputs=[image, label], outputs=x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_b7_1024 = create_model_b7_1024()\nmodel_b7_1024.load_weights('../input/glr2-sub-model/Acc-0.994-efnb7-model.h5')\nmodel_b7_1024.trainable=False\nmodel_b7_1024 = tf.keras.Model(\n inputs=model_b7_1024.get_layer('input/image').input,\n outputs=model_b7_1024.get_layer('head/bn_0').output)\n\nmodel_b7_1024.summary()\n\nmodel_b6_512 = create_model_b6_512()\nmodel_b6_512.load_weights('../input/glr2-sub-model/Acc-0.997-effb6-512-model.h5')\nmodel_b6_512.trainable=False\nmodel_b6_512 = tf.keras.Model(\n inputs=model_b6_512.get_layer('input/image').input,\n outputs=model_b6_512.get_layer('head/bn_1').output)\n\nmodel_b6_512.summary()\n\nmodel_b7_512 = create_model_b7_512()\nmodel_b7_512.load_weights('../input/glr2-sub-model/Acc-0.985-efnb7-512-model.h5')\nmodel_b7_512.trainable=False\nmodel_b7_512 = tf.keras.Model(\n inputs=model_b7_512.get_layer('input/image').input,\n outputs=model_b7_512.get_layer('head/bn_1').output)\n\nmodel_b7_512.summary()\n\nmodel_b7_2560 = create_model_b7_2560()\nmodel_b7_2560.load_weights('../input/glr2-sub-model/Acc-0.996-efnb7-2560-model.h5')\nmodel_b7_2560.trainable=False\nmodel_b7_2560 = tf.keras.Model(\n inputs=model_b7_2560.get_layer('input/image').input,\n outputs=model_b7_2560.get_layer('head/pooling').output)\n\nmodel_b7_2560.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_test_file():\n files_paths = glob.glob('../input/landmark-recognition-2020' + '/test/*/*/*/*')\n mapping = {}\n for path in files_paths:\n mapping[path.split('/')[-1].split('.')[0]] = path\n df = pd.read_csv('../input/landmark-recognition-2020/sample_submission.csv')\n df['path'] = df['id'].map(mapping)\n return df\n\ndf=read_test_file()\nprint(len(df))\n\ndef load_labelmap():\n with open('../input/landmark-recognition-2020/train.csv', mode='r') as csv_file:\n csv_reader = csv.DictReader(csv_file)\n labelmap = {row['id']: row['landmark_id'] for row in csv_reader}\n\n return labelmap\n\ndef read_train_file():\n labelmap=load_labelmap()\n \n files_paths = glob.glob('../input/landmark-recognition-2020' + '/train/*/*/*/*')\n\n mapping = {}\n for path in files_paths:\n image_id=path.split('/')[-1].split('.')[0]\n mapping[path] = image_id\n return mapping\n\ndf_train=read_train_file()\nprint(len(df_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\ndef read_image(img_id,image_path):\n image = tf.io.read_file(image_path)\n image = tf.image.decode_jpeg(image, channels=3)\n image = tf.image.resize(image, config['input_size'][0:2], method='bilinear')\n image = tf.cast(image, tf.float32)\n image /= 255.\n return image,img_id\n\ndef create_dataset(df,batch_size):\n image_paths= df.path\n img_id=df.id\n dataset = tf.data.Dataset.from_tensor_slices((img_id,image_paths))\n dataset = dataset.map(read_image,tf.data.experimental.AUTOTUNE)\n dataset = dataset.batch(batch_size)\n dataset = dataset.prefetch(AUTO)\n return dataset\n\ndef create_train_dataset(df,batch_size):\n image_paths= list(df_train.keys())\n img_id=list(df_train.values())\n dataset = tf.data.Dataset.from_tensor_slices((img_id,image_paths))\n dataset = dataset.map(read_image,tf.data.experimental.AUTOTUNE)\n dataset = dataset.batch(batch_size)\n dataset = dataset.prefetch(AUTO)\n return dataset\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dataset parameters:\nINPUT_DIR = os.path.join('..', 'input')\n\nDATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')\nTEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')\nTRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')\nTRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')\n\n# DEBUGGING PARAMS:\nNUM_PUBLIC_TRAIN_IMAGES = 1580470 # Used to detect if in session or re-run.\nMAX_NUM_EMBEDDINGS = -1 # Set to > 1 to subsample dataset while debugging.\n\n# Retrieval & re-ranking parameters:\nNUM_TO_RERANK = 3\nTOP_K = 3 #Number of retrieved images used to make prediction for a test image.\n\n# RANSAC parameters:\nMAX_INLIER_SCORE = 35\nMAX_REPROJECTION_ERROR = 7.0\nMAX_RANSAC_ITERATIONS = 70000\nHOMOGRAPHY_CONFIDENCE = 0.99","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #############################1_model\n# @tf.function(input_signature=[tf.TensorSpec(shape=[None, None, None, 3],dtype=tf.float32,name='input_image')])\n# def serving_train(input_image):\n# outputs0 = model_b7_1024(input_image)\n# outputs0 = tf.math.l2_normalize(outputs0,axis=-1)\n\n# return outputs0\n\n# def get_train_ids_labels_and_scores():\n\n# train_dataset=create_train_dataset(df_train,16)\n\n# features_train=[]\n# ID_marks=[]\n# train_dataset = tqdm.tqdm(train_dataset)\n# for k, inputs in enumerate(train_dataset):\n\n# ID_marks+=list(inputs[1].numpy())\n# out=serving_train(inputs[0])\n# features_train.append(out)\n\n# #if k==20:break\n# del inputs\n# del out\n# gc.collect()\n \n# del train_dataset\n# gc.collect()\n \n# features_train=tf.concat(features_train,axis=0)\n# ID_marks=np.array(ID_marks)\n \n# W=tf.concat(features_train,axis=0)\n# W = tf.math.l2_normalize(W,axis=-1)\n# W=tf.transpose(W)\n# @tf.function(input_signature=[tf.TensorSpec(shape=[None,None, None, 3],dtype=tf.float32,name='input_image')])\n# def serving(input_image):\n# outputs = model_b7_1024(input_image)\n# outputs = tf.math.l2_normalize(outputs,axis=-1)\n# res=tf.matmul(outputs,W)\n# return res\n\n\n# num_k=NUM_TO_RERANK\n# test_dataset=create_dataset(df,16)\n# ID_test=list(df.id)\n\n# test_dataset = tqdm.tqdm(test_dataset)\n# train_ids_labels_and_scores=[]\n# labelmap=load_labelmap()\n# for k, inputs in enumerate(test_dataset):\n# out=serving(inputs[0]).numpy()\n# for j in range(inputs[1].shape[0]):\n# top_k = np.argpartition(out[j], -num_k)[-num_k:]\n# top_k_score=out[j][top_k]\n# top_k_id=[str(x)[2:-1] for x in ID_marks[top_k]]\n# top_k_label=[labelmap[x] for x in top_k_id] \n# train_ids_labels_and_scores.append(list(zip(top_k_id,top_k_label,top_k_score)))\n# #if k ==3: break\n\n# del inputs\n# del out\n# gc.collect()\n# del W\n# del test_dataset\n# gc.collect()\n# return ID_test[0:len(train_ids_labels_and_scores)],train_ids_labels_and_scores#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #############################2_model\n# @tf.function(input_signature=[tf.TensorSpec(shape=[None, None, None, 3],dtype=tf.float32,name='input_image')])\n# def serving_train(input_image):\n# outputs0 = model_b7_1024(input_image)\n# outputs0 = tf.math.l2_normalize(outputs0,axis=-1)\n \n# outputs1 = model_b7_512(input_image)\n# outputs1 = tf.math.l2_normalize(outputs1,axis=-1)\n# return outputs0,outputs1\n\n# def get_train_ids_labels_and_scores():\n\n# train_dataset=create_train_dataset(df_train,16)\n\n# features_b7_1024=[]\n# features_b7_512=[]\n# ID_marks=[]\n# train_dataset = tqdm.tqdm(train_dataset)\n# for k, inputs in enumerate(train_dataset):\n\n# ID_marks+=list(inputs[1].numpy())\n# out=serving_train(inputs[0])\n# features_b7_1024.append(out[0])\n# features_b7_512.append(out[1])\n \n# #if k==100:break\n\n# del inputs\n# del out\n# gc.collect()\n \n# del train_dataset\n# gc.collect()\n \n# features_b7_1024=tf.concat(features_b7_1024,axis=0)\n# features_b7_512=tf.concat(features_b7_512,axis=0)\n# ID_marks=np.array(ID_marks)\n \n# W_b7_1024=tf.concat(features_b7_1024,axis=0)\n# W_b7_1024=tf.math.l2_normalize(W_b7_1024,axis=-1)\n# W_b7_1024=tf.transpose(W_b7_1024)\n# print(W_b7_1024.shape)\n \n# W_b7_512=tf.concat(features_b7_512,axis=0)\n# W_b7_512=tf.math.l2_normalize(W_b7_512,axis=-1)\n# W_b7_512=tf.transpose(W_b7_512)\n# print(W_b7_512.shape)\n \n# del features_b7_1024\n# del features_b7_512\n# gc.collect()\n \n# @tf.function(input_signature=[tf.TensorSpec(shape=[None, None, None, 3],dtype=tf.float32,name='input_image')])\n# def serving(input_image):\n# outputs0 = model_b7_1024(input_image)\n# outputs0 = tf.math.l2_normalize(outputs0,axis=-1)\n# res0=tf.matmul(outputs0,W_b7_1024)\n \n# outputs1 = model_b7_512(input_image)\n# outputs1 = tf.math.l2_normalize(outputs1,axis=-1)\n# res1=tf.matmul(outputs1,W_b7_512)\n# return res0,res1\n\n# num_k=NUM_TO_RERANK\n# labelmap=load_labelmap()\n \n \n# test_dataset=create_dataset(df,16)\n# ID_test=list(df.id)\n\n# test_dataset = tqdm.tqdm(test_dataset)\n# train_ids_labels_and_scores=[]\n \n# for k, inputs in enumerate(test_dataset):\n# out=serving(inputs[0])\n# out=(out[0].numpy(),out[1].numpy())\n# for j in range(inputs[1].shape[0]): \n# # top_k=list(np.argpartition(out[0][j], -num_k)[-num_k:])+list(np.argpartition(out[1][j], -num_k)[-num_k:])\n# # top_k=list(set(top_k)) \n# # top_k_score=np.max(np.stack([out[0][j][top_k],out[1][j][top_k]]),axis=0)\n\n# top_k0=list(np.argpartition(out[0][j], -num_k)[-num_k:])\n# top_k1=list(np.argpartition(out[1][j], -num_k)[-num_k:])\n# top_k=top_k0+top_k1 \n \n# top_k_score=list(out[0][j][top_k0])+list(out[1][j][top_k1]) \n# top_k_id=[str(x)[2:-1] for x in ID_marks[top_k]]\n# top_k_label=[labelmap[x] for x in top_k_id]\n \n# # aggregate_scores = {}\n# # for i in range(len(top_k_id)):\n# # if top_k_id[i] not in aggregate_scores:\n# # aggregate_scores[top_k_id[i]] = 0\n# # aggregate_scores[top_k_id[i]] += top_k_score[i]\n \n# # top_k_id=list(aggregate_scores.keys())\n# # top_k_label=[labelmap[x] for x in top_k_id]\n# # top_k_score=list(aggregate_scores.values())\n \n# res_out=list(zip(top_k_id,top_k_label,top_k_score))\n# train_ids_labels_and_scores.append(res_out)\n# #if k ==50: break\n\n# del inputs\n# del out\n# gc.collect()\n# del W_b7_1024\n# del W_b7_512\n# del test_dataset\n# gc.collect()\n# return ID_test[0:len(train_ids_labels_and_scores)],train_ids_labels_and_scores#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #############################2_merge_model\n# @tf.function(input_signature=[tf.TensorSpec(shape=[None, None, None, 3],dtype=tf.float32,name='input_image')])\n# def serving_train(input_image):\n# outputs0 = model_b7_1024(input_image)\n# outputs0 = tf.math.l2_normalize(outputs0,axis=-1)\n \n# outputs1 = model_b7_512(input_image)\n# outputs1 = tf.math.l2_normalize(outputs1,axis=-1)\n# return outputs0,outputs1\n\n# def get_train_ids_labels_and_scores():\n\n# train_dataset=create_train_dataset(df_train,16)\n\n# features_b7_1024=[]\n# features_b7_512=[]\n# ID_marks=[]\n# train_dataset = tqdm.tqdm(train_dataset)\n# for k, inputs in enumerate(train_dataset):\n\n# ID_marks+=list(inputs[1].numpy())\n# out=serving_train(inputs[0])\n# features_b7_1024.append(out[0])\n# features_b7_512.append(out[1])\n \n# #if k==100:break\n\n# del inputs\n# del out\n# gc.collect()\n \n# del train_dataset\n# gc.collect()\n \n# features_b7_1024=tf.concat(features_b7_1024,axis=0)\n# features_b7_512=tf.concat(features_b7_512,axis=0)\n# ID_marks=np.array(ID_marks)\n \n# W_b7_1024=tf.concat(features_b7_1024,axis=0)\n# W_b7_1024=tf.transpose(W_b7_1024)\n# print(W_b7_1024.shape)\n \n# W_b7_512=tf.concat(features_b7_512,axis=0)\n# W_b7_512=tf.transpose(W_b7_512)\n# print(W_b7_512.shape)\n \n# del features_b7_1024\n# del features_b7_512\n# gc.collect()\n \n# @tf.function(input_signature=[tf.TensorSpec(shape=[None, None, None, 3],dtype=tf.float32,name='input_image')])\n# def serving(input_image):\n# outputs0 = model_b7_1024(input_image)\n# outputs0 = tf.math.l2_normalize(outputs0,axis=-1)\n# res0=tf.matmul(outputs0,W_b7_1024)\n \n# outputs1 = model_b7_512(input_image)\n# outputs1 = tf.math.l2_normalize(outputs1,axis=-1)\n# res1=tf.matmul(outputs1,W_b7_512)\n# return res0,res1\n\n# num_k=NUM_TO_RERANK\n# labelmap=load_labelmap()\n \n \n# test_dataset=create_dataset(df,16)\n# ID_test=list(df.id)\n\n# test_dataset = tqdm.tqdm(test_dataset)\n# train_ids_labels_and_scores=[]\n \n# for k, inputs in enumerate(test_dataset):\n# out=serving(inputs[0])\n# out=(out[0].numpy(),out[1].numpy())\n# for j in range(inputs[1].shape[0]): \n# # top_k=list(np.argpartition(out[0][j], -num_k)[-num_k:])+list(np.argpartition(out[1][j], -num_k)[-num_k:])\n# # top_k=list(set(top_k)) \n# # top_k_score=np.max(np.stack([out[0][j][top_k],out[1][j][top_k]]),axis=0)\n\n# top_k0=list(np.argpartition(out[0][j], -num_k)[-num_k:])\n# top_k1=list(np.argpartition(out[1][j], -num_k)[-num_k:])\n# top_k=top_k0+top_k1 \n \n# top_k_score=list(out[0][j][top_k0])+list(out[1][j][top_k1]) \n# top_k_id=[str(x)[2:-1] for x in ID_marks[top_k]]\n# top_k_label=[labelmap[x] for x in top_k_id]\n \n# # aggregate_scores = {}\n# # for i in range(len(top_k_id)):\n# # if top_k_id[i] not in aggregate_scores:\n# # aggregate_scores[top_k_id[i]] = 0\n# # aggregate_scores[top_k_id[i]] += top_k_score[i]\n \n# # top_k_id=list(aggregate_scores.keys())\n# # top_k_label=[labelmap[x] for x in top_k_id]\n# # top_k_score=list(aggregate_scores.values())\n \n# res_out=list(zip(top_k_id,top_k_label,top_k_score))\n# train_ids_labels_and_scores.append(res_out)\n# #if k ==50: break\n\n# del inputs\n# del out\n# gc.collect()\n# del W_b7_1024\n# del W_b7_512\n# del test_dataset\n# gc.collect()\n# return ID_test[0:len(train_ids_labels_and_scores)],train_ids_labels_and_scores#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #############################3_model\n# @tf.function(input_signature=[tf.TensorSpec(shape=[None, None, None, 3],dtype=tf.float32,name='input_image')])\n# def serving_train(input_image):\n# outputs0 = model_b7_1024(input_image)\n# outputs0 = tf.math.l2_normalize(outputs0,axis=-1)\n \n# outputs1 = model_b6_512(input_image)\n# outputs1 = tf.math.l2_normalize(outputs1,axis=-1)\n \n# outputs2 = model_b7_512(input_image)\n# outputs2 = tf.math.l2_normalize(outputs2,axis=-1)\n# return outputs0,outputs1,outputs2\n\n# def get_train_ids_labels_and_scores():\n\n# train_dataset=create_train_dataset(df_train,16)\n\n# features_b7_1024=[]\n# features_b6_512=[]\n# features_b7_512=[]\n \n# ID_marks=[]\n# train_dataset = tqdm.tqdm(train_dataset)\n# for k, inputs in enumerate(train_dataset):\n\n# ID_marks+=list(inputs[1].numpy())\n# out=serving_train(inputs[0])\n# features_b7_1024.append(out[0])\n# features_b6_512.append(out[1])\n# features_b7_512.append(out[2])\n \n# #if k==100:break\n\n# del inputs\n# del out\n# gc.collect()\n \n# del train_dataset\n# gc.collect()\n \n# features_b7_1024=tf.concat(features_b7_1024,axis=0)\n# features_b6_512=tf.concat(features_b6_512,axis=0)\n# features_b7_512=tf.concat(features_b7_512,axis=0)\n \n# ID_marks=np.array(ID_marks)\n \n# W_b7_1024=tf.concat(features_b7_1024,axis=0)\n# W_b7_1024=tf.math.l2_normalize(W_b7_1024,axis=-1)\n# W_b7_1024=tf.transpose(W_b7_1024)\n# print(W_b7_1024.shape)\n \n# W_b6_512=tf.concat(features_b6_512,axis=0)\n# W_b6_512=tf.math.l2_normalize(W_b6_512,axis=-1)\n# W_b6_512=tf.transpose(W_b6_512)\n# print(W_b6_512.shape)\n \n# W_b7_512=tf.concat(features_b7_512,axis=0)\n# W_b7_512=tf.math.l2_normalize(W_b7_512,axis=-1)\n# W_b7_512=tf.transpose(W_b7_512)\n# print(W_b7_512.shape)\n \n# del features_b7_1024\n# del features_b6_512\n# del features_b7_512\n# gc.collect()\n \n# @tf.function(input_signature=[tf.TensorSpec(shape=[None, None, None, 3],dtype=tf.float32,name='input_image')])\n# def serving(input_image):\n# outputs0 = model_b7_1024(input_image)\n# outputs0 = tf.math.l2_normalize(outputs0,axis=-1)\n# res0=tf.matmul(outputs0,W_b7_1024)\n \n# outputs1 = model_b6_512(input_image)\n# outputs1 = tf.math.l2_normalize(outputs1,axis=-1)\n# res1=tf.matmul(outputs1,W_b6_512)\n \n# outputs2 = model_b7_512(input_image)\n# outputs2 = tf.math.l2_normalize(outputs2,axis=-1)\n# res2=tf.matmul(outputs2,W_b7_512)\n# return res0,res1,res2\n\n# num_k=NUM_TO_RERANK\n# labelmap=load_labelmap()\n \n \n# test_dataset=create_dataset(df,16)\n# ID_test=list(df.id)\n\n# test_dataset = tqdm.tqdm(test_dataset)\n# train_ids_labels_and_scores=[]\n \n# for k, inputs in enumerate(test_dataset):\n# out=serving(inputs[0])\n# out=(out[0].numpy(),out[1].numpy(),out[2].numpy())\n# for j in range(inputs[1].shape[0]): \n\n# top_k0=list(np.argpartition(out[0][j], -num_k)[-num_k:])\n# top_k1=list(np.argpartition(out[1][j], -num_k)[-num_k:])\n# top_k2=list(np.argpartition(out[2][j], -num_k)[-num_k:])\n# top_k=top_k0+top_k1+top_k2 \n \n# top_k_score=list(out[0][j][top_k0])+list(0.8*out[1][j][top_k1])+list(out[2][j][top_k2]) \n# top_k_id=[str(x)[2:-1] for x in ID_marks[top_k]]\n \n# # aggregate_scores = {}\n# # for i in range(len(top_k_id)):\n# # if top_k_id[i] not in aggregate_scores:\n# # aggregate_scores[top_k_id[i]] = 0\n# # aggregate_scores[top_k_id[i]] += top_k_score[i]\n \n# # top_k_id=list(aggregate_scores.keys()) \n# # top_k_score=list(aggregate_scores.values())\n \n# top_k_label=[labelmap[x] for x in top_k_id]\n# res_out=list(zip(top_k_id,top_k_label,top_k_score))\n# train_ids_labels_and_scores.append(res_out)\n# #if k ==100: break\n\n# del inputs\n# del out\n# gc.collect()\n \n# del W_b7_1024\n# del W_b6_512\n# del W_b7_512\n# del test_dataset\n# gc.collect()\n# return ID_test[0:len(train_ids_labels_and_scores)],train_ids_labels_and_scores#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #############################3_merge_model\n# @tf.function(input_signature=[tf.TensorSpec(shape=[None, None, None, 3],dtype=tf.float32,name='input_image')])\n# def serving_train(input_image):\n# outputs0 = model_b7_1024(input_image)\n# outputs0 = tf.math.l2_normalize(outputs0,axis=-1)\n \n# outputs1 = model_b6_512(input_image)\n# outputs1 = tf.math.l2_normalize(outputs1,axis=-1)\n \n# outputs2 = model_b7_512(input_image)\n# outputs2 = tf.math.l2_normalize(outputs2,axis=-1)\n \n# outputs=tf.concat([outputs0,outputs1,outputs2],axis=-1)\n# return outputs\n\n# def get_train_ids_labels_and_scores():\n\n# train_dataset=create_train_dataset(df_train,16)\n# features_2048=[] \n# ID_marks=[]\n# train_dataset = tqdm.tqdm(train_dataset)\n# for k, inputs in enumerate(train_dataset):\n\n# ID_marks+=list(inputs[1].numpy())\n# out=serving_train(inputs[0])\n# features_2048.append(out)\n \n# if k==5:break\n\n# del inputs\n# del out\n# gc.collect()\n \n# del train_dataset\n# gc.collect()\n \n# features_2048=tf.concat(features_2048,axis=0) \n# ID_marks=np.array(ID_marks)\n \n# features_2048=tf.concat(features_2048,axis=0)\n# features_2048=tf.transpose(features_2048)\n# print(features_2048.shape)\n \n# @tf.function(input_signature=[tf.TensorSpec(shape=[None, None, None, 3],dtype=tf.float32,name='input_image')])\n# def serving(input_image):\n# outputs0 = model_b7_1024(input_image)\n# outputs0 = tf.math.l2_normalize(outputs0,axis=-1)\n \n# outputs1 = model_b6_512(input_image)\n# outputs1 = 0.7*tf.math.l2_normalize(outputs1,axis=-1)\n \n# outputs2 = model_b7_512(input_image)\n# outputs2 = tf.math.l2_normalize(outputs2,axis=-1)\n \n# outputs=tf.concat([outputs0,outputs1,outputs2],axis=-1)\n# outputs=tf.matmul(outputs,features_2048)\n# return outputs\n\n# num_k=NUM_TO_RERANK\n# labelmap=load_labelmap()\n \n \n# test_dataset=create_dataset(df,16)\n# ID_test=list(df.id)\n\n# test_dataset = tqdm.tqdm(test_dataset)\n# train_ids_labels_and_scores=[]\n \n# for k, inputs in enumerate(test_dataset):\n# out=serving(inputs[0]).numpy()\n# for j in range(inputs[1].shape[0]):\n# top_k = np.argpartition(out[j], -num_k)[-num_k:]\n# top_k_score=out[j][top_k]/3.0\n# top_k_id=[str(x)[2:-1] for x in ID_marks[top_k]]\n# top_k_label=[labelmap[x] for x in top_k_id] \n# train_ids_labels_and_scores.append(list(zip(top_k_id,top_k_label,top_k_score)))\n# if k ==3: break\n\n# del inputs\n# del out\n# gc.collect()\n \n# del features_2048\n# del test_dataset\n# gc.collect()\n# return ID_test[0:len(train_ids_labels_and_scores)],train_ids_labels_and_scores#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#############################3_merge_model\n@tf.function(input_signature=[tf.TensorSpec(shape=[None, None, None, 3],dtype=tf.float32,name='input_image')])\ndef serving_train(input_image):\n outputs0 = model_b7_1024(input_image)\n outputs0 = tf.math.l2_normalize(outputs0,axis=-1)\n \n outputs1 = model_b6_512(input_image)\n outputs1 = tf.math.l2_normalize(outputs1,axis=-1)\n \n outputs2 = model_b7_512(input_image)\n outputs2 = tf.math.l2_normalize(outputs2,axis=-1)\n \n outputs3 = model_b7_2560(input_image)\n outputs3 = tf.math.l2_normalize(outputs3,axis=-1)\n \n outputs=tf.concat([outputs0,outputs1,outputs2,outputs3],axis=-1)\n return outputs\n\ndef get_train_ids_labels_and_scores():\n\n train_dataset=create_train_dataset(df_train,8)\n features_2048=[] \n ID_marks=[]\n train_dataset = tqdm.tqdm(train_dataset)\n for k, inputs in enumerate(train_dataset):\n\n ID_marks+=list(inputs[1].numpy())\n out=serving_train(inputs[0])\n features_2048.append(out)\n \n #if k==20:break\n\n del inputs\n del out\n gc.collect()\n \n del train_dataset\n gc.collect()\n \n features_2048=tf.concat(features_2048,axis=0) \n ID_marks=np.array(ID_marks)\n \n features_2048=tf.concat(features_2048,axis=0)\n features_2048=tf.transpose(features_2048)\n print(features_2048.shape)\n \n @tf.function(input_signature=[tf.TensorSpec(shape=[None, None, None, 3],dtype=tf.float32,name='input_image')])\n def serving(input_image):\n outputs0 = model_b7_1024(input_image)\n outputs0 = tf.math.l2_normalize(outputs0,axis=-1)\n \n outputs1 = model_b6_512(input_image)\n outputs1 = 0.7*tf.math.l2_normalize(outputs1,axis=-1)\n \n outputs2 = model_b7_512(input_image)\n outputs2 = tf.math.l2_normalize(outputs2,axis=-1)\n \n outputs3 = model_b7_2560(input_image)\n outputs3 = tf.math.l2_normalize(outputs3,axis=-1)\n \n outputs=tf.concat([outputs0,outputs1,outputs2,outputs3],axis=-1)\n outputs=tf.matmul(outputs,features_2048)\n return outputs\n\n num_k=NUM_TO_RERANK\n labelmap=load_labelmap()\n \n \n test_dataset=create_dataset(df,8)\n ID_test=list(df.id)\n\n test_dataset = tqdm.tqdm(test_dataset)\n train_ids_labels_and_scores=[]\n \n for k, inputs in enumerate(test_dataset):\n out=serving(inputs[0]).numpy()\n for j in range(inputs[1].shape[0]):\n top_k = np.argpartition(out[j], -num_k)[-num_k:]\n top_k_score=out[j][top_k]/4.0\n top_k_id=[str(x)[2:-1] for x in ID_marks[top_k]]\n top_k_label=[labelmap[x] for x in top_k_id] \n train_ids_labels_and_scores.append(list(zip(top_k_id,top_k_label,top_k_score)))\n #if k ==20: break\n\n del inputs\n del out\n gc.collect()\n \n del features_2048\n del test_dataset\n gc.collect()\n return ID_test[0:len(train_ids_labels_and_scores)],train_ids_labels_and_scores#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DELG model:\nSAVED_MODEL_DIR = '../input/delg-saved-models/local_and_global'\nDELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)\nDELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])\nDELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)\nDELG_INPUT_TENSOR_NAMES = [\n 'input_image:0', 'input_scales:0', 'input_abs_thres:0'\n]\n\n# Global feature extraction:\nGLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,['global_descriptors:0'])\n\n# Local feature extraction:\nLOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)\nLOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(\n DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],\n ['boxes:0', 'features:0'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_image_path(subset, image_id):\n name =image_id\n return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],'{}.jpg'.format(name))\n\n\ndef load_image_tensor(image_path):\n return tf.convert_to_tensor(\n np.array(PIL.Image.open(image_path).convert('RGB')))\n\n\ndef extract_local_features(image_path):\n \"\"\"Extracts local features for the given `image_path`.\"\"\"\n image_tensor = load_image_tensor(image_path) \n #print(image_tensor.shape)\n features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,\n DELG_SCORE_THRESHOLD_TENSOR,\n LOCAL_FEATURE_NUM_TENSOR)\n #print(features[0].shape, features[1].shape)\n # Shape: (N, 2)\n keypoints = tf.divide(\n tf.add(\n tf.gather(features[0], [0, 1], axis=1),\n tf.gather(features[0], [2, 3], axis=1)), 2.0).numpy()\n\n # Shape: (N, 128)\n descriptors = tf.nn.l2_normalize(\n features[1], axis=1, name='l2_normalization').numpy()\n \n #print(keypoints.shape, descriptors.shape)\n return keypoints, descriptors\n\n\ndef get_putative_matching_keypoints(test_keypoints,\n test_descriptors,\n train_keypoints,\n train_descriptors,\n max_distance=0.99):\n \"\"\"Finds matches from `test_descriptors` to KD-tree of `train_descriptors`.\"\"\"\n \n train_descriptor_tree = spatial.cKDTree(train_descriptors)\n _, matches = train_descriptor_tree.query(\n test_descriptors, distance_upper_bound=max_distance)\n\n test_kp_count = test_keypoints.shape[0]\n train_kp_count = train_keypoints.shape[0]\n\n test_matching_keypoints = np.array([\n test_keypoints[i,]\n for i in range(test_kp_count)\n if matches[i] != train_kp_count\n ])\n train_matching_keypoints = np.array([\n train_keypoints[matches[i],]\n for i in range(test_kp_count)\n if matches[i] != train_kp_count\n ])\n \n return test_matching_keypoints, train_matching_keypoints\n\n\ndef get_num_inliers(test_keypoints, test_descriptors, train_keypoints,\n train_descriptors):\n \"\"\"Returns the number of RANSAC inliers.\"\"\"\n\n test_match_kp, train_match_kp = get_putative_matching_keypoints(\n test_keypoints, test_descriptors, train_keypoints, train_descriptors)\n\n if test_match_kp.shape[0] <= 4: # Min keypoints supported by `pydegensac.findHomography()`\n return 0\n\n try:\n _, mask = pydegensac.findHomography(test_match_kp, train_match_kp,\n MAX_REPROJECTION_ERROR,\n HOMOGRAPHY_CONFIDENCE,\n MAX_RANSAC_ITERATIONS)\n except np.linalg.LinAlgError: # When det(H)=0, can't invert matrix.\n return 0\n\n return int(copy.deepcopy(mask).astype(np.float32).sum())\n\n\ndef get_total_score(num_inliers, global_score):\n local_score = min(num_inliers, MAX_INLIER_SCORE) / MAX_INLIER_SCORE\n return local_score + 0.8*global_score\n\n\ndef rescore_and_rerank_by_num_inliers(test_image_id,\n train_ids_labels_and_scores):\n \"\"\"Returns rescored and sorted training images by local feature extraction.\"\"\"\n\n test_image_path = get_image_path('test', test_image_id)\n test_keypoints, test_descriptors = extract_local_features(test_image_path)\n\n for i in range(len(train_ids_labels_and_scores)):\n train_image_id, label, global_score = train_ids_labels_and_scores[i]\n\n train_image_path = get_image_path('train', train_image_id)\n train_keypoints, train_descriptors = extract_local_features(\n train_image_path)\n\n num_inliers = get_num_inliers(test_keypoints, test_descriptors,\n train_keypoints, train_descriptors)\n total_score = get_total_score(num_inliers, global_score)\n train_ids_labels_and_scores[i] = (train_image_id, label, total_score)\n\n train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)\n\n return train_ids_labels_and_scores\n\n\ndef get_prediction_map(test_ids, train_ids_labels_and_scores):\n \"\"\"Makes dict from test ids and ranked training ids, labels, scores.\"\"\"\n\n prediction_map = dict()\n\n for test_index, test_id in enumerate(test_ids):\n hex_test_id = test_id\n\n aggregate_scores = {}\n for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:\n if label not in aggregate_scores:\n aggregate_scores[label] = 0\n aggregate_scores[label] += score\n\n label, score = max(aggregate_scores.items(), key=operator.itemgetter(1))\n\n prediction_map[hex_test_id] = {'score': score, 'class': label}\n\n return prediction_map\n\n\ndef get_predictions(labelmap):\n\n \n test_ids, train_ids_labels_and_scores=get_train_ids_labels_and_scores()\n pre_verification_predictions = get_prediction_map(test_ids, train_ids_labels_and_scores)\n\n #return None, pre_verification_predictions########################################\n #print(test_ids,train_ids_labels_and_scores)\n \n test_ids = tqdm.tqdm(test_ids)\n for test_index, test_id in enumerate(test_ids):\n train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(\n test_id, train_ids_labels_and_scores[test_index])\n \n #print(test_ids,train_ids_labels_and_scores)\n post_verification_predictions = get_prediction_map(\n test_ids, train_ids_labels_and_scores)\n\n return pre_verification_predictions, post_verification_predictions\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_submission_csv(predictions=None):\n \n if predictions is None:\n # Dummy submission!\n shutil.copyfile(\n os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')\n return\n\n with open('submission.csv', 'w') as submission_csv:\n csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])\n csv_writer.writeheader()\n for image_id, prediction in predictions.items():\n label = prediction['class']\n score = prediction['score']\n if score>0.3:\n csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})\n else:\n csv_writer.writerow({'id': image_id, 'landmarks': ''})\n\ndef main():\n pass\n\n labelmap = load_labelmap()\n num_training_images = len(labelmap.keys())\n print(f'Found {num_training_images} training images.')\n\n if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:\n print(\n f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images. Copying sample submission.'\n )\n save_submission_csv()\n return\n \n _, post_verification_predictions = get_predictions(labelmap)\n save_submission_csv(post_verification_predictions)\n\n\n\nif __name__ == '__main__':\n main()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}