{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Comments\n\n* This is just an experiment where I used DELG global feature embddings to build a DNN. For faster training I filter the data to use only the images that have 15 or more sample in 15 tf records, i will leave the coude commented on how i build them\n\n* In the next step I trained a DNN with 2 layers, the code i also commented below.\n\nThe experiment is a failure, public leaderboard is very bad. I believe the main reason for this is the filter of 15 or more samples for each class. Nevertheless it was a fun experiment and I wanted to share it :).","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# TF Record Extractor\n\nHere is the code to extract global embedding features and save them as tf records. \n\n* I created a csv file and filter the classes that have 14 or less samples\n* Stratified the data by class, this will give you 15 groups, use this groups to build tf records\n* You need to run different sessions to extract each tf record because it is very slow\n\nHere are the stafs of the final dataset I use for train\n\nThe number of images in our train set is: 1580470\n\nThe number of classes in out train set is: 81313\n\nThe number of classes that has more or equal than 15 observations is: 27756 which corresponds to 34.1348% of the total classes\n\nThe number of train images filter by the classes that has more or equal than 15 observations is: 1223195 which corresponds to 77.3944% of the total dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# import csv\n# import os\n# import pathlib\n# import pandas as pd\n# import numpy as np\n# from sklearn import preprocessing\n# from sklearn.model_selection import StratifiedKFold\n# from tqdm import tqdm\n# import tensorflow as tf\n# import PIL\n\n# def _bytes_feature(value):\n#     \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n#     if isinstance(value, type(tf.constant(0))):\n#         value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n#     return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n# def _float_feature(value):\n#     \"\"\"Returns a float_list from a float / double.\"\"\"\n#     return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\n# def _int64_feature(value):\n#     \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n#     return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n# def serialize_example(feature0, feature1):\n#     feature = {\n#         'embeddings': _bytes_feature(feature0),\n#         'target': _int64_feature(feature1)\n#     }\n#     example_proto = tf.train.Example(features = tf.train.Features(feature = feature))\n#     return example_proto.SerializeToString()\n\n# DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')\n# TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')\n# TRAIN = os.path.join(INPUT_DIR, 'landmark-train-encoded/train_encoded.csv')\n\n# # DELG model:\n# SAVED_MODEL_DIR = '../input/delg-saved-models/local_and_global'\n# DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)\n# DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])\n# DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)\n# DELG_INPUT_TENSOR_NAMES = [\n#     'input_image:0', 'input_scales:0', 'input_abs_thres:0'\n# ]\n\n# # Global feature extraction:\n# NUM_EMBEDDING_DIMENSIONS = 2048\n# GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,\n#                                                 ['global_descriptors:0'])\n\n# def to_hex(image_id) -> str:\n#     return '{0:0{1}x}'.format(image_id, 16)\n\n# # load an image to a tf tensor\n# def load_image_tensor(image_path):\n#     return tf.convert_to_tensor(\n#         np.array(PIL.Image.open(image_path).convert('RGB')))\n\n# def get_tf_records(record = 0):\n#     df = pd.read_csv(TRAIN)\n#     # get image paths\n#     image_paths = [x for x in pathlib.Path(TRAIN_IMAGE_DIR).rglob('*.jpg')]\n#     # get only one group, this is a slow process so we need to make 15 different sessions\n#     df = df[df['group'] == record]\n#     # reset index \n#     df.reset_index(drop = True, inplace = True)\n#     # get a list of ids\n#     ids_list = list(df['id'].unique())\n#     # write tf records\n#     with tf.io.TFRecordWriter('train_{}.tfrec'.format(record)) as writer:\n#         for image_path in tqdm(image_paths):\n#             image_id = int(image_path.name.split('.')[0], 16)\n#             image_id = to_hex(image_id)\n#             if image_id in ids_list:\n#                 # target\n#                 target = df[df['id'] == image_id]['landmark_id_encode']\n#                 image_tensor = load_image_tensor(image_path)\n#                 features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,\n#                                                         DELG_IMAGE_SCALES_TENSOR,\n#                                                         DELG_SCORE_THRESHOLD_TENSOR)\n#                 embedding = tf.nn.l2_normalize(\n#                     tf.reduce_sum(features[0], axis = 0, name = 'sum_pooling'),\n#                     axis = 0,\n#                     name = 'final_l2_normalization').numpy()\n#                 # transform numpy array to bytes\n#                 embedding = embedding.tobytes()\n#                 example = serialize_example(\n#                     embedding,\n#                     target.values[0]\n#                 )\n#                 writer.write(example)\n                \n# get_tf_records(record = 0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Simple DNN Model\n\nHere I trained in colab gpu a simple DNN model for 10 epochs with one fold of validation. \n\nValidation val_categorical_accuracy for the best epoch is 0.7878 and a global average precision of 0.7670, nevertheless this is not reflected on the leadearboard.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# import re\n# import os\n# import numpy\n# import pandas as pd\n# import numpy as np\n# import random\n# import math\n# from sklearn import metrics\n# from sklearn.model_selection import KFold\n# import tensorflow as tf\n# from tensorflow.keras import backend as K\n# !pip install gcsfs\n\n# # For tf.dataset\n# AUTO = tf.data.experimental.AUTOTUNE\n\n# # Data access\n# GCS_PATH = 'gs://kds-2048c0f014df07f1ef48ea726c68e902f57cdd083fbf9f4bbb46c2b2'\n# # Dictionary acces\n# DICT_PATH = 'gs://kds-13db2280942fc707e90594cc5c29d055a3fc72594eff7c85cc3ab006/train_encoded.csv'\n\n# # Configurations\n# EPOCHS = 10\n# BATCH_SIZE = 32\n# # Seed for deterministic results\n# SEED = 123\n# # Learning rate\n# LR = 0.001\n\n# # Training filenames directory\n# TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\n\n# # Seed everything for deterministic results\n# def seed_everything(seed):\n#     random.seed(seed)\n#     np.random.seed(seed)\n#     os.environ['PYTHONHASHSEED'] = str(seed)\n#     tf.random.set_seed(seed)\n\n# # Parse tf records, also decode bytes embeddings and one hot target vector\n# def read_tfrecord(example):\n#     tfrec_format = {\n#         'embeddings': tf.io.FixedLenFeature([], tf.string),\n#         'target': tf.io.FixedLenFeature([], tf.int64)\n#     }\n#     # Parse the data\n#     example = tf.io.parse_single_example(example, tfrec_format)\n#     # Decode raw bytes data to float\n#     embeddings = tf.io.decode_raw(input_bytes = example['embeddings'], out_type = float)\n#     # One hot encode target label, we extracted the amount of classes on another kernel\n#     target = tf.cast(tf.one_hot(example['target'], 27756), tf.int32)\n#     embeddings = tf.cast(embeddings, tf.float32)\n#     return embeddings, target\n\n# # Load dataset for training and evaluating model\n# def load_dataset(filenames, ordered = False):\n#     # Read from TFRecords. For optimal performance, reading from multiple files at once and\n#     # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n#     ignore_order = tf.data.Options()\n#     if not ordered:\n#         # Disable order, increase speed\n#         ignore_order.experimental_deterministic = False\n#     # Automatically interleaves reads from multiple files\n#     dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)\n#     # Use data as soon as it streams in, rather than in its original order\n#     dataset = dataset.with_options(ignore_order)\n#     # Returns a dataset of (image, label) pairs\n#     dataset = dataset.map(read_tfrecord, num_parallel_calls = AUTO)\n#     return dataset\n\n# # Training pipeline\n# def get_training_dataset(filenames, ordered = False):\n#     dataset = load_dataset(filenames, ordered = ordered)\n#     # The training dataset must repeat for several epochs\n#     dataset = dataset.repeat()\n#     dataset = dataset.shuffle(2048)\n#     dataset = dataset.batch(BATCH_SIZE)\n#     # Prefetch next batch while trianing\n#     dataset = dataset.prefetch(AUTO)\n#     return dataset\n\n# # Evaluation pipeline\n# def get_validation_dataset(filenames, ordered = True):\n#     dataset = load_dataset(filenames, ordered = ordered)\n#     dataset = dataset.batch(BATCH_SIZE)\n#     # Prefetch next batch while evaluating\n#     dataset = dataset.prefetch(AUTO)\n#     return dataset\n\n# # Count the number of observations with the tabular csv\n# def count_data_items(filenames):\n#     records = [int(re.compile(r\"_([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n#     df = pd.read_csv(DICT_PATH)\n#     n = df[df['group'].isin(records)].shape[0]\n#     return n\n\n# NUM_TOTAL_OBSERVATIONS = count_data_items(TRAINING_FILENAMES)\n# print(f'Dataset has {NUM_TOTAL_OBSERVATIONS} observations')\n\n# def gap_vector(pred, conf, true, return_x = False):\n#     '''\n#     Compute Global Average Precision (aka micro AP), the metric for the\n#     Google Landmark Recognition competition. \n#     This function takes predictions, labels and confidence scores as vectors.\n#     In both predictions and ground-truth, use None/np.nan for \"no label\".\n\n#     Args:\n#         pred: vector of integer-coded predictions\n#         conf: vector of probability or confidence scores for pred\n#         true: vector of integer-coded labels for ground truth\n#         return_x: also return the data frame used in the calculation\n\n#     Returns:\n#         GAP score\n#     '''\n#     x = pd.DataFrame({'pred': pred, 'conf': conf, 'true': true})\n#     x.sort_values('conf', ascending=False, inplace=True, na_position='last')\n#     x['correct'] = (x.true == x.pred).astype(int)\n#     x['prec_k'] = x.correct.cumsum() / (np.arange(len(x)) + 1)\n#     x['term'] = x.prec_k * x.correct\n#     gap = x.term.sum() / x.true.count()\n#     if return_x:\n#         return gap, x\n#     else:\n#         return gap\n\n\n# # Simple baseline with 27756 target classes and a softmax output\n# def get_model():\n  \n#     inp = tf.keras.layers.Input(shape = (2048))\n#     x = tf.keras.layers.Dense(1024, activation = 'relu')(inp)\n#     x = tf.keras.layers.BatchNormalization()(x)\n#     x = tf.keras.layers.Dropout(0.2)(x)\n#     x = tf.keras.layers.Dense(512, activation = 'relu')(inp)\n#     x = tf.keras.layers.BatchNormalization()(x)\n#     x = tf.keras.layers.Dropout(0.2)(x)\n#     output = tf.keras.layers.Dense(27756, activation = 'softmax', name = 'out')(x)\n\n#     model = tf.keras.models.Model(inputs = [inp], outputs = [output])\n\n#     opt = tf.keras.optimizers.Adam(learning_rate = LR)\n\n#     model.compile(\n#       optimizer = opt,\n#       loss = [tf.keras.losses.CategoricalCrossentropy()],\n#       metrics = [tf.keras.metrics.CategoricalAccuracy()]\n#     ) \n\n#     return model\n\n\n# def train_and_predict():\n\n#     # Out of folds confidence list\n#     oof_confidence = []\n#     # Out of folds target\n#     oof_target = []\n#     # Ground truth target\n#     target = []\n\n#     # Seed everything\n#     seed_everything(SEED)\n\n#     print('\\n')\n#     print('-'*50)\n#     train_dataset = get_training_dataset(TRAINING_FILENAMES[0:8], ordered = False)\n#     val_dataset = get_validation_dataset(TRAINING_FILENAMES[8], ordered = True)\n#     STEPS_PER_EPOCH = count_data_items(TRAINING_FILENAMES[0:8]) // BATCH_SIZE\n#     K.clear_session()\n#     model = get_model()\n#     # using early stopping using val loss\n#     checkpoint = tf.keras.callbacks.ModelCheckpoint(f'baseline_model.h5', monitor = 'val_loss', save_best_only = True, save_weights_only = False)\n#     # lr scheduler\n#     cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss', mode = 'min', factor = 0.5, patience = 1, verbose = 1, min_delta = 0.0001)\n#     history = model.fit(train_dataset, \n#                       steps_per_epoch = STEPS_PER_EPOCH,\n#                       epochs = EPOCHS,\n#                       callbacks = [checkpoint, cb_lr_schedule],\n#                       validation_data = val_dataset,\n#                       verbose = 1)\n\n#     print('Getting ground truth target')\n#     # Get ground truth target for the fold\n#     val_target = val_dataset.map(lambda embeddings, target: target)\n#     val_target = val_target.as_numpy_iterator()\n#     print('Stacking')\n#     val_target = np.vstack(list(val_target))\n#     val_target = np.argmax(val_target, axis = 1)\n#     target.extend(list(val_target))\n\n#     # Predictions\n#     val_embeddings = val_dataset.map(lambda embeddings, target: embeddings)\n#     print('Predicting validation embeddings')\n#     val_embeddings = list(val_embeddings.as_numpy_iterator())\n#     val_emb_len = len(val_embeddings)\n#     iterations = math.ceil(val_emb_len / 300)\n#     predictions = []\n#     # for some reason using model.predict with the tf.data.dataset generator burn out the memory of the gpu\n#     for i in range(iterations):\n#     prediction = model.predict(np.vstack(val_embeddings[300 * i : 300 * (i + 1)]))\n#     predictions.append(prediction)\n#     predictions = np.vstack(predictions)\n#     print('Get max indices')\n#     target_prediction = np.argmax(predictions, axis = -1)\n#     print('Get max confidence')\n#     target_confidence = np.max(predictions, axis = -1)\n#     print('Extend predictions')\n#     oof_target.extend(list(target_prediction))\n#     print('Extend confidence')\n#     oof_confidence.extend(list(target_confidence))\n\n#     # Calculate global average precision for the fold\n#     gap = gap_vector(list(target_prediction), list(target_confidence), list(val_target))\n#     accuracy_score = metrics.accuracy_score(list(val_target), list(target_prediction))\n#     print(f'Our global average precision for is {gap}')\n#     print(f'Our accuracy score for is {accuracy_score}')\n\n\n# train_and_predict()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference\n\nHere I load DELG model and the previous DNN model to make inference","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import csv\nimport gc\nimport os\nimport math\n\nimport shutil\nimport pathlib\nimport pandas as pd\nimport numpy as np\nimport PIL\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DATASET_DIR = '../input/landmark-train-encoded/train_encoded.csv'\nTEST_IMAGE_DIR = '../input/landmark-recognition-2020/test'\n\n# DEBUGGING PARAMS:\nNUM_PUBLIC_TEST_IMAGES = 10345 # Used to detect if in session or re-run.\nMAX_NUM_EMBEDDINGS = -1  # Set to > 1 to subsample dataset while debugging.\n\n# DNN model\nDNN_MODEL = tf.keras.models.load_model('../input/landmark-baseline-model/baseline_model.h5')\n\n# DELG model:\nSAVED_MODEL_DIR = '../input/delg-saved-models/local_and_global'\nDELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)\nDELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])\nDELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)\nDELG_INPUT_TENSOR_NAMES = [\n    'input_image:0', 'input_scales:0', 'input_abs_thres:0'\n]\n\n# Global feature extraction:\nNUM_EMBEDDING_DIMENSIONS = 2048\nGLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,\n                                                ['global_descriptors:0'])\n\ndef to_hex(image_id) -> str:\n    return '{0:0{1}x}'.format(image_id, 16)\n\n# load an image to a tf tensor\ndef load_image_tensor(image_path):\n    return tf.convert_to_tensor(\n        np.array(PIL.Image.open(image_path).convert('RGB')))\n\n# function to extract the global features using delg_model\ndef extract_global_features(image_root_dir):\n    \"\"\"Extracts embeddings for all the images in given `image_root_dir`.\"\"\"\n    \n    # get the path for all the training or test images\n    image_paths = [x for x in pathlib.Path(image_root_dir).rglob('*.jpg')]\n    num_embeddings = len(image_paths)\n    if MAX_NUM_EMBEDDINGS > 0:\n        num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)\n        \n    ids = num_embeddings * [None]\n    embeddings = np.empty((num_embeddings, NUM_EMBEDDING_DIMENSIONS))\n    \n    for i, image_path in enumerate(image_paths):\n        if i >= num_embeddings:\n            break\n            \n        ids[i] = to_hex(int(image_path.name.split('.')[0], 16))\n        image_tensor = load_image_tensor(image_path)\n        features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,\n                                                DELG_IMAGE_SCALES_TENSOR,\n                                                DELG_SCORE_THRESHOLD_TENSOR)\n        \n        embeddings[i, :] = tf.nn.l2_normalize(\n            tf.reduce_sum(features[0], axis=0, name='sum_pooling'),\n            axis=0,\n            name='final_l2_normalization').numpy()\n        \n    return ids, embeddings","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_and_save_submission_csv(test_path, train_csv):\n    image_paths = [x for x in pathlib.Path(test_path).rglob('*.jpg')]\n    test_len = len(image_paths)\n    if test_len == NUM_PUBLIC_TEST_IMAGES:\n        # Dummy submission\n        shutil.copyfile('../input/landmark-recognition-2020/sample_submission.csv', 'submission.csv')\n        return\n    else:\n        # Predict\n        test_ids, test_embeddings = extract_global_features(test_path)\n        embeddings_len = len(test_embeddings)\n        steps = math.ceil(embeddings_len / 9600)\n        predictions = []\n        # Predict in batches of 9600\n        for i in range(steps):\n            prediction = DNN_MODEL.predict(test_embeddings[i * 9600 : (i + 1) * 9600])\n            predictions.append(prediction)\n        predictions = np.vstack(predictions)\n        target = np.argmax(predictions, axis = -1)\n        confidence = np.max(predictions, axis = -1)\n        final = pd.DataFrame({'id': list(test_ids), 'target': list(target), 'confidence': list(confidence)})\n        # Get target dictionary\n        df = pd.read_csv(train_csv)\n        df = df[['landmark_id', 'landmark_id_encode']]\n        df.set_index('landmark_id_encode', inplace = True)\n        df = df.to_dict()['landmark_id']\n        final['landmarks'] = final['target'].map(df).astype(str) + ' ' + final['confidence'].astype(str)\n        final[['id', 'landmarks']].to_csv('submission.csv', index = False)\n\ninference_and_save_submission_csv(TEST_IMAGE_DIR, DATASET_DIR)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I hope you enjoy the reading, and the failed method (:","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}