{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Hello Fellow Kagglers,\n\nThis notebook demonstrates the inference process for the [Happy Whale EfficientNetV2-XL DOLG Training TPU](https://www.kaggle.com/markwijkhuizen/happy-whale-efficientnetv2-xl-dolg-training-tpu) training notebook. For both the training and test images a descriptor of size 2048 is predicted and matched by cosine similarity. The 35 percentile of maximum correlation between test and train images is used a threshold for adding the new individual id.\n\n**Update V3**\n\nFixed inference mistake where duplicate individual_ids could be predicted instead of 5 unique individual_ids.\n\n**Update V5**\n\nUpgraded Tensorflow from 2.4 -> 2.6 which speeds up inference loop by a factor ~30 and fixed bug to always to predict 5 unique individual_ids","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/efficientnetv2-pretrained-imagenet21k-weights/brain_automl/')\nsys.path.append('/kaggle/input/efficientnetv2-pretrained-imagenet21k-weights/brain_automl/efficientnetv2/')","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:29:45.751791Z","iopub.execute_input":"2022-04-04T09:29:45.752332Z","iopub.status.idle":"2022-04-04T09:29:45.784755Z","shell.execute_reply.started":"2022-04-04T09:29:45.752238Z","shell.execute_reply":"2022-04-04T09:29:45.78395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:29:45.786686Z","iopub.execute_input":"2022-04-04T09:29:45.786958Z","iopub.status.idle":"2022-04-04T09:29:45.792321Z","shell.execute_reply.started":"2022-04-04T09:29:45.7869Z","shell.execute_reply":"2022-04-04T09:29:45.791486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nfrom multiprocessing import cpu_count\n\nimport imageio\nimport cv2\nimport os\nimport glob\nimport math\nimport time\nimport pickle\nimport gc\nimport joblib\nimport sys\nimport effnetv2_model\n\nprint(f'tensorflow version: {tf.__version__}')\nprint(f'tensorflow keras version: {tf.keras.__version__}')\nprint(f'python version: P{sys.version}')","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:29:45.79409Z","iopub.execute_input":"2022-04-04T09:29:45.794374Z","iopub.status.idle":"2022-04-04T09:29:52.081418Z","shell.execute_reply.started":"2022-04-04T09:29:45.794338Z","shell.execute_reply":"2022-04-04T09:29:52.080063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model Configuration\nEFNV2_SIZE = 'xl'\nIMG_SIZE = 640\nN_CHANNELS = 3\n\n# Image Dimensions\nINPUT_SHAPE = (IMG_SIZE, IMG_SIZE, N_CHANNELS)\nAUTO = tf.data.experimental.AUTOTUNE\nBATCH_SIZE = 32\nSEED = 42\n\n# Percentile to introduce new individual class to prediction\nNEW_INDIVIDUAL_PERCENTILE = 35\n\nTTA = 1\nDOLG_SIZE = 1024\nEMBEDDING_SIZE = 2048\n\nRECOMPUTE_EMBEDDINGS = True\n\nIMAGENET_MEAN = tf.constant([0.485, 0.456, 0.406], dtype=tf.float32)\nIMAGENET_STD = tf.constant([0.229, 0.224, 0.225], dtype=tf.float32)\n\nEPS = tf.keras.backend.epsilon()","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:29:52.083168Z","iopub.execute_input":"2022-04-04T09:29:52.083422Z","iopub.status.idle":"2022-04-04T09:29:54.512129Z","shell.execute_reply.started":"2022-04-04T09:29:52.083376Z","shell.execute_reply":"2022-04-04T09:29:54.511406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hardware Configuration","metadata":{}},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    TPU = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', TPU.master())\nexcept ValueError:\n    print('Running on GPU')\n    TPU = None\n\nif TPU:\n    tf.config.experimental_connect_to_cluster(TPU)\n    tf.tpu.experimental.initialize_tpu_system(TPU)\n    strategy = tf.distribute.experimental.TPUStrategy(TPU)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:29:54.513431Z","iopub.execute_input":"2022-04-04T09:29:54.513811Z","iopub.status.idle":"2022-04-04T09:29:54.526432Z","shell.execute_reply.started":"2022-04-04T09:29:54.513773Z","shell.execute_reply":"2022-04-04T09:29:54.525531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DOLG\n\nDOLG: Single-Stage Image Retrieval with Deep Orthogonal Fusion of Local and Global Features ([paper](https://arxiv.org/pdf/2108.02927.pdf))\n\nImplementation based on the [DOLG Pytorch implementation](https://github.com/ChristofHenkel/kaggle-landmark-2021-1st-place/blob/main/models/ch_mdl_dolg_efficientnet.py) by [Christof Henkel](https://www.kaggle.com/christofhenkel) for the Google Landmark Recognition 2021 competition","metadata":{}},{"cell_type":"code","source":"class GeM(tf.keras.layers.Layer):\n    def __init__(self, init_norm=3.0, **kwargs):\n        super(GeM, self).__init__(**kwargs)\n        self.init_norm = init_norm\n        self.gap2d = tf.keras.layers.GlobalAveragePooling2D()\n\n    def build(self, input_shape):\n        super(GeM, self).build(input_shape)\n        feature_size = input_shape[-1]\n        self.p = self.add_weight(\n                name = \"norms\",\n                shape = feature_size,\n                initializer = tf.keras.initializers.constant(self.init_norm),\n                trainable = True,\n            )\n\n    def call(self, inputs):\n        x = tf.math.maximum(inputs, EPS)\n        x = tf.pow(x, self.p)\n\n        x = self.gap2d(x)\n        x = tf.pow(x, 1.0 / self.p)\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:29:54.530164Z","iopub.execute_input":"2022-04-04T09:29:54.531767Z","iopub.status.idle":"2022-04-04T09:29:54.539284Z","shell.execute_reply.started":"2022-04-04T09:29:54.53174Z","shell.execute_reply":"2022-04-04T09:29:54.53847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Multi-Atrous Branch\nclass MultiAtrous(tf.keras.layers.Layer):\n    def __init__(self, dolg_s, upsampling=1, kernel_size=3, padding=\"same\",  **kwargs):\n        super(MultiAtrous, self).__init__(**kwargs)\n        self.d0 = tf.keras.layers.Conv2D(dolg_s // 2, 3, dilation_rate=(3,3), padding='same')\n        self.d1 = tf.keras.layers.Conv2D(dolg_s // 2, 3, dilation_rate=(6,6), padding='same')\n        self.d2 = tf.keras.layers.Conv2D(dolg_s // 2, 3, dilation_rate=(9,9), padding='same')\n        self.conv1 = tf.keras.layers.Conv2D(dolg_s, kernel_size=1)\n        \n    @tf.function()\n    def call(self, inputs, training=None, **kwargs):\n        x0 = self.d0(inputs)\n        x1 = self.d1(inputs)\n        x2 = self.d2(inputs)\n        x = tf.keras.layers.Concatenate(axis=3)([x0,x1,x2])\n        x = self.conv1(x)\n        x = tf.keras.activations.relu(x)\n        return x\n            \n    def get_config(self):\n        config = {\n            'dilation_rates': self.dilation_rates,\n            'kernel_size'   : self.kernel_size,\n            'padding'       : self.padding,\n            'upsampling'    : self.upsampling\n        }\n        base_config = super(MultiAtrous, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:29:54.540478Z","iopub.execute_input":"2022-04-04T09:29:54.540934Z","iopub.status.idle":"2022-04-04T09:29:54.554763Z","shell.execute_reply.started":"2022-04-04T09:29:54.540875Z","shell.execute_reply":"2022-04-04T09:29:54.554124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SpatialAttention2d(tf.keras.layers.Layer):\n    def __init__(self, dolg_s, **kwargs):\n        super(SpatialAttention2d, self).__init__(**kwargs)\n        self.conv1 = tf.keras.layers.Conv2D(dolg_s, 1)\n        self.bn = tf.keras.layers.BatchNormalization()\n        self.conv2 = tf.keras.layers.Conv2D(1, 1)\n\n    @tf.function()\n    def call(self, x):\n        '''\n        x : spatial feature map. (b x c x w x h)\n        att : softplus attention score \n        '''\n        x = self.conv1(x)\n        x = self.bn(x)\n        \n        feature_map_norm, _ = tf.linalg.normalize(x, ord=2, axis=3)\n        \n        x = tf.keras.activations.relu(x)\n        x = self.conv2(x)\n        \n        att_score = tf.keras.activations.softplus(x)\n        \n        x = att_score * feature_map_norm\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:29:54.55598Z","iopub.execute_input":"2022-04-04T09:29:54.556285Z","iopub.status.idle":"2022-04-04T09:29:54.566079Z","shell.execute_reply.started":"2022-04-04T09:29:54.55625Z","shell.execute_reply":"2022-04-04T09:29:54.565365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class OrthogonalFusion(tf.keras.layers.Layer):\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n        \n    @tf.function()\n    def call(self, inputs):\n        fl, fg = inputs\n        fl = tf.transpose(fl, [0,3,1,2])\n        \n        bs, c, w, h = fl.shape\n        \n        fl_b = tf.reshape(fl, [tf.shape(fl)[0],c,w*h])\n        fl_dot_fg = tf.matmul(fg[:,tf.newaxis,:] ,fl_b)\n       \n        # fl_dot_fg = fl_dot_fg.reshape(bs,1,w,h)\n        fl_dot_fg = tf.reshape(fl_dot_fg, [tf.shape(fl_dot_fg)[0],1,w,h])\n        # fg_norm = torch.norm(fg, dim=1)\n        \n        fg_norm = tf.norm(fg, ord=2, axis=1)\n        \n        fl_proj = (fl_dot_fg / fg_norm[:,tf.newaxis,tf.newaxis,tf.newaxis]) * fg[:,:,tf.newaxis,tf.newaxis]\n        fl_orth = fl - fl_proj\n        \n        fg_rep = tf.tile(fg[:,:,tf.newaxis,tf.newaxis], multiples=(1,1,w,h))\n        f_fused = tf.keras.layers.Concatenate(axis=1)([fl_orth, fg_rep])\n        \n        # Transpose\n        f_fused = tf.transpose(f_fused, [0,2,3,1])\n        \n        return f_fused","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:29:54.567339Z","iopub.execute_input":"2022-04-04T09:29:54.567691Z","iopub.status.idle":"2022-04-04T09:29:54.579347Z","shell.execute_reply.started":"2022-04-04T09:29:54.567659Z","shell.execute_reply":"2022-04-04T09:29:54.578659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GlobalBranch(tf.keras.layers.Layer):\n    def __init__(self, dolg_s, **kwargs):\n        super().__init__(**kwargs)\n        self.conv2d = tf.keras.layers.Conv2D(dolg_s, 1, name='global_conv2d')\n        self.bn = tf.keras.layers.BatchNormalization()\n        self.pool = tf.keras.layers.GlobalAveragePooling2D()\n        \n    @tf.function()\n    def call(self, inputs):\n        x = self.conv2d(inputs)\n        x = self.bn(x)\n        x = tf.nn.silu(x)\n        x = self.pool(x)\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:29:54.580627Z","iopub.execute_input":"2022-04-04T09:29:54.580872Z","iopub.status.idle":"2022-04-04T09:29:54.589369Z","shell.execute_reply.started":"2022-04-04T09:29:54.580839Z","shell.execute_reply":"2022-04-04T09:29:54.588559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DolgBranch(tf.keras.layers.Layer):\n    def __init__(self, dolg_s, idx, **kwargs):\n        super().__init__(name=f'dolg_branch_{idx}', **kwargs)\n        dolg_s = int(dolg_s)\n        # Local\n        self.mam = MultiAtrous(dolg_s, name=f'mam_{idx}')\n        self.sa2d = SpatialAttention2d(dolg_s, name=f'sa2d_{idx}')\n        # Global\n        self.global_branch = GlobalBranch(dolg_s, name=f'g_{idx}')\n        # Orthogonal Fusion\n        self.orthogonal_fusion = OrthogonalFusion()\n        # Pooling\n        self.pool = tf.keras.layers.GlobalAveragePooling2D()\n        \n    @tf.function()\n    def call(self, inputs):\n        inputs_l, inputs_g = inputs\n        # Local\n        l = self.mam(inputs_l)\n        l = self.sa2d(l)\n        # Global\n        g = self.global_branch(inputs_g)\n        # Orthogonal Fusion\n        f = self.orthogonal_fusion([l, g])\n        # Pooling\n        descriptor = self.pool(f)\n        \n        return descriptor","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:29:54.590829Z","iopub.execute_input":"2022-04-04T09:29:54.591124Z","iopub.status.idle":"2022-04-04T09:29:54.601441Z","shell.execute_reply.started":"2022-04-04T09:29:54.591064Z","shell.execute_reply":"2022-04-04T09:29:54.600824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def get_model():\n    tf.keras.backend.clear_session()\n    # enable XLA optmizations\n    tf.config.optimizer.set_jit(True)\n\n    # Input\n    image = tf.keras.layers.Input(INPUT_SHAPE, name='image', dtype=tf.float32)\n    individual_id = tf.keras.layers.Input([], name='individual_id_input', dtype=tf.int32)\n\n\n    # EfficientNetV2 CNN\n    cnn = effnetv2_model.get_model(f'efficientnetv2-{EFNV2_SIZE}', include_top=False, weights=None)\n\n    # Input\n    image = tf.keras.layers.Input(INPUT_SHAPE, name='image', dtype=tf.float32)\n\n    # CNN Outputs\n    embedding, fm5, fm4, fm3, fm2, fm1 = cnn(image, with_endpoints=True)\n    print(f'embedding: {embedding.shape}, fm5: {fm5.shape}, fm4: {fm4.shape}, fm3: {fm3.shape}, fm2: {fm2.shape}, fm1: {fm1.shape}')\n\n    # DOLG Branches\n    descriptor = DolgBranch(DOLG_SIZE, 1)([fm2, fm1])\n\n    # Concatenate Descriptors\n\n    # Dense Layer\n    outputs = tf.keras.layers.Dense(EMBEDDING_SIZE)(descriptor)\n\n    model = tf.keras.models.Model(inputs=image, outputs=outputs)\n    model.trainable = False\n\n    # Load Weights\n    model.load_weights('/kaggle/input/happy-whale-efficientnetv2xl-dolg-training-tpu/model_embedding.h5')\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:29:54.604744Z","iopub.execute_input":"2022-04-04T09:29:54.604993Z","iopub.status.idle":"2022-04-04T09:29:54.614603Z","shell.execute_reply.started":"2022-04-04T09:29:54.604961Z","shell.execute_reply":"2022-04-04T09:29:54.612556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model()","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:29:54.615935Z","iopub.execute_input":"2022-04-04T09:29:54.616176Z","iopub.status.idle":"2022-04-04T09:30:06.448462Z","shell.execute_reply.started":"2022-04-04T09:29:54.616144Z","shell.execute_reply":"2022-04-04T09:30:06.447777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:30:06.449516Z","iopub.execute_input":"2022-04-04T09:30:06.449746Z","iopub.status.idle":"2022-04-04T09:30:06.475531Z","shell.execute_reply.started":"2022-04-04T09:30:06.449713Z","shell.execute_reply":"2022-04-04T09:30:06.474795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The model outputs a descriptor of size 2048\ntf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:30:06.476658Z","iopub.execute_input":"2022-04-04T09:30:06.476881Z","iopub.status.idle":"2022-04-04T09:30:07.211581Z","shell.execute_reply.started":"2022-04-04T09:30:06.476847Z","shell.execute_reply":"2022-04-04T09:30:07.210609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{}},{"cell_type":"code","source":"def show_batch(dataset, rows=5, cols=4):\n    imgs, ids = next(iter(dataset))\n    fig, axes = plt.subplots(nrows=rows, ncols=cols, figsize=(cols*6, rows*6))\n    for r in range(rows):\n        for c in range(cols):\n            idx = r*cols+c\n            img = imgs[idx].numpy().astype(np.float32)\n            img += abs(img.min())\n            img /= img.max()\n            axes[r, c].imshow(img)\n            image_name = ids[idx]\n            axes[r, c].set_title(f'image_name: {image_name}', size=12)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:30:07.213087Z","iopub.execute_input":"2022-04-04T09:30:07.213307Z","iopub.status.idle":"2022-04-04T09:30:07.220344Z","shell.execute_reply.started":"2022-04-04T09:30:07.213281Z","shell.execute_reply":"2022-04-04T09:30:07.219447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_tfrecord(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'image_name': tf.io.FixedLenFeature([], tf.string),\n        'image': tf.io.FixedLenFeature([], tf.string),\n    })\n\n    image_name = tf.cast(features['image_name'], tf.string)\n    \n    image = tf.io.decode_jpeg(features['image'])\n    # Resize Image\n    image = tf.image.resize(image, [IMG_SIZE, IMG_SIZE])\n    # Explicit reshape needed for TPU, tell cimpiler dimensions of image\n    image = tf.reshape(image, INPUT_SHAPE)\n        \n    # ImageNet Normalization\n    image = tf.cast(image, tf.float32)  / 255.0\n    image = (image - IMAGENET_MEAN) / IMAGENET_STD\n    \n    return image, image_name","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:30:07.221823Z","iopub.execute_input":"2022-04-04T09:30:07.222358Z","iopub.status.idle":"2022-04-04T09:30:07.233824Z","shell.execute_reply.started":"2022-04-04T09:30:07.222277Z","shell.execute_reply":"2022-04-04T09:30:07.23308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function(experimental_compile=True)\ndef predict_on_batch(images):\n        return model(images, training=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:30:07.235377Z","iopub.execute_input":"2022-04-04T09:30:07.235664Z","iopub.status.idle":"2022-04-04T09:30:07.246102Z","shell.execute_reply.started":"2022-04-04T09:30:07.235621Z","shell.execute_reply":"2022-04-04T09:30:07.245293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Dataset","metadata":{}},{"cell_type":"code","source":"def get_train_dataset(center_cutout=False):\n    FNAMES_TRAIN_TFRECORDS = tf.io.gfile.glob(f'/kaggle/input/backfintfrecords/*train*.tfrec')\n    \n    train_dataset = tf.data.TFRecordDataset(FNAMES_TRAIN_TFRECORDS, num_parallel_reads=1)\n        \n    train_dataset = train_dataset.map(decode_tfrecord, num_parallel_calls=cpu_count())\n    train_dataset = train_dataset.batch(BATCH_SIZE, drop_remainder=False)\n    train_dataset = train_dataset.prefetch(1)\n    \n    return train_dataset","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:30:07.247481Z","iopub.execute_input":"2022-04-04T09:30:07.247873Z","iopub.status.idle":"2022-04-04T09:30:07.255762Z","shell.execute_reply.started":"2022-04-04T09:30:07.247834Z","shell.execute_reply":"2022-04-04T09:30:07.255147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, image_ids = next(iter(get_train_dataset()))\nprint(f'images shape: {images.shape}, images dtype: {images.dtype}')\nprint(f'image_ids shape: {image_ids.shape}, image_ids dtype: {image_ids.dtype}')\nimg0 = images[0].numpy().astype(np.float32)\ntrain_imgs_info = (img0.mean(), img0.std(), img0.min(), img0.max())\nprint('train img 0 mean: %.3f, 0 std: %.3f, min: %.3f, max: %.3f' % train_imgs_info)\nprint(f'first 5 image ids: {image_ids[:3].numpy().astype(str)}')","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:30:07.257154Z","iopub.execute_input":"2022-04-04T09:30:07.25742Z","iopub.status.idle":"2022-04-04T09:30:07.816636Z","shell.execute_reply.started":"2022-04-04T09:30:07.257384Z","shell.execute_reply":"2022-04-04T09:30:07.815896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_batch(get_train_dataset())","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:30:07.818206Z","iopub.execute_input":"2022-04-04T09:30:07.818735Z","iopub.status.idle":"2022-04-04T09:30:12.43636Z","shell.execute_reply.started":"2022-04-04T09:30:07.818693Z","shell.execute_reply":"2022-04-04T09:30:12.43547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Embeddings\n\nUse the model to generate descriptors for all training images","metadata":{}},{"cell_type":"code","source":"N_TRAIN_SAMPLES = 41574\nprint(f'N_TRAIN_SAMPLES: {N_TRAIN_SAMPLES}, EMBEDDING_SIZE: {EMBEDDING_SIZE}')","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:30:12.437461Z","iopub.execute_input":"2022-04-04T09:30:12.437715Z","iopub.status.idle":"2022-04-04T09:30:12.444215Z","shell.execute_reply.started":"2022-04-04T09:30:12.437678Z","shell.execute_reply":"2022-04-04T09:30:12.443453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if RECOMPUTE_EMBEDDINGS:\n    TRAIN_EMBEDDINGS = np.zeros(shape=[N_TRAIN_SAMPLES, EMBEDDING_SIZE], dtype=np.float32)\n    TRAIN_EMBEDDINGS_IDS = np.empty(shape=N_TRAIN_SAMPLES, dtype=object)\n    \n    total = math.ceil(N_TRAIN_SAMPLES / BATCH_SIZE)\n    offset = 0\n\n    for idx, (images, file_names) in enumerate(tqdm(get_train_dataset(), total=total)):\n        start = offset\n        end = offset + len(images)\n        TRAIN_EMBEDDINGS[start:end] = predict_on_batch(images)\n        TRAIN_EMBEDDINGS_IDS[start:end] = file_names.numpy().astype(str).tolist()\n        offset += len(images)\nelse:\n    TRAIN_EMBEDDINGS = np.load('/kaggle/input/happy-whale-2022-efficientnetv2-xl-dolg-inference/TRAIN_EMBEDDINGS.npy')\n    TRAIN_EMBEDDINGS_IDS = np.load('/kaggle/input/happy-whale-2022-efficientnetv2-xl-dolg-inference/TRAIN_EMBEDDINGS_IDS.npy')\n    \nprint(f'TRAIN_EMBEDDINGS shape: {TRAIN_EMBEDDINGS.shape}, dtype: {TRAIN_EMBEDDINGS.dtype}')\nprint(f'TRAIN_EMBEDDINGS_IDS shape: {TRAIN_EMBEDDINGS_IDS.shape}, dtype: {TRAIN_EMBEDDINGS_IDS.dtype}')","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:30:12.445817Z","iopub.execute_input":"2022-04-04T09:30:12.446335Z","iopub.status.idle":"2022-04-04T09:37:34.460602Z","shell.execute_reply.started":"2022-04-04T09:30:12.446277Z","shell.execute_reply":"2022-04-04T09:37:34.459882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_EMBEDDINGS_TF = tf.constant(TRAIN_EMBEDDINGS)\nTRAIN_EMBEDDINGS_IDS_TF = tf.constant(TRAIN_EMBEDDINGS_IDS)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:37:34.462102Z","iopub.execute_input":"2022-04-04T09:37:34.462363Z","iopub.status.idle":"2022-04-04T09:37:34.852153Z","shell.execute_reply.started":"2022-04-04T09:37:34.462326Z","shell.execute_reply":"2022-04-04T09:37:34.851409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save Train Embeddings\nnp.save('TRAIN_EMBEDDINGS.npy', TRAIN_EMBEDDINGS)\nnp.save('TRAIN_EMBEDDINGS_IDS.npy', TRAIN_EMBEDDINGS_IDS)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:37:34.853671Z","iopub.execute_input":"2022-04-04T09:37:34.854007Z","iopub.status.idle":"2022-04-04T09:37:35.1135Z","shell.execute_reply.started":"2022-04-04T09:37:34.853967Z","shell.execute_reply":"2022-04-04T09:37:35.112718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Embeddings Analysis","metadata":{}},{"cell_type":"code","source":"TRAIN_EMBEDDINGS_SERIES = pd.Series(TRAIN_EMBEDDINGS.flatten())","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:37:35.115044Z","iopub.execute_input":"2022-04-04T09:37:35.115342Z","iopub.status.idle":"2022-04-04T09:37:35.398047Z","shell.execute_reply.started":"2022-04-04T09:37:35.115302Z","shell.execute_reply":"2022-04-04T09:37:35.397271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(TRAIN_EMBEDDINGS_SERIES.describe().to_frame(name='Value'))","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:37:35.399948Z","iopub.execute_input":"2022-04-04T09:37:35.400389Z","iopub.status.idle":"2022-04-04T09:37:38.24456Z","shell.execute_reply.started":"2022-04-04T09:37:35.400348Z","shell.execute_reply":"2022-04-04T09:37:38.242968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nplt.title('Train Embedding Statistics', size=24)\nTRAIN_EMBEDDINGS_SERIES.plot(kind='hist', bins=32)\nplt.xticks(size=12)\nplt.yticks(size=12)\nplt.xlabel('Value', size=16)\nplt.ylabel('Frequency', size=16)\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:37:38.245717Z","iopub.execute_input":"2022-04-04T09:37:38.246069Z","iopub.status.idle":"2022-04-04T09:37:52.425312Z","shell.execute_reply.started":"2022-04-04T09:37:38.246029Z","shell.execute_reply":"2022-04-04T09:37:52.424613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean Up Numpy Train Arrays\ndel TRAIN_EMBEDDINGS, TRAIN_EMBEDDINGS_IDS, TRAIN_EMBEDDINGS_SERIES\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:37:52.426452Z","iopub.execute_input":"2022-04-04T09:37:52.427387Z","iopub.status.idle":"2022-04-04T09:37:52.733489Z","shell.execute_reply.started":"2022-04-04T09:37:52.427339Z","shell.execute_reply":"2022-04-04T09:37:52.732812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Dataset","metadata":{}},{"cell_type":"code","source":"def get_test_dataset(center_cutout=False):\n    FNAMES_TEST_TFRECORDS = tf.io.gfile.glob(f'/kaggle/input/backfintfrecords/*test*.tfrec')\n    \n    test_dataset = tf.data.TFRecordDataset(FNAMES_TEST_TFRECORDS, num_parallel_reads=1)\n        \n    test_dataset = test_dataset.map(decode_tfrecord, num_parallel_calls=cpu_count())\n    test_dataset = test_dataset.batch(BATCH_SIZE, drop_remainder=False)\n    test_dataset = test_dataset.prefetch(1)\n    \n    return test_dataset","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:37:52.73825Z","iopub.execute_input":"2022-04-04T09:37:52.738454Z","iopub.status.idle":"2022-04-04T09:37:52.743746Z","shell.execute_reply.started":"2022-04-04T09:37:52.738429Z","shell.execute_reply":"2022-04-04T09:37:52.742941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, image_ids = next(iter(get_test_dataset()))\nprint(f'images shape: {images.shape}, images dtype: {images.dtype}')\nimg0 = images[0].numpy().astype(np.float32)\ntrain_imgs_info = (img0.mean(), img0.std(), img0.min(), img0.max())\nprint('train img 0 mean: %.3f, 0 std: %.3f, min: %.3f, max: %.3f' % train_imgs_info)\nprint(f'first 5 image ids: {image_ids[:3].numpy().astype(str)}')","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:37:52.745247Z","iopub.execute_input":"2022-04-04T09:37:52.745662Z","iopub.status.idle":"2022-04-04T09:37:53.344187Z","shell.execute_reply.started":"2022-04-04T09:37:52.745626Z","shell.execute_reply":"2022-04-04T09:37:53.343438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_batch(get_test_dataset())","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:37:53.345649Z","iopub.execute_input":"2022-04-04T09:37:53.346201Z","iopub.status.idle":"2022-04-04T09:37:58.576054Z","shell.execute_reply.started":"2022-04-04T09:37:53.346161Z","shell.execute_reply":"2022-04-04T09:37:58.575191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Test Embeddings\n\nGenerate descriptors for all teste images","metadata":{}},{"cell_type":"code","source":"N_TEST_SAMPLES = len(glob.glob('/kaggle/input/happy-whale-and-dolphin/test_images/*.jpg'))\nTEST_EMBEDDINGS = np.zeros(shape=[N_TEST_SAMPLES, EMBEDDING_SIZE], dtype=np.float32)\nTEST_EMBEDDINGS_IDS = np.empty(shape=[N_TEST_SAMPLES], dtype=object)\n\ntotal = math.ceil(N_TEST_SAMPLES / BATCH_SIZE)\noffset = 0\n\nfor idx, (images, image_ids) in enumerate(tqdm(get_test_dataset(), total=total)):\n    start = offset\n    end = offset + len(images)\n    TEST_EMBEDDINGS[start:end] = predict_on_batch(images)\n    TEST_EMBEDDINGS_IDS[start:end] = image_ids.numpy().astype(str)\n    offset += len(images)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:37:58.577395Z","iopub.execute_input":"2022-04-04T09:37:58.577896Z","iopub.status.idle":"2022-04-04T09:43:21.131611Z","shell.execute_reply.started":"2022-04-04T09:37:58.577856Z","shell.execute_reply":"2022-04-04T09:43:21.130874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Add Missing Images\n\nNot all test images are included in the TFRecords. Missing test images are added with a fin crop given in [this](https://www.kaggle.com/c/happy-whale-and-dolphin/discussion/310153) discussion.","metadata":{}},{"cell_type":"code","source":"def get_bbox(bbox_str):\n    if type(bbox_str) is not str:\n        return []\n    \n    bbox = [int(s) for s in bbox_str.replace('[[', '').replace(']]', '').strip().split(' ') if len(s) > 0]\n    \n    return np.array([\n        bbox[:2],\n        [bbox[2], bbox[1]],\n        bbox[2:],\n        [bbox[0], bbox[3]],\n    ])","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:43:21.132858Z","iopub.execute_input":"2022-04-04T09:43:21.133203Z","iopub.status.idle":"2022-04-04T09:43:21.140072Z","shell.execute_reply.started":"2022-04-04T09:43:21.133164Z","shell.execute_reply":"2022-04-04T09:43:21.139095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_bbox_crop(img, bbox, scale=2, debug=False):\n    if len(bbox) == 0:\n        return img\n    \n    img_h, img_w = img.shape[:2]\n    img_min = min(img_h, img_w)\n    \n    bbox_h0 = (bbox[2,1] - bbox[0,1])\n    bbox_h = min(bbox_h0 * scale, img_min)\n    bbox_w0 = (bbox[1,0] - bbox[0,0])\n    bbox_w = min(bbox_w0 * scale, img_min)\n    bbox_s = min(img_min, max(bbox_h, bbox_w))\n    \n    # Update bbox dimensions\n    bbox_y = bbox[0,1]\n    bbox_x = bbox[0,0]\n\n    crop_y = max(0, bbox_y - (bbox_s - bbox_h0) // 2)\n    crop_x = max(0, bbox_x - (bbox_s - bbox_w0) // 2)\n    \n    return img[crop_y:crop_y+bbox_s, crop_x:crop_x+bbox_s]","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:43:21.14136Z","iopub.execute_input":"2022-04-04T09:43:21.142011Z","iopub.status.idle":"2022-04-04T09:43:21.153006Z","shell.execute_reply.started":"2022-04-04T09:43:21.141971Z","shell.execute_reply":"2022-04-04T09:43:21.152291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_image(file_path, bbox):\n    img = imageio.imread(file_path)\n    # Grayscale to RGB\n    if len(img.shape) == 2:\n        img = np.stack([img] * 3, axis=2)\n        \n    # Crop\n    img = get_bbox_crop(img, bbox)\n    \n    h, w, _ = img.shape\n\n    r = IMG_SIZE / min(w, h)\n    \n    img = cv2.resize(img, (IMG_SIZE, IMG_SIZE), interpolation=cv2.INTER_LANCZOS4)\n    \n    img = img.astype(np.float32)  / 255.0\n    img = (img - IMAGENET_MEAN) / IMAGENET_STD\n    \n    return img","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:43:21.154106Z","iopub.execute_input":"2022-04-04T09:43:21.154438Z","iopub.status.idle":"2022-04-04T09:43:21.16895Z","shell.execute_reply.started":"2022-04-04T09:43:21.154386Z","shell.execute_reply":"2022-04-04T09:43:21.167767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_fin_annotations = pd.read_csv('/kaggle/input/happy-whale-2022-fin-bounding-boxes/test_fin_annotations.csv')\n\n# Add Bounding Box Coordinates\ntest_fin_annotations['bbox_np'] = test_fin_annotations['bbox'].apply(get_bbox)\n\nTEST_EMBEDDINGS_IDS_SET = set(TEST_EMBEDDINGS_IDS)\n\nfor idx, row in tqdm(test_fin_annotations.iterrows(), total=N_TEST_SAMPLES):\n    image_id = row['image_id']\n    bbox_np = row['bbox_np']\n    if image_id not in TEST_EMBEDDINGS_IDS_SET:\n        print(f'Added image_id: {image_id}, bbox_np: {bbox_np}')\n        image = process_image(row['image_path'], bbox_np)\n        \n        TEST_EMBEDDINGS[offset] = predict_on_batch(image[np.newaxis,:])\n        TEST_EMBEDDINGS_IDS[offset] = image_id\n        \n        offset += 1","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:43:21.17024Z","iopub.execute_input":"2022-04-04T09:43:21.170965Z","iopub.status.idle":"2022-04-04T09:43:33.671923Z","shell.execute_reply.started":"2022-04-04T09:43:21.170922Z","shell.execute_reply":"2022-04-04T09:43:33.670975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Test Tensorflow Tensor","metadata":{}},{"cell_type":"code","source":"print(f'TEST_EMBEDDINGS shape: {TEST_EMBEDDINGS.shape}, dtype: {TEST_EMBEDDINGS.dtype}')\nprint(f'TEST_EMBEDDINGS_IDS shape: {TEST_EMBEDDINGS_IDS.shape}, dtype: {TEST_EMBEDDINGS_IDS.dtype}')","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:43:33.673432Z","iopub.execute_input":"2022-04-04T09:43:33.673692Z","iopub.status.idle":"2022-04-04T09:43:33.678919Z","shell.execute_reply.started":"2022-04-04T09:43:33.673657Z","shell.execute_reply":"2022-04-04T09:43:33.67825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_EMBEDDINGS_TF = tf.constant(TEST_EMBEDDINGS)\nTEST_EMBEDDINGS_IDS_TF = tf.constant(TEST_EMBEDDINGS_IDS)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:43:33.680298Z","iopub.execute_input":"2022-04-04T09:43:33.681086Z","iopub.status.idle":"2022-04-04T09:43:33.939845Z","shell.execute_reply.started":"2022-04-04T09:43:33.680978Z","shell.execute_reply":"2022-04-04T09:43:33.938831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Embeddings Analysis","metadata":{}},{"cell_type":"code","source":"TEST_EMBEDDINGS_SERIES = pd.Series(TEST_EMBEDDINGS.flatten())","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:43:33.941263Z","iopub.execute_input":"2022-04-04T09:43:33.941589Z","iopub.status.idle":"2022-04-04T09:43:34.121342Z","shell.execute_reply.started":"2022-04-04T09:43:33.94155Z","shell.execute_reply":"2022-04-04T09:43:34.120626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(TEST_EMBEDDINGS_SERIES.describe().to_frame(name='Value'))","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:43:34.122694Z","iopub.execute_input":"2022-04-04T09:43:34.122955Z","iopub.status.idle":"2022-04-04T09:43:35.968509Z","shell.execute_reply.started":"2022-04-04T09:43:34.122898Z","shell.execute_reply":"2022-04-04T09:43:35.967785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nplt.title('Test Embedding Statistics', size=24)\nTEST_EMBEDDINGS_SERIES.plot(kind='hist', bins=32)\nplt.xticks(size=12)\nplt.yticks(size=12)\nplt.xlabel('Value', size=16)\nplt.ylabel('Frequency', size=16)\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:43:35.969799Z","iopub.execute_input":"2022-04-04T09:43:35.970363Z","iopub.status.idle":"2022-04-04T09:43:45.930112Z","shell.execute_reply.started":"2022-04-04T09:43:35.970321Z","shell.execute_reply":"2022-04-04T09:43:45.929366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del TEST_EMBEDDINGS, TEST_EMBEDDINGS_IDS, TEST_EMBEDDINGS_SERIES\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:43:45.931296Z","iopub.execute_input":"2022-04-04T09:43:45.932062Z","iopub.status.idle":"2022-04-04T09:43:46.192233Z","shell.execute_reply.started":"2022-04-04T09:43:45.932022Z","shell.execute_reply":"2022-04-04T09:43:46.191527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# File Name to Individual Id","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/happy-whale-and-dolphin/train.csv')\ndisplay(train.head())","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:43:46.193995Z","iopub.execute_input":"2022-04-04T09:43:46.194247Z","iopub.status.idle":"2022-04-04T09:43:46.272454Z","shell.execute_reply.started":"2022-04-04T09:43:46.194212Z","shell.execute_reply":"2022-04-04T09:43:46.271797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We need to predict the individual_id, not the file name\n# This dictionary maps a file name to an individual id\nfile_name2individual_id = train[['image', 'individual_id']].set_index('image').squeeze().to_dict()","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:43:46.273741Z","iopub.execute_input":"2022-04-04T09:43:46.274148Z","iopub.status.idle":"2022-04-04T09:43:46.33775Z","shell.execute_reply.started":"2022-04-04T09:43:46.27411Z","shell.execute_reply":"2022-04-04T09:43:46.337122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark Predictions and Confidence","metadata":{}},{"cell_type":"code","source":"@tf.function(experimental_compile=False)\ndef get_pred(query, k=1000):\n    # Expand dims and tile test descriptor\n    query_tile = tf.expand_dims(query, axis=0)\n    query_tile = tf.tile(input=query_tile, multiples=[N_TRAIN_SAMPLES, 1])\n    # Compute cosine similarity between test descriptor and all train descriptors\n    cors = -1.0 * tf.keras.losses.cosine_similarity(query_tile, TRAIN_EMBEDDINGS_TF)\n    # Retrieve the indices and correlations of the top 100 train descriptors\n    cors_pred, indices = tf.math.top_k(cors, k=k)\n    # Get the file names of the top 100 most similar train images\n    labels_pred = tf.gather(TRAIN_EMBEDDINGS_IDS_TF, indices)\n    \n    return labels_pred, cors_pred","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:45:42.974155Z","iopub.execute_input":"2022-04-04T09:45:42.97442Z","iopub.status.idle":"2022-04-04T09:45:42.98221Z","shell.execute_reply.started":"2022-04-04T09:45:42.97439Z","shell.execute_reply":"2022-04-04T09:45:42.980787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"# Raw labels and correlations to actual individual_id prediction string\ndef get_nunique(labels_pred, cors_pred, k=5):\n    # Dictionary to keep track of individual_ids and summed correlation\n    individual_ids_dict = {}\n    # Loop over labels and correlations\n    for l, c in zip(labels_pred, cors_pred):\n        # Map label to individual_id\n        individual_id = file_name2individual_id.get(l)\n        # If individual_id is not in dictionary add it with correlation\n        if individual_id not in individual_ids_dict:\n            individual_ids_dict[individual_id] = c\n        # If individual_id is in dictionary add correlation\n        else:\n            individual_ids_dict[individual_id] += c\n        \n        # If 5 unique individual_ids are found stop\n        if len(individual_ids_dict.keys()) == k:\n            break\n        \n    # Make prediction string\n    predictions = ' '.join(list(individual_ids_dict.keys()))\n    # Make list of summed correlations per individual_id\n    individual_id_cors = list(individual_ids_dict.values())\n            \n    return predictions, individual_id_cors","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:45:43.258526Z","iopub.execute_input":"2022-04-04T09:45:43.258843Z","iopub.status.idle":"2022-04-04T09:45:43.266866Z","shell.execute_reply.started":"2022-04-04T09:45:43.258811Z","shell.execute_reply":"2022-04-04T09:45:43.266129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Actual Prediction Loop","metadata":{}},{"cell_type":"code","source":"df_dicts = []\nscore_list = []\ntest_nl_score_list = []\ncors_max_list = []\nfor idx, (query, image_id) in tqdm(enumerate(zip(TEST_EMBEDDINGS_TF, TEST_EMBEDDINGS_IDS_TF)), total=N_TEST_SAMPLES):\n    # Top 1000 most similar train file names and cosine similarities\n    labels_pred, cors_pred = get_pred(query)\n    # Convert labels and correlations to prediction string and summed correlation\n    labels_pred = labels_pred.numpy().astype(str)\n    \n    # Maximum Correlation\n    cors_pred = cors_pred.numpy()\n    c_max = cors_pred.max()\n    cors_max_list.append(c_max)\n    \n    # Get top 5 unique predictions and summed correlation\n    predictions, individual_id_cors = get_nunique(labels_pred, cors_pred)\n    \n    # image_id to string\n    image_id = image_id.numpy().decode()\n\n    # Append Prediction dictionary\n    df_dicts.append({\n        'image': image_id,\n        'predictions': predictions,\n        'correlations': individual_id_cors,\n        'cors_max': c_max,\n    })","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:45:43.891515Z","iopub.execute_input":"2022-04-04T09:45:43.892216Z","iopub.status.idle":"2022-04-04T09:47:15.119458Z","shell.execute_reply.started":"2022-04-04T09:45:43.892172Z","shell.execute_reply":"2022-04-04T09:47:15.118745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Correlation Analysis","metadata":{}},{"cell_type":"code","source":"# Maximum correlation distribution\nplt.figure(figsize=(15,8))\nplt.title('Highest Correlation Distribution', size=24)\npd.Series(cors_max_list).plot(kind='hist', bins=32)\nplt.xlim(0,1)\nplt.xticks(np.arange(0, 1.1, 0.1), size=12)\nplt.yticks(size=12)\nplt.xlabel('Value', size=16)\nplt.ylabel('Frequency', size=16)\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:47:15.121234Z","iopub.execute_input":"2022-04-04T09:47:15.121642Z","iopub.status.idle":"2022-04-04T09:47:15.394536Z","shell.execute_reply.started":"2022-04-04T09:47:15.121602Z","shell.execute_reply":"2022-04-04T09:47:15.393853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Introduce New Individuals","metadata":{}},{"cell_type":"code","source":"# Threshold for adding \"new individual\" is based on N-th percentile of maximum correlation\ncors_percentile = np.percentile(cors_max_list, NEW_INDIVIDUAL_PERCENTILE)\nprint(f'cors_percentile: {cors_percentile:.2f}')","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:47:15.395876Z","iopub.execute_input":"2022-04-04T09:47:15.396136Z","iopub.status.idle":"2022-04-04T09:47:15.405803Z","shell.execute_reply.started":"2022-04-04T09:47:15.3961Z","shell.execute_reply":"2022-04-04T09:47:15.404937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add new individual if maximum correlation is below threshold\nfor v in df_dicts:\n    if v['cors_max'] < cors_percentile:\n        v['predictions'] = 'new_individual ' + ' '.join(v['predictions'].split(' ')[:-1])","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:47:15.408113Z","iopub.execute_input":"2022-04-04T09:47:15.408498Z","iopub.status.idle":"2022-04-04T09:47:15.492873Z","shell.execute_reply.started":"2022-04-04T09:47:15.408458Z","shell.execute_reply":"2022-04-04T09:47:15.49224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save Submission","metadata":{}},{"cell_type":"code","source":"# Create submission Pandas DataFrame\nsubmission = pd.DataFrame.from_dict(df_dicts).sort_values('image')","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:47:15.493979Z","iopub.execute_input":"2022-04-04T09:47:15.494215Z","iopub.status.idle":"2022-04-04T09:47:15.572851Z","shell.execute_reply.started":"2022-04-04T09:47:15.49418Z","shell.execute_reply":"2022-04-04T09:47:15.572136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(submission.head(10))","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:47:15.574108Z","iopub.execute_input":"2022-04-04T09:47:15.574367Z","iopub.status.idle":"2022-04-04T09:47:15.590706Z","shell.execute_reply.started":"2022-04-04T09:47:15.57433Z","shell.execute_reply":"2022-04-04T09:47:15.589945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(submission.info())","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:47:15.59196Z","iopub.execute_input":"2022-04-04T09:47:15.592278Z","iopub.status.idle":"2022-04-04T09:47:15.616434Z","shell.execute_reply.started":"2022-04-04T09:47:15.592241Z","shell.execute_reply":"2022-04-04T09:47:15.615735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save Submission as CSV\nsubmission[['image', 'predictions']].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:47:15.617635Z","iopub.execute_input":"2022-04-04T09:47:15.617867Z","iopub.status.idle":"2022-04-04T09:47:15.745884Z","shell.execute_reply.started":"2022-04-04T09:47:15.617834Z","shell.execute_reply":"2022-04-04T09:47:15.745138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_unique_predictions = submission['predictions'].str.split().apply(set).apply(len)\n\n# Count of Unique Individual IDs per Prediction [test whether all 5]\ndisplay(n_unique_predictions.value_counts().sort_index().to_frame(name='count'))\n# Ratio of Unique Individual IDs per Prediction [test whether 100% 5]\ndisplay(n_unique_predictions.value_counts(normalize=True).sort_index().to_frame(name='ratio'))","metadata":{"execution":{"iopub.status.busy":"2022-04-04T09:47:15.74862Z","iopub.execute_input":"2022-04-04T09:47:15.748815Z","iopub.status.idle":"2022-04-04T09:47:15.85439Z","shell.execute_reply.started":"2022-04-04T09:47:15.74879Z","shell.execute_reply":"2022-04-04T09:47:15.853699Z"},"trusted":true},"execution_count":null,"outputs":[]}]}