{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T02:41:10.632346Z","iopub.execute_input":"2022-03-29T02:41:10.632707Z","iopub.status.idle":"2022-03-29T02:41:16.279285Z","shell.execute_reply.started":"2022-03-29T02:41:10.63261Z","shell.execute_reply":"2022-03-29T02:41:16.278363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -U keras-cv-attention-models\n!pip install -q efficientnet\n!pip install tensorflow_addons\n\nfrom kaggle_datasets import KaggleDatasets\nfrom keras_cv_attention_models import nfnets, efficientnet\nimport re\nimport os\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold, train_test_split\nfrom tensorflow.keras import backend as K\nimport tensorflow_addons as tfa\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nimport pickle\nimport json\nimport tensorflow_hub as tfhub\nfrom datetime import datetime","metadata":{"execution":{"iopub.status.busy":"2022-03-29T02:41:16.281Z","iopub.execute_input":"2022-03-29T02:41:16.281223Z","iopub.status.idle":"2022-03-29T02:41:42.675792Z","shell.execute_reply.started":"2022-03-29T02:41:16.28119Z","shell.execute_reply":"2022-03-29T02:41:42.674891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"class config:\n    \n    bnneck = False\n    gem = False\n    MIX_UP = False\n    PSEUDO = True\n    TEST = True\n    EVALUATE = True\n    \n    SEED = 42\n    FOLD_TO_RUN = 1\n    FOLDS = 10\n    DEBUG = False\n    RESUME = False\n    RESUME_EPOCH = None\n    \n    \n    ### Dataset\n    BATCH_SIZE = 16 * strategy.num_replicas_in_sync\n    IMAGE_SIZE = 512\n    # IMAGE_SIZE = 32\n    N_CLASSES = 15587\n    \n    ### Model\n    model_type = 'effnetv1'  # nfnet effnetv1 effnetv2\n    EFF_NET = 7\n    # EFF_NET = 0\n    \n    EFF_NETV2 = 's-21k-ft1k'\n    FREEZE_BATCH_NORM = True\n    head = 'arcface' # arcface  curricular\n    EPOCHS = 35\n    LR = 0.001\n    \n    ### Augmentations\n    CUTOUT = False\n    \n    ### Save-Directory\n    save_dir = '.'\n\n    \ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)\n\n# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)\n    \ndef is_interactive():\n    return 'runtime'    in get_ipython().config.IPKernelApp.connection_file\nIS_INTERACTIVE = is_interactive()\nprint(IS_INTERACTIVE)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T02:41:42.677163Z","iopub.execute_input":"2022-03-29T02:41:42.677475Z","iopub.status.idle":"2022-03-29T02:41:42.69067Z","shell.execute_reply.started":"2022-03-29T02:41:42.677431Z","shell.execute_reply":"2022-03-29T02:41:42.689733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_NAME = config.model_type\nif config.model_type == 'effnetv1':\n    MODEL_NAME = f'effnetv1_b{config.EFF_NET}'\nelif config.model_type == 'effnetv2':\n    MODEL_NAME = f'effnetv2_{config.EFF_NETV2}'\n\nconfig.MODEL_NAME = MODEL_NAME\nprint(MODEL_NAME)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T02:41:42.692574Z","iopub.execute_input":"2022-03-29T02:41:42.69282Z","iopub.status.idle":"2022-03-29T02:41:42.706392Z","shell.execute_reply.started":"2022-03-29T02:41:42.692793Z","shell.execute_reply":"2022-03-29T02:41:42.705458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(config.save_dir+'/config.json', 'w') as fp:\n    json.dump({x:dict(config.__dict__)[x] for x in dict(config.__dict__) if not x.startswith('_')}, fp)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T02:41:42.707737Z","iopub.execute_input":"2022-03-29T02:41:42.708308Z","iopub.status.idle":"2022-03-29T02:41:42.717626Z","shell.execute_reply.started":"2022-03-29T02:41:42.708269Z","shell.execute_reply":"2022-03-29T02:41:42.71698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_PATH = 'gs://kds-d916c3252bf3bc5b3500b904f05f51ce57c8df85221d11b7711bcda9'  # Get GCS Path from kaggle notebook if GCS Path is expired\n# if not IS_COLAB:\n#     GCS_PATH1 = KaggleDatasets().get_gcs_path('randomdatasetc')\n    \nGCS_PATH1 = 'gs://kds-c021fa6ca054971f5d9333f955bf9fc12fa28d1ba64b2a91ae9fdf8e'\ntrain_files = np.sort(np.array(tf.io.gfile.glob(GCS_PATH1 + '/happywhale-2022-train*.tfrec')))\ntest_files = np.sort(np.array(tf.io.gfile.glob(GCS_PATH1 + '/happywhale-2022-test*.tfrec')))\nprint(GCS_PATH)\nprint(len(train_files),len(test_files),count_data_items(train_files),count_data_items(test_files))","metadata":{"execution":{"iopub.status.busy":"2022-03-29T00:00:48.423239Z","iopub.execute_input":"2022-03-29T00:00:48.423591Z","iopub.status.idle":"2022-03-29T00:00:48.604646Z","shell.execute_reply.started":"2022-03-29T00:00:48.423547Z","shell.execute_reply":"2022-03-29T00:00:48.603732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"code","source":"def cutmix(posting_id, image, label, matches, PROBABILITY):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = config.IMAGE_SIZE\n    CLASSES = config.N_CLASSES\n    AUG_BATCH = config.BATCH_SIZE\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n\n        lab1 = label[j,]\n        lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return posting_id, image2,label2, matches\n\n\ndef mixup(posting_id, image, label, matches, PROBABILITY):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = config.IMAGE_SIZE\n    CLASSES = config.N_CLASSES\n    AUG_BATCH = config.BATCH_SIZE\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        # a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n        a = np.random.beta(0.5, 0.5) * P\n        # MAKE MIXUP IMAGE        \n        img1 = image[j,]\n        img2 = image[k,]\n        \n        imgs.append((1-a)*img1 + a*img2)\n\n        lab1 = label[j,]\n        lab2 = label[k,]\n        \n        labs.append((1-a)*lab1 + a*lab2)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return posting_id, image2, label2, matches\n\n\ndef transform(posting_id, image, label, matches):\n    # THIS FUNCTION APPLIES BOTH CUTMIX AND MIXUP\n    DIM = config.IMAGE_SIZE\n    CLASSES = config.N_CLASSES\n    AUG_BATCH = config.BATCH_SIZE\n    SWITCH = 0\n    CUTMIX_PROB = 0.5\n    MIXUP_PROB = 0.5\n    # FOR SWITCH PERCENT OF TIME WE DO CUTMIX AND (1-SWITCH) WE DO MIXUP\n    _, image2, label2, _ = cutmix(posting_id, image, label, matches, CUTMIX_PROB)\n    _, image3, label3, _ = mixup(posting_id, image, label, matches, MIXUP_PROB)\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        P = tf.cast( tf.random.uniform([],0,1)<=SWITCH, tf.float32)\n        imgs.append(P*image2[j,]+(1-P)*image3[j,])\n        labs.append(P*label2[j,]+(1-P)*label3[j,])\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return posting_id, image4, label4, matches\n\n\n\ndef onehot(posting_id, image, label, matches):\n    return posting_id, image,tf.one_hot(label, config.N_CLASSES), matches\n\n\ndef arcface_format(posting_id, image, label_group, matches):\n    return posting_id, {'inp1': image, 'inp2': label_group}, label_group, matches\n\ndef arcface_inference_format(posting_id, image, label_group, matches):\n    return image,posting_id\n\ndef arcface_eval_format(posting_id, image, label_group, matches):\n    return image,label_group\n\n# Data augmentation function\ndef data_augment(posting_id, image, label_group, matches):\n\n    ### CUTOUT\n    if tf.random.uniform([])>0.5 and config.CUTOUT:\n      N_CUTOUT = 6\n      for cutouts in range(N_CUTOUT):\n        if tf.random.uniform([])>0.5:\n           DIM = config.IMAGE_SIZE\n           CUTOUT_LENGTH = DIM//8\n           x1 = tf.cast( tf.random.uniform([],0,DIM-CUTOUT_LENGTH),tf.int32)\n           x2 = tf.cast( tf.random.uniform([],0,DIM-CUTOUT_LENGTH),tf.int32)\n           filter_ = tf.concat([tf.zeros((x1,CUTOUT_LENGTH)),tf.ones((CUTOUT_LENGTH,CUTOUT_LENGTH)),tf.zeros((DIM-x1-CUTOUT_LENGTH,CUTOUT_LENGTH))],axis=0)\n           filter_ = tf.concat([tf.zeros((DIM,x2)),filter_,tf.zeros((DIM,DIM-x2-CUTOUT_LENGTH))],axis=1)\n           cutout = tf.reshape(1-filter_,(DIM,DIM,1))\n           image = cutout*image\n\n    image = tf.image.random_flip_left_right(image)\n    # image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_hue(image, 0.01)\n    image = tf.image.random_saturation(image, 0.70, 1.30)\n    image = tf.image.random_contrast(image, 0.80, 1.20)\n    image = tf.image.random_brightness(image, 0.10)\n    return posting_id, image, label_group, matches\n\ndef data_augment_test(posting_id, image, label_group, matches):\n\n    return posting_id, image, label_group, matches\n\n# Function to decode our images\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels = 3)\n    image = tf.image.resize(image, [config.IMAGE_SIZE,config.IMAGE_SIZE])\n    image = tf.cast(image, tf.float32) / 255.0\n    return image\n\n# This function parse our images and also get the target variable\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64),\n#         \"matches\": tf.io.FixedLenFeature([], tf.string)\n    }\n\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    posting_id = example['image_name']\n    image = decode_image(example['image'])\n#     label_group = tf.one_hot(tf.cast(example['label_group'], tf.int32), depth = N_CLASSES)\n    label_group = tf.cast(example['target'], tf.int32)\n#     matches = example['matches']\n    matches = 1\n    return posting_id, image, label_group, matches\n\n# This function loads TF Records and parse them into tensors\ndef load_dataset(filenames, ordered = False):\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False \n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)\n#     dataset = dataset.cache()\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls = AUTO) \n    return dataset\n\n# This function is to get our training tensors\ndef get_training_dataset(filenames):\n    dataset = load_dataset(filenames, ordered = False)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.map(onehot, num_parallel_calls=AUTO)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    if config.MIX_UP:\n        dataset = dataset.map(transform, num_parallel_calls=AUTO)\n    dataset = dataset.map(arcface_format, num_parallel_calls = AUTO)\n    dataset = dataset.map(lambda posting_id, image, label_group, matches: (image, label_group))\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n# This function is to get our training tensors\ndef get_val_dataset(filenames):\n    dataset = load_dataset(filenames, ordered = True)\n    dataset = dataset.map(onehot, num_parallel_calls=AUTO)\n    dataset = dataset.map(data_augment_test, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_format, num_parallel_calls = AUTO)\n    dataset = dataset.map(lambda posting_id, image, label_group, matches: (image, label_group))\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2022-03-29T00:00:48.606109Z","iopub.execute_input":"2022-03-29T00:00:48.60684Z","iopub.status.idle":"2022-03-29T00:00:48.661132Z","shell.execute_reply.started":"2022-03-29T00:00:48.606802Z","shell.execute_reply":"2022-03-29T00:00:48.660124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"# class CurricularFace(tf.keras.layers.Layer):\n#     def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n#                  ls_eps=0.0, **kwargs):\n#         super(CurricularFace, self).__init__(**kwargs)\n\n#         self.n_classes = n_classes\n#         self.s = s\n#         self.m = m\n#         self.ls_eps = ls_eps\n#         self.easy_margin = easy_margin\n#         self.cos_m = tf.math.cos(m)\n#         self.sin_m = tf.math.sin(m)\n#         self.th = tf.math.cos(math.pi - m)\n#         self.mm = tf.math.sin(math.pi - m) * m\n#         self._USE_V2_BEHAVIOR = True\n\n#     def _assign_new_value(self, variable, value):\n#         with K.name_scope('AssignNewValue') as scope:\n#           if tf.compat.v1.executing_eagerly_outside_functions():\n#             return variable.assign(value, name=scope)\n#           else:\n#             with tf.compat.v1.colocate_with(variable):  # pylint: disable=protected-access\n#               return tf.compat.v1.assign(variable, value, name=scope)\n\n\n#     def _get_training_value(self, training=None):\n#         if training is None:\n#           training = K.learning_phase()\n#         if self._USE_V2_BEHAVIOR:\n#           if isinstance(training, int):\n#             training = bool(training)\n#           if not self.trainable:\n#             # When the layer is not trainable, it overrides the value passed from\n#             # model.\n#             training = False\n#         return training\n\n\n#     def get_config(self):\n\n#         config = super().get_config().copy()\n#         config.update({\n#             'n_classes': self.n_classes,\n#             's': self.s,\n#             'm': self.m,\n#             'ls_eps': self.ls_eps,\n#             'easy_margin': self.easy_margin,\n#         })\n#         return config\n\n#     def build(self, input_shape):\n#         super(CurricularFace, self).build(input_shape[0])\n\n#         self.W = self.add_weight(\n#             name='W',\n#             shape=(int(input_shape[0][-1]), self.n_classes),\n#             initializer='glorot_uniform',\n#             dtype='float32',\n#             trainable=True,\n#             regularizer=None)\n        \n#         self.t = self.add_weight(\n#             name='t',\n#             shape=(1),\n#             initializer=tf.zeros_initializer(),\n#             dtype='float32',\n#             trainable=False,\n#             regularizer=None,\n#             aggregation=tf.VariableAggregation.MEAN,\n#             experimental_autocast=False,\n#             synchronization=tf.VariableSynchronization.ON_READ)\n        \n#     def call(self, inputs, training=None):\n#         X, y = inputs\n#         y = tf.cast(y, dtype=tf.int32)\n\n#         do_training = self._get_training_value(training)\n\n#         if do_training:\n#             cosine = tf.matmul(\n#                 tf.math.l2_normalize(X, axis=1),\n#                 tf.math.l2_normalize(self.W, axis=0)\n#             )\n#             sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n#             phi = cosine * self.cos_m - sine * self.sin_m\n\n#             target_logit = tf.reduce_sum(cosine * tf.cast(tf.one_hot(y, depth=self.n_classes),dtype=cosine.dtype), axis=-1)\n#             sin_theta = tf.math.sqrt(1.0 - tf.math.pow(target_logit, 2))\n#             cos_theta_m = target_logit * self.cos_m - sin_theta * self.sin_m\n\n#             phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n#             one_hot = tf.cast(\n#                 tf.one_hot(y, depth=self.n_classes),\n#                 dtype=cosine.dtype\n#             )\n        \n#             t = tf.reduce_mean(target_logit) * 0.01 + (1 - 0.01) * self.t\n#             self._assign_new_value(self.t, t)\n#             cosine = tf.where(cosine > tf.expand_dims(cos_theta_m, axis=-1), cosine*(self.t+cosine), cosine)\n\n#             if self.ls_eps > 0:\n#                 one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n#             output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n#             output *= self.s\n\n#         else:\n#             output = tf.matmul(\n#                 tf.math.l2_normalize(X, axis=1),\n#                 tf.math.l2_normalize(self.W, axis=0)\n#             )\n\n#         return output\n    \n    \n# # Arcmarginproduct class keras layer\n# class ArcMarginProduct(tf.keras.layers.Layer):\n#     '''\n#     Implements large margin arc distance.\n\n#     Reference:\n#         https://arxiv.org/pdf/1801.07698.pdf\n#         https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n#             blob/master/src/modeling/metric_learning.py\n#     '''\n#     def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n#                  ls_eps=0.0, **kwargs):\n\n#         super(ArcMarginProduct, self).__init__(**kwargs)\n\n#         self.n_classes = n_classes\n#         self.s = s\n#         self.m = m\n#         self.ls_eps = ls_eps\n#         self.easy_margin = easy_margin\n#         self.cos_m = tf.math.cos(m)\n#         self.sin_m = tf.math.sin(m)\n#         self.th = tf.math.cos(math.pi - m)\n#         self.mm = tf.math.sin(math.pi - m) * m\n\n#     def get_config(self):\n\n#         config = super().get_config().copy()\n#         config.update({\n#             'n_classes': self.n_classes,\n#             's': self.s,\n#             'm': self.m,\n#             'ls_eps': self.ls_eps,\n#             'easy_margin': self.easy_margin,\n#         })\n#         return config\n\n#     def build(self, input_shape):\n#         super(ArcMarginProduct, self).build(input_shape[0])\n\n#         self.W = self.add_weight(\n#             name='W',\n#             shape=(int(input_shape[0][-1]), self.n_classes),\n#             initializer='glorot_uniform',\n#             dtype='float32',\n#             trainable=True,\n#             regularizer=None)\n\n#     def call(self, inputs):\n#         X, y = inputs\n#         y = tf.cast(y, dtype=tf.int32)\n#         cosine = tf.matmul(\n#             tf.math.l2_normalize(X, axis=1),\n#             tf.math.l2_normalize(self.W, axis=0)\n#         )\n#         sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n#         phi = cosine * self.cos_m - sine * self.sin_m\n#         if self.easy_margin:\n#             phi = tf.where(cosine > 0, phi, cosine)\n#         else:\n#             phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n#         one_hot = tf.cast(\n#             tf.one_hot(y, depth=self.n_classes),\n#             dtype=cosine.dtype\n#         )\n#         if self.ls_eps > 0:\n#             one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n#         output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n#         output *= self.s\n#         return output\n    \n# class GeMPoolingLayer(tf.keras.layers.Layer):\n#     def __init__(self, p=1., train_p=False):\n#         super().__init__()\n#         if train_p:\n#             self.p = tf.Variable(p, dtype=tf.float32)\n#         else:\n#             self.p = p\n#         self.eps = 1e-6\n\n#     def call(self, inputs: tf.Tensor, **kwargs):\n#         inputs = tf.clip_by_value(inputs, clip_value_min=1e-6, clip_value_max=tf.reduce_max(inputs))\n#         inputs = tf.pow(inputs, self.p)\n#         inputs = tf.reduce_mean(inputs, axis=[1, 2], keepdims=False)\n#         inputs = tf.pow(inputs, 1./self.p)\n#         return inputs\n\nclass CurricularFace(tf.keras.layers.Layer):\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n        super(CurricularFace, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n        self._USE_V2_BEHAVIOR = True\n\n    def _assign_new_value(self, variable, value):\n        with K.name_scope('AssignNewValue') as scope:\n          if tf.compat.v1.executing_eagerly_outside_functions():\n            return variable.assign(value, name=scope)\n          else:\n            with tf.compat.v1.colocate_with(variable):  # pylint: disable=protected-access\n              return tf.compat.v1.assign(variable, value, name=scope)\n\n\n    def _get_training_value(self, training=None):\n        if training is None:\n          training = K.learning_phase()\n        if self._USE_V2_BEHAVIOR:\n          if isinstance(training, int):\n            training = bool(training)\n          if not self.trainable:\n            # When the layer is not trainable, it overrides the value passed from\n            # model.\n            training = False\n        return training\n\n\n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(CurricularFace, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n        \n        self.t = self.add_weight(\n            name='t',\n            shape=(1),\n            initializer=tf.zeros_initializer(),\n            dtype='float32',\n            trainable=False,\n            regularizer=None,\n            aggregation=tf.VariableAggregation.MEAN,\n            experimental_autocast=False,\n            synchronization=tf.VariableSynchronization.ON_READ)\n        \n    def call(self, inputs, training=None):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n\n        do_training = self._get_training_value(training)\n\n        if do_training:\n            cosine = tf.matmul(\n                tf.math.l2_normalize(X, axis=1),\n                tf.math.l2_normalize(self.W, axis=0)\n            )\n            sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n            phi = cosine * self.cos_m - sine * self.sin_m\n\n            # target_logit = tf.reduce_sum(cosine * tf.cast(tf.one_hot(y, depth=self.n_classes),dtype=cosine.dtype), axis=-1)\n            target_logit = tf.reduce_sum(cosine * tf.cast(y, dtype=cosine.dtype), axis=-1)\n            sin_theta = tf.math.sqrt(1.0 - tf.math.pow(target_logit, 2))\n            cos_theta_m = target_logit * self.cos_m - sin_theta * self.sin_m\n\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n#             one_hot = tf.cast(\n#                 tf.one_hot(y, depth=self.n_classes),\n#                 dtype=cosine.dtype\n#             )\n            one_hot = tf.cast(\n            y,\n            dtype=cosine.dtype\n        ) \n            t = tf.reduce_mean(target_logit) * 0.01 + (1 - 0.01) * self.t\n            self._assign_new_value(self.t, t)\n            cosine = tf.where(cosine > tf.expand_dims(cos_theta_m, axis=-1), cosine*(self.t+cosine), cosine)\n\n            if self.ls_eps > 0:\n                one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n            output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n            output *= self.s\n\n        else:\n            output = tf.matmul(\n                tf.math.l2_normalize(X, axis=1),\n                tf.math.l2_normalize(self.W, axis=0)\n            )\n\n        return output\n    \n# Arcmarginproduct class keras layer\nclass ArcMarginProduct(tf.keras.layers.Layer):\n    '''\n    Implements large margin arc distance.\n\n    Reference:\n        https://arxiv.org/pdf/1801.07698.pdf\n        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n            blob/master/src/modeling/metric_learning.py\n    '''\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n\n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n\n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n#         one_hot = tf.cast(\n#             tf.one_hot(y, depth=self.n_classes),\n#             dtype=cosine.dtype\n#         )\n        one_hot = tf.cast(\n            y,\n            dtype=cosine.dtype\n        ) \n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output\n    \n    \nclass GeMPoolingLayer(tf.keras.layers.Layer):\n    def __init__(self, p=1., train_p=False):\n        super().__init__()\n        if train_p:\n            self.p = tf.Variable(p, dtype=tf.float32)\n        else:\n            self.p = p\n        self.eps = 1e-6\n\n    def call(self, inputs: tf.Tensor, **kwargs):\n        inputs = tf.clip_by_value(inputs, clip_value_min=1e-6, clip_value_max=tf.reduce_max(inputs))\n        inputs = tf.pow(inputs, self.p)\n        inputs = tf.reduce_mean(inputs, axis=[1, 2], keepdims=False)\n        inputs = tf.pow(inputs, 1./self.p)\n        return inputs","metadata":{"execution":{"iopub.status.busy":"2022-03-29T00:00:48.663238Z","iopub.execute_input":"2022-03-29T00:00:48.663867Z","iopub.status.idle":"2022-03-29T00:00:48.713934Z","shell.execute_reply.started":"2022-03-29T00:00:48.663812Z","shell.execute_reply":"2022-03-29T00:00:48.713117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\n\ndef freeze_BN(model):\n    # Unfreeze layers while leaving BatchNorm layers frozen\n    for layer in model.layers:\n        if not isinstance(layer, tf.keras.layers.BatchNormalization):\n            layer.trainable = True\n        else:\n            layer.trainable = False\n\n# Function to create our EfficientNetB3 model\ndef get_model():\n\n    if config.head=='arcface':\n        head = ArcMarginProduct\n    elif config.head=='curricular':\n        head = CurricularFace\n    else:\n        assert 1==2, \"INVALID HEAD\"\n    \n    with strategy.scope():\n        \n        margin = head(\n            n_classes = config.N_CLASSES, \n            s = 30, \n            m = 0.3, \n            name=f'head/{config.head}', \n            dtype='float32'\n            )\n\n        inp = tf.keras.layers.Input(shape = [config.IMAGE_SIZE, config.IMAGE_SIZE, 3], name = 'inp1')\n        label = tf.keras.layers.Input(shape = (), name = 'inp2')\n        \n        if config.model_type == 'effnetv1':\n            x = EFNS[config.EFF_NET](weights = 'noisy-student', include_top = False)(inp)\n            if config.gem:\n                embed = GeMPoolingLayer(train_p=False)(x)\n            else:\n                embed = tf.keras.layers.GlobalAveragePooling2D()(x)\n            \n        elif config.model_type == 'effnetv2':\n            x = efficientnet.EfficientNetV2M(input_shape=[config.IMAGE_SIZE, config.IMAGE_SIZE, 3], pretrained=\"imagenet\", num_classes=0)(inp)\n            if config.gem:\n                embed = GeMPoolingLayer(train_p=False)(x)\n            else:\n                embed = tf.keras.layers.GlobalAveragePooling2D()(x)\n            \n        elif config.model_type == 'nfnet':\n            x = nfnets.NFNetL0(input_shape=[config.IMAGE_SIZE, config.IMAGE_SIZE, 3], pretrained=\"imagenet\", num_classes=0)(inp)\n            if config.gem:\n                embed = GeMPoolingLayer(train_p=False)(x)\n            else:\n                embed = tf.keras.layers.GlobalAveragePooling2D()(x)\n            \n        \n        if config.bnneck:\n            bnneck = tf.keras.layers.BatchNormalization()(embed)\n            embed = tf.keras.layers.Dense(512, use_bias=False)(bnneck)\n        else:\n            embed = tf.keras.layers.Dropout(0.2)(embed)\n            embed = tf.keras.layers.Dense(512)(embed)\n        x = margin([embed, label])\n        \n        output = tf.keras.layers.Softmax(dtype='float32')(x)\n        \n        model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])\n        embed_model = tf.keras.models.Model(inputs = inp, outputs = embed)  \n        \n        opt = tf.keras.optimizers.Adam(learning_rate = config.LR)\n        # opt = tfa.optimizers.AdamW(learning_rate = config.LR, weight_decay=0.01)\n        if config.FREEZE_BATCH_NORM:\n            freeze_BN(model)\n\n        model.compile(\n            optimizer = opt,\n#             loss = [tf.keras.losses.SparseCategoricalCrossentropy()],\n#             metrics = [tf.keras.metrics.SparseCategoricalAccuracy(),tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5)]\n \n            loss = [tf.keras.losses.CategoricalCrossentropy()],\n            metrics = [tf.keras.metrics.CategoricalAccuracy(),tf.keras.metrics.TopKCategoricalAccuracy(k=5)]\n        )\n        return model,embed_model","metadata":{"execution":{"iopub.status.busy":"2022-03-29T00:00:48.715244Z","iopub.execute_input":"2022-03-29T00:00:48.715738Z","iopub.status.idle":"2022-03-29T00:00:48.739902Z","shell.execute_reply.started":"2022-03-29T00:00:48.715663Z","shell.execute_reply":"2022-03-29T00:00:48.738903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(plot=False):\n    lr_start   = 0.000001\n    lr_max     = 0.000005 * config.BATCH_SIZE  \n    lr_min     = 0.000001\n    lr_ramp_ep = 4\n    lr_sus_ep  = 0\n    lr_decay   = 0.9\n   \n    def lrfn(epoch):\n        if config.RESUME:\n            epoch = epoch + config.RESUME_EPOCH\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n        \n    if plot:\n        epochs = list(range(config.EPOCHS))\n        learning_rates = [lrfn(x) for x in epochs]\n        plt.scatter(epochs,learning_rates)\n        plt.show()\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\n\nget_lr_callback(plot=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T00:00:48.741186Z","iopub.execute_input":"2022-03-29T00:00:48.741903Z","iopub.status.idle":"2022-03-29T00:00:49.01018Z","shell.execute_reply.started":"2022-03-29T00:00:48.741868Z","shell.execute_reply":"2022-03-29T00:00:49.009315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Snapshot(tf.keras.callbacks.Callback):\n    \n    def __init__(self,fold,snapshot_epochs=[]):\n        super(Snapshot, self).__init__()\n        self.snapshot_epochs = snapshot_epochs\n        self.fold = fold\n        \n        \n    def on_epoch_end(self, epoch, logs=None):\n        # logs is a dictionary\n#         print(f\"epoch: {epoch}, train_acc: {logs['acc']}, valid_acc: {logs['val_acc']}\")\n        if epoch in self.snapshot_epochs: # your custom condition         \n            self.model.save_weights(config.save_dir+f\"/EF{config.MODEL_NAME}_epoch{epoch}.h5\")\n        self.model.save_weights(config.save_dir+f\"/{config.MODEL_NAME}_last.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-03-29T00:00:49.012555Z","iopub.execute_input":"2022-03-29T00:00:49.012899Z","iopub.status.idle":"2022-03-29T00:00:49.02027Z","shell.execute_reply.started":"2022-03-29T00:00:49.012867Z","shell.execute_reply":"2022-03-29T00:00:49.019135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"TRAINING_FILENAMES = [x for i,x in enumerate(train_files) if i%config.FOLDS!=config.FOLD_TO_RUN]\nVALIDATION_FILENAMES = [x for i,x in enumerate(train_files) if i%config.FOLDS==config.FOLD_TO_RUN]\nprint(len(TRAINING_FILENAMES),len(VALIDATION_FILENAMES),count_data_items(TRAINING_FILENAMES),count_data_items(VALIDATION_FILENAMES))\n\nGCS_PATH2 = 'gs://kds-d79d083c75d25699e49640c8c5dcb58dd05c8b23a3d0ada2fe03b96e'\ntrain_files2 = np.sort(np.array(tf.io.gfile.glob(GCS_PATH2 + '/happywhale-2022-train*.tfrec')))\nTRAINING_FILENAMES2 = np.concatenate((TRAINING_FILENAMES, train_files2))","metadata":{"execution":{"iopub.status.busy":"2022-03-29T00:01:43.39521Z","iopub.execute_input":"2022-03-29T00:01:43.395569Z","iopub.status.idle":"2022-03-29T00:01:43.552097Z","shell.execute_reply.started":"2022-03-29T00:01:43.395532Z","shell.execute_reply":"2022-03-29T00:01:43.550851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config.DEBUG:\n    TRAINING_FILENAMES = [TRAINING_FILENAMES[0]]\n    VALIDATION_FILENAMES = [VALIDATION_FILENAMES[0]]\n    print(len(TRAINING_FILENAMES),len(VALIDATION_FILENAMES),count_data_items(TRAINING_FILENAMES),count_data_items(VALIDATION_FILENAMES))\n    test_files = [test_files[0]]","metadata":{"execution":{"iopub.status.busy":"2022-03-29T00:02:56.20345Z","iopub.execute_input":"2022-03-29T00:02:56.203815Z","iopub.status.idle":"2022-03-29T00:02:56.209086Z","shell.execute_reply.started":"2022-03-29T00:02:56.203781Z","shell.execute_reply":"2022-03-29T00:02:56.20816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(config.SEED)\nVERBOSE = 1\nif config.PSEUDO:\n    train_dataset = get_training_dataset(TRAINING_FILENAMES2)\n    STEPS_PER_EPOCH = count_data_items(TRAINING_FILENAMES2) // config.BATCH_SIZE\n\nelse:\n    train_dataset = get_training_dataset(TRAINING_FILENAMES)\n    STEPS_PER_EPOCH = count_data_items(TRAINING_FILENAMES) // config.BATCH_SIZE\n\nval_dataset = get_val_dataset(VALIDATION_FILENAMES)\ntrain_logger = tf.keras.callbacks.CSVLogger(config.save_dir+'/training-log-fold-%i.h5.csv'%config.FOLD_TO_RUN)\n# SAVE BEST MODEL EACH FOLD        \nsv_loss = tf.keras.callbacks.ModelCheckpoint(\n    config.save_dir+f\"/{config.MODEL_NAME}_loss.h5\", monitor='val_loss', verbose=0, save_best_only=True,\n    save_weights_only=True, mode='min', save_freq='epoch')\n# BUILD MODEL\nK.clear_session()\nmodel,embed_model = get_model()\nsnap = Snapshot(fold=config.FOLD_TO_RUN,snapshot_epochs=[5,8])\nmodel.summary()\n\nif config.RESUME:   \n    model.load_weights(config.resume_model_wts)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T00:02:56.769914Z","iopub.execute_input":"2022-03-29T00:02:56.770417Z","iopub.status.idle":"2022-03-29T00:03:37.953763Z","shell.execute_reply.started":"2022-03-29T00:02:56.770383Z","shell.execute_reply":"2022-03-29T00:03:37.952912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('#### Image Size %i with EfficientNet B%i and batch_size %i'%\n      (config.IMAGE_SIZE,config.EFF_NET,config.BATCH_SIZE))\n\nhistory = model.fit(train_dataset,\n                validation_data = val_dataset,\n                steps_per_epoch = STEPS_PER_EPOCH,\n                epochs = config.EPOCHS,\n                callbacks = [snap,get_lr_callback(),train_logger,sv_loss], \n                verbose = VERBOSE)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T00:03:49.120168Z","iopub.execute_input":"2022-03-29T00:03:49.120492Z","iopub.status.idle":"2022-03-29T00:08:28.640885Z","shell.execute_reply.started":"2022-03-29T00:03:49.120456Z","shell.execute_reply":"2022-03-29T00:08:28.639597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This function is to get our training tensors\ndef get_eval_dataset(filenames, get_targets = True):\n    dataset = load_dataset(filenames, ordered = True)\n    dataset = dataset.map(data_augment_test, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_eval_format, num_parallel_calls = AUTO)\n    if not get_targets:\n        dataset = dataset.map(lambda image, target: image)\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n# This function is to get our training tensors\ndef get_test_dataset(filenames, get_names = True):\n    dataset = load_dataset(filenames, ordered = True)\n    dataset = dataset.map(data_augment_test, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_inference_format, num_parallel_calls = AUTO)\n    if not get_names:\n        dataset = dataset.map(lambda image, posting_id: image)\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_ids(filename):\n    ds = get_test_dataset([filename],get_names=True).map(lambda image, image_name: image_name).unbatch()\n    NUM_IMAGES = count_data_items([filename])\n    ids = next(iter(ds.batch(NUM_IMAGES))).numpy().astype('U')\n    return ids\n\ndef get_targets(filename):\n    ds = get_eval_dataset([filename],get_targets=True).map(lambda image, target: target).unbatch()\n    NUM_IMAGES = count_data_items([filename])\n    ids = next(iter(ds.batch(NUM_IMAGES))).numpy()\n    return ids\n\ndef get_embeddings(filename):\n    ds = get_test_dataset([filename],get_names=False)\n    embeddings = np.mean(np.stack([embed_models[x][1].predict(ds,verbose=0) for x in range(len(embed_models))]), axis=0)\n    return embeddings\n\ndef get_predictions(test_df,threshold=0.2):\n    predictions = {}\n    for i,row in tqdm(test_df.iterrows()):\n        if row.image in predictions:\n            if len(predictions[row.image])==5:\n                continue\n            predictions[row.image].append(row.target)\n        elif row.confidence>threshold:\n            predictions[row.image] = [row.target,'new_individual']\n        else:\n            predictions[row.image] = ['new_individual',row.target]\n\n    for x in tqdm(predictions):\n        if len(predictions[x])<5:\n            remaining = [y for y in sample_list if y not in predictions]\n            predictions[x] = predictions[x]+remaining\n            predictions[x] = predictions[x][:5]\n\n    return predictions\n\ndef map_per_image(label, predictions):\n    \"\"\"Computes the precision score of one image.\n\n    Parameters\n    ----------\n    label : string\n            The true label of the image\n    predictions : list\n            A list of predicted elements (order does matter, 5 predictions allowed per image)\n\n    Returns\n    -------\n    score : double\n    \"\"\"    \n    try:\n        return 1 / (predictions[:5].index(label) + 1)\n    except ValueError:\n        return 0.0\n\n\nif config.EVALUATE:\n\n    embed_models=[]\n    model,embed_model = get_model()\n    embed_models.append((model.load_weights(f\"{config.MODEL_NAME}_loss.h5\"),embed_model))\n    len(embed_models)\n\n    target_encodings = np.load('../input/happywhaleyolo/individual_id_map_backfins.npy', allow_pickle=True).item()\n    sample_list = ['938b7e931166', '5bf17305f073', '7593d2aee842', '7362d7a01d00','956562ff2888']\n\n    train_targets = []\n    train_embeddings = []\n    train_data_list=[] \n\n    for filename in tqdm(TRAINING_FILENAMES):\n        embeddings = get_embeddings(filename)\n        with open(config.save_dir+f'train_{filename.split(\"/\")[-1]}_{config.FOLD_TO_RUN}.npy', 'wb') as f:\n            np.save(f, embeddings)    \n        targets = get_targets(filename)\n        train_embeddings.append(embeddings)\n        train_targets.append(targets)\n        train_data_list.append([filename,embeddings])\n    train_embeddings_df = pd.DataFrame(train_data_list, columns=['filename', 'embeddings'])\n    train_embeddings_df['FOLD_TO_RUN']=config.FOLD_TO_RUN\n    train_embeddings_df.to_csv(config.save_dir+f\"/train_embeddings_{config.FOLD_TO_RUN}.csv\",index=False)\n    train_embeddings = np.concatenate(train_embeddings)\n    train_targets = np.concatenate(train_targets)\n    from sklearn.neighbors import NearestNeighbors\n    neigh = NearestNeighbors(n_neighbors=50,metric='cosine')\n    neigh.fit(train_embeddings)\n\n\n    test_ids = []\n    test_nn_distances = []\n    test_nn_idxs = []\n    val_targets = []\n    val_embeddings = []\n    val_data_list=[] \n    for filename in tqdm(VALIDATION_FILENAMES):\n        embeddings = get_embeddings(filename)\n        with open(config.save_dir+f'val_{filename.split(\"/\")[-1]}_{config.FOLD_TO_RUN}.npy', 'wb') as f:\n            np.save(f, embeddings) \n        val_data_list.append([filename,embeddings])\n        targets = get_targets(filename)\n        ids = get_ids(filename)\n        distances,idxs = neigh.kneighbors(embeddings, 50, return_distance=True)\n        test_ids.append(ids)\n        test_nn_idxs.append(idxs)\n        test_nn_distances.append(distances)\n        val_embeddings.append(embeddings)\n        val_targets.append(targets)\n    val_embeddings_df = pd.DataFrame(val_data_list, columns=['filename', 'embeddings'])\n    val_embeddings_df['FOLD_TO_RUN']=config.FOLD_TO_RUN\n    val_embeddings_df.to_csv(config.save_dir+f\"/val_embeddings_{config.FOLD_TO_RUN}.csv\",index=False)\n\n    test_nn_distances = np.concatenate(test_nn_distances)\n    test_nn_idxs = np.concatenate(test_nn_idxs)\n    test_ids = np.concatenate(test_ids)\n    val_embeddings = np.concatenate(val_embeddings)\n    val_targets = np.concatenate(val_targets)\n\n    allowed_targets = set([target_encodings[x] for x in np.unique(train_targets)])\n    val_targets_df = pd.DataFrame(np.stack([test_ids,val_targets],axis=1),columns=['image','target'])\n    val_targets_df['target'] = val_targets_df['target'].astype(int).map(target_encodings)\n    val_targets_df.loc[~val_targets_df.target.isin(allowed_targets),'target'] = 'new_individual'\n    val_targets_df.target.value_counts()\n\n    test_df = []\n    for i in tqdm(range(len(test_ids))):\n        id_ = test_ids[i]\n        targets = train_targets[test_nn_idxs[i]]\n        distances = test_nn_distances[i]\n        subset_preds = pd.DataFrame(np.stack([targets,distances],axis=1),columns=['target','distances'])\n        subset_preds['image'] = id_\n        test_df.append(subset_preds)\n    test_df = pd.concat(test_df).reset_index(drop=True)\n    test_df['confidence'] = 1-test_df['distances']\n    test_df = test_df.groupby(['image','target']).confidence.max().reset_index()\n    test_df = test_df.sort_values('confidence',ascending=False).reset_index(drop=True)\n    test_df['target'] = test_df['target'].map(target_encodings)\n    test_df.to_csv('val_neighbors.csv')\n    test_df.image.value_counts().value_counts()\n\n    ## Compute CV\n    best_th = 0\n    best_cv = 0\n    for th in [0.02 * x for x in range(20, 33)]:\n        all_preds = get_predictions(test_df,threshold=th)\n        cv = 0\n        for i,row in val_targets_df.iterrows():\n            target = row.target\n            preds = all_preds[row.image]\n            val_targets_df.loc[i,th] = map_per_image(target,preds)\n        cv = val_targets_df[th].mean()\n        print(f\"CV at threshold {th}: {cv}\")\n        if cv>best_cv:\n            best_th = th\n            best_cv = cv\n\n    print(\"Best threshold\",best_th)\n    print(\"Best cv\",best_cv)\n    val_targets_df.describe()\n\n    ## Adjustment: Since Public lb has nearly 10% 'new_individual' (Be Careful for private LB)\n    val_targets_df['is_new_individual'] = val_targets_df.target=='new_individual'\n    print(val_targets_df.is_new_individual.value_counts().to_dict())\n    val_scores = val_targets_df.groupby('is_new_individual').mean().T\n    val_scores['adjusted_cv'] = val_scores[True]*0.1+val_scores[False]*0.9\n    best_threshold_adjusted = val_scores['adjusted_cv'].idxmax()\n    print(\"best_threshold\",best_threshold_adjusted)\n    print(val_scores)\n    \n    \nif config.TEST:\n    test_ids = []\n    test_nn_distances = []\n    test_nn_idxs = []\n    test_data_list=[] \n    for filename in tqdm(test_files):\n        embeddings = get_embeddings(filename)\n        with open(config.save_dir+f'test_{filename.split(\"/\")[-1]}_{config.FOLD_TO_RUN}.npy', 'wb') as f:\n            np.save(f, embeddings)","metadata":{"execution":{"iopub.status.busy":"2022-03-29T00:08:28.642392Z","iopub.status.idle":"2022-03-29T00:08:28.642771Z","shell.execute_reply.started":"2022-03-29T00:08:28.642558Z","shell.execute_reply":"2022-03-29T00:08:28.642582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Arc + mixup nfnetl0\n# Epoch 25/25\n# 146/146 [==============================] - 163s 1s/step \n# - loss: 0.7047 - categorical_accuracy: 0.9365 - top_k_categorical_accuracy: 0.9892 \n# - val_loss: 9.7082 - val_categorical_accuracy: 0.3736 - val_top_k_categorical_accuracy: 0.4502\n\n# curriclar nfnetl0\n# Epoch 25/25\n# 146/146 [==============================] - 115s 790ms/step \n# - loss: 0.0095 - sparse_categorical_accuracy: 1.0000 - sparse_top_k_categorical_accuracy: 1.0000 \n# - val_loss: 9.1595 - val_sparse_categorical_accuracy: 0.6060 - val_sparse_top_k_categorical_accuracy: 0.6606","metadata":{"execution":{"iopub.status.busy":"2022-03-29T00:00:49.041357Z","iopub.status.idle":"2022-03-29T00:00:49.04172Z","shell.execute_reply.started":"2022-03-29T00:00:49.04152Z","shell.execute_reply":"2022-03-29T00:00:49.041543Z"},"trusted":true},"execution_count":null,"outputs":[]}]}