{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q efficientnet\n!pip install -q tensorflow_addons\n#!pip install tfimm","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-04-17T15:45:53.644391Z","iopub.execute_input":"2022-04-17T15:45:53.64497Z","iopub.status.idle":"2022-04-17T15:46:10.612528Z","shell.execute_reply.started":"2022-04-17T15:45:53.644881Z","shell.execute_reply":"2022-04-17T15:46:10.61173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:#75B7BF;letter-spacing:0.1px;overflow:hidden\"><p style=\"padding:20px;color:white;overflow:hidden;margin:0;font-size:100%;text-align:center\">Import</p></div>","metadata":{}},{"cell_type":"code","source":"import re\nimport os\nimport numpy as np\n# numpy and matplotlib defaults\nnp.set_printoptions(threshold=15, linewidth=80)\nimport pandas as pd\nimport random\nimport math\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.neighbors import NearestNeighbors\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.layers import BatchNormalization, Input, Conv2D, GlobalAveragePooling1D, Dropout, Dense, Softmax, LayerNormalization\nfrom tensorflow.keras.losses import SparseCategoricalCrossentropy\nfrom tensorflow.keras.metrics import SparseCategoricalAccuracy, SparseTopKCategoricalAccuracy\nfrom tensorflow.keras.callbacks import LearningRateScheduler, CSVLogger, ModelCheckpoint\nfrom tensorflow.keras.utils import plot_model\nimport tensorflow_addons as tfa\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nimport pickle\nimport json\nimport tensorflow_hub as tfhub\nfrom datetime import datetime\n\nfrom kaggle_datasets import KaggleDatasets","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-04-17T15:46:10.61613Z","iopub.execute_input":"2022-04-17T15:46:10.616351Z","iopub.status.idle":"2022-04-17T15:46:15.930684Z","shell.execute_reply.started":"2022-04-17T15:46:10.616323Z","shell.execute_reply":"2022-04-17T15:46:15.929891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"\\n... ACCELERATOR SETUP STARTING ...\\n\")\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    TPU = tf.distribute.cluster_resolver.TPUClusterResolver()  \nexcept ValueError:\n    TPU = None\n\nif TPU:\n    print(f\"\\n... RUNNING ON TPU - {TPU.master()}...\")\n    tf.config.experimental_connect_to_cluster(TPU)\n    tf.tpu.experimental.initialize_tpu_system(TPU)\n    strategy = tf.distribute.experimental.TPUStrategy(TPU)\nelse:\n    print(f\"\\n... RUNNING ON CPU/GPU ...\")\n    # Yield the default distribution strategy in Tensorflow\n    #   --> Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy() \n\n# What Is a Replica?\n#    --> A single Cloud TPU device consists of FOUR chips, each of which has TWO TPU cores. \n#    --> Therefore, for efficient utilization of Cloud TPU, a program should make use of each of the EIGHT (4x2) cores. \n#    --> Each replica is essentially a copy of the training graph that is run on each core and \n#        trains a mini-batch containing 1/8th of the overall batch size\nN_REPLICAS = strategy.num_replicas_in_sync\n    \nprint(f\"... # OF REPLICAS: {N_REPLICAS} ...\\n\")\n\nAUTO = tf.data.experimental.AUTOTUNE\n\nprint(f\"\\n... ACCELERATOR SETUP COMPLTED ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:15.93319Z","iopub.execute_input":"2022-04-17T15:46:15.933784Z","iopub.status.idle":"2022-04-17T15:46:15.946633Z","shell.execute_reply.started":"2022-04-17T15:46:15.933745Z","shell.execute_reply":"2022-04-17T15:46:15.945752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"\\n... XLA OPTIMIZATIONS STARTING ...\\n\")\n\nprint(f\"\\n... CONFIGURE JIT (JUST IN TIME) COMPILATION ...\\n\")\n# enable XLA optmizations (10% speedup when using @tf.function calls)\ntf.config.optimizer.set_jit(True)\n\nprint(f\"\\n... XLA OPTIMIZATIONS COMPLETED ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:15.949391Z","iopub.execute_input":"2022-04-17T15:46:15.949701Z","iopub.status.idle":"2022-04-17T15:46:15.956266Z","shell.execute_reply.started":"2022-04-17T15:46:15.949663Z","shell.execute_reply":"2022-04-17T15:46:15.955416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:#75B7BF;letter-spacing:0.1px;overflow:hidden\"><p style=\"padding:20px;color:white;overflow:hidden;margin:0;font-size:100%;text-align:center\">Configuration and Versions</p></div>","metadata":{}},{"cell_type":"code","source":"run_ts = datetime.now().strftime('%Y%m%d-%H%M%S')\nprint(run_ts)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:15.957924Z","iopub.execute_input":"2022-04-17T15:46:15.958632Z","iopub.status.idle":"2022-04-17T15:46:15.964122Z","shell.execute_reply.started":"2022-04-17T15:46:15.958582Z","shell.execute_reply":"2022-04-17T15:46:15.963269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    \n    SEED = 42\n    FOLD_TO_RUN = 0   # To seperate train/validation dataset\n    FOLDS = 10 # If FOLDS == FOLD_TO_RUN, using all data for trainning\n    DEBUG = False  # If True, get out some data to run model\n    EVALUATE = True\n    \n    ### Dataset\n    GCS_PATH = 'backfintfrecords'\n    BATCH_SIZE = 16 * N_REPLICAS\n    IMAGE_SIZE = [384, 384]\n    N_CLASSES = 15587\n    \n    ### Model\n    model_type = f'swin'\n    EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n            efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\n    EFF_NET = 7 # choose EfficientV1\n    EFF_NETV2 = f'efficientnetv2-xl-21k-ft1k'\n    FREEZE_BATCH_NORM = False # Choose inference or \n    head = 'arcface' # head layer in model\n    EPOCHS = 30\n    LR = 0.001\n    message='baseline'\n    RESUME = False # Resume learning from the checkpoint\n    \n    ### Augmentations\n    CUTOUT = False\n    N_CUTOUT = 6\n    ### Inference\n    KNN = 100\n    \n    ###Learning Rate Scheduler\n    RESUME_EPOCH = 9\n    \ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)\n    \n# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)\n    \ndef is_interactive():\n    return 'runtime'    in get_ipython().config.IPKernelApp.connection_file\nIS_INTERACTIVE = is_interactive()\nprint(IS_INTERACTIVE)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:15.965721Z","iopub.execute_input":"2022-04-17T15:46:15.966338Z","iopub.status.idle":"2022-04-17T15:46:15.978507Z","shell.execute_reply.started":"2022-04-17T15:46:15.966305Z","shell.execute_reply":"2022-04-17T15:46:15.977689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_NAME = None\nif CFG.model_type == 'effnetv1':\n    MODEL_NAME = f'effnetv1_b{CFG.EFF_NET}'\nelif CFG.model_type == 'effnetv2':\n    MODEL_NAME = f'effnetv2_{CFG.EFF_NETV2}'\nelif CFG.model_type =='swin':\n    MODEL_NAME = f'SwinTransformer'\n\nCFG.MODEL_NAME = MODEL_NAME\nprint(MODEL_NAME)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:15.979852Z","iopub.execute_input":"2022-04-17T15:46:15.980127Z","iopub.status.idle":"2022-04-17T15:46:15.990063Z","shell.execute_reply.started":"2022-04-17T15:46:15.980095Z","shell.execute_reply":"2022-04-17T15:46:15.989286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\n... DATA ACCESS SETUP STARTED ...\\n\")\n# Choose the \nGCS_PATH = KaggleDatasets().get_gcs_path(CFG.GCS_PATH)\ntrain_files = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/happywhale-2022-train*.tfrec')))\ntest_files = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/happywhale-2022-test*.tfrec')))\n\nprint(count_data_items(train_files),count_data_items(test_files))\n    \nprint(\"\\n\\n... DATA ACCESS SETUP COMPLETED ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:15.991347Z","iopub.execute_input":"2022-04-17T15:46:15.991662Z","iopub.status.idle":"2022-04-17T15:46:18.687372Z","shell.execute_reply.started":"2022-04-17T15:46:15.991629Z","shell.execute_reply":"2022-04-17T15:46:18.686531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:#75B7BF;letter-spacing:0.1px;overflow:hidden\"><p style=\"padding:20px;color:white;overflow:hidden;margin:0;font-size:100%;text-align:center\">Dataset</p></div>","metadata":{}},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:#75B7BF;letter-spacing:0.1px;overflow:hidden\"><p style=\"padding:20px;color:white;overflow:hidden;margin:0;font-size:70%;text-align:left\">EXTRA DATASET</p></div>\n****\n## 1. happywhale-splits\nThis dataset is created by @ks2019 with code:\n```\ntrain_df = pd.read_csv('../input/happy-whale-and-dolphin/train.csv')\ntrain_df.species.replace({\"globis\": \"short_finned_pilot_whale\",\n                          \"pilot_whale\": \"short_finned_pilot_whale\",\n                          \"kiler_whale\": \"killer_whale\",\n                          \"bottlenose_dolpin\": \"bottlenose_dolphin\"}, inplace=True)\ntrain_df.to_csv('train_fixed.csv',index=False)\n\ntrain_df = pd.read_csv('train_fixed.csv')\ntrain_df['individual_id'] = train_df['individual_id'].map(individual_ids)\ntrain_df['species'] = train_df['species'].map(species)\nskf = StratifiedKFold(n_splits=5,random_state=123)\nfor fold,(train_index, test_index) in enumerate(skf.split(train_df, train_df.species)):\n    train_df.loc[test_index,'fold'] = fold\nprint(train_df.groupby('fold').individual_id.nunique().to_dict())\nprint(train_df.groupby('fold').species.nunique().to_dict())\nprint(train_df.groupby('fold').image.nunique().to_dict())\ntrain_df.to_csv('skf_species_5folds.csv',index=False)\n```\n\n\n## 2. TFRecords \nhttps://www.kaggle.com/datasets/phanttan/happywhale-cropped-removebackground-tfrecords-v1\n","metadata":{}},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:#75B7BF;letter-spacing:0.1px;overflow:hidden\"><p style=\"padding:20px;color:white;overflow:hidden;margin:0;font-size:100%;text-align:center\">HELPER FUNCTIONS</p></div>","metadata":{}},{"cell_type":"markdown","source":"## ArcFace","metadata":{}},{"cell_type":"code","source":"def arcface_format(posting_id, image, label_group, matches):\n    return posting_id, {'Input1': image, 'Input2': label_group}, label_group, matches\n\ndef arcface_inference_format(posting_id, image, label_group, matches):\n    return image,posting_id\n\ndef arcface_eval_format(posting_id, image, label_group, matches):\n    return image,label_group","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.688513Z","iopub.execute_input":"2022-04-17T15:46:18.688971Z","iopub.status.idle":"2022-04-17T15:46:18.695045Z","shell.execute_reply.started":"2022-04-17T15:46:18.688933Z","shell.execute_reply":"2022-04-17T15:46:18.694153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Augmentation ","metadata":{}},{"cell_type":"code","source":"# data augmentation\ndef add_gaussian_noise(image):\n    # image must be scaled in [0, 1]\n    with tf.name_scope('Add_gaussian_noise'):\n        noise = tf.random.normal(shape=tf.shape(image), mean=0.0, stddev=(25)/(255), dtype=tf.float32)\n        noise_img = image + noise\n        noise_img = tf.clip_by_value(noise_img, 0.0, 1.0)\n    return noise_img\n\ndef add_gaussian_blue(image):\n    image = tfa.image.gaussian_filter2d(image)\n    return image\n\ndef random_lighting(image):\n    \"\"\"\n    Applys random augmentations related to lighting\n    \"\"\"\n    #apply random brightness\n    image = tf.image.random_brightness(image, 0.3,)\n    #apply random contrast\n    image = tf.image.random_contrast(image, 0.6, 1.4,)\n    #apply random hue\n    image = tf.image.random_hue(image, 0.002)\n    #apply random saturation\n    image = tf.image.random_saturation(image, 0.3, 1.7,)\n    return image\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    if tf.random.uniform([])>0.5:\n        image = tf.image.random_flip_left_right(image)\n        \n    if tf.random.uniform([])>0.5:\n        image = random_lighting(image)\n        \n    if tf.random.uniform([])>0.7:\n        image = add_gaussian_noise(image)\n        \n    if tf.random.uniform([])>0.7:\n        image = add_gaussian_blue(image)\n    return image, label  \n\n# Function to decode our images\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels = 3)\n    image = tf.image.resize(image, [CFG.IMAGE_SIZE[0],CFG.IMAGE_SIZE[0]])\n    image = tf.cast(image, tf.float32) / 255.0\n    return image\n\n# This function parse our images and also get the target variable\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['image_name']\n    return image, idnum # returns a dataset of image(s)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.699011Z","iopub.execute_input":"2022-04-17T15:46:18.69921Z","iopub.status.idle":"2022-04-17T15:46:18.71644Z","shell.execute_reply.started":"2022-04-17T15:46:18.699188Z","shell.execute_reply":"2022-04-17T15:46:18.715748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TF Records to Tensors Functions","metadata":{}},{"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered = False):\n    \"\"\"Loading Dataset from dataset: happywhale-tfrecords-v1.\n\n    Parameters\n    ----------\n    label : string\n            The true label of the image\n    predictions : list\n            A list of predicted elements (order does matter, 5 predictions allowed per image)\n\n    Returns\n    -------\n    score : double\n    \"\"\"    \n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset \n\n# Get Dataset with some configurations for Training\ndef get_training_dataset(TRAINING_FILENAMES):\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(CFG.BATCH_SIZE, drop_remainder=True)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\n# Get Dataset with some configurations for Validation\ndef get_validation_dataset(VALIDATION_FILENAMES, ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.batch(CFG.BATCH_SIZE, drop_remainder=True)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\n# Get Dataset with some configurations for Prediction\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.717896Z","iopub.execute_input":"2022-04-17T15:46:18.71843Z","iopub.status.idle":"2022-04-17T15:46:18.731939Z","shell.execute_reply.started":"2022-04-17T15:46:18.71838Z","shell.execute_reply":"2022-04-17T15:46:18.731251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Arcmarginproduct class keras layer\nclass ArcMarginProduct(tf.keras.layers.Layer):\n    '''\n    Implements large margin arc distance.\n\n    Reference:\n        https://arxiv.org/pdf/1801.07698.pdf\n        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n            blob/master/src/modeling/metric_learning.py\n    '''\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n\n    def get_config(self):\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n\n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n        one_hot = tf.cast(\n            tf.one_hot(y, depth=self.n_classes),\n            dtype=cosine.dtype\n        )\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.733583Z","iopub.execute_input":"2022-04-17T15:46:18.734165Z","iopub.status.idle":"2022-04-17T15:46:18.750356Z","shell.execute_reply.started":"2022-04-17T15:46:18.734128Z","shell.execute_reply":"2022-04-17T15:46:18.749669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# BatchNormalization Mode Function\ndef freeze_BatchNorm(model):\n    # Unfreeze all Layers except for Batch Norm\n    for layer in model.layers:\n        if not isinstance(layer, BatchNormalization):\n            layer.trainable = True \n        else:\n            layer.trainable = False # in inference mode\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.751654Z","iopub.execute_input":"2022-04-17T15:46:18.752109Z","iopub.status.idle":"2022-04-17T15:46:18.762345Z","shell.execute_reply.started":"2022-04-17T15:46:18.752075Z","shell.execute_reply":"2022-04-17T15:46:18.761553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## For Evaluation ","metadata":{}},{"cell_type":"code","source":"## Find the name of image in each filename. Exp : 00021adfb725ed.jpg \ndef get_id(filename):\n    ds = get_test_dataset([filename],get_names=True).map(lambda image, image_name: image_name).unbatch()\n    NUM_IMAGES = count_data_items([filename])\n    ids = next(iter(ds.batch(NUM_IMAGES))).numpy().astype('U')\n    return ids\n## Find the Target Encoding (defined in happywhale-splits/individual_ids.json)\ndef get_target(filename):\n    ds = get_eval_dataset([filename],get_targets=True).map(lambda image, target: target).unbatch()\n    NUM_IMAGES = count_data_items([filename])\n    ids = next(iter(ds.batch(NUM_IMAGES))).numpy()\n    return ids\n## Find the average of predictions in the best 5-models\ndef get_embedding(filename):\n    ds = get_test_dataset([filename],get_names=False)\n    embedding = np.mean(np.stack([embed_models[x][1].predict(ds,verbose=0) for x in range(len(embed_models))]), axis=0)\n    return embedding\n\ndef get_prediction(test_df,threshold=0.2):\n    predictions = {}\n    for i,row in tqdm(test_df.iterrows()):\n        if row.image in predictions:\n            if len(predictions[row.image])==5:\n                continue\n            predictions[row.image].append(row.target)\n        elif row.confidence>threshold:\n            predictions[row.image] = [row.target,'new_individual']\n        else:\n            predictions[row.image] = ['new_individual',row.target]\n\n    return predictions\n\ndef map_per_image(label, predictions):\n    \"\"\"Computes the precision score of one image.\n\n    Parameters\n    ----------\n    label : string\n            The true label of the image\n    predictions : list\n            A list of predicted elements (order does matter, 5 predictions allowed per image)\n\n    Returns\n    -------\n    score : double\n    \"\"\"    \n    try:\n        return 1 / (predictions[:5].index(label) + 1)\n    except ValueError:\n        return 0.0","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.765077Z","iopub.execute_input":"2022-04-17T15:46:18.765344Z","iopub.status.idle":"2022-04-17T15:46:18.780075Z","shell.execute_reply.started":"2022-04-17T15:46:18.765313Z","shell.execute_reply":"2022-04-17T15:46:18.779344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n    \ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n\ndef display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()\n    \ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.783007Z","iopub.execute_input":"2022-04-17T15:46:18.783506Z","iopub.status.idle":"2022-04-17T15:46:18.807458Z","shell.execute_reply.started":"2022-04-17T15:46:18.783471Z","shell.execute_reply":"2022-04-17T15:46:18.806618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:#75B7BF;letter-spacing:0.1px;overflow:hidden\"><p style=\"padding:20px;color:white;overflow:hidden;margin:0;font-size:100%;text-align:center\">BUILD MODEL</p></div>","metadata":{}},{"cell_type":"code","source":"CFGS = {\n    'swin_tiny_224': dict(input_size=(224, 224), window_size=7, embed_dim=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24]),\n    'swin_small_224': dict(input_size=(224, 224), window_size=7, embed_dim=96, depths=[2, 2, 18, 2], num_heads=[3, 6, 12, 24]),\n    'swin_base_224': dict(input_size=(224, 224), window_size=7, embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32]),\n    'swin_base_384': dict(input_size=(384, 384), window_size=12, embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32]),\n    'swin_large_224': dict(input_size=(224, 224), window_size=7, embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48]),\n    'swin_large_384': dict(input_size=(384, 384), window_size=12, embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48])\n}","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.808827Z","iopub.execute_input":"2022-04-17T15:46:18.809317Z","iopub.status.idle":"2022-04-17T15:46:18.819418Z","shell.execute_reply.started":"2022-04-17T15:46:18.809254Z","shell.execute_reply":"2022-04-17T15:46:18.818702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Mlp(tf.keras.layers.Layer):\n    def __init__(self, in_features, hidden_features=None, out_features=None, drop=0., prefix=''):\n        super().__init__()\n        out_features = out_features or in_features      \n        hidden_features = hidden_features or in_features\n        self.fc1 = Dense(hidden_features, name=f'{prefix}/mlp/fc1')\n        self.fc2 = Dense(out_features, name=f'{prefix}/mlp/fc2')\n        self.drop = Dropout(drop)\n\n    def call(self, x):\n        x = self.fc1(x)\n        x = tf.keras.activations.gelu(x)\n        x = self.drop(x)\n        x = self.fc2(x)\n        x = self.drop(x)\n        return x\n\n\ndef window_partition(x, window_size):\n    B, H, W, C = x.get_shape().as_list()\n    x = tf.reshape(x, shape=[-1, H // window_size,\n                   window_size, W // window_size, window_size, C])\n    x = tf.transpose(x, perm=[0, 1, 3, 2, 4, 5])\n    windows = tf.reshape(x, shape=[-1, window_size, window_size, C])\n    return windows\n\n\ndef window_reverse(windows, window_size, H, W, C):\n    x = tf.reshape(windows, shape=[-1, H // window_size,\n                   W // window_size, window_size, window_size, C])\n    x = tf.transpose(x, perm=[0, 1, 3, 2, 4, 5])\n    x = tf.reshape(x, shape=[-1, H, W, C])\n    return x","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.820729Z","iopub.execute_input":"2022-04-17T15:46:18.821263Z","iopub.status.idle":"2022-04-17T15:46:18.833504Z","shell.execute_reply.started":"2022-04-17T15:46:18.821225Z","shell.execute_reply":"2022-04-17T15:46:18.832826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class WindowAttention(tf.keras.layers.Layer):\n    def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0., prefix=''):\n        super().__init__()\n        self.dim = dim\n        self.window_size = window_size\n        self.num_heads = num_heads\n        head_dim = dim // num_heads\n        self.scale = qk_scale or head_dim ** -0.5\n        self.prefix = prefix\n\n        self.qkv = Dense(dim * 3, use_bias=qkv_bias,\n                         name=f'{self.prefix}/attn/qkv')\n        self.attn_drop = Dropout(attn_drop)\n        self.proj = Dense(dim, name=f'{self.prefix}/attn/proj')\n        self.proj_drop = Dropout(proj_drop)\n\n    def build(self, input_shape):\n        self.relative_position_bias_table = self.add_weight(f'{self.prefix}/attn/relative_position_bias_table',\n                                                            shape=(\n                                                                (2 * self.window_size[0] - 1) * (2 * self.window_size[1] - 1), self.num_heads),\n                                                            initializer=tf.initializers.Zeros(), trainable=True)\n\n        coords_h = np.arange(self.window_size[0])\n        coords_w = np.arange(self.window_size[1])\n        coords = np.stack(np.meshgrid(coords_h, coords_w, indexing='ij'))\n        coords_flatten = coords.reshape(2, -1)\n        relative_coords = coords_flatten[:, :,\n                                         None] - coords_flatten[:, None, :]\n        relative_coords = relative_coords.transpose([1, 2, 0])\n        relative_coords[:, :, 0] += self.window_size[0] - 1\n        relative_coords[:, :, 1] += self.window_size[1] - 1\n        relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1\n        relative_position_index = relative_coords.sum(-1).astype(np.int64)\n        self.relative_position_index = tf.Variable(initial_value=tf.convert_to_tensor(\n            relative_position_index), trainable=False, name=f'{self.prefix}/attn/relative_position_index')\n        self.built = True\n\n    def call(self, x, mask=None):\n        B_, N, C = x.get_shape().as_list()\n        qkv = tf.transpose(tf.reshape(self.qkv(\n            x), shape=[-1, N, 3, self.num_heads, C // self.num_heads]), perm=[2, 0, 3, 1, 4])\n        q, k, v = qkv[0], qkv[1], qkv[2]\n\n        q = q * self.scale\n        attn = (q @ tf.transpose(k, perm=[0, 1, 3, 2]))\n        relative_position_bias = tf.gather(self.relative_position_bias_table, tf.reshape(\n            self.relative_position_index, shape=[-1]))\n        relative_position_bias = tf.reshape(relative_position_bias, shape=[\n                                            self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1])\n        relative_position_bias = tf.transpose(\n            relative_position_bias, perm=[2, 0, 1])\n        attn = attn + tf.expand_dims(relative_position_bias, axis=0)\n\n        if mask is not None:\n            nW = mask.get_shape()[0]  # tf.shape(mask)[0]\n            attn = tf.reshape(attn, shape=[-1, nW, self.num_heads, N, N]) + tf.cast(\n                tf.expand_dims(tf.expand_dims(mask, axis=1), axis=0), attn.dtype)\n            attn = tf.reshape(attn, shape=[-1, self.num_heads, N, N])\n            attn = tf.nn.softmax(attn, axis=-1)\n        else:\n            attn = tf.nn.softmax(attn, axis=-1)\n\n        attn = self.attn_drop(attn)\n\n        x = tf.transpose((attn @ v), perm=[0, 2, 1, 3])\n        x = tf.reshape(x, shape=[-1, N, C])\n        x = self.proj(x)\n        x = self.proj_drop(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.836765Z","iopub.execute_input":"2022-04-17T15:46:18.83703Z","iopub.status.idle":"2022-04-17T15:46:18.858455Z","shell.execute_reply.started":"2022-04-17T15:46:18.837Z","shell.execute_reply":"2022-04-17T15:46:18.857798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drop_path(inputs, drop_prob, is_training):\n    if (not is_training) or (drop_prob == 0.):\n        return inputs\n\n    # Compute keep_prob\n    keep_prob = 1.0 - drop_prob\n\n    # Compute drop_connect tensor\n    random_tensor = keep_prob\n    shape = (tf.shape(inputs)[0],) + (1,) * \\\n        (len(tf.shape(inputs)) - 1)\n    random_tensor += tf.random.uniform(shape, dtype=inputs.dtype)\n    binary_tensor = tf.floor(random_tensor)\n    output = tf.math.divide(inputs, keep_prob) * binary_tensor\n    return output","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.859685Z","iopub.execute_input":"2022-04-17T15:46:18.860353Z","iopub.status.idle":"2022-04-17T15:46:18.868126Z","shell.execute_reply.started":"2022-04-17T15:46:18.860318Z","shell.execute_reply":"2022-04-17T15:46:18.867451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DropPath(tf.keras.layers.Layer):\n    def __init__(self, drop_prob=None):\n        super().__init__()\n        self.drop_prob = drop_prob\n\n    def call(self, x, training=None):\n        return drop_path(x, self.drop_prob, training)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.869285Z","iopub.execute_input":"2022-04-17T15:46:18.869631Z","iopub.status.idle":"2022-04-17T15:46:18.876414Z","shell.execute_reply.started":"2022-04-17T15:46:18.869597Z","shell.execute_reply":"2022-04-17T15:46:18.87556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SwinTransformerBlock(tf.keras.layers.Layer):\n    def __init__(self, dim, input_resolution, num_heads, window_size=7, shift_size=0, mlp_ratio=4.,\n                 qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path_prob=0., norm_layer=LayerNormalization, prefix=''):\n        super().__init__()\n        self.dim = dim\n        self.input_resolution = input_resolution\n        self.num_heads = num_heads\n        self.window_size = window_size\n        self.shift_size = shift_size\n        self.mlp_ratio = mlp_ratio\n        if min(self.input_resolution) <= self.window_size:\n            self.shift_size = 0\n            self.window_size = min(self.input_resolution)\n        assert 0 <= self.shift_size < self.window_size, \"shift_size must in 0-window_size\"\n        self.prefix = prefix\n\n        self.norm1 = norm_layer(epsilon=1e-5, name=f'{self.prefix}/norm1')\n        self.attn = WindowAttention(dim, window_size=(self.window_size, self.window_size), num_heads=num_heads,\n                                    qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop, prefix=self.prefix)\n        self.drop_path = DropPath(\n            drop_path_prob if drop_path_prob > 0. else 0.)\n        self.norm2 = norm_layer(epsilon=1e-5, name=f'{self.prefix}/norm2')\n        mlp_hidden_dim = int(dim * mlp_ratio)\n        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim,\n                       drop=drop, prefix=self.prefix)\n\n    def build(self, input_shape):\n        if self.shift_size > 0:\n            H, W = self.input_resolution\n            img_mask = np.zeros([1, H, W, 1])\n            h_slices = (slice(0, -self.window_size),\n                        slice(-self.window_size, -self.shift_size),\n                        slice(-self.shift_size, None))\n            w_slices = (slice(0, -self.window_size),\n                        slice(-self.window_size, -self.shift_size),\n                        slice(-self.shift_size, None))\n            cnt = 0\n            for h in h_slices:\n                for w in w_slices:\n                    img_mask[:, h, w, :] = cnt\n                    cnt += 1\n\n            img_mask = tf.convert_to_tensor(img_mask)\n            mask_windows = window_partition(img_mask, self.window_size)\n            mask_windows = tf.reshape(\n                mask_windows, shape=[-1, self.window_size * self.window_size])\n            attn_mask = tf.expand_dims(\n                mask_windows, axis=1) - tf.expand_dims(mask_windows, axis=2)\n            attn_mask = tf.where(attn_mask != 0, -100.0, attn_mask)\n            attn_mask = tf.where(attn_mask == 0, 0.0, attn_mask)\n            self.attn_mask = tf.Variable(\n                initial_value=attn_mask, trainable=False, name=f'{self.prefix}/attn_mask')\n        else:\n            self.attn_mask = None\n\n        self.built = True\n\n    def call(self, x):\n        H, W = self.input_resolution\n        B, L, C = x.get_shape().as_list()\n        assert L == H * W, \"input feature has wrong size\"\n\n        shortcut = x\n        x = self.norm1(x)\n        x = tf.reshape(x, shape=[-1, H, W, C])\n\n        # cyclic shift\n        if self.shift_size > 0:\n            shifted_x = tf.roll(\n                x, shift=[-self.shift_size, -self.shift_size], axis=[1, 2])\n        else:\n            shifted_x = x\n\n        # partition windows\n        x_windows = window_partition(shifted_x, self.window_size)\n        x_windows = tf.reshape(\n            x_windows, shape=[-1, self.window_size * self.window_size, C])\n\n        # W-MSA/SW-MSA\n        attn_windows = self.attn(x_windows, mask=self.attn_mask)\n\n        # merge windows\n        attn_windows = tf.reshape(\n            attn_windows, shape=[-1, self.window_size, self.window_size, C])\n        shifted_x = window_reverse(attn_windows, self.window_size, H, W, C)\n\n        # reverse cyclic shift\n        if self.shift_size > 0:\n            x = tf.roll(shifted_x, shift=[\n                        self.shift_size, self.shift_size], axis=[1, 2])\n        else:\n            x = shifted_x\n        x = tf.reshape(x, shape=[-1, H * W, C])\n\n        # FFN\n        x = shortcut + self.drop_path(x)\n        x = x + self.drop_path(self.mlp(self.norm2(x)))\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.878315Z","iopub.execute_input":"2022-04-17T15:46:18.878697Z","iopub.status.idle":"2022-04-17T15:46:18.900614Z","shell.execute_reply.started":"2022-04-17T15:46:18.878663Z","shell.execute_reply":"2022-04-17T15:46:18.899968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PatchMerging(tf.keras.layers.Layer):\n    def __init__(self, input_resolution, dim, norm_layer=LayerNormalization, prefix=''):\n        super().__init__()\n        self.input_resolution = input_resolution\n        self.dim = dim\n        self.reduction = Dense(2 * dim, use_bias=False,\n                               name=f'{prefix}/downsample/reduction')\n        self.norm = norm_layer(epsilon=1e-5, name=f'{prefix}/downsample/norm')\n\n    def call(self, x):\n        H, W = self.input_resolution\n        B, L, C = x.get_shape().as_list()\n        assert L == H * W, \"input feature has wrong size\"\n        assert H % 2 == 0 and W % 2 == 0, f\"x size ({H}*{W}) are not even.\"\n\n        x = tf.reshape(x, shape=[-1, H, W, C])\n\n        x0 = x[:, 0::2, 0::2, :]  # B H/2 W/2 C\n        x1 = x[:, 1::2, 0::2, :]  # B H/2 W/2 C\n        x2 = x[:, 0::2, 1::2, :]  # B H/2 W/2 C\n        x3 = x[:, 1::2, 1::2, :]  # B H/2 W/2 C\n        x = tf.concat([x0, x1, x2, x3], axis=-1)\n        x = tf.reshape(x, shape=[-1, (H // 2) * (W // 2), 4 * C])\n\n        x = self.norm(x)\n        x = self.reduction(x)\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.902097Z","iopub.execute_input":"2022-04-17T15:46:18.90255Z","iopub.status.idle":"2022-04-17T15:46:18.914346Z","shell.execute_reply.started":"2022-04-17T15:46:18.902516Z","shell.execute_reply":"2022-04-17T15:46:18.913656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BasicLayer(tf.keras.layers.Layer):\n    def __init__(self, dim, input_resolution, depth, num_heads, window_size,\n                 mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0.,\n                 drop_path_prob=0., norm_layer=LayerNormalization, downsample=None, use_checkpoint=False, prefix=''):\n        super().__init__()\n        self.dim = dim\n        self.input_resolution = input_resolution\n        self.depth = depth\n        self.use_checkpoint = use_checkpoint\n\n        # build blocks\n        self.blocks = tf.keras.Sequential([SwinTransformerBlock(dim=dim, input_resolution=input_resolution,\n                                           num_heads=num_heads, window_size=window_size,\n                                           shift_size=0 if (\n                                               i % 2 == 0) else window_size // 2,\n                                           mlp_ratio=mlp_ratio,\n                                           qkv_bias=qkv_bias, qk_scale=qk_scale,\n                                           drop=drop, attn_drop=attn_drop,\n                                           drop_path_prob=drop_path_prob[i] if isinstance(\n                                               drop_path_prob, list) else drop_path_prob,\n                                           norm_layer=norm_layer,\n                                           prefix=f'{prefix}/blocks{i}') for i in range(depth)])\n        if downsample is not None:\n            self.downsample = downsample(\n                input_resolution, dim=dim, norm_layer=norm_layer, prefix=prefix)\n        else:\n            self.downsample = None\n\n    def call(self, x):\n        x = self.blocks(x)\n\n        if self.downsample is not None:\n            x = self.downsample(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.916309Z","iopub.execute_input":"2022-04-17T15:46:18.916776Z","iopub.status.idle":"2022-04-17T15:46:18.927481Z","shell.execute_reply.started":"2022-04-17T15:46:18.916743Z","shell.execute_reply":"2022-04-17T15:46:18.926805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PatchEmbed(tf.keras.layers.Layer):\n    def __init__(self, img_size=(224, 224), patch_size=(4, 4), in_chans=3, embed_dim=96, norm_layer=None):\n        super().__init__(name='patch_embed')\n        patches_resolution = [img_size[0] //\n                              patch_size[0], img_size[1] // patch_size[1]]\n        self.img_size = img_size\n        self.patch_size = patch_size\n        self.patches_resolution = patches_resolution\n        self.num_patches = patches_resolution[0] * patches_resolution[1]\n\n        self.in_chans = in_chans\n        self.embed_dim = embed_dim\n\n        self.proj = Conv2D(embed_dim, kernel_size=patch_size,\n                           strides=patch_size, name='proj')\n        if norm_layer is not None:\n            self.norm = norm_layer(epsilon=1e-5, name='norm')\n        else:\n            self.norm = None\n\n    def call(self, x):\n        B, H, W, C = x.get_shape().as_list()\n        assert H == self.img_size[0] and W == self.img_size[1], \\\n            f\"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]}).\"\n        x = self.proj(x)\n        x = tf.reshape(\n            x, shape=[-1, (H // self.patch_size[0]) * (W // self.patch_size[0]), self.embed_dim])\n        if self.norm is not None:\n            x = self.norm(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.928677Z","iopub.execute_input":"2022-04-17T15:46:18.929385Z","iopub.status.idle":"2022-04-17T15:46:18.941107Z","shell.execute_reply.started":"2022-04-17T15:46:18.929276Z","shell.execute_reply":"2022-04-17T15:46:18.940318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SwinTransformerModel(tf.keras.Model):\n    def __init__(self, model_name='swin_tiny_patch4_window7_224', include_top=False,\n                 img_size=(224, 224), patch_size=(4, 4), in_chans=3, num_classes=1000,\n                 embed_dim=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24],\n                 window_size=7, mlp_ratio=4., qkv_bias=True, qk_scale=None,\n                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0.1,\n                 norm_layer=LayerNormalization, ape=False, patch_norm=True,\n                 use_checkpoint=False, **kwargs):\n        super().__init__(name=model_name)\n\n        self.include_top = include_top\n\n        self.num_classes = num_classes\n        self.num_layers = len(depths)\n        self.embed_dim = embed_dim\n        self.ape = ape\n        self.patch_norm = patch_norm\n        self.num_features = int(embed_dim * 2 ** (self.num_layers - 1))\n        self.mlp_ratio = mlp_ratio\n\n        # split image into non-overlapping patches\n        self.patch_embed = PatchEmbed(img_size=img_size, \n                                      patch_size=patch_size, \n                                      in_chans=in_chans, \n                                      embed_dim=embed_dim,\n                                      norm_layer=norm_layer if self.patch_norm else None\n                                     )\n        num_patches = self.patch_embed.num_patches\n        patches_resolution = self.patch_embed.patches_resolution\n        self.patches_resolution = patches_resolution\n\n        # absolute postion embedding\n        if self.ape:\n            self.absolute_pos_embed = self.add_weight('absolute_pos_embed',\n                                                      shape=(1, num_patches, embed_dim),\n                                                      initializer=tf.initializers.Zeros()\n                                                     )\n        self.pos_drop = Dropout(drop_rate)\n\n        # stochastic depth\n        dpr = [x for x in np.linspace(0., drop_path_rate, sum(depths))]\n\n        # build layers\n        self.basic_layers = tf.keras.Sequential([BasicLayer(dim=int(embed_dim * 2 ** i_layer),\n                                                input_resolution=(patches_resolution[0] // (2 ** i_layer),\n                                                                  patches_resolution[1] // (2 ** i_layer)),\n                                                depth=depths[i_layer],\n                                                num_heads=num_heads[i_layer],\n                                                window_size=window_size,\n                                                mlp_ratio=self.mlp_ratio,\n                                                qkv_bias=qkv_bias, qk_scale=qk_scale,\n                                                drop=drop_rate, attn_drop=attn_drop_rate,\n                                                drop_path_prob=dpr[sum(depths[:i_layer]):sum(\n                                                    depths[:i_layer + 1])],\n                                                norm_layer=norm_layer,\n                                                downsample=PatchMerging if (\n                                                    i_layer < self.num_layers - 1) else None,\n                                                use_checkpoint=use_checkpoint,\n                                                prefix=f'layers{i_layer}') for i_layer in range(self.num_layers)]\n                                               )\n        self.norm = norm_layer(epsilon=1e-5, name='norm')\n        self.avgpool = GlobalAveragePooling1D()\n        if self.include_top:\n            self.head = Dense(num_classes, name='head')\n        else:\n            self.head = None\n\n    def forward_features(self, x):\n        x = self.patch_embed(x)\n        if self.ape:\n            x = x + self.absolute_pos_embed\n        x = self.pos_drop(x)\n\n        x = self.basic_layers(x)\n        x = self.norm(x)\n        x = self.avgpool(x)\n        return x\n\n    def call(self, x):\n        x = self.forward_features(x)\n        if self.include_top:\n            x = self.head(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.943928Z","iopub.execute_input":"2022-04-17T15:46:18.944843Z","iopub.status.idle":"2022-04-17T15:46:18.963554Z","shell.execute_reply.started":"2022-04-17T15:46:18.944642Z","shell.execute_reply":"2022-04-17T15:46:18.962934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def SwinTransformer(model_name='swin_tiny_224', \n                    num_classes=1000, \n                    include_top=True, \n                    pretrained=True, \n                    use_tpu=False, \n                    cfgs=CFGS):\n    cfg = cfgs[model_name]\n    net = SwinTransformerModel(model_name=model_name, \n                               include_top=include_top, \n                               num_classes=num_classes, \n                               img_size=cfg['input_size'], \n                               window_size=cfg['window_size'], \n                               embed_dim=cfg['embed_dim'], \n                               depths=cfg['depths'],\n                               num_heads=cfg['num_heads']\n                                )\n    net(tf.keras.Input(shape=(cfg['input_size'][0], cfg['input_size'][1], 3)))\n    if pretrained is True:\n        url = f'https://github.com/rishigami/Swin-Transformer-TF/releases/download/v0.1-tf-swin-weights/{model_name}.tgz'\n        pretrained_ckpt = tf.keras.utils.get_file(model_name, url, untar=True)\n    else:\n        pretrained_ckpt = pretrained\n\n    if pretrained_ckpt:\n        if tf.io.gfile.isdir(pretrained_ckpt):\n            pretrained_ckpt = f'{pretrained_ckpt}/{model_name}.ckpt'\n\n        if use_tpu:\n            load_locally = tf.saved_model.LoadOptions(\n                experimental_io_device='/job:localhost')\n            net.load_weights(pretrained_ckpt, options=load_locally)\n        else:\n            net.load_weights(pretrained_ckpt)\n\n    return net","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.964902Z","iopub.execute_input":"2022-04-17T15:46:18.965343Z","iopub.status.idle":"2022-04-17T15:46:18.975683Z","shell.execute_reply.started":"2022-04-17T15:46:18.965308Z","shell.execute_reply":"2022-04-17T15:46:18.974909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def freeze_BN(model):\n    # Unfreeze layers while leaving BatchNorm layers frozen\n    for layer in model.layers:\n        if not isinstance(layer, tf.keras.layers.BatchNormalization):\n            layer.trainable = True\n        else:\n            layer.trainable = False","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.977043Z","iopub.execute_input":"2022-04-17T15:46:18.977477Z","iopub.status.idle":"2022-04-17T15:46:18.985404Z","shell.execute_reply.started":"2022-04-17T15:46:18.977445Z","shell.execute_reply":"2022-04-17T15:46:18.984756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to create our EfficientNetB3 model\ndef get_model():\n\n    EMB_DIM = 512\n    N_CLASSES_MODEL = CFG.N_CLASSES\n\n    with strategy.scope():\n        inp = tf.keras.layers.Input(shape=(*CFG.IMAGE_SIZE, 3), name=\"Input1\")\n        label = tf.keras.layers.Input(shape=(), name=\"Input2\")\n\n        model_feat = SwinTransformer('swin_large_384', \n                                     num_classes=N_CLASSES_MODEL, \n                                     include_top=False, \n                                     pretrained=True, \n                                     use_tpu=True)\n\n        embed = model_feat(inp)\n        embed = tf.keras.layers.BatchNormalization()(embed) # batch norm or L2\n        embed = tf.keras.layers.Dropout(0.2)(embed)\n        embed = tf.keras.layers.Dense(EMB_DIM, \n                                      name=\"dense_before_arcface\", \n                                      kernel_initializer=\"he_normal\")(embed)\n        x = ArcMarginProduct(n_classes=N_CLASSES_MODEL,\n                             s=30,\n                             m=0.3,\n                             name=\"head/arc_margin\",\n                             dtype=\"float32\")([embed, label])\n\n        output = tf.keras.layers.Softmax(dtype=\"float32\")(x)\n\n        model = tf.keras.Model(inputs=[inp, label], outputs=[output])\n        embed_model = tf.keras.Model(inputs = inp, outputs = embed)\n\n        model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3, epsilon=1e-5),\n                      loss     = [ tf.keras.losses.SparseCategoricalCrossentropy()],\n                      metrics  = [tf.keras.metrics.SparseCategoricalAccuracy(),\n                                  tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5)]\n                    )\n        model.summary()\n        \n        return model,embed_model","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:18.990184Z","iopub.execute_input":"2022-04-17T15:46:18.991785Z","iopub.status.idle":"2022-04-17T15:46:19.001931Z","shell.execute_reply.started":"2022-04-17T15:46:18.991759Z","shell.execute_reply":"2022-04-17T15:46:19.001136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(plot=False):\n    lr_start   = 0.000001\n    lr_max     = 0.000005 * CFG.BATCH_SIZE  \n    lr_min     = 0.000001\n    lr_ramp_ep = 6\n    lr_sus_ep  = 0\n    lr_decay   = 0.9\n   \n    def lrfn(epoch):\n        if CFG.RESUME:\n            epoch = epoch + CFG.RESUME_EPOCH\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n        \n    if plot:\n        epochs = list(range(CFG.EPOCHS))\n        learning_rates = [lrfn(x) for x in epochs]\n        plt.scatter(epochs,learning_rates)\n        plt.show()\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:19.003092Z","iopub.execute_input":"2022-04-17T15:46:19.003619Z","iopub.status.idle":"2022-04-17T15:46:19.013399Z","shell.execute_reply.started":"2022-04-17T15:46:19.003585Z","shell.execute_reply":"2022-04-17T15:46:19.012778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Snapshot(tf.keras.callbacks.Callback):\n    \n    def __init__(self,fold,snapshot_epochs=[]):\n        super(Snapshot, self).__init__()\n        self.snapshot_epochs = snapshot_epochs\n        self.fold = fold\n        \n        \n    def on_epoch_end(self, epoch, logs=None):\n        if epoch in self.snapshot_epochs: # your custom condition         \n            self.model.save_weights(f\"{CFG.MODEL_NAME}_epoch{epoch}.h5\")\n        self.model.save_weights(f\"{CFG.MODEL_NAME}_last.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:19.015996Z","iopub.execute_input":"2022-04-17T15:46:19.016532Z","iopub.status.idle":"2022-04-17T15:46:19.022923Z","shell.execute_reply.started":"2022-04-17T15:46:19.016494Z","shell.execute_reply":"2022-04-17T15:46:19.022275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:#75B7BF;letter-spacing:0.1px;overflow:hidden\"><p style=\"padding:20px;color:white;overflow:hidden;margin:0;font-size:100%;text-align:center\">TRAINING</p></div>","metadata":{}},{"cell_type":"code","source":"training_filenames = [x for i,x in enumerate(train_files) if i%CFG.FOLDS!=CFG.FOLD_TO_RUN]\nvalidation_filenames = [x for i,x in enumerate(train_files) if i%CFG.FOLDS==CFG.FOLD_TO_RUN]\ndel train_files","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:19.024195Z","iopub.execute_input":"2022-04-17T15:46:19.02463Z","iopub.status.idle":"2022-04-17T15:46:19.030838Z","shell.execute_reply.started":"2022-04-17T15:46:19.024581Z","shell.execute_reply":"2022-04-17T15:46:19.030148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Callbacks","metadata":{}},{"cell_type":"code","source":"seed_everything(CFG.SEED)\n# Training Tensor\ntrain_dataset = get_training_dataset(training_filenames)\ntrain_dataset = train_dataset.map(lambda image, label: ({'Input1': image, 'Input2': label}, label))\n# Validation Tensor\nval_dataset = get_validation_dataset(validation_filenames)\nval_dataset = val_dataset.map(lambda image, label: ({'Input1': image, 'Input2': label}, label))\n\nSTEPS_PER_EPOCH = count_data_items(training_filenames)//CFG.BATCH_SIZE\n\n## Logging\ntrain_logger = CSVLogger('./training-log-fold-%i.h5.csv'%CFG.FOLD_TO_RUN)\n\n# SAVE BEST MODEL EACH FOLD\nsv_loss = ModelCheckpoint(f\"./{CFG.MODEL_NAME}_loss_{CFG.FOLD_TO_RUN}.h5\", \n                          monitor='val_loss', \n                          verbose=0, \n                          save_best_only=True, \n                          save_weight_only=True, mode='min', save_freq='epoch')\n# Snapshot\nsnap = Snapshot(fold=CFG.FOLD_TO_RUN,snapshot_epochs=[25,26,27,28,29,30,31])\n# Learning Rate Scheduler\nget_lr_callback(plot=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T15:46:19.032731Z","iopub.execute_input":"2022-04-17T15:46:19.03335Z","iopub.status.idle":"2022-04-17T15:46:22.523401Z","shell.execute_reply.started":"2022-04-17T15:46:19.033318Z","shell.execute_reply":"2022-04-17T15:46:22.522743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build Model\nK.clear_session()\nmodel, embed_model = get_model()\nplot_model(model)","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2022-04-17T15:48:45.991691Z","iopub.execute_input":"2022-04-17T15:48:45.992293Z","iopub.status.idle":"2022-04-17T15:49:43.432623Z","shell.execute_reply.started":"2022-04-17T15:48:45.992249Z","shell.execute_reply":"2022-04-17T15:49:43.431849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####-------------In the last, we trained model and save the log data in Server---------- ###\nhistory = model.fit(train_dataset, \n                    validation_data=val_dataset, \n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    epochs=CFG.EPOCHS, \n                    callbacks=[snap, get_lr_callback(), train_logger, sv_loss], \n                    verbose=1)\n###-------------Loading the previous results----------------###","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"color:white;display:fill;border-radius:5px;background-color:#75B7BF;letter-spacing:0.1px;overflow:hidden\"><p style=\"padding:20px;color:white;overflow:hidden;margin:0;font-size:100%;text-align:center\">REFERENCE</p></div>","metadata":{}},{"cell_type":"markdown","source":"https://www.kaggle.com/code/aikhmelnytskyy/happywhale-effnet-b7-fork-with-detic-training\n\nhttps://pythonrepo.com/repo/rishigami-Swin-Transformer-TF\n\nhttps://www.kaggle.com/code/rishigami/tpu-swin-transformer-tensorflow/","metadata":{}}]}