{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Intro\n\nThe goal of this notebook is to extend upon the work of [MANOJ PRABHAKAR](https://www.kaggle.com/manojprabhaakr) in the [EFFNET B6 WHALE COMP](https://www.kaggle.com/manojprabhaakr/effnet-b6-whale-comp), utilising the [Detic bounding boxes](https://www.kaggle.com/c/happy-whale-and-dolphin/discussion/305503) to precrop the images.\n\nI have created a new set of TFRecords which includes the bounding boxes in [this](https://www.kaggle.com/lextoumbourou/happywhale-tfrecords-with-bounding-boxes) repo.\n\nAt the time of creation, this notebook achieves a top 100 score. Given there's still 2 months left in the competition and that this data is all publically available, I felt it to be reasonable to release.","metadata":{"id":"0793a0ed"}},{"cell_type":"markdown","source":"# Google drive connection","metadata":{"id":"wYGAK_lnl-EK"}},{"cell_type":"code","source":"#effv2\n# https://www.kaggle.com/c/happy-whale-and-dolphin/discussion/307995\n# !pip install kaggle_datasets\n# from kaggle_datasets import KaggleDatasets\n# GCS_DS_PATH=KaggleDatasets().get_gcs_path('efficientnetv2-tfhub-weight-files')\n# print('GCS_DS_PATH', GCS_DS_PATH)\nEFFNETV2_ROOT = 'gs://kds-b9810ad22af72f1e7eacba31a60bffc027d24e6ec7c54b863acea0d3'\n\n","metadata":{"id":"NS5MgmplD_kO","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\n#small val datasets\n# #GCS_DS_PATH=KaggleDatasets().get_gcs_path('dorsalfinv10valval')\n# GCS_DS_PATH gs://kds-4e353a1a78207392ebd5556217aa8fa5b6c7c68e656b23ac0dffc999\n# #GCS_DS_PATH=KaggleDatasets().get_gcs_path('dorsalfinv105valval')\n# GCS_DS_PATH gs://kds-dd609279d91a722b5abf450bb5d97e83983045fcc1e36ef16c665a17\n# #GCS_DS_PATH=KaggleDatasets().get_gcs_path('dorsalfinv11valval')\n# GCS_DS_PATH_VAL = 'gs://kds-17738bb688029b9512a51f01ae2807cf1e6101a9beca572a9d360697'\n# #GCS_DS_PATH=KaggleDatasets().get_gcs_path('dorsalfinv10val')\n# GCS_DS_PATH gs://kds-43f32c5ce3c6907085249e42958ada0574574303970d4786c4b81635\n# #GCS_DS_PATH=KaggleDatasets().get_gcs_path('dorsalfinv105val')\n# GCS_DS_PATH gs://kds-1b1813a3ca80ffbd81c5e4f43410ea012569fc3f47f0238d005f6a8d\n# #GCS_DS_PATH=KaggleDatasets().get_gcs_path('dorsalfinv11val')\n# GCS_DS_PATH = 'gs://kds-1fe71031b93156a76a3db9e65877e9e9b960810e79ff4f1997e39ea2'\n\nGCS_DS_PATH=KaggleDatasets().get_gcs_path('vbodyv10vf8')\n# GCS_DS_PATH = 'gs://kds-8f7821012e27f398d3a41dc4abe6afdb308dadc13352daa3cb2c83b5'\n\n#Pseudo\nGCS_DS_PATH_PSEUDO=KaggleDatasets().get_gcs_path('vbodyv10addvfinal')\n# GCS_DS_PATH_PSEUDO = 'gs://kds-f14272d94700b908b711fa4acee2ef96ec88f23c71e26fcacf6fb98a'\n#GCS_DS_PATH=KaggleDatasets().get_gcs_path('dorsalfinv10addv3')\n#GCS_DS_PATH gs://kds-111a945510244e5eb5b3bca82d31a3c3dd7de0bbbd3fc5868470c38d\n","metadata":{"id":"XxNEBDXH-mM-","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # FEATURE_VECTOR = f'{EFFNETV2_ROOT}/tfhub_models/efficientnetv2-{config.EFF_NETV2}/feature_vector'\n# train_files = np.sort(np.array(tf.io.gfile.glob(FEATURE_VECTOR + '/*.*')))\n# train_files","metadata":{"id":"u4t3c68DaW41","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unzip the download dataset\n# !mkdir /kaggle\n# !mkdir /kaggle/input\n# !mkdir /kaggle/input/happywhale-splits\n# !mkdir /kaggle/logs\n# !mkdir /kaggle/models\n","metadata":{"id":"ksfj8g2bR1K1","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"id":"6c2ba5a5","outputId":"9ae0fa57-c550-4da7-af33-3a8c8f97110b","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# from tensorflow.python.profiler import profiler_client\n\n# tpu_profile_service_address = os.environ['COLAB_TPU_ADDR'].replace('8470', '8466')\n# print(profiler_client.monitor(tpu_profile_service_address, 100, 2))","metadata":{"id":"aoxxkO_nM85l","outputId":"d786c6ea-88a1-4b14-a22e-1d9696f1d83a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q efficientnet\n!pip install tensorflow_addons\nimport re\nimport os\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold, train_test_split\nfrom tensorflow.keras import backend as K\nimport tensorflow_addons as tfa\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nimport pickle\nimport json\nimport tensorflow_hub as tfhub\nfrom datetime import datetime","metadata":{"id":"38034709","outputId":"49de4a5c-c384-480a-f262-11f573102a6c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{"id":"6a86723a"}},{"cell_type":"code","source":"train_files = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH + '/happywhale-2022-train*.tfrec')))\ntrain_files","metadata":{"id":"3P7cUta7ITNy","outputId":"5335da00-2f2c-47c2-ce44-f14b8a1db234","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class config:\n    \n    \n    SEED = 42\n    FOLD_TO_RUN = 10 #0\n    FOLDS = 11 #5\n    DEBUG = False\n    EVALUATE = True\n    RESUME = False\n    RESUME_EPOCH = 9\n    \n    \n    ### Dataset\n    BATCH_SIZE = 8 * strategy.num_replicas_in_sync\n    IMAGE_SIZE = 1024 #608\n    N_CLASSES = 15587\n    \n    ### Model\n    model_type = 'effnetv1' #'effnetv1'  \n    EFF_NET = 5\n    EFF_NETV2 = 'xl-21k-ft1k' #'s-21k-ft1k'\n    FREEZE_BATCH_NORM = False\n    head = 'arcface' \n    EPOCHS = 3\n    LR = 0.001\n    message='baseline'\n    \n    ### Augmentations\n    CUTOUT = False\n    \n    ### Save-Directory\n    save_dir = '.'#save_dir\n    \n    ### Inference\n    KNN = 100\n    \ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)\n\n# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)\n    \ndef is_interactive():\n    return 'runtime'    in get_ipython().config.IPKernelApp.connection_file\nIS_INTERACTIVE = is_interactive()\nprint(IS_INTERACTIVE)","metadata":{"id":"386693ee","outputId":"a9594af4-dc9e-4c94-ad16-6aac060a5018","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_NAME = None\nif config.model_type == 'effnetv1':\n    MODEL_NAME = f'effnetv1_b{config.EFF_NET}'\nelif config.model_type == 'effnetv2':\n    MODEL_NAME = f'effnetv2_{config.EFF_NETV2}'\n\nconfig.MODEL_NAME = MODEL_NAME\nprint(MODEL_NAME)","metadata":{"id":"78ac11c5","outputId":"305f2827-297e-4f31-95b0-8ec3c3b78cde","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_dir = '.'\n# save_dir = '/content/gdrive/MyDrive/kaggle/Whale/output/temp'\n\nEXPERIMENT = 1\n# run_ts = datetime.now().strftime('%Y%m%d-%H%M%S')\n# print(run_ts)\n# run_ts = run_ts + MODEL_NAME + '_rev' + str(config.IMAGE_SIZE)\n# if IS_COLAB:\n#     save_dir = f'/content/gdrive/MyDrive/kaggle/Whale/output/{run_ts}'\n#     !mkdir -p {save_dir}\n\nprint(save_dir)","metadata":{"id":"225a7c6c","outputId":"ebedaec4-d523-4e67-cc98-577256d3cb37","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(config.save_dir+'/config.json', 'w') as fp:\n    json.dump({x:dict(config.__dict__)[x] for x in dict(config.__dict__) if not x.startswith('_')}, fp)","metadata":{"id":"c1c27e1c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH + '/happywhale-2022-train*.tfrec')))\ntest_files = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH + '/happywhale-2022-test*.tfrec')))\n# val_files = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH_VAL + '/happywhale-2022-train*.tfrec')))\npseudo_files = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH_PSEUDO + '/happywhale-2022-train*.tfrec')))\nprint(GCS_DS_PATH)\n# print(GCS_DS_PATH_VAL)\nprint(GCS_DS_PATH_PSEUDO)\nprint(len(train_files),len(test_files),count_data_items(train_files),count_data_items(test_files))\n# print(len(train_files),len(val_files),len(test_files),count_data_items(train_files),count_data_items(val_files),count_data_items(test_files))\nprint(len(pseudo_files),count_data_items(pseudo_files))","metadata":{"id":"96109027","outputId":"29adc3b7-6e76-42e8-fbd2-c0860d6b303b","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data","metadata":{"id":"c6ca4617"}},{"cell_type":"code","source":"SATURATION  = (0.9, 1.1)\nCONTRAST = (0.9, 1.1)\nBRIGHTNESS  =  0.1\nROTATION    = 10.0\nSHEAR    = 10.0\nHZOOM  = 8.0\nWZOOM  = 4.0\nHSHIFT = 4.0\nWSHIFT = 4.0","metadata":{"id":"SrW4WBVcjJmk","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = config.IMAGE_SIZE\n    XDIM = DIM%2\n    \n    rot = ROTATION * tf.random.normal([1], dtype='float32')\n    shr = SHEAR * tf.random.normal([1], dtype='float32')\n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM\n    h_shift = HSHIFT * tf.random.normal([1], dtype='float32')\n    w_shift = WSHIFT * tf.random.normal([1], dtype='float32')\n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot, shr, h_zoom, w_zoom, h_shift, w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","metadata":{"id":"5EdKWdaqjcJ6","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def arcface_format(posting_id, image, label_group, matches):\n    return posting_id, {'inp1': image, 'inp2': label_group}, label_group, matches\n\ndef arcface_inference_format(posting_id, image, label_group, matches):\n    return image,posting_id\n\ndef arcface_eval_format(posting_id, image, label_group, matches):\n    return image,label_group\n\n# Data augmentation function\ndef data_augment(posting_id, image, label_group, matches):\n\n    ### CUTOUT\n    if tf.random.uniform([])>0.5 and config.CUTOUT:\n      N_CUTOUT = 6\n      for cutouts in range(N_CUTOUT):\n        if tf.random.uniform([])>0.5:\n           DIM = config.IMAGE_SIZE\n           CUTOUT_LENGTH = DIM//32\n           x1 = tf.cast( tf.random.uniform([],0,DIM-CUTOUT_LENGTH),tf.int32)\n           x2 = tf.cast( tf.random.uniform([],0,DIM-CUTOUT_LENGTH),tf.int32)\n           filter_ = tf.concat([tf.zeros((x1,CUTOUT_LENGTH)),tf.ones((CUTOUT_LENGTH,CUTOUT_LENGTH)),tf.zeros((DIM-x1-CUTOUT_LENGTH,CUTOUT_LENGTH))],axis=0)\n           filter_ = tf.concat([tf.zeros((DIM,x2)),filter_,tf.zeros((DIM,DIM-x2-CUTOUT_LENGTH))],axis=1)\n           cutout = tf.reshape(1-filter_,(DIM,DIM,1))\n           image = cutout*image\n\n    image = tf.image.random_flip_left_right(image)\n    # image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_hue(image, 0.01)\n    image = tf.image.random_saturation(image, 0.70, 1.30)\n    # image = tf.image.random_saturation(image, SATURATION[0], SATURATION[1])\n    image = tf.image.random_contrast(image, 0.80, 1.20)\n    # image = tf.image.random_contrast(image, CONTRAST[0], CONTRAST[1])\n    image = tf.image.random_brightness(image, 0.10)\n    # image = tf.image.random_brightness(image, BRIGHTNESS)\n    if tf.random.uniform([])>0.5:\n        image = transform(image)\n    return posting_id, image, label_group, matches\n\n# Function to decode our images\n# Updated to include crops.\n# def decode_image(image_data, box):\n#     if box is not None and box[0] != -1:\n#         left, top, right, bottom = box[0], box[1], box[2], box[3]\n#         bbs = tf.convert_to_tensor([top, left, bottom - top, right - left])\n#         image = tf.io.decode_and_crop_jpeg(image_data, bbs, channels=3)\n#     else:\n#         image = tf.image.decode_jpeg(image_data, channels = 3)\n\n#     image = tf.image.resize(image, [config.IMAGE_SIZE,config.IMAGE_SIZE])\n#     image = tf.cast(image, tf.float32) / 255.0\n#     return image\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels = 3)\n    image = tf.image.resize(image, [config.IMAGE_SIZE,config.IMAGE_SIZE])\n    image = tf.cast(image, tf.float32) / 255.0\n    return image\n\n# This function parse our images and also get the target variable\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64),\n        # 'detic_box': tf.io.FixedLenFeature([4], tf.int64),\n         # 'yolov5_box': tf.io.FixedLenFeature([4], tf.int64),\n    }\n\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    posting_id = example['image_name']\n    # bb = tf.cast(example['detic_box'], tf.int32)\n    # image = decode_image(example['image'], bb)\n    image = decode_image(example['image'])\n#     label_group = tf.one_hot(tf.cast(example['label_group'], tf.int32), depth = N_CLASSES)\n    label_group = tf.cast(example['target'], tf.int32)\n#     matches = example['matches']\n    matches = 1\n    return posting_id, image, label_group, matches\n\n# This function loads TF Records and parse them into tensors\ndef load_dataset(filenames, ordered = False):\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False \n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)\n#     dataset = dataset.cache()\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls = AUTO) \n    return dataset\n\n# This function is to get our training tensors\ndef get_training_dataset(filenames):\n    dataset = load_dataset(filenames, ordered = False)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_format, num_parallel_calls = AUTO)\n    dataset = dataset.map(lambda posting_id, image, label_group, matches: (image, label_group))\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n# This function is to get our training tensors\ndef get_val_dataset(filenames):\n    dataset = load_dataset(filenames, ordered = True)\n    # dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_format, num_parallel_calls = AUTO)\n    dataset = dataset.map(lambda posting_id, image, label_group, matches: (image, label_group))\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n# This function is to get our training tensors\ndef get_eval_dataset(filenames, get_targets = True):\n    dataset = load_dataset(filenames, ordered = True)\n    # dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_eval_format, num_parallel_calls = AUTO)\n    if not get_targets:\n        dataset = dataset.map(lambda image, target: image)\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n# This function is to get our training tensors\ndef get_test_dataset(filenames, get_names = True):\n    dataset = load_dataset(filenames, ordered = True)\n    # dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_inference_format, num_parallel_calls = AUTO)\n    if not get_names:\n        dataset = dataset.map(lambda image, posting_id: image)\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","metadata":{"id":"f9538d3f","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# row = 10; col = 8;\n# row = min(row,config.BATCH_SIZE//col)\n# N_TRAIN = count_data_items(train_files)\n# print(N_TRAIN)\n# ds = get_training_dataset(train_files)\n\n# for (sample,label) in ds:\n#     img = sample['inp1']\n#     plt.figure(figsize=(25,int(25*row/col)))\n#     for j in range(row*col):\n#         plt.subplot(row,col,j+1)\n#         plt.title(label[j].numpy())\n#         plt.axis('off')\n#         plt.imshow(img[j,])\n#     plt.show()\n#     break\n# print(img.shape)","metadata":{"id":"df50c1f2","outputId":"54493d48-35ba-437e-f4e1-c38d7672c54e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# img.shape","metadata":{"id":"d-cshFSfOUbO","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# row = 10; col = 8;\n# row = min(row,config.BATCH_SIZE//col)\n# N_TEST = count_data_items(test_files)\n# print(N_TEST)\n# ds = get_test_dataset(test_files)\n\n# for (img,label) in ds:\n#     plt.figure(figsize=(25,int(25*row/col)))\n#     for j in range(row*col):\n#         plt.subplot(row,col,j+1)\n#         plt.title(label[j].numpy())\n#         plt.axis('off')\n#         plt.imshow(img[j,])\n#     plt.show()\n#     break\n# print(img.shape)","metadata":{"id":"a8b06d5e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{"id":"fb235915"}},{"cell_type":"code","source":"# Arcmarginproduct class keras layer\nclass ArcMarginProduct(tf.keras.layers.Layer):\n    '''\n    Implements large margin arc distance.\n\n    Reference:\n        https://arxiv.org/pdf/1801.07698.pdf\n        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n            blob/master/src/modeling/metric_learning.py\n    '''\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n\n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n\n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n        one_hot = tf.cast(\n            tf.one_hot(y, depth=self.n_classes),\n            dtype=cosine.dtype\n        )\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output","metadata":{"id":"17c671a3","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arr = [\n         'blue_whale',\n         'bottlenose_dolphin',\n         'brydes_whale',\n         'commersons_dolphin',\n         'common_dolphin',\n         'cuviers_beaked_whale',\n         'dusky_dolphin',\n         'false_killer_whale',\n         'fin_whale',\n         'frasiers_dolphin',\n         'globis',\n         'humpback_whale',\n         'killer_whale',\n         'long_finned_pilot_whale',\n         'melon_headed_whale',\n         'minke_whale',\n         'pantropic_spotted_dolphin',\n         'pilot_whale',\n         'pygmy_killer_whale',\n         'rough_toothed_dolphin',\n         'sei_whale',\n         'short_finned_pilot_whale',\n         'spinner_dolphin',\n         'spotted_dolphin',\n         'white_sided_dolphin'\n    \n        ]\n\n","metadata":{"id":"o4VCs0CVA9dS","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get adaptive margin\ndf_marg = pd.read_csv('/kaggle/input/happy-whale-and-dolphin/train.csv')\n# df_marg = pd.read_csv('/kaggle/input/happywhale-splits/skf_species_10folds.csv')\n# df_marg = df_marg[df_marg[\"species_x\"].isin(arr)]\n# df_marg.species.replace({\"globis\": \"short_finned_pilot_whale\",\n#                           \"pilot_whale\": \"short_finned_pilot_whale\",\n#                           \"kiler_whale\": \"killer_whale\",\n#                           \"bottlenose_dolpin\": \"bottlenose_dolphin\"}, inplace=True)\n# df_marg = df_marg[df_marg[\"species\"].isin(arr)]\ntmp = np.sqrt(1 / np.sqrt(df_marg['individual_id'].value_counts().sort_index().values))\nmargins = (tmp - tmp.min()) / (tmp.max() - tmp.min()) * 0.55 + 0.05\n# margins_remain = np.repeat(1e-9, config.N_CLASSES)\n# margins = np.concatenate([margins, margins_remain], 0)[:config.N_CLASSES]\nmargins = margins.astype(np.float32)\nmargins.shape","metadata":{"id":"HGmBkdNgoJK4","outputId":"7778e0de-5452-4f45-987f-6538e887ac44","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(margins).value_counts()","metadata":{"id":"B01pDDCcXqdm","outputId":"020ce4b9-d0c6-4ec6-9755-878a6794d24e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n#         efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\n\n# def freeze_BN(model):\n#     # Unfreeze layers while leaving BatchNorm layers frozen\n#     for layer in model.layers:\n#         if not isinstance(layer, tf.keras.layers.BatchNormalization):\n#             layer.trainable = True\n#         else:\n#             layer.trainable = False\n\n# # Function to create our EfficientNetB3 model\n# def get_model():\n\n#     if config.head=='arcface':\n#         head = ArcMarginProduct\n#     else:\n#         assert 1==2, \"INVALID HEAD\"\n    \n#     with strategy.scope():\n        \n#         margin = head(\n#             n_classes = config.N_CLASSES, \n#             s = 30, \n#             m = margins, #0.3, \n#             name=f'head/{config.head}', \n#             dtype='float32'\n#             )\n\n#         inp = tf.keras.layers.Input(shape = [config.IMAGE_SIZE, config.IMAGE_SIZE, 3], name = 'inp1')\n#         label = tf.keras.layers.Input(shape = (), name = 'inp2')\n        \n#         if config.model_type == 'effnetv1':\n#             x = EFNS[config.EFF_NET](weights = 'noisy-student', include_top = False)(inp)\n#             embed = tf.keras.layers.GlobalAveragePooling2D()(x)\n#             # inp = EFNS[config.EFF_NET](weights = 'noisy-student', include_top = False,input_shape = [config.IMAGE_SIZE, config.IMAGE_SIZE, 3])\n#             # inp.layers[0]._name = 'inp1'\n#             #inp.summary()\n#             # x1=tf.keras.layers.GlobalAveragePooling2D()(inp.layers[-1].output)\n#             # x2=tf.keras.layers.GlobalAveragePooling2D()(inp.layers[-5].output)\n#             # x3=tf.keras.layers.GlobalAveragePooling2D()(inp.layers[-7].output)\n#             # x4=tf.keras.layers.GlobalAveragePooling2D()(inp.layers[-13].output)\n#             # embed =  tf.concat([x1,x2,x3,x4],axis = 1)\n\n#         elif config.model_type == 'effnetv2':\n#             FEATURE_VECTOR = f'{EFFNETV2_ROOT}/tfhub_models/efficientnetv2-{config.EFF_NETV2}/feature_vector'\n#             embed = tfhub.KerasLayer(FEATURE_VECTOR, trainable=True)(inp)\n            \n#         embed = tf.keras.layers.Dropout(0.2)(embed)\n#         # embed = tf.keras.layers.Dense(512)(embed)\n#         embed = tf.keras.layers.Dense(512,kernel_regularizer=tf.keras.regularizers.l2(0.0001))(embed)\n#         # embed = tf.keras.layers.Dense(config.N_CLASSES,kernel_regularizer=tf.keras.regularizers.l2(0.0001))(embed)       \n#         # embed = tf.keras.layers.Dropout(0.3)(embed)\n#         # embed = tf.keras.layers.Dense(2048)(embed)\n\n#         x = margin([embed, label])\n        \n#         output = tf.keras.layers.Softmax(dtype='float32')(x)\n        \n#         model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])\n#         embed_model = tf.keras.models.Model(inputs = inp, outputs = embed)\n#         # model = tf.keras.models.Model(inputs = [inp.input, label], outputs = [output])\n#         # embed_model = tf.keras.models.Model(inputs = inp.input, outputs = embed)    \n        \n#         # opt = tf.keras.optimizers.Adam(learning_rate = config.LR)\n#         # opt = tf.keras.optimizers.SGD(learning_rate=config.LR, momentum=0.9,decay=0.0005)\n#         opt = tf.keras.optimizers.SGD(learning_rate=config.LR, momentum=0.9)\n#         if config.FREEZE_BATCH_NORM:\n#             freeze_BN(model)\n\n#         model.compile(\n#             optimizer = opt,\n#             loss = [tf.keras.losses.SparseCategoricalCrossentropy()],\n#             metrics = [tf.keras.metrics.SparseCategoricalAccuracy(),tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5)]\n#             ) \n        \n#         return model,embed_model","metadata":{"id":"8f5fa3b1","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\n\ndef freeze_BN(model):\n    # Unfreeze layers while leaving BatchNorm layers frozen\n    for layer in model.layers:\n        if not isinstance(layer, tf.keras.layers.BatchNormalization):\n            layer.trainable = True\n        else:\n            layer.trainable = False\n\n# Function to create our EfficientNetB3 model\ndef get_model():\n\n    if config.head=='arcface':\n        head = ArcMarginProduct\n    else:\n        assert 1==2, \"INVALID HEAD\"\n    \n    with strategy.scope():\n        \n        margin = head(\n            n_classes = config.N_CLASSES, \n            s = 30, \n            m = margins, #0.3, \n            name=f'head/{config.head}', \n            dtype='float32'\n            )\n\n        # inp = tf.keras.layers.Input(shape = [config.IMAGE_SIZE, config.IMAGE_SIZE, 3], name = 'inp1')\n        label = tf.keras.layers.Input(shape = (), name = 'inp2')\n        \n        if config.model_type == 'effnetv1':\n            # x = EFNS[config.EFF_NET](weights = 'noisy-student', include_top = False)(inp)\n            # embed = tf.keras.layers.GlobalAveragePooling2D()(x)\n            inp = EFNS[config.EFF_NET](weights = 'noisy-student', include_top = False,input_shape = [config.IMAGE_SIZE, config.IMAGE_SIZE, 3])\n            inp.layers[0]._name = 'inp1'\n            embed = tf.keras.layers.GlobalAveragePooling2D()(inp.layers[-1].output)\n\n        elif config.model_type == 'effnetv2':\n            FEATURE_VECTOR = f'{EFFNETV2_ROOT}/tfhub_models/efficientnetv2-{config.EFF_NETV2}/feature_vector'\n            embed = tfhub.KerasLayer(FEATURE_VECTOR, trainable=True)(inp.output)\n            \n        embed = tf.keras.layers.BatchNormalization()(embed) # batch norm or L2\n        embed = tf.keras.layers.Dropout(0.2)(embed)\n        embed = tf.keras.layers.Dense(512, kernel_initializer=\"he_normal\",kernel_regularizer=tf.keras.regularizers.l2(0.0001))(embed)\n\n        x = margin([embed, label])\n        \n        output = tf.keras.layers.Softmax(dtype='float32')(x)\n        \n        # model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])\n        # embed_model = tf.keras.models.Model(inputs = inp, outputs = embed)\n        model = tf.keras.models.Model(inputs = [inp.input, label], outputs = [output])\n        embed_model = tf.keras.models.Model(inputs = inp.input, outputs = embed)\n        for layer in inp.layers:\n          if isinstance(layer, tf.keras.layers.BatchNormalization):\n              layer.trainable = False\n        \n        opt = tf.keras.optimizers.Adam(learning_rate = config.LR)\n        # opt = tf.keras.optimizers.SGD(learning_rate=config.LR, momentum=0.9,decay=0.0005)\n        # opt = tf.keras.optimizers.SGD(learning_rate=config.LR, momentum=0.9)\n        if config.FREEZE_BATCH_NORM:\n            freeze_BN(model)\n\n        model.compile(\n            optimizer = opt,\n            loss = [tf.keras.losses.SparseCategoricalCrossentropy()],\n            metrics = [tf.keras.metrics.SparseCategoricalAccuracy(),tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5)]\n            ) \n        \n        return model,embed_model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(plot=False):\n    lr_start   = 0.0001 #0.000001\n    lr_max     = 0.0002 #0.000002 * config.BATCH_SIZE  \n    lr_min     = 0.000001 #0.000001\n    lr_ramp_ep = 2\n    lr_sus_ep  = 0\n    lr_decay   = 0.96\n   \n    def lrfn(epoch):\n        if config.RESUME:\n            epoch = epoch + config.RESUME_EPOCH\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n        \n    if plot:\n        epochs = list(range(config.EPOCHS))\n        learning_rates = [lrfn(x) for x in epochs]\n        plt.scatter(epochs,learning_rates)\n        plt.show()\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\n\nget_lr_callback(plot=True)","metadata":{"id":"7c1c5eb8","outputId":"84d3cc57-a95c-4c00-c11b-9f59b3870271","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Snapshot(tf.keras.callbacks.Callback):\n    \n    def __init__(self,fold,snapshot_epochs=[]):\n        super(Snapshot, self).__init__()\n        self.snapshot_epochs = snapshot_epochs\n        self.fold = fold\n        \n        \n    def on_epoch_end(self, epoch, logs=None):\n        # logs is a dictionary\n#         print(f\"epoch: {epoch}, train_acc: {logs['acc']}, valid_acc: {logs['val_acc']}\")\n        if epoch in self.snapshot_epochs: # your custom condition         \n            self.model.save_weights(config.save_dir+f\"/EF{config.MODEL_NAME}_epoch{epoch}.h5\")\n        self.model.save_weights(config.save_dir+f\"/{config.MODEL_NAME}_last.h5\")","metadata":{"id":"81b505b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train","metadata":{"id":"550a9719"}},{"cell_type":"code","source":"# # TRAINING_FILENAMES = [x for i,x in enumerate(train_files) if i%config.FOLDS!=config.FOLD_TO_RUN]\n# # VALIDATION_FILENAMES = [x for i,x in enumerate(train_files) if i%config.FOLDS==config.FOLD_TO_RUN]\n# val_filename = GCS_DS_PATH + \"/happywhale-2022-train-10-645.tfrec\"\n# TRAINING_FILENAMES = [x for i,x in enumerate(train_files)]\n# TRAINING_FILENAMES.remove(val_filename)\n# VALIDATION_FILENAMES = [val_filename]\n# print(len(TRAINING_FILENAMES),len(VALIDATION_FILENAMES),count_data_items(TRAINING_FILENAMES),count_data_items(VALIDATION_FILENAMES))\n# PSEUDO_FILENAMES = [x for i,x in enumerate(pseudo_files)]\n# print(len(PSEUDO_FILENAMES),count_data_items(PSEUDO_FILENAMES))\nTRAINING_FILENAMES = [x for i,x in enumerate(train_files)]\nVALIDATION_FILENAMES = []\nPSEUDO_FILENAMES = [x for i,x in enumerate(pseudo_files)]\nprint(len(TRAINING_FILENAMES),len(VALIDATION_FILENAMES),count_data_items(TRAINING_FILENAMES),count_data_items(VALIDATION_FILENAMES))\nprint(len(PSEUDO_FILENAMES),count_data_items(PSEUDO_FILENAMES))\n","metadata":{"id":"bb8e0750","outputId":"0957be43-5fa7-43b1-94d1-394cfbe72d90","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAINING_FILENAMES = TRAINING_FILENAMES+PSEUDO_FILENAMES\nprint(len(TRAINING_FILENAMES),len(VALIDATION_FILENAMES),count_data_items(TRAINING_FILENAMES),count_data_items(VALIDATION_FILENAMES))","metadata":{"id":"4j1i7yajNrfW","outputId":"4fda62a3-295e-4d13-cd16-8dff144bfc2a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAINING_FILENAMES","metadata":{"id":"RdzOqj1Gc4xn","outputId":"99c6c950-d4d7-4cd7-f7c1-93d99132364a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VALIDATION_FILENAMES","metadata":{"id":"bbA937kzi_R-","outputId":"433230ac-3a15-423c-f26c-50ff4198f0d5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config.DEBUG:\n    TRAINING_FILENAMES = [TRAINING_FILENAMES[0]]\n    VALIDATION_FILENAMES = [VALIDATION_FILENAMES[0]]\n    print(len(TRAINING_FILENAMES),len(VALIDATION_FILENAMES),count_data_items(TRAINING_FILENAMES),count_data_items(VALIDATION_FILENAMES))\n    test_files = [test_files[0]]","metadata":{"id":"f2fb5064","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(config.SEED)\nVERBOSE = 1\ntrain_dataset = get_training_dataset(TRAINING_FILENAMES)\nval_dataset = get_val_dataset(VALIDATION_FILENAMES)\nSTEPS_PER_EPOCH = count_data_items(TRAINING_FILENAMES) // config.BATCH_SIZE\ntrain_logger = tf.keras.callbacks.CSVLogger(config.save_dir+'/training-log-fold-%i.h5.csv'%config.FOLD_TO_RUN)\n# SAVE BEST MODEL EACH FOLD        \nsv_loss = tf.keras.callbacks.ModelCheckpoint(\n    config.save_dir+f\"/{config.MODEL_NAME}_loss.h5\", monitor='val_loss', verbose=0, save_best_only=True,\n    save_weights_only=True, mode='min', save_freq='epoch')\n# BUILD MODEL\nK.clear_session()\nmodel,embed_model = get_model()\nsnap = Snapshot(fold=config.FOLD_TO_RUN,snapshot_epochs=[10,15,16,17,18,19,20,21,24,27,30,35,40,45,48,53,57])\nmodel.summary()\n\n# if config.RESUME:   \n#     model.load_weights(config.resume_model_wts)","metadata":{"id":"bfefe3f6","outputId":"f039e25f-90af-4dcb-b130-8b414235a011","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"id":"_0Uhd-RUKJqj","outputId":"61faced9-ec3c-4f86-fd72-8e11b91ddc59","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#use body data set pretrain model\n# model.load_weights('/kaggle/input/pretrainedbodyb7ep21/EFeffnetv1_b7_epoch21.h5')\nmodel.load_weights('/kaggle/input/bodyvfinal/effnetv1_b5_loss.h5')\n","metadata":{"id":"zQLaR1t4wEW3","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('#### Image Size %i with EfficientNet B%i and batch_size %i'%\n      (config.IMAGE_SIZE,config.EFF_NET,config.BATCH_SIZE))\n\nhistory = model.fit(train_dataset,\n                validation_data = val_dataset,\n                steps_per_epoch = STEPS_PER_EPOCH,\n                epochs = config.EPOCHS,\n                callbacks = [snap,get_lr_callback(),train_logger,sv_loss], \n                verbose = VERBOSE)","metadata":{"id":"c997cfb4","outputId":"c79e946d-24e6-4955-cb24-63e4b7f61fdf","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}