{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q efficientnet\n!pip install -q tensorflow_addons\n#!pip install tfimm","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-04-05T11:12:27.484146Z","iopub.execute_input":"2022-04-05T11:12:27.484446Z","iopub.status.idle":"2022-04-05T11:12:43.038563Z","shell.execute_reply.started":"2022-04-05T11:12:27.484417Z","shell.execute_reply":"2022-04-05T11:12:43.037481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Import

","metadata":{}},{"cell_type":"code","source":"import re\nimport os\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.neighbors import NearestNeighbors\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.layers import BatchNormalization, Input, GlobalAveragePooling2D, Dropout, Dense, Softmax\nfrom tensorflow.keras.losses import SparseCategoricalCrossentropy\nfrom tensorflow.keras.metrics import SparseCategoricalAccuracy, SparseTopKCategoricalAccuracy\nfrom tensorflow.keras.callbacks import LearningRateScheduler, CSVLogger, ModelCheckpoint\nfrom tensorflow.keras.utils import plot_model\nimport tensorflow_addons as tfa\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nimport pickle\nimport json\nimport tensorflow_hub as tfhub\nfrom datetime import datetime\n\nfrom kaggle_datasets import KaggleDatasets","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-04-05T11:12:43.040967Z","iopub.execute_input":"2022-04-05T11:12:43.041225Z","iopub.status.idle":"2022-04-05T11:12:43.053823Z","shell.execute_reply.started":"2022-04-05T11:12:43.041194Z","shell.execute_reply":"2022-04-05T11:12:43.052872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"\\n... ACCELERATOR SETUP STARTING ...\\n\")\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n TPU = tf.distribute.cluster_resolver.TPUClusterResolver() \nexcept ValueError:\n TPU = None\n\nif TPU:\n print(f\"\\n... RUNNING ON TPU - {TPU.master()}...\")\n tf.config.experimental_connect_to_cluster(TPU)\n tf.tpu.experimental.initialize_tpu_system(TPU)\n strategy = tf.distribute.experimental.TPUStrategy(TPU)\nelse:\n print(f\"\\n... RUNNING ON CPU/GPU ...\")\n # Yield the default distribution strategy in Tensorflow\n # --> Works on CPU and single GPU.\n strategy = tf.distribute.get_strategy() \n\n# What Is a Replica?\n# --> A single Cloud TPU device consists of FOUR chips, each of which has TWO TPU cores. \n# --> Therefore, for efficient utilization of Cloud TPU, a program should make use of each of the EIGHT (4x2) cores. \n# --> Each replica is essentially a copy of the training graph that is run on each core and \n# trains a mini-batch containing 1/8th of the overall batch size\nN_REPLICAS = strategy.num_replicas_in_sync\n \nprint(f\"... # OF REPLICAS: {N_REPLICAS} ...\\n\")\n\nAUTO = tf.data.experimental.AUTOTUNE\n\nprint(f\"\\n... ACCELERATOR SETUP COMPLTED ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:43.056647Z","iopub.execute_input":"2022-04-05T11:12:43.056963Z","iopub.status.idle":"2022-04-05T11:12:49.584783Z","shell.execute_reply.started":"2022-04-05T11:12:43.056923Z","shell.execute_reply":"2022-04-05T11:12:49.583959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"\\n... XLA OPTIMIZATIONS STARTING ...\\n\")\n\nprint(f\"\\n... CONFIGURE JIT (JUST IN TIME) COMPILATION ...\\n\")\n# enable XLA optmizations (10% speedup when using @tf.function calls)\ntf.config.optimizer.set_jit(True)\n\nprint(f\"\\n... XLA OPTIMIZATIONS COMPLETED ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:49.586698Z","iopub.execute_input":"2022-04-05T11:12:49.586959Z","iopub.status.idle":"2022-04-05T11:12:49.592535Z","shell.execute_reply.started":"2022-04-05T11:12:49.586932Z","shell.execute_reply":"2022-04-05T11:12:49.591709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Configuration and Versions

","metadata":{}},{"cell_type":"code","source":"run_ts = datetime.now().strftime('%Y%m%d-%H%M%S')\nprint(run_ts)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:49.594083Z","iopub.execute_input":"2022-04-05T11:12:49.594375Z","iopub.status.idle":"2022-04-05T11:12:49.606392Z","shell.execute_reply.started":"2022-04-05T11:12:49.594338Z","shell.execute_reply":"2022-04-05T11:12:49.605425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n \n SEED = 42\n FOLD_TO_RUN = 0 # To seperate train/validation dataset\n FOLDS = 5 # If FOLDS == FOLD_TO_RUN, using all data for trainning\n DEBUG = False # If True, get out some data to run model\n EVALUATE = True\n \n ### Dataset\n GCS_PATH = 'happywhale-cropped-removebackground-tfrecords-v1'\n BATCH_SIZE = 8 * N_REPLICAS\n IMAGE_SIZE = 768\n N_CLASSES = 15587\n \n ### Model\n model_type = f'effnetv2'\n EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\n EFF_NET = 7 # choose EfficientV1\n EFF_NETV2 = f'efficientnetv2-xl-21k-ft1k'\n FREEZE_BATCH_NORM = False # Choose inference or \n head = 'arcface' # head layer in model\n EPOCHS = 25\n LR = 0.001\n message='baseline'\n RESUME = False # Resume learning from the checkpoint\n \n ### Augmentations\n CUTOUT = False\n N_CUTOUT = 6\n ### Inference\n KNN = 100\n \n ###Learning Rate Scheduler\n RESUME_EPOCH = 9\n \ndef count_data_items(filenames):\n n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n for filename in filenames]\n return np.sum(n)\n \n# Function to seed everything\ndef seed_everything(seed):\n random.seed(seed)\n np.random.seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed)\n tf.random.set_seed(seed)\n \ndef is_interactive():\n return 'runtime' in get_ipython().config.IPKernelApp.connection_file\nIS_INTERACTIVE = is_interactive()\nprint(IS_INTERACTIVE)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:49.607931Z","iopub.execute_input":"2022-04-05T11:12:49.608475Z","iopub.status.idle":"2022-04-05T11:12:49.622494Z","shell.execute_reply.started":"2022-04-05T11:12:49.608435Z","shell.execute_reply":"2022-04-05T11:12:49.621657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_NAME = None\nif CFG.model_type == 'effnetv1':\n MODEL_NAME = f'effnetv1_b{CFG.EFF_NET}'\nelif CFG.model_type == 'effnetv2':\n MODEL_NAME = f'effnetv2_{CFG.EFF_NETV2}'\n\nCFG.MODEL_NAME = MODEL_NAME\nprint(MODEL_NAME)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:49.624178Z","iopub.execute_input":"2022-04-05T11:12:49.624745Z","iopub.status.idle":"2022-04-05T11:12:49.63559Z","shell.execute_reply.started":"2022-04-05T11:12:49.624708Z","shell.execute_reply":"2022-04-05T11:12:49.634777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\n... DATA ACCESS SETUP STARTED ...\\n\")\n# Choose the \nGCS_PATH = KaggleDatasets().get_gcs_path(CFG.GCS_PATH)\ntrain_files = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/happywhale-2022-train*.tfrec')))\ntest_files = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/happywhale-2022-test*.tfrec')))\n\nprint(count_data_items(train_files),count_data_items(test_files))\n \nprint(\"\\n\\n... DATA ACCESS SETUP COMPLETED ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:49.637193Z","iopub.execute_input":"2022-04-05T11:12:49.637741Z","iopub.status.idle":"2022-04-05T11:12:50.189199Z","shell.execute_reply.started":"2022-04-05T11:12:49.637705Z","shell.execute_reply":"2022-04-05T11:12:50.188323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Dataset

","metadata":{}},{"cell_type":"markdown","source":"#

EXTRA DATASET

\n****\n## 1. happywhale-splits\nThis dataset is created by @ks2019 with code:\n```\ntrain_df = pd.read_csv('../input/happy-whale-and-dolphin/train.csv')\ntrain_df.species.replace({\"globis\": \"short_finned_pilot_whale\",\n \"pilot_whale\": \"short_finned_pilot_whale\",\n \"kiler_whale\": \"killer_whale\",\n \"bottlenose_dolpin\": \"bottlenose_dolphin\"}, inplace=True)\ntrain_df.to_csv('train_fixed.csv',index=False)\n\ntrain_df = pd.read_csv('train_fixed.csv')\ntrain_df['individual_id'] = train_df['individual_id'].map(individual_ids)\ntrain_df['species'] = train_df['species'].map(species)\nskf = StratifiedKFold(n_splits=5,random_state=123)\nfor fold,(train_index, test_index) in enumerate(skf.split(train_df, train_df.species)):\n train_df.loc[test_index,'fold'] = fold\nprint(train_df.groupby('fold').individual_id.nunique().to_dict())\nprint(train_df.groupby('fold').species.nunique().to_dict())\nprint(train_df.groupby('fold').image.nunique().to_dict())\ntrain_df.to_csv('skf_species_5folds.csv',index=False)\n```\n\n\n## 2. TFRecords \nhttps://www.kaggle.com/datasets/phanttan/happywhale-cropped-removebackground-tfrecords-v1\n","metadata":{}},{"cell_type":"markdown","source":"#

HELPER FUNCTIONS

","metadata":{}},{"cell_type":"markdown","source":"## ArcFace","metadata":{}},{"cell_type":"code","source":"def arcface_format(posting_id, image, label_group, matches):\n return posting_id, {'Input1': image, 'Input2': label_group}, label_group, matches\n\ndef arcface_inference_format(posting_id, image, label_group, matches):\n return image,posting_id\n\ndef arcface_eval_format(posting_id, image, label_group, matches):\n return image,label_group","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:50.190256Z","iopub.execute_input":"2022-04-05T11:12:50.190445Z","iopub.status.idle":"2022-04-05T11:12:50.19593Z","shell.execute_reply.started":"2022-04-05T11:12:50.190423Z","shell.execute_reply":"2022-04-05T11:12:50.195138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Augmentation ","metadata":{}},{"cell_type":"code","source":"def data_augment(posting_id, image, label_group, matches):\n ### CUTOUT\n if tf.random.uniform([])>0.5 and CFG.CUTOUT:\n N_CUTOUT = 6\n for cutouts in range(N_CUTOUT):\n if tf.random.uniform([])>0.5:\n DIM = CFG.IMAGE_SIZE\n CUTOUT_LENGTH = DIM//8\n x1 = tf.cast( tf.random.uniform([],0,DIM-CUTOUT_LENGTH),tf.int32)\n x2 = tf.cast( tf.random.uniform([],0,DIM-CUTOUT_LENGTH),tf.int32)\n filter_ = tf.concat([tf.zeros((x1,CUTOUT_LENGTH)),tf.ones((CUTOUT_LENGTH,CUTOUT_LENGTH)),tf.zeros((DIM-x1-CUTOUT_LENGTH,CUTOUT_LENGTH))],axis=0)\n filter_ = tf.concat([tf.zeros((DIM,x2)),filter_,tf.zeros((DIM,DIM-x2-CUTOUT_LENGTH))],axis=1)\n cutout = tf.reshape(1-filter_,(DIM,DIM,1))\n image = cutout*image\n\n image = tf.image.random_flip_left_right(image)\n # image = tf.image.random_flip_up_down(image)\n image = tf.image.random_hue(image, 0.01)\n image = tf.image.random_saturation(image, 0.70, 1.30)\n image = tf.image.random_contrast(image, 0.80, 1.20)\n image = tf.image.random_brightness(image, 0.10)\n return posting_id, image, label_group, matches\n\n# Function to decode our images\ndef decode_image(image_data):\n image = tf.image.decode_jpeg(image_data, channels = 3)\n image = tf.image.resize(image, [CFG.IMAGE_SIZE,CFG.IMAGE_SIZE])\n image = tf.cast(image, tf.float32) / 255.0\n return image\n\n# This function parse our images and also get the target variable\ndef read_labeled_tfrecord(example):\n LABELED_TFREC_FORMAT = {\n \"image_name\": tf.io.FixedLenFeature([], tf.string),\n \"image\": tf.io.FixedLenFeature([], tf.string),\n \"target\": tf.io.FixedLenFeature([], tf.int64),\n# \"matches\": tf.io.FixedLenFeature([], tf.string)\n }\n\n example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n posting_id = example['image_name']\n image = decode_image(example['image'])\n# label_group = tf.one_hot(tf.cast(example['label_group'], tf.int32), depth = N_CLASSES)\n label_group = tf.cast(example['target'], tf.int32)\n# matches = example['matches']\n matches = 1\n return posting_id, image, label_group, matches","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:50.198075Z","iopub.execute_input":"2022-04-05T11:12:50.198423Z","iopub.status.idle":"2022-04-05T11:12:50.214689Z","shell.execute_reply.started":"2022-04-05T11:12:50.198396Z","shell.execute_reply":"2022-04-05T11:12:50.213899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TF Records to Tensors Functions","metadata":{}},{"cell_type":"code","source":"def load_dataset(filenames, ordered = False):\n \"\"\"Loading Dataset from dataset: happywhale-tfrecords-v1.\n\n Parameters\n ----------\n label : string\n The true label of the image\n predictions : list\n A list of predicted elements (order does matter, 5 predictions allowed per image)\n\n Returns\n -------\n score : double\n \"\"\" \n \n ignore_order = tf.data.Options()\n if not ordered:\n ignore_order.experimental_deterministic = False \n \n dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)\n# dataset = dataset.cache()\n dataset = dataset.with_options(ignore_order)\n dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls = AUTO) \n return dataset\n\n# Get Dataset with some configurations for Training\ndef get_training_dataset(filenames):\n dataset = load_dataset(filenames, ordered = False)\n dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n dataset = dataset.map(arcface_format, num_parallel_calls = AUTO)\n dataset = dataset.map(lambda posting_id, image, label_group, matches: (image, label_group))\n dataset = dataset.repeat()\n dataset = dataset.shuffle(2048)\n dataset = dataset.batch(CFG.BATCH_SIZE)\n dataset = dataset.prefetch(AUTO)\n return dataset\n\n# Get Dataset with some configurations for Validation\ndef get_val_dataset(filenames):\n dataset = load_dataset(filenames, ordered = True)\n dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n dataset = dataset.map(arcface_format, num_parallel_calls = AUTO)\n dataset = dataset.map(lambda posting_id, image, label_group, matches: (image, label_group))\n dataset = dataset.batch(CFG.BATCH_SIZE)\n dataset = dataset.prefetch(AUTO)\n return dataset\n\n# This function is to get our training tensors\ndef get_eval_dataset(filenames, get_targets = True):\n dataset = load_dataset(filenames, ordered = True)\n dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n dataset = dataset.map(arcface_eval_format, num_parallel_calls = AUTO)\n if not get_targets:\n dataset = dataset.map(lambda image, target: image)\n dataset = dataset.batch(CFG.BATCH_SIZE)\n dataset = dataset.prefetch(AUTO)\n return dataset\n\n# Get Dataset with some configurations for Prediction\ndef get_test_dataset(filenames, get_names = True):\n dataset = load_dataset(filenames, ordered = True)\n dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n dataset = dataset.map(arcface_inference_format, num_parallel_calls = AUTO)\n if not get_names:\n dataset = dataset.map(lambda image, posting_id: image)\n dataset = dataset.batch(CFG.BATCH_SIZE)\n dataset = dataset.prefetch(AUTO)\n return dataset","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:50.21595Z","iopub.execute_input":"2022-04-05T11:12:50.216348Z","iopub.status.idle":"2022-04-05T11:12:50.231902Z","shell.execute_reply.started":"2022-04-05T11:12:50.216318Z","shell.execute_reply":"2022-04-05T11:12:50.231081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Arcmarginproduct class keras layer\nclass ArcMarginProduct(tf.keras.layers.Layer):\n '''\n Implements large margin arc distance.\n\n Reference:\n https://arxiv.org/pdf/1801.07698.pdf\n https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n blob/master/src/modeling/metric_learning.py\n '''\n def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n ls_eps=0.0, **kwargs):\n\n super(ArcMarginProduct, self).__init__(**kwargs)\n\n self.n_classes = n_classes\n self.s = s\n self.m = m\n self.ls_eps = ls_eps\n self.easy_margin = easy_margin\n self.cos_m = tf.math.cos(m)\n self.sin_m = tf.math.sin(m)\n self.th = tf.math.cos(math.pi - m)\n self.mm = tf.math.sin(math.pi - m) * m\n\n def get_config(self):\n config = super().get_config().copy()\n config.update({\n 'n_classes': self.n_classes,\n 's': self.s,\n 'm': self.m,\n 'ls_eps': self.ls_eps,\n 'easy_margin': self.easy_margin,\n })\n return config\n\n def build(self, input_shape):\n super(ArcMarginProduct, self).build(input_shape[0])\n self.W = self.add_weight(\n name='W',\n shape=(int(input_shape[0][-1]), self.n_classes),\n initializer='glorot_uniform',\n dtype='float32',\n trainable=True,\n regularizer=None)\n\n def call(self, inputs):\n X, y = inputs\n y = tf.cast(y, dtype=tf.int32)\n cosine = tf.matmul(\n tf.math.l2_normalize(X, axis=1),\n tf.math.l2_normalize(self.W, axis=0)\n )\n sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n phi = cosine * self.cos_m - sine * self.sin_m\n if self.easy_margin:\n phi = tf.where(cosine > 0, phi, cosine)\n else:\n phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n one_hot = tf.cast(\n tf.one_hot(y, depth=self.n_classes),\n dtype=cosine.dtype\n )\n if self.ls_eps > 0:\n one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n output *= self.s\n return output","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:50.233315Z","iopub.execute_input":"2022-04-05T11:12:50.233853Z","iopub.status.idle":"2022-04-05T11:12:50.2502Z","shell.execute_reply.started":"2022-04-05T11:12:50.233813Z","shell.execute_reply":"2022-04-05T11:12:50.249407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# BatchNormalization Mode Function\ndef freeze_BatchNorm(model):\n # Unfreeze all Layers except for Batch Norm\n for layer in model.layers:\n if not isinstance(layer, BatchNormalization):\n layer.trainable = True \n else:\n layer.trainable = False # in inference mode\n return model\n","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:50.25148Z","iopub.execute_input":"2022-04-05T11:12:50.25173Z","iopub.status.idle":"2022-04-05T11:12:50.264612Z","shell.execute_reply.started":"2022-04-05T11:12:50.251703Z","shell.execute_reply":"2022-04-05T11:12:50.263825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## For Evaluation ","metadata":{}},{"cell_type":"code","source":"## Find the name of image in each filename. Exp : 00021adfb725ed.jpg \ndef get_id(filename):\n ds = get_test_dataset([filename],get_names=True).map(lambda image, image_name: image_name).unbatch()\n NUM_IMAGES = count_data_items([filename])\n ids = next(iter(ds.batch(NUM_IMAGES))).numpy().astype('U')\n return ids\n## Find the Target Encoding (defined in happywhale-splits/individual_ids.json)\ndef get_target(filename):\n ds = get_eval_dataset([filename],get_targets=True).map(lambda image, target: target).unbatch()\n NUM_IMAGES = count_data_items([filename])\n ids = next(iter(ds.batch(NUM_IMAGES))).numpy()\n return ids\n## Find the average of predictions in the best 5-models\ndef get_embedding(filename):\n ds = get_test_dataset([filename],get_names=False)\n embedding = np.mean(np.stack([embed_models[x][1].predict(ds,verbose=0) for x in range(len(embed_models))]), axis=0)\n return embedding\n\ndef get_prediction(test_df,threshold=0.2):\n predictions = {}\n for i,row in tqdm(test_df.iterrows()):\n if row.image in predictions:\n if len(predictions[row.image])==5:\n continue\n predictions[row.image].append(row.target)\n elif row.confidence>threshold:\n predictions[row.image] = [row.target,'new_individual']\n else:\n predictions[row.image] = ['new_individual',row.target]\n\n return predictions\n\ndef map_per_image(label, predictions):\n \"\"\"Computes the precision score of one image.\n\n Parameters\n ----------\n label : string\n The true label of the image\n predictions : list\n A list of predicted elements (order does matter, 5 predictions allowed per image)\n\n Returns\n -------\n score : double\n \"\"\" \n try:\n return 1 / (predictions[:5].index(label) + 1)\n except ValueError:\n return 0.0","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:50.266184Z","iopub.execute_input":"2022-04-05T11:12:50.266486Z","iopub.status.idle":"2022-04-05T11:12:50.281324Z","shell.execute_reply.started":"2022-04-05T11:12:50.266449Z","shell.execute_reply":"2022-04-05T11:12:50.280694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

BUILD MODEL

","metadata":{}},{"cell_type":"code","source":"def freeze_BN(model):\n # Unfreeze layers while leaving BatchNorm layers frozen\n for layer in model.layers:\n if not isinstance(layer, tf.keras.layers.BatchNormalization):\n layer.trainable = True\n else:\n layer.trainable = False\n\n\n# Create EfficientNetB6 Model\ndef get_model():\n if CFG.head == \"arcface\":\n head = ArcMarginProduct\n else:\n assert 2021==2022 , \"INVALID HEAD IN MODEL\"\n \n with strategy.scope():\n margin = head(n_classes=CFG.N_CLASSES, \n s=30, \n m=0.3, \n name=f'head/{CFG.head}', \n dtype='float32'\n )\n inp = tf.keras.layers.Input(shape = [CFG.IMAGE_SIZE, CFG.IMAGE_SIZE, 3], name = 'Input1')\n label = Input(shape=(), name='Input2')\n if CFG.model_type == 'effnetv1':\n x = CFG.EFNS[CFG.EFF_NET](weights = 'noisy-student', include_top = False)(inp)\n embed = tf.keras.layers.GlobalAveragePooling2D()(x)\n elif CFG.model_type == 'effnetv2':\n GCS_WEIGHTS_PATH = KaggleDatasets().get_gcs_path('efficientnetv2-tfhub-weight-files')\n FEATURE_VECTOR = f'{GCS_WEIGHTS_PATH}/tfhub_models/{CFG.EFF_NETV2}/feature_vector'\n embed = tfhub.KerasLayer(FEATURE_VECTOR, trainable=True)(inp)\n else:\n embed = tfimm.create_model(CFG.model_type, pretrained=\"timm\")(inp)\n\n embed = tf.keras.layers.Dropout(0.3)(embed)\n embed = tf.keras.layers.Dense(1024)(embed)\n x = margin([embed, label])\n \n output = tf.keras.layers.Softmax(dtype='float32')(x)\n \n model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])\n embed_model = tf.keras.models.Model(inputs = inp, outputs = embed) \n \n opt = tf.keras.optimizers.Adam(learning_rate = CFG.LR)\n if CFG.FREEZE_BATCH_NORM:\n freeze_BN(model)\n\n model.compile(\n optimizer = opt,\n loss = [tf.keras.losses.SparseCategoricalCrossentropy()],\n metrics = [tf.keras.metrics.SparseCategoricalAccuracy(),tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5)]\n ) \n \n return model,embed_model","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:50.282462Z","iopub.execute_input":"2022-04-05T11:12:50.283145Z","iopub.status.idle":"2022-04-05T11:12:50.29803Z","shell.execute_reply.started":"2022-04-05T11:12:50.28311Z","shell.execute_reply":"2022-04-05T11:12:50.29733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(plot=False):\n lr_start = 0.000001\n lr_max = 0.000005 * CFG.BATCH_SIZE \n lr_min = 0.000001\n lr_ramp_ep = 4\n lr_sus_ep = 0\n lr_decay = 0.9\n \n def lrfn(epoch):\n if CFG.RESUME:\n epoch = epoch + CFG.RESUME_EPOCH\n if epoch < lr_ramp_ep:\n lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n \n elif epoch < lr_ramp_ep + lr_sus_ep:\n lr = lr_max\n \n else:\n lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n \n return lr\n \n if plot:\n epochs = list(range(CFG.EPOCHS))\n learning_rates = [lrfn(x) for x in epochs]\n plt.scatter(epochs,learning_rates)\n plt.show()\n\n lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n return lr_callback","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:50.299028Z","iopub.execute_input":"2022-04-05T11:12:50.299724Z","iopub.status.idle":"2022-04-05T11:12:50.311104Z","shell.execute_reply.started":"2022-04-05T11:12:50.299692Z","shell.execute_reply":"2022-04-05T11:12:50.31047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Snapshot(tf.keras.callbacks.Callback):\n \n def __init__(self,fold,snapshot_epochs=[]):\n super(Snapshot, self).__init__()\n self.snapshot_epochs = snapshot_epochs\n self.fold = fold\n \n \n def on_epoch_end(self, epoch, logs=None):\n if epoch in self.snapshot_epochs: # your custom condition \n self.model.save_weights(f\"./EF{CFG.MODEL_NAME}_epoch{epoch}.h5\")\n self.model.save_weights(f\"./{CFG.MODEL_NAME}_last.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:50.312144Z","iopub.execute_input":"2022-04-05T11:12:50.312795Z","iopub.status.idle":"2022-04-05T11:12:50.322088Z","shell.execute_reply.started":"2022-04-05T11:12:50.312763Z","shell.execute_reply":"2022-04-05T11:12:50.321528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

TRAINING

","metadata":{}},{"cell_type":"code","source":"training_filenames = [x for i,x in enumerate(train_files) if i%CFG.FOLDS!=CFG.FOLD_TO_RUN]\nvalidation_filenames = [x for i,x in enumerate(train_files) if i%CFG.FOLDS==CFG.FOLD_TO_RUN]\ndel train_files","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:50.323476Z","iopub.execute_input":"2022-04-05T11:12:50.323976Z","iopub.status.idle":"2022-04-05T11:12:50.332108Z","shell.execute_reply.started":"2022-04-05T11:12:50.323935Z","shell.execute_reply":"2022-04-05T11:12:50.331279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Callbacks","metadata":{}},{"cell_type":"code","source":"seed_everything(CFG.SEED)\ntrain_dataset = get_training_dataset(training_filenames)\nval_dataset = get_val_dataset(validation_filenames)\nSTEPS_PER_EPOCH = count_data_items(training_filenames)//CFG.BATCH_SIZE\n\n## Logging\ntrain_logger = CSVLogger('./training-log-fold-%i.h5.csv'%CFG.FOLD_TO_RUN)\n\n# SAVE BEST MODEL EACH FOLD\nsv_loss = ModelCheckpoint(f\"./{CFG.MODEL_NAME}_loss_{CFG.FOLD_TO_RUN}.h5\", \n monitor='val_loss', \n verbose=0, \n save_best_only=True, \n save_weight_only=True, mode='min', save_freq='epoch')\n# Snapshot\nsnap = Snapshot(fold=CFG.FOLD_TO_RUN,snapshot_epochs=[5,8])\n# Learning Rate Scheduler\nget_lr_callback(plot=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:12:50.333155Z","iopub.execute_input":"2022-04-05T11:12:50.333778Z","iopub.status.idle":"2022-04-05T11:12:51.327661Z","shell.execute_reply.started":"2022-04-05T11:12:50.33375Z","shell.execute_reply":"2022-04-05T11:12:51.326797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build Model\nK.clear_session()\nmodel, embed_model = get_model()\nplot_model(model)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-04-05T11:12:51.32908Z","iopub.execute_input":"2022-04-05T11:12:51.329505Z","iopub.status.idle":"2022-04-05T11:13:58.100465Z","shell.execute_reply.started":"2022-04-05T11:12:51.32947Z","shell.execute_reply":"2022-04-05T11:13:58.099481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG.BATCH_SIZE","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:14:45.638489Z","iopub.execute_input":"2022-04-05T11:14:45.638808Z","iopub.status.idle":"2022-04-05T11:14:45.644215Z","shell.execute_reply.started":"2022-04-05T11:14:45.638779Z","shell.execute_reply":"2022-04-05T11:14:45.643291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"--- Size Image %i with %str and Batchsize %i ---\"%(CFG.IMAGE_SIZE, CFG.EFF_NETV2, CFG.BATCH_SIZE))\n####-------------In the last, we trained model and save the log data in Server---------- ###\nhistory = model.fit(train_dataset, \n validation_data=val_dataset, \n steps_per_epoch=STEPS_PER_EPOCH, \n epochs=CFG.EPOCHS, \n callbacks=[snap, get_lr_callback(), train_logger, sv_loss], \n verbose=1)\n###-------------Loading the previous results----------------###","metadata":{"execution":{"iopub.status.busy":"2022-04-05T11:16:11.262089Z","iopub.execute_input":"2022-04-05T11:16:11.26263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

REFERENCE

","metadata":{}},{"cell_type":"markdown","source":"https://www.kaggle.com/code/aikhmelnytskyy/happywhale-effnet-b7-fork-with-detic-training","metadata":{}}]}