{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Intro\n\nIn this notebook, we integrate;\n- Albumentations augmentations\n- CutMix augmentation\n- TFRecords\n- Multi-GPU pipeline\n\nThese implementations will speed up training and allow more opportunities for generalization."},{"metadata":{},"cell_type":"markdown","source":"# SEED Everything"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport os\nimport numpy as np\nimport random\n\nSEED = 42\n\ndef set_seeds(seed=SEED):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    tf.random.set_seed(seed)\n    np.random.seed(seed)\n\ndef set_global_determinism(seed=SEED):\n    set_seeds(seed=seed)\n\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n    os.environ['TF_CUDNN_DETERMINISTIC'] = '1'\n    \n    tf.config.threading.set_inter_op_parallelism_threads(1)\n    tf.config.threading.set_intra_op_parallelism_threads(1)\n\n    print(\"Random seed initialized.\")\n\nset_global_determinism(seed=SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport json, cv2, re, math, ast\nimport seaborn as sns\nimport tqdm.notebook as tqdm\nimport matplotlib.pyplot as plt\nsns.set(style='darkgrid')\n\nfrom sklearn.model_selection import KFold\nfrom tensorflow.keras.models import Model, load_model, Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, GlobalAvgPool2D, Input, BatchNormalization\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, LearningRateScheduler, ModelCheckpoint, CSVLogger, TensorBoard, LearningRateScheduler\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.optimizers.schedules import ExponentialDecay\nfrom tensorflow.keras.losses import CategoricalCrossentropy\nfrom tensorflow.keras.applications import *\nfrom tensorflow.keras.mixed_precision import Policy, set_global_policy, LossScaleOptimizer\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.python.client import device_lib\nimport tensorflow_addons as tfa\n\nimport albumentations as A\nfrom functools import partial\n\n# Filter all Tensorflow logs except FATAL errors\ntf.get_logger().setLevel('FATAL') #DEBUG,ERROR,FATAL,INFO,WARN\n\n# Mixed Precision\nset_global_policy(Policy('mixed_float16'))\n\ngpus = tf.config.list_physical_devices('GPU')\nif gpus:\n  try:\n    # Currently, memory growth needs to be the same across GPUs\n    for gpu in gpus:\n      tf.config.experimental.set_memory_growth(gpu, True)\n    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n  except RuntimeError as e:\n    # Memory growth must be set before GPUs have been initialized\n    print(e)\n\n# Mirrored Strategy\nstrategy = tf.distribute.MirroredStrategy(\n    devices=[\"/gpu:0\", \"/gpu:1\"], \n    cross_device_ops=tf.distribute.HierarchicalCopyAllReduce()\n)\n\n# Initialize paths\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nPATH = '../input/plant-pathology-2021-fgvc8'\nns_weights = '../input/keras-efficientnetb3-noisy-student/noisy_student_efficientnet-b1.h5'\n\nfor x in device_lib.list_local_devices():\n    if x.device_type == 'GPU':\n        print(x.physical_device_desc)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Due to the compute capability of Kaggle's in-built GPUs, we will not be able to;\n1. Activate mixed precision\n2. Increase our batch size per replica"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initialize variables\nREPLICAS = strategy.num_replicas_in_sync\nBATCH_SIZE_PER_REPLICA = 16\nBUFFER_SIZE = 512\nIMAGE_SIZE = [224, 224]\nn_train_augments = 3\nCLASSES = 12\nGLOBAL_BATCH_SIZE = BATCH_SIZE_PER_REPLICA * REPLICAS\nprint('Number of replicas:', REPLICAS)\nprint('Global batch size:', GLOBAL_BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Images from both 2020 and 2021 training sets have been merged into train.csv."},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(r'../input/train-data2/train.csv')\nIMG_PATH = r'../input/plant-pathology-2021-fgvc8/train_images'\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(y=train['labels'], order=train['labels'].value_counts().index)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Duplicate Detection\n\n## Image hashing"},{"metadata":{"trusted":true},"cell_type":"code","source":"# funcs = [\n#         imagehash.average_hash,\n#         imagehash.phash,\n#         imagehash.dhash,\n#         imagehash.whash,\n#     ]\n# image_ids = []\n# hashes = []\n\n# for path in tqdm(train['image_id'], desc='Hashing images'):\n#     image = Image.open(os.path.join(IMG_PATH, path))\n#     image_id = os.path.basename(path)\n#     image_ids.append(image_id)\n#     hashes.append(np.array([f(image).hash for f in funcs]).reshape(256))\n\n# hashes_all = np.array(hashes)\n# hashes_all = torch.Tensor(hashes_all.astype(int))\n# sims = np.array([(hashes_all[i] == hashes_all).sum(dim=1).numpy()/256 for i in tqdm(range(hashes_all.shape[0]), desc='Calculating similarities')])\n\n# indices1 = np.where(sims > 0.9)\n# indices2 = np.where(indices1[0] != indices1[1])\n# image_ids1 = [image_ids[i] for i in indices1[0][indices2]]\n# image_ids2 = [image_ids[i] for i in indices1[1][indices2]]\n# dups = {tuple(sorted([image_id1,image_id2])):True for image_id1, image_id2 in zip(image_ids1, image_ids2)}\n# duplicate_image_ids = sorted(list(dups))\n# print('Found %d duplicates' % len(duplicate_image_ids))\n\n# # Remove duplicates from external data\n# imgs_to_remove = [x[1] for x in duplicate_image_ids]\n\n# duplicates = pd.DataFrame(duplicate_image_ids)\n# duplicates.columns = ['image0', 'image1']\n\nduplicates = pd.read_csv('../input/train-data/duplicates.csv')\nduplicates","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nrows = 5; ncols=2\nfig, axes = plt.subplots(nrows, ncols, figsize=(6*ncols, 4*nrows))\nfor i, row in enumerate(duplicates.to_numpy()[-nrows:]):\n    image0 = cv2.imread(os.path.join(IMG_PATH, row[0]))\n    axes[i][0].axis('off')\n    axes[i][0].set_title(row[0])\n    axes[i][0].imshow(image0)\n\n    image1 = cv2.imread(os.path.join(IMG_PATH, row[1]))\n    axes[i][1].axis('off')\n    axes[i][1].set_title(row[1])\n    axes[i][1].imshow(image1)\n\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[train['image_id'].isin(duplicates['image1'].tolist())]['labels'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train[~train['image_id'].isin(duplicates['image1'].tolist())]\ntrain","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reshape Resize"},{"metadata":{"trusted":true},"cell_type":"code","source":"# def image_shape(x):\n#     return cv2.imread(os.path.join(IMG_PATH, x)).shape\n\n# tqdm.pandas(desc='Getting shapes')\n# train['shape'] = train['image_id'].progress_apply(image_shape)\n\ntrain = pd.read_csv('../input/train-data/train_shape.csv')\ntrain['shape'] = train['shape'].apply(ast.literal_eval)\ntrain['shape'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_resize(x):\n    image = cv2.imread(os.path.join(IMG_PATH, x))\n    if image.shape[0] > image.shape[1]:\n        image = cv2.transpose(image)\n    else:\n        image = image   \n    return cv2.resize(image, dsize=(512, 512))\n\nimage = image_resize(train['image_id'][5000])\nplt.axis('off')\nplt.imshow(image)\nplt.title(image.shape)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"trainV1.csv is the finalized dataset with duplicates removed."},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train[['image_id', 'labels']]\n\n# def image_shape(x):\n#     return cv2.imread(os.path.join(SAVE_PATH, x)).shape\n\n# tqdm.pandas(desc='Getting shapes')\n# train['shape'] = train['image_id'].progress_apply(image_shape)\n\ntrain = pd.read_csv(r\"../input/train-data/trainV1.csv\")\ntrain['shape'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Label encode\ntrain['labels_codes'] = pd.Categorical(train['labels']).codes\npd.DataFrame({\"Categories\": pd.Categorical(train['labels']).categories})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Convert images to TFREC shards"},{"metadata":{"trusted":true},"cell_type":"code","source":"# The following functions can be used to convert a value to a type compatible\n# with tf.train.Example.\n\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def serialize_example(feature0, feature1, feature2):\n  \"\"\"\n  Creates a tf.train.Example message ready to be written to a file.\n  \"\"\"\n  # Create a dictionary mapping the feature name to the tf.train.Example-compatible\n  # data type.\n  feature = {\n      'image': _bytes_feature(feature0),\n      'target': _int64_feature(feature1),\n      'image_name': _bytes_feature(feature2),\n  }\n\n  # Create a Features message using tf.train.Example.\n\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Obtain the multiple of training set size for even splitting\nfold_list = []\nprint('Possible number of folds:')\nfor n in range(1,30):\n    if np.divmod(train.shape[0], n)[1]==0:\n        print('{} folds - {} images per fold'.format(n, int(train.shape[0]/n)))\n        fold_list.append(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# N_FILES = fold_list[1]\n# IMG_QUALITY = 96 # Avoiding 100% Quality as it increases file size\n# IMAGE_SIZE = (512, 512)\n# train['shard'] = 0\n\n# # Stratify the shards\n# skf = StratifiedKFold(n_splits=N_FILES, shuffle=True, random_state=42)\n# for fold, (train_idx, val_idx) in enumerate(skf.split(train, train['labels'])):\n#     train.loc[val_idx, 'shard'] = fold\n\n# # Rewrite the TFRecords after stratification and resize\n# for tfrec_num in tqdm(range(N_FILES), desc='Writing TFRecords'):\n#     samples = train[train['shard'] == tfrec_num]\n#     n_samples = len(samples)\n#     with tf.io.TFRecordWriter('train-%.2i-%i.tfrec'%(tfrec_num, n_samples)) as writer:\n#         for row in tqdm(samples.itertuples(), desc=('Fold {}'.format(tfrec_num+1)), total=int(train.shape[0]/N_FILES)):\n#             label = row.labels_codes\n#             image_name = row.image_id\n#             img = image_resize(image_name)\n#             img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, IMG_QUALITY))[1].tobytes()\n#             example = serialize_example(img, label, str.encode(image_name))\n#             writer.write(example)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading tfrec shards"},{"metadata":{"trusted":true},"cell_type":"code","source":"TFR_PATH = '../input/plant-pathology-20202021-tfrec'\nFILENAMES = tf.io.gfile.glob(TFR_PATH + \"/*.tfrec\")\nprint(\"Number of shards:\", len(FILENAMES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32)/255.0\n    image = tf.image.resize(image, size=IMAGE_SIZE, method='nearest')\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\ndef read_tfrecord(example, labeled):\n    labeled_map = {\"image\": tf.io.FixedLenFeature([], tf.string),\n                    \"target\": tf.io.FixedLenFeature([], tf.int64)}\n    unlabeled_map = {\"image\": tf.io.FixedLenFeature([], tf.string)}\n\n    tfrecord_format = (labeled_map if labeled else unlabeled_map)\n    \n    example = tf.io.parse_single_example(serialized=example,\n                                  features=tfrecord_format)\n\n    image = decode_image(example[\"image\"])\n\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    return image\n\ndef load_dataset(filenames, labeled=True):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled),\n        num_parallel_calls=AUTOTUNE\n        )\n    return dataset\n\ndef one_hot(image, label, CLASSES=CLASSES):\n    return image, tf.one_hot(indices=label, \n                             depth=CLASSES,\n                             dtype=tf.float32)\n\ndef count_data_items(filenames):\n    \"\"\"Obtaining total number of images in dataset from\n    the tfrecord shards.\"\"\"\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1))\n         for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentation\n\nWe emphasize on the duplication of our augments to increase image counts. This is replicative of Keras' ImageDataGenerator API.\n\nFor demonstration purposes, only basic augmentations were performed."},{"metadata":{"trusted":true},"cell_type":"code","source":"def album_augment(image, label):\n    transforms = A.Compose([\n        A.RandomResizedCrop(*IMAGE_SIZE, p=1),\n        # A.Transpose(p=1),\n        A.ShiftScaleRotate(\n            shift_limit=0.0625,\n            scale_limit=0.1,\n            rotate_limit=45,\n            p=1),\n        # A.RandomBrightnessContrast(\n        #     brightness_limit=0.2,\n        #     constrast_limit=0.2,\n        #     p=1\n        # ),\n        # A.HueSaturationValue(\n        #     hue_shift_limit=20,\n        #     sat_shift_limit=30,\n        #     val_shift_limit=20,\n        #     p=1),\n        A.Flip(p=1),\n        A.Cutout(num_holes=8,\n            max_h_size=8,\n            max_w_size=8,\n            p=1)\n        ], p=1)\n\n    images = [image]\n    labels = [label]\n\n    for _ in range(0, n_train_augments):\n        aug_image = transforms(image=image)['image']\n        aug_image = tf.cast(x=aug_image, dtype=tf.float32)\n        images.append(aug_image)\n        labels.append(label)\n\n    return images, labels    \n\ndef train_augment(image, label):\n    aug_func = tf.numpy_function(func=album_augment, inp=[image, label], Tout=[tf.float32, tf.int32])\n    return aug_func","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cutmix(image, label, PROBABILITY=0.5):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = IMAGE_SIZE[0]\n\n    imgs = []; labs = []\n    for j in range(GLOBAL_BATCH_SIZE):\n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast( tf.random.uniform([],0,GLOBAL_BATCH_SIZE),tf.int32)\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n\n        lab1 = label[j,]\n        lab2 = label[k,]\n        \n        labs.append((1-a)*lab1 + a*lab2)\n            \n    image2 = tf.reshape(tf.stack(imgs),(GLOBAL_BATCH_SIZE,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(GLOBAL_BATCH_SIZE,CLASSES))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_dataset(filenames, labeled=True):\n    dataset = load_dataset(filenames, labeled=labeled)\n    dataset = dataset.map(partial(train_augment), num_parallel_calls=AUTOTUNE)\n    dataset = dataset.unbatch()\n    dataset = dataset.shuffle(BUFFER_SIZE*n_train_augments)\n    dataset = dataset.repeat()\n    dataset = dataset.map(one_hot, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.batch(GLOBAL_BATCH_SIZE)\n    dataset = dataset.map(cutmix, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.unbatch()\n    dataset = dataset.shuffle(BUFFER_SIZE)\n    dataset = dataset.batch(GLOBAL_BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def augm_dataset(filenames, labeled=True):\n    dataset = load_dataset(filenames, labeled=labeled)\n    dataset = dataset.map(partial(train_augment), num_parallel_calls=AUTOTUNE)\n    # dataset = dataset.batch(BATCH_SIZE_PER_REPLICA)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Validation set"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"repeat() required to increase validation_steps\"\"\"\ndef get_val_dataset(filenames, labeled=True):\n    dataset = load_dataset(filenames, labeled=labeled)\n    dataset = dataset.map(one_hot, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.shuffle(BUFFER_SIZE)\n    dataset = dataset.batch(GLOBAL_BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Augment Visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_image(dataset):\n    dataset = dataset.unbatch().batch(16)\n    images, labels = next(iter(dataset))\n    col = 4; row = 4\n    plt.figure(figsize=(10, 10))\n    for i in range(row*col):\n        plt.subplot(row, col, i+1)\n        plt.axis('off')\n        image = tf.image.convert_image_dtype(images[i], 'uint8')\n        plt.imshow(image)\n\n    plt.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Albumentations Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"display_image(augm_dataset(FILENAMES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Albumentations + CutMix Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"display_image(get_train_dataset(FILENAMES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Raw images"},{"metadata":{"trusted":true},"cell_type":"code","source":"display_image(get_val_dataset(FILENAMES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model training\n\nMacro-averaging F1-score is used here as it is stricter on imbalanced datasets."},{"metadata":{},"cell_type":"markdown","source":"### Wrapper Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Set from_logits=False when using Dense Softmax output layer\"\"\"\ndef create_model(model, learning_rate, dropout_rate, \nlabel_smoothing, weights):\n    with strategy.scope():\n        inputs = Input(shape=(*IMAGE_SIZE, 3)) \n        base_model = model(\n                include_top=False, \n                weights=weights,\n                pooling='avg',\n                )(inputs)\n\n        hidden = Dropout(dropout_rate)(base_model)\n        outputs = Dense(12, activation='softmax', dtype='float32')(hidden)\n        model = Model(inputs=inputs, outputs=outputs)\n\n        model.compile(\n            optimizer=LossScaleOptimizer(\n                Adam(learning_rate=learning_rate)),\n            loss=CategoricalCrossentropy(\n                from_logits=False, \n                label_smoothing=label_smoothing),\n            metrics=[\n                'categorical_accuracy',\n                tfa.metrics.F1Score(\n                    num_classes=CLASSES,\n                    average='macro',\n                    name='f1_score'\n                )]\n            )\n        return model\n\ndef model_training(model, model_name, weights, learning_rate, \n  dropout_rate, label_smoothing, fold, n_splits, epochs):\n    cv = KFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n\n    lr_schedule = LearningRateScheduler(ExponentialDecay(\n        initial_learning_rate=1e-4,\n        decay_steps=2,\n        decay_rate=0.5,\n        staircase=True), verbose=1)\n\n    # lr_reduce = ReduceLROnPlateau(\n    #     monitor='val_categorical_accuracy',\n    #     factor=0.9,\n    #     patience=1,\n    #     verbose=1,\n    #     mode='max',\n    #     cooldown=2,\n    #     min_lr=1e-9\n    # )\n\n    # early_stopping = EarlyStopping(\n    #     monitor='val_categorical_accuracy',\n    #     patience=10,\n    #     verbose=1,\n    #     mode='max',\n    #     restore_best_weights=False\n    # )\n\n    for i, (train_idx, valid_idx) in enumerate(cv.split(FILENAMES)): \n        if i == (fold-1):\n            model_name = model_name\n            filepath = model_name + '-fold' + str(i+1) + '-epoch{epoch:02d}-f1{f1_score:.4f}-valf1{val_f1_score:.4f}.h5'\n            model_save = ModelCheckpoint(\n                filepath=filepath,\n                save_best_only=False,\n                save_freq='epoch',\n                save_weights_only=True,\n                monitor='val_f1_score',\n                mode='max',\n                verbose=1)\n\n            filename = model_name + '-fold' + str(i+1) + '.csv'\n            csv_logger = CSVLogger(\n                filename=filename, \n                separator=',', \n                append=False)\n\n            TRAIN_FILENAMES = [FILENAMES[id] for id in train_idx]\n            VALID_FILENAMES = [FILENAMES[id] for id in valid_idx]\n            \n            N_TRAIN_IMAGES = count_data_items(TRAIN_FILENAMES)\n            N_VALID_IMAGES = count_data_items(VALID_FILENAMES)\n\n            print('Number of training images:', N_TRAIN_IMAGES)\n            print('Number of validation images:', N_VALID_IMAGES)\n\n            print(f'\\n-------------- FOLD {i+1}/{n_splits} --------------')\n\n            train = get_train_dataset(TRAIN_FILENAMES)\n            val = get_val_dataset(VALID_FILENAMES)\n\n            model = create_model(\n                model=model,\n                weights=weights,\n                dropout_rate=dropout_rate,\n                learning_rate=learning_rate,\n                label_smoothing=label_smoothing\n            )\n            \n            history = model.fit(\n                x=train,\n                verbose=1,\n                epochs=epochs,\n                validation_data=val,\n                batch_size=GLOBAL_BATCH_SIZE,\n                steps_per_epoch=(N_TRAIN_IMAGES//GLOBAL_BATCH_SIZE),\n                validation_steps=(N_VALID_IMAGES//GLOBAL_BATCH_SIZE),\n                shuffle=True,\n                callbacks=[\n                    # lr_reduce,\n                    lr_schedule,\n                    # early_stopping,\n                    model_save,\n                    csv_logger]\n                )     ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Learning Schedule\n\nOther learning schedules can be implemented here, like cosine annealling with warm-up phase or simply using Tensorflow's in-built ReduceOnLRPlateau callback."},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_schedule = ExponentialDecay(\n    initial_learning_rate=1e-4,\n    decay_steps=2,\n    decay_rate=0.5,\n    staircase=True)\n\nepochs = 35\n\ny = [lr_schedule(x) for x in range(epochs)]\n\nfor epoch, lr in enumerate(y):\n  if epoch==0 or epoch==(len(y)-1):\n    print(epoch, lr)\n\nplt.figure(figsize=(10,5))\nplt.plot(range(epochs), y)\nplt.xticks(range(epochs))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### EfficientNet\n\nFor demonstration purposes, we will only train for 10 epochs without any dropout or label smoothing incorporated."},{"metadata":{"trusted":true},"cell_type":"code","source":"model_training(\n    model=EfficientNetB0,\n    model_name='EFNB0',\n    weights='imagenet',\n    learning_rate=1e-4,\n    dropout_rate=0,\n    label_smoothing=0,\n    n_splits=5,\n    fold=5,\n    epochs=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = pd.read_csv('./EFNB0-fold5.csv')\nhistory","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15,5))\n\nsns.lineplot(\n    x='epoch', \n    y='loss', \n    data=history, \n    label='loss', \n    ax=ax1)\n\nsns.lineplot(\n    x='epoch', \n    y='val_loss', \n    data=history, \n    label='val_loss',\n    ax=ax1)\n\nsns.lineplot(\n    x='epoch',\n    y='f1_score',\n    data=history,\n    label='f1_score',\n    ax=ax2)\n\nsns.lineplot(\n    x='epoch', \n    y='val_f1_score',\n    data=history,\n    label='val_f1_score',\n    ax=ax2)\n\nax1.set_ylabel('loss')\nax2.set_ylabel('score')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test time Augmentation\n\nTo specify n_test_augments and insert your augmentation functions for TTA."},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_transform(image, label):\n    transforms = A.Compose([\n        A.CenterCrop(*IMAGE_SIZE),\n        A.Resize(\n            *IMAGE_SIZE, \n            interpolation=cv2.INTER_CUBIC)\n    ])\n\n    images = [image]\n    labels = [label]\n\n    for _ in range(0, n_test_augments):\n        aug_image = transforms(image=image)['image']\n        aug_image = tf.cast(x=aug_image, dtype=tf.float32)\n        images.append(aug_image)\n        labels.append(label)\n\n    return images, labels\n\ndef test_augment(image, label):\n    aug_image = tf.numpy_function(func=test_transform, inp=[image, label], Tout=[tf.float32, tf.int32])\n    return aug_image, label\n\ndef get_test_dataset(filenames, labeled=False):\n    dataset = load_dataset(filenames, labeled=labeled)\n    dataset = dataset.map(partial(test_augment))\n    dataset = dataset.repeat()\n    dataset = dataset.batch(GLOBAL_BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Findings\n\n- Increasing n_train_augments to about 6 or 7 will reduce generalization difference\n- CutMix augmentation tends to cause model to underfit, reducing n_train_augments might help\n- The raw images have resolutions of up to 2000 by 4000 pixels, using larger image sizes like 1024 might increase scores but requires higher GPU memory"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}