{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import math, re, os\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow import keras\nfrom functools import partial\nfrom sklearn.model_selection import train_test_split\nfrom kaggle_datasets import KaggleDatasets\nprint(\"Tensorflow version \" + tf.__version__)\n# Keras Core\nfrom keras.layers.convolutional import MaxPooling2D, Convolution2D, AveragePooling2D\nfrom keras.layers import Input, Dropout, Dense, Flatten, Activation\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers.merge import concatenate\nfrom keras import regularizers\nfrom keras import initializers\nfrom keras.models import Model\n# Backend\nfrom keras import backend as K\nfrom keras.applications import imagenet_utils\nfrom keras import layers\nimport tensorflow_addons as tfa\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = 600\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def auto_select_accelerator():\n    \"\"\"\n    Reference: \n        * https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu\n        * https://www.kaggle.com/xhlulu/ranzcr-efficientnet-tpu-training\n    \"\"\"\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    return strategy\n\n\n\ndef build_decoder(with_labels=True, target_size=(IMAGE_SIZE, IMAGE_SIZE), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(img, label):\n    #seed = RNG.make_seeds(2)[0]\n\n    def augment(img):\n        img = tf.image.resize_with_crop_or_pad(img, IMAGE_SIZE + 6, IMAGE_SIZE + 6)\n        #tf.print(\"seed is:\", seed)\n        # Random crop back to the original size\n        img = tf.image.random_crop(img, size=[IMAGE_SIZE, IMAGE_SIZE, 3])\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        img = tf.image.random_contrast(img, lower=0.95, upper=1.1)\n        img = tf.image.random_brightness(img, max_delta=0.05)\n        img = tf.image.random_saturation(img, 0.95, 1.05)\n        img = tf.image.random_jpeg_quality(img, 90, 100)\n        img = tf.clip_by_value(img, 0, 1)\n        img = tf.image.resize(img, (IMAGE_SIZE, IMAGE_SIZE))\n        return img\n                \n    return augment(img), label\n\n\ndef build_dataset(paths, labels=None, bsize=32, cache=True,\n                  decode_fn=None, augment_fn=build_augmenter,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    if augment:\n        dset = dset.map(augment_fn, num_parallel_calls=AUTO) \n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO)\n    \n    return dset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nCOMPETITION_NAME = \"ranzcr-clip-catheter-line-classification\"\nstrategy = auto_select_accelerator()\nPER_REPLICA_BATCH_SIZE = 16\nGLOBAL_BATCH_SIZE = strategy.num_replicas_in_sync * PER_REPLICA_BATCH_SIZE\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(COMPETITION_NAME)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_datasets(curr_fold, batch_size):\n    load_dir = f\"/kaggle/input/{COMPETITION_NAME}/\"\n    df = pd.read_csv(\"../input/ranzcr-clip-catheter-line-classification/train.csv\") \n\n    paths = GCS_DS_PATH + \"/train/\" + df['StudyInstanceUID'] + '.jpg'\n\n    sub_df = pd.read_csv(load_dir + 'sample_submission.csv')\n\n    test_paths = GCS_DS_PATH + \"/test/\" + sub_df['StudyInstanceUID'] + '.jpg'\n\n    # Get the multi-labels\n    label_cols = sub_df.columns[1:]\n    labels = df[label_cols]\n\n\n    df = pd.read_csv(\"../input/how-to-properly-split-folds/train_folds.csv\") \n    valid_idx = df[\"fold\"] == curr_fold \n\n    train_paths = paths[~valid_idx].values.tolist()\n    train_labels = labels[~valid_idx].values.tolist()\n\n    valid_paths = paths[valid_idx].values.tolist()\n    valid_labels = labels[valid_idx].values.tolist()\n    \n\n    decoder = build_decoder(with_labels=True, target_size=(IMAGE_SIZE, IMAGE_SIZE))\n    test_decoder = build_decoder(with_labels=False, target_size=(IMAGE_SIZE, IMAGE_SIZE))\n\n    train_dataset = build_dataset(\n        train_paths, train_labels, bsize=batch_size, decode_fn=decoder, cache=True\n    )\n\n    valid_dataset = build_dataset(\n        valid_paths, valid_labels, bsize=batch_size, decode_fn=decoder, cache=True,\n            repeat=False, shuffle=False, augment=False\n    )\n    \n    size_train_dataset = len(train_paths)\n    return train_dataset, valid_dataset, size_train_dataset\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MODEL\n\nModel from paper \"Inception-v4, Inception-ResNet and the Impact of Residual Connections on Learning\" https://arxiv.org/pdf/1602.07261.pdf\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_NORM_EPSILON = 0.001\nBATCH_NORM_DECAY = 0.99\nWEIGHT_DECAY = 1e-6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#function from https://github.com/keras-team/keras/blob/master/keras/applications/inception_v3.py\n\"\"\"Utility function to apply conv + BN. \n  Args:\n    x: input tensor.\n    filters: filters in `Conv2D`.\n    num_row: height of the convolution kernel.\n    num_col: width of the convolution kernel.\n    padding: padding mode in `Conv2D`.\n    strides: strides in `Conv2D`.\n    name: name of the ops; will become `name + '_conv'`\n      for the convolution and `name + '_bn'` for the\n      batch norm layer.\n  Returns:\n    Output tensor after applying `Conv2D` and `BatchNormalization`.\n  \"\"\"\ndef conv2d_bn(x, filters, num_row, num_col, padding='same', strides=(1, 1), name=None):\n    if name is not None:\n        bn_name = name + '_bn'\n        conv_name = name + '_conv'\n    else:\n        bn_name = None\n        conv_name = None\n    if K.image_data_format() == 'channels_first':\n        bn_axis = 1\n    else:\n        bn_axis = 3\n    x = layers.Conv2D(\n        filters, (num_row, num_col),\n        strides=strides,\n        padding=padding,\n        use_bias=False,\n        kernel_regularizer = tf.keras.regularizers.L2(l2 = WEIGHT_DECAY),\n        name=conv_name)(\n          x)\n    x = tf.keras.layers.experimental.SyncBatchNormalization(\n                                  axis=bn_axis,\n                                  momentum = BATCH_NORM_DECAY,\n                                  epsilon = BATCH_NORM_EPSILON,\n                                  scale=False,\n                                  name=bn_name)(x)\n    \n    x = layers.Activation('relu', name=name)(x)\n    return x\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def stem(x):\n    # shape of x is 229 x 229 x 3 (channel first)\n    # shape of x is 3 x 229 x 229 (channel last)\n    if K.image_data_format() == 'channels_first':\n        channel_axis = 1\n    else:\n        channel_axis = -1\n        \n    x = conv2d_bn(x, 32, 3, 3, padding=\"valid\", strides=(2,2))\n    x = conv2d_bn(x, 32, 3, 3, padding=\"valid\")\n    x = conv2d_bn(x, 64, 3, 3)\n    \n    branch_0 = MaxPooling2D((3,3),strides=(2,2),padding=\"valid\")(x)\n    branch_1 = conv2d_bn(x, 96, 3, 3, strides=(2,2),padding=\"valid\")\n    \n    x = concatenate([branch_0, branch_1], axis=channel_axis)\n    \n    branch_0 = conv2d_bn(x, 64, 1, 1)\n    branch_0 = conv2d_bn(branch_0, 96, 3, 3, padding=\"valid\")\n    \n    branch_1 = conv2d_bn(x, 64, 1, 1)\n    branch_1 = conv2d_bn(branch_1, 64, 7, 1)\n    branch_1 = conv2d_bn(branch_1, 64, 1, 7)\n    branch_1 = conv2d_bn(branch_1, 96, 3, 3, padding=\"valid\")\n    \n    x = concatenate([branch_0, branch_1], axis=channel_axis)\n    \n    branch_0 = conv2d_bn(x, 192, 3, 3, strides=(2,2), padding=\"valid\")\n    branch_1 = MaxPooling2D((3,3),strides=(2,2),padding=\"valid\")(x)\n    \n    x = concatenate([branch_0, branch_1], axis=channel_axis)\n    return x\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inception_a(x):\n    if K.image_data_format() == 'channels_first':\n        channel_axis = 1\n    else:\n        channel_axis = -1\n        \n    branch_0 = AveragePooling2D((3,3), strides=(1,1), padding=\"same\")(x)\n    branch_0 = conv2d_bn(branch_0, 96, 1, 1)\n    \n    branch_1 = conv2d_bn(x, 96, 1, 1)\n    \n    branch_2 = conv2d_bn(x, 64, 1, 1)\n    branch_2 = conv2d_bn(branch_2, 96, 3, 3)\n\n    branch_3 = conv2d_bn(x, 64, 1, 1)\n    branch_3 = conv2d_bn(branch_3, 96, 3, 3)\n    branch_3 = conv2d_bn(branch_3, 96, 3, 3)\n    \n    x = concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis)\n    return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduction_a(x):\n    if K.image_data_format() == 'channels_first':\n        channel_axis = 1\n    else:\n        channel_axis = -1\n    \n    branch_0 = MaxPooling2D((3,3), strides=(2,2), padding=\"valid\")(x)\n    \n    branch_1 = conv2d_bn(x, 384, 3, 3, strides=(2,2), padding=\"valid\")\n    \n    branch_2 = conv2d_bn(x, 192, 1, 1)\n    branch_2 = conv2d_bn(branch_2, 224, 3, 3)\n    branch_2 = conv2d_bn(branch_2, 256, 3, 3, strides=(2,2), padding=\"valid\")\n    \n    x = concatenate([branch_0, branch_1, branch_2], axis=channel_axis)\n    return x\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inception_b(x):\n    if K.image_data_format() == 'channels_first':\n        channel_axis = 1\n    else:\n        channel_axis = -1\n    branch_0 = AveragePooling2D((3,3), strides=(1,1), padding=\"same\")(x)\n    branch_0 = conv2d_bn(branch_0, 128, 1, 1)\n    \n    branch_1 = conv2d_bn(x, 384, 1, 1)\n    \n    branch_2 = conv2d_bn(x, 192, 1, 1)\n    branch_2 = conv2d_bn(branch_2, 224, 1, 7)\n    branch_2 = conv2d_bn(branch_2, 256, 7, 1)\n    \n    branch_3 = conv2d_bn(x, 192, 1, 1)\n    branch_3 = conv2d_bn(branch_3, 192, 1, 7)\n    branch_3 = conv2d_bn(branch_3, 224, 7, 1)\n    branch_3 = conv2d_bn(branch_3, 224, 1, 7)\n    branch_3 = conv2d_bn(branch_3, 256, 7, 1)\n    \n    x = concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduction_b(x):\n    if K.image_data_format() == 'channels_first':\n        channel_axis = 1\n    else:\n        channel_axis = -1\n    branch_0 = MaxPooling2D((3,3), strides=(2,2), padding=\"valid\")(x)\n    \n    branch_1 = conv2d_bn(x, 192, 1, 1)\n    branch_1 = conv2d_bn(branch_1, 192, 3, 3, strides=(2,2), padding=\"valid\")\n    \n    branch_2 = conv2d_bn(x, 256, 1, 1)\n    branch_2 = conv2d_bn(branch_2, 256, 1, 7)\n    branch_2 = conv2d_bn(branch_2, 320, 7, 1)\n    branch_2 = conv2d_bn(branch_2, 320, 3, 3, strides=(2,2), padding=\"valid\")\n    \n    x = concatenate([branch_0, branch_1, branch_2], channel_axis)\n    return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inception_c(x):\n    if K.image_data_format() == 'channels_first':\n        channel_axis = 1\n    else:\n        channel_axis = -1\n        \n    branch_0 = AveragePooling2D((3,3), strides=(1,1), padding=\"same\")(x)\n    branch_0 = conv2d_bn(branch_0, 256, 1, 1)\n    \n    branch_1 = conv2d_bn(x, 256, 1, 1)\n    \n    branch_2 = conv2d_bn(x, 384, 1, 1)\n    branch_2a = conv2d_bn(branch_2, 256, 1, 3)\n    branch_2b = conv2d_bn(branch_2, 256, 3, 1)\n    \n    branch_3 = conv2d_bn(x, 384, 1, 1)\n    branch_3 = conv2d_bn(branch_3, 448, 1, 3)\n    branch_3 = conv2d_bn(branch_3, 512, 3, 1)\n    branch_3a = conv2d_bn(branch_3, 256, 1, 3)\n    branch_3b = conv2d_bn(branch_3, 256, 3, 1)\n    \n    x = concatenate([branch_0, branch_1, branch_2a,\n                     branch_2b, branch_3a, branch_3b], channel_axis)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def InceptionV4(input_tensor=None, input_shape=None, classes=1000):\n      #input_tensor: Optional Keras tensor (i.e. output of `layers.Input()`)\n      #to use as image input for the model. `input_tensor` is useful for sharing\n      #inputs between multiple different networks. Default to None.\n    \n    \n    if input_tensor is None:\n        img_input = layers.Input(shape=input_shape)\n    \n    else:\n        if not K.is_keras_tensor(input_tensor):\n            img_input = layers.Input(tensor=input_tensor, shape=input_shape)\n        else:\n            img_input = input_tensor\n            \n    if K.image_data_format() == 'channels_first':\n        channel_axis = 1\n    else:\n        channel_axis = 3\n        \n    \n    x = stem(img_input)\n    for i in range(4):\n        x = inception_a(x)\n        \n    x = reduction_a(x)\n    for i in range(7):\n        x = inception_b(x)\n        \n    x = reduction_b(x)\n    for i in range(3):\n        x = inception_c(x)\n    \n    x = layers.GlobalAveragePooling2D()(x)\n    x = Dropout(rate=0.2)(x)\n    \n    x = Dense(classes,\n              kernel_regularizer = tf.keras.regularizers.L2(l2 = WEIGHT_DECAY),\n              activation=\"sigmoid\")(x)\n    \n    # Ensure that the model takes into account\n    # any potential predecessors of `input_tensor`.\n    if input_tensor is not None:\n        inputs = layer_utils.get_source_inputs(input_tensor)\n    else:\n        inputs = img_input\n    # Create model.\n    model = Model(inputs, x, name='inception_v4')\n    return model\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TRAINING\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# reduces the learning rate of the optimizer by factor after \"patience\" epochs\n# in which the validation AUC has not improved\nclass ReduceLRScheduler():\n    def __init__(self, optimizer, patience, factor, min_lr):\n        self.optimizer = optimizer\n        self.patience = patience\n        self.factor = factor\n        self.min_lr = min_lr\n        self.best_auc = None\n        self.epochs_no_improvement = 0\n    \n    def on_epoch_end(self, val_auc):\n        # called first epoch\n        if self.best_auc is None:\n            self.best_auc = val_auc\n        # improvement\n        elif val_auc > self.best_auc:\n            self.epochs_no_improvement = 0\n        # no improvement\n        else:\n            self.epochs_no_improvement += 1\n            if self.epochs_no_improvement >= self.patience:\n                self.optimizer.learning_rate = max(self.min_lr, self.optimizer.learning_rate * self.factor)\n                print(\"reducing learning rate to:\", self.optimizer.learning_rate)\n        \n\nclass EarlyStopping():\n    def __init__(self, patience):\n        self.epochs_no_improvement = 0\n        self.best_auc = None\n        self.patience = patience\n        \n    def hasToStop(self, val_auc):\n        if self.best_auc is None:\n            self.best_auc = val_auc\n            return False\n        \n        elif val_auc > self.best_auc:\n            self.best_auc = val_auc\n            self.epochs_no_improvement = 0\n            return False\n        \n        else:\n            self.epochs_no_improvement += 1\n            return self.epochs_no_improvement >= self.patience\n        \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##### TODO: Early Stopping Epochs no improv = 15\n#####       change number of epochs to 200\n#####   try removing weight decay..?\n#####   put a safety condition: if val_auc < 1e-3: interrupt training\n#####   to not waste TPU","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_fold(n_fold):\n    n_labels = 11\n    tf.keras.backend.clear_session() #clear memory\n    \n    train_dataset, valid_dataset, size_train_dataset = get_train_datasets(curr_fold=n_fold, batch_size=GLOBAL_BATCH_SIZE)\n    \n    train_dist_dataset = strategy.experimental_distribute_dataset(train_dataset)\n    valid_dist_dataset = strategy.experimental_distribute_dataset(valid_dataset)\n    \n    steps_per_epoch = size_train_dataset // GLOBAL_BATCH_SIZE\n\n\n    EPOCHS = 200\n\n    \n    with strategy.scope():\n        ############ DEFINE LOSS ####################################\n        # Set reduction to `none` so we can do the reduction afterwards and divide by\n        # global batch size.\n        loss_object = tf.keras.losses.BinaryCrossentropy(reduction=tf.keras.losses.Reduction.NONE)\n        def compute_loss(model, labels, predictions):\n            per_example_loss = loss_object(labels, predictions)\n\n            # Compute loss that is scaled by sample_weight and by global batch size.\n            loss = tf.nn.compute_average_loss(\n                                per_example_loss,\n                                global_batch_size=GLOBAL_BATCH_SIZE)\n\n            # Add scaled regularization losses.\n            loss += tf.nn.scale_regularization_loss(model.losses)\n            return loss\n        \n        ############ METRICS #########################\n        train_auc = tf.keras.metrics.AUC(name=\"train_auc\", multi_label=True)\n        val_auc = tf.keras.metrics.AUC(name=\"val_auc\", multi_label=True)\n        \n        ############ MODEL #######################\n        \n        model = InceptionV4(input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3), classes = n_labels)\n\n        optimizer = tf.keras.optimizers.RMSprop(\n                    learning_rate=0.001, rho=0.9, momentum=0.9, epsilon=1e-7, centered=False)\n    \n        optimizer  = tfa.optimizers.MovingAverage(optimizer)\n        \n        lr_scheduler = ReduceLRScheduler(optimizer, patience=3, factor=5, min_lr=1e-6)\n        early_stopping = EarlyStopping(patience = 15)\n        \n    ############ TRAINING LOOP #######################\n        \n    def train_step(inputs):\n        images, labels = inputs\n        with tf.GradientTape() as tape:\n            predictions = model(images, training=True)\n            loss = compute_loss(model, labels, predictions)\n        \n        gradients = tape.gradient(loss, model.trainable_variables)\n        optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n        \n        train_auc.update_state(labels, predictions)\n        return loss\n    \n    def test_step(inputs):\n        images, labels = inputs\n        predictions = model(images, training=False)\n        loss = compute_loss(model, labels, predictions)\n        val_auc.update_state(labels, predictions)\n        return loss\n        \n    # `run` replicates the provided computation and runs it\n    # with the distributed input.\n    @tf.function\n    def distributed_train_step(dataset_inputs):\n        per_replica_losses = strategy.run(train_step, args=(dataset_inputs,))\n        return strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_losses,\n                         axis=None)\n\n    @tf.function\n    def distributed_test_step(dataset_inputs):\n        per_replica_losses = strategy.run(test_step, args=(dataset_inputs,))\n        return strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_losses,\n                         axis=None)\n    \n    val_aucs = []\n    train_aucs = []\n\n    for epoch in range(EPOCHS):\n        print(\"Fold:\", n_fold, \"Epoch:\", epoch+1, \"Training\")\n        \n        ######## TRAIN LOOP ###############\n        total_loss = 0.0\n        num_batches = 0\n        train_iter = iter(train_dist_dataset)\n        for i in tqdm(range(steps_per_epoch)):    \n            total_loss += distributed_train_step(next(train_iter))\n            num_batches += 1\n\n        train_loss = total_loss / num_batches\n        \n        ######## TEST LOOP ###############\n        # loading averaged weights\n        with strategy.scope():\n            optimizer.swap_weights()\n            \n\n        total_loss = 0.0\n        num_batches = 0\n        for x in valid_dist_dataset:\n            total_loss += distributed_test_step(x)\n            num_batches += 1\n            \n        val_loss = total_loss / num_batches\n            \n        # loading back original weights\n        with strategy.scope():\n            optimizer.swap_weights()\n        \n        template = (\"Epoch: {}, Loss: {:.4f}, train_auc: {:.4f}, val_loss: {:.4f}, val_auc: {:.4f}\")\n        print(template.format(epoch+1, train_loss, \n                              train_auc.result(), val_loss, val_auc.result()))\n        \n        val_aucs.append(tf.cast(val_auc.result(), tf.float32))\n        train_aucs.append(tf.cast(train_auc.result(), tf.float32))\n        \n        ############ SAVE BEST MODEL ##############\n        if val_auc.result() >= max(val_aucs):\n            print(\"Saving best model with validation AUC:\", str(tf.cast(val_auc.result(), tf.float32)))\n            with strategy.scope():\n                optimizer.swap_weights()\n            model.save(\"model_fold_\" + str(n_fold) + \".h5\")\n            with strategy.scope():\n                optimizer.swap_weights()\n        \n        # Sanity Check, problems due to weight regularization\n        if val_auc.result() < 1e-3:\n            print(\"Numerical problem... Validation AUC is too low. Interrupting training\")\n            break\n            \n        # CallBacks \n        if early_stopping.hasToStop(val_auc.result()):\n            print(\"No improvement after\", early_stopping.patience, \"epochs. Interrupting training\")\n            break\n            \n        lr_scheduler.on_epoch_end(val_auc.result())\n        \n        train_auc.reset_states()\n        val_auc.reset_states()\n        \n    data = {\"val_auc\": val_aucs, \"train_auc\" : train_aucs}\n    history = pd.DataFrame(data=data)\n    history.to_csv(\"history_\" + str(n_fold) + \".csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_folds():\n    for i in range(5):\n        train_fold(n_fold=i)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_fold(0)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}