{"cells":[{"metadata":{},"cell_type":"markdown","source":"# **Libraries and Configurations**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"The notebook is forked from https://www.kaggle.com/itsuki9180/birdcall-using-tpu-train/notebook","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd, numpy as np, gc\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DEVICE = \"TPU\" # \"TPU\" or \"GPU\"\nIMG_SIZE = [128, 313]\nFOLDS = 5\nBATCH_SIZE = 512\nEPOCHS = [50]*FOLDS\nAUG_BATCH = BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Directories and Classes","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS; GCS_PATH3 = [None]*FOLDS; GCS_PATH4 = [None]*FOLDS; GCS_PATH5 = [None]*FOLDS\nfor i in range(FOLDS):\n    GCS_PATH[i] = KaggleDatasets().get_gcs_path('birdsongab')\n    GCS_PATH2[i] = KaggleDatasets().get_gcs_path('birdsongcf')\n    GCS_PATH3[i] = KaggleDatasets().get_gcs_path('birdsonggm')\n    GCS_PATH4[i] = KaggleDatasets().get_gcs_path('birdsongnr')\n    GCS_PATH5[i] = KaggleDatasets().get_gcs_path('birdsongsy')\n\nfiles_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/*.tfrec')))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset and Augmentation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [128, 313, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled = True, ordered = False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # use data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO) # returns a dataset of (image, label) pairs if labeled = True or (image, id) pair if labeld = False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    #image = tf.image.random_flip_left_right(image)\n    return image, label   \n\ndef get_training_dataset(dataset, do_aug=True):\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.batch(AUG_BATCH)\n    if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO) # note we put AFTER batching\n    dataset = dataset.unbatch()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(dataset, do_onehot=True):\n    dataset = dataset.batch(BATCH_SIZE)\n    if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTO) # we must use one hot like augmented train data\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = int( count_data_items(files_train) * (FOLDS-1.)/FOLDS )\nNUM_VALIDATION_IMAGES = int( count_data_items(files_train) * (1./FOLDS) )\n\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\n#print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def onehot(image,label):\n    CLASSES = 264\n    return image,tf.one_hot(label,CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mixup(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    #DIM = IMAGE_SIZE[0]\n    CLASSES = 264\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n        b = 1-a\n        #a = tf.sqrt(a)\n        #b = tf.sqrt(b)\n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append(b*img1 + a*img2)\n        # MAKE CUTMIX LABEL\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        \n        \n        labs.append(tf.math.minimum(b*lab1 + a*lab2,1))\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,128,313,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform(image,label):\n    # THIS FUNCTION APPLIES BOTH CUTMIX AND MIXUP\n    CLASSES = 264\n    SWITCH = 0.5\n    CUTMIX_PROB = 0.666\n    MIXUP_PROB = 0.\n    # FOR SWITCH PERCENT OF TIME WE DO CUTMIX AND (1-SWITCH) WE DO MIXUP\n    #image2, label2 = cutmix(image, label, CUTMIX_PROB)\n    image3, label3 = mixup(image, label, MIXUP_PROB)\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        imgs.append(image3[j,])\n        labs.append(label3[j,])\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,128,313,3))\n    label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image4,label4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"row = 2; col = 2;\nrow = min(row,AUG_BATCH//col)\nall_elements = get_training_dataset(load_dataset(files_train),do_aug=False).unbatch()\naugmented_element = all_elements.repeat().batch(AUG_BATCH).map(transform)\n\nfor (img,label) in augmented_element:\n    print(label)\n    plt.figure(figsize=(15,int(15*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build and Train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import Callback\ndef total_acc(y_true, y_pred):\n    pred = K.cast(K.greater_equal(y_pred, 0.5), \"float\")\n    flag = K.cast(K.equal(y_true, pred), \"float\")\n    return K.prod(flag, axis=-1)\n\ndef binary_acc(y_true, y_pred):\n    pred = K.cast(K.greater_equal(y_pred, 0.5), \"float\")\n    flag = K.cast(K.equal(y_true, pred), \"float\")\n    return K.mean(flag, axis=-1)\n\nclass F1Callback(Callback):\n    def __init__(self):\n        self.f1s = []\n\n    def on_epoch_end(self, epoch, logs):\n        eps = np.finfo(np.float32).eps\n        recall = logs[\"val_true_positives\"] / (logs[\"val_possible_positives\"] + eps)\n        precision = logs[\"val_true_positives\"] / (logs[\"val_predicted_positives\"] + eps)\n        f1 = 2*precision*recall / (precision+recall+eps)\n        print(\"f1_val (from log) =\", f1)\n        self.f1s.append(f1)\n\ndef true_positives(y_true, y_pred):\n    return K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n\ndef possible_positives(y_true, y_pred):\n    return K.sum(K.round(K.clip(y_true, 0, 1)))\n\ndef predicted_positives(y_true, y_pred):\n    return K.sum(K.round(K.clip(y_pred, 0, 1)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f1cb = F1Callback()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def binary_loss(y_true, y_pred):\n    bce = K.binary_crossentropy(y_true, y_pred)\n    return K.sum(bce, axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend as K\n\nimport dill\n\n\ndef binary_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n\n        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\n    return binary_focal_loss_fixed\n\n\ndef categorical_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Softmax version of focal loss.\n           m\n      FL = ∑  -alpha * (1 - p_o,c)^gamma * y_o,c * log(p_o,c)\n          c=1\n      where m = number of classes, c = class and o = observation\n    Parameters:\n      alpha -- the same as weighing factor in balanced cross entropy\n      gamma -- focusing parameter for modulating factor (1-p)\n    Default value:\n      gamma -- 2.0 as mentioned in the paper\n      alpha -- 0.25 as mentioned in the paper\n    References:\n        Official paper: https://arxiv.org/pdf/1708.02002.pdf\n        https://www.tensorflow.org/api_docs/python/tf/keras/backend/categorical_crossentropy\n    Usage:\n     model.compile(loss=[categorical_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def categorical_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred: A tensor resulting from a softmax\n        :return: Output tensor.\n        \"\"\"\n        epsilon = K.epsilon()\n        # Scale predictions so that the class probas of each sample sum to 1\n        y_pred /= K.sum(y_pred+epsilon, axis=-1, keepdims=True)\n\n        # Clip the prediction value to prevent NaN's and Inf's\n        epsilon = K.epsilon()\n        y_pred = K.clip(y_pred, epsilon, 1. - epsilon)\n\n        # Calculate Cross Entropy\n        cross_entropy = -y_true * K.log(y_pred)\n\n        # Calculate Focal Loss\n        loss = alpha * K.pow(1 - y_pred, gamma) * cross_entropy\n\n        # Sum the losses in mini_batch\n        return K.sum(loss, axis=1)\n\n    return categorical_focal_loss_fixed\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import efficientnet.keras as efn \n# from tensorflow import keras\n# model = efn.EfficientNetB0(weights='imagenet')  # or weights='noisy-student'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #!pip uninstall tensorflow\n# !pip install tensorflow==1.15.2\n\n# #!pip uninstall Keras\n# !pip install Keras==2.3.1\n\n# #!pip uninstall Keras-Applications\n# !pip install Keras-Applications==1.0.8\n\n# #!pip uninstall Keras-Preprocessing\n# !pip install Keras-Preprocessing==1.1.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # import efficientnet.tfkeras as efn\n# import tensorflow.keras.layers as L\n\n# def build_model():\n#     #inp = tf.keras.layers.Input(shape=(128,313,3))\n    \n#     model = tf.keras.Sequential([\n#         #tf.keras.applications.EfficientNetB5(\n#         tf.keras.applications.DenseNet121(\n            \n#             input_shape=(*IMG_SIZE, 3),\n#             weights='imagenet',\n#             #weights='noisy-student',\n#             include_top=False\n#         ),\n#         L.GlobalAveragePooling2D(),\n#         L.Dense(1024, activation = 'relu'), \n#         L.Dropout(0.3), \n#         L.Dense(512, activation= 'relu'), \n#         L.Dropout(0.2), \n#         L.Dense(512, activation='relu'), \n        \n#         L.Dropout(0.1), \n#         L.Dense(264, activation='sigmoid')\n#     ])\n#     #model = tf.keras.Model(inputs=inp,outputs=x)\n#     opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n#     model.compile(optimizer=opt,loss=binary_focal_loss(),metrics=[total_acc, true_positives, possible_positives, predicted_positives])\n#     return model\n\n# # model = build_model()\n# # model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef build_model():\n    inp = tf.keras.layers.Input(shape=(128,313,3))\n    base = tf.keras.applications.DenseNet121(include_top=False, weights='imagenet')\n    x = base(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(1024,activation='swish')(x)\n    x = tf.keras.layers.Dropout(0.25)(x)\n    x = tf.keras.layers.Dense(512,activation='swish')(x)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    x = tf.keras.layers.Dense(384,activation='swish')(x)\n    x = tf.keras.layers.Dropout(0.25)(x)\n    x = tf.keras.layers.Dense(264,activation='sigmoid')(x)\n    model = tf.keras.Model(inputs=inp,outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    model.compile(optimizer=opt,loss=binary_focal_loss(),metrics=[total_acc, true_positives, possible_positives, predicted_positives])\n    return model\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr_callback(batch_size=8):\n    lr_start   = 0.0005\n    lr_max     = 0.001\n    lr_min     = 0.00001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 5\n    lr_decay   = 0.9\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = KFold(n_splits=FOLDS,shuffle=True,random_state=12)\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(5))):\n    if fold==(FOLDS-1):\n        idxTT = idxT; idxVV = idxV\n        print('### Using fold',fold,'for experiments')\n    print('Fold',fold,'has TRAIN:',idxT,'VALID:',idxV)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold in range(FOLDS):\n    if fold>0:\n        break;\n    # REPEAT SAME FOLD OVER AND OVER\n    idxT = idxTT\n    idxV = idxVV\n    \n    # DISPLAY FOLD INFO\n    if DEVICE=='TPU':\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print('#'*25); print('#### EXPERIMENT',fold+1)\n\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train-*-%.2i*.tfrec'%x for x in idxT])\n    files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train-*-%.2i*.tfrec'%x for x in idxT])\n    files_train += tf.io.gfile.glob([GCS_PATH3[fold] + '/train-*-%.2i*.tfrec'%x for x in idxT])\n    files_train += tf.io.gfile.glob([GCS_PATH4[fold] + '/train-*-%.2i*.tfrec'%x for x in idxT])\n    files_train += tf.io.gfile.glob([GCS_PATH5[fold] + '/train-*-%.2i*.tfrec'%x for x in idxT])\n    print('#### all trains',len(files_train))\n        \n    files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train-*-%.2i*.tfrec'%x for x in idxV])\n    files_valid += tf.io.gfile.glob([GCS_PATH2[fold] + '/train-*-%.2i*.tfrec'%x for x in idxV])\n    files_valid += tf.io.gfile.glob([GCS_PATH3[fold] + '/train-*-%.2i*.tfrec'%x for x in idxV])\n    files_valid += tf.io.gfile.glob([GCS_PATH4[fold] + '/train-*-%.2i*.tfrec'%x for x in idxV])\n    files_valid += tf.io.gfile.glob([GCS_PATH5[fold] + '/train-*-%.2i*.tfrec'%x for x in idxV])\n    print('#### all valids',len(files_valid))\n    \n    NUM_TRAINING_IMAGES = int( count_data_items(files_train))\n    NUM_VALIDATION_IMAGES = int( count_data_items(files_valid) )\n    STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n    print('Dataset: {} training images, {} validation images,'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))\n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():\n        model = build_model()\n        \n    # SAVE BEST MODEL EACH FOLD\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n    es = tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss', min_delta=0, patience=10, verbose=0, mode='auto',\n        baseline=None, restore_best_weights=False\n    )\n    # TRAIN\n    train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': files_train}).loc[:]['TRAINING_FILENAMES']), labeled = True)\n    val_dataset = load_dataset(list(pd.DataFrame({'VALIDATION_FILENAMES': files_valid}).loc[:]['VALIDATION_FILENAMES']), labeled = True, ordered = True)\n    print(model.summary())\n    print('Training...')\n    history = model.fit(\n            get_training_dataset(train_dataset), \n            steps_per_epoch = STEPS_PER_EPOCH,\n            epochs = EPOCHS[fold],\n            callbacks = [sv,get_lr_callback(BATCH_SIZE),f1cb,es],\n            validation_data = get_validation_dataset(val_dataset),\n            verbose=1\n        )\n    model.save_weights('fold-%if.h5'%fold)\n        \n    del model; z = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}