{"cells":[{"metadata":{"_uuid":"5e0c8a1e-8bd8-4943-b684-44ac9cdc6620","_cell_guid":"786eeb32-4ef5-4c81-9c87-cd079077ce2e","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n    #for filename in filenames:\n        #print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U tensorflow==2.3.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --quiet /kaggle/input/kerasapplications\n!pip install --quiet /kaggle/input/efficientnet-git","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"update TPU server tensorflow version...\")\n\n!pip install cloud-tpu-client\nimport tensorflow as tf \nfrom cloud_tpu_client import Client\nprint(tf.__version__)\nClient().configure_tpu_version(tf.__version__, restart_type='ifNeeded')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport matplotlib.pyplot as plt\nimport pandas as pd\n# import seaborn as sns\nimport os\nimport cv2\nimport re\nimport math\nimport datetime\nimport time\nfrom collections import namedtuple\nfrom functools import partial\nfrom tensorflow import keras\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import confusion_matrix, classification_report\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.losses import CategoricalCrossentropy\nimport tensorflow as tf\n\nfrom tensorflow.keras.layers import Dense, Dropout,\\\n        Flatten,GlobalAveragePooling2D,BatchNormalization, Activation\n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras import layers\n\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\n\nfrom kaggle_datasets import KaggleDatasets\n\nimport efficientnet.keras as efn ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"main_dir = '../input/cassava-leaf-disease-classification/'\nos.listdir(main_dir) \ntrain_img_path = '../input/cassava-leaf-disease-classification/train_images'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_PATH = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification')\n\nGCS_NewTFPure512 = KaggleDatasets().get_gcs_path('cassava-leaf-disease-tfrecords-512x512')\nGCS_NewTFCenter512 = KaggleDatasets().get_gcs_path('cassava-leaf-disease-tfrecords-center-512x512')\n        \n\nIMAGE_SIZE = [512,512]\nCLASSES = ['0', '1', '2', '3', '4']\n\nN_FOLDS = 5\nPATIENCE = 15 \nPATIENT = 10\n\nSEED = 100\nMODEL_NAME = \"EfficentNetB3\"\n\nHEIGHT = 512\nWIDTH = 512\nHEIGHT_RS = 512\nWIDTH_RS = 512\nCHANNELS = 3\nN_CLASSES = 5\n\n#BiTemperedLoss\nT_1 = 0.6\nT_2 = 1.4\nSMOOTH_FRACTION = 0.1\nN_ITER = 5\n\n#Model variables\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nEPOCHS = 30\ninput_shape = (512,512,3)\ndropout_rate = 0.2\n\nMODEL_SAVE_PATH = \"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"custom_files = tf.io.gfile.glob(GCS_NewTFPure512 + '/Id_train*.tfrec')\n\nDATASET_FILENAMES = custom_files\n\nNUM_FILES = len(DATASET_FILENAMES)\nnp.random.shuffle(DATASET_FILENAMES)\ndisplay(DATASET_FILENAMES)\n\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test_tfrecords/ld_test00-1.tfrec')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Functions for datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\n# CutOut\ndef data_augment_cutout(image, min_mask_size=(int(HEIGHT * .1), int(HEIGHT * .1)), \n                        max_mask_size=(int(HEIGHT * .125), int(HEIGHT * .125))):\n    p_cutout = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    if p_cutout > .85: # 10~15 cut outs\n        n_cutout = tf.random.uniform([], 10, 15, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    elif p_cutout > .6: # 5~10 cut outs\n        n_cutout = tf.random.uniform([], 5, 10, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    elif p_cutout > .25: # 2~5 cut outs\n        n_cutout = tf.random.uniform([], 2, 5, dtype=tf.int32)\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=n_cutout)\n    else: # 1 cut out\n        image = random_cutout(image, HEIGHT, WIDTH, \n                              min_mask_size=min_mask_size, max_mask_size=max_mask_size, k=1)\n\n    return image\n\ndef random_cutout(image, height, width, channels=3, min_mask_size=(10, 10), max_mask_size=(80, 80), k=1):\n    assert height > min_mask_size[0]\n    assert width > min_mask_size[1]\n    assert height > max_mask_size[0]\n    assert width > max_mask_size[1]\n\n    for i in range(k):\n      mask_height = tf.random.uniform(shape=[], minval=min_mask_size[0], maxval=max_mask_size[0], dtype=tf.int32)\n      mask_width = tf.random.uniform(shape=[], minval=min_mask_size[1], maxval=max_mask_size[1], dtype=tf.int32)\n\n      pad_h = height - mask_height\n      pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)\n      pad_bottom = pad_h - pad_top\n\n      pad_w = width - mask_width\n      pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)\n      pad_right = pad_w - pad_left\n\n      cutout_area = tf.zeros(shape=[mask_height, mask_width, channels], dtype=tf.uint8)\n\n      cutout_mask = tf.pad([cutout_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)\n      cutout_mask = tf.squeeze(cutout_mask, axis=0)\n      image = tf.multiply(tf.cast(image, tf.float32), tf.cast(cutout_mask, tf.float32))\n\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_cutout = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n#     # Shear\n#     if p_shear > .2:\n#         if p_shear > .6:\n#             image = transform_shear(image, HEIGHT, shear=20.)\n#         else:\n#             image = transform_shear(image, HEIGHT, shear=-20.)\n            \n    # Rotation\n    if p_rotation > .2:\n        if p_rotation > .6:\n            image = transform_rotation(image, HEIGHT, rotation=45.)\n        else:\n            image = transform_rotation(image, HEIGHT, rotation=-45.)\n            \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n#     if p_spatial > .75:\n#         image = tf.image.transpose(image)\n        \n#     # Rotates\n#     if p_rotate > .75:\n#         image = tf.image.rot90(image, k=3) # rotate 270º\n#     elif p_rotate > .5:\n#         image = tf.image.rot90(image, k=2) # rotate 180º\n#     elif p_rotate > .25:\n#         image = tf.image.rot90(image, k=1) # rotate 90º\n        \n    # Pixel-level transforms\n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n        \n#     # Crops\n#     if p_crop > .6:\n#         if p_crop > .9:\n#             image = tf.image.central_crop(image, central_fraction=.5)\n#         elif p_crop > .8:\n#             image = tf.image.central_crop(image, central_fraction=.6)\n#         elif p_crop > .7:\n#             image = tf.image.central_crop(image, central_fraction=.7)\n#         else:\n#             image = tf.image.central_crop(image, central_fraction=.8)\n#     elif p_crop > .3:\n#         crop_size = tf.random.uniform([], int(HEIGHT*.6), HEIGHT, dtype=tf.int32)\n#         image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n            \n    image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n\n    if p_cutout > .5:\n        image = data_augment_cutout(image)\n        \n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def one_hot(image, label):\n    label = tf.one_hot(label, len(CLASSES), dtype = tf.float32)\n    return image,label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Decode the data\n#turn the images into tensors\n#normalize the image (get every pixel to have a value between 0 and 1)\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float64) / 255.0\n    image = tf.image.resize(image, [512, 512])\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#setting up variables X and y; in this case image and prediction (for images with no label)\ndef read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# the following code will load the dataset using the TPU\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#The following functions will be used to load our training, validation, and test datasets, as well as print out the number of images in each dataset.\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)  \n    dataset = dataset.map(one_hot, num_parallel_calls=AUTOTUNE)  \n    dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE)  \n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=ordered) \n    dataset = dataset.map(one_hot, num_parallel_calls=AUTOTUNE) \n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.map(one_hot, num_parallel_calls=AUTOTUNE) \n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\n\n# NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\n# NUM_VALIDATION_IMAGES = count_data_items(VALID_FILENAMES)\n# NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\n\n\n# print('Dataset: {} training images, {} validation images, {} (unlabeled) test images'.format(\n#     NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# #adding in augmentations\n# def data_augment(image, label):\n#     # Thanks to the dataset.prefetch(AUTO) statement in the following function this happens essentially for free on TPU. \n#     # Data pipeline code is executed on the \"CPU\" part of the TPU while the TPU itself is computing gradients.\n#     image = tf.image.random_flip_left_right(image)\n#     image = tf.image.random_contrast(image, 0.2,0.5)\n# #     image = tf.image.random_jpeg_quality(image, 50,100)\n#     return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import module we'll need to import our custom module\nfrom shutil import copyfile\n\n# copy our file into the working directory (make sure it has .py suffix)\ncopyfile(src = \"../input/bitempered-logistic-loss-tensorflow-v2/bi_tempered_loss.py\", dst = \"../working/loss.py\")\n\n# import all our functions\nfrom loss import bi_tempered_logistic_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BiTemperedLogisticLoss(tf.keras.losses.Loss):\n  def __init__(self, t1, t2, lbl_smth, n_iter):\n    super(BiTemperedLogisticLoss, self).__init__()\n    self.t1 = t1\n    self.t2 = t2\n    self.lbl_smth = lbl_smth\n    self.n_iter = n_iter\n\n  def call(self, y_true, y_pred):\n    return bi_tempered_logistic_loss(y_pred, y_true, self.t1, self.t2, self.lbl_smth, self.n_iter)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.optimizers import RMSprop, Adam\nfrom keras.layers import DepthwiseConv2D, Reshape, Activation\nimport efficientnet.keras as efn\nfrom tensorflow.keras import models\nfrom tensorflow.keras import layers\n\nwith strategy.scope():\n    class LRSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n        def __call__(self, step):\n            return lrfn(epoch=step//STEPS_PER_EPOCH)\n        \n    def get_model():\n        conv_base = efn.EfficientNetB3(weights='noisy-student', include_top = False, input_shape = input_shape)\n        conv_base.trainable = True\n    \n        model = models.Sequential()\n        model.add(conv_base)\n        model.add(layers.GlobalAveragePooling2D(name=\"gap\"))\n        model.add(layers.Dense(16, activation=\"relu\", name=\"intermediate\"))\n        model.add(layers.Dropout(dropout_rate, name=\"dropout_out\"))\n        model.add(layers.Dense(5, activation=\"softmax\", name=\"fc_out\"))\n        \n        optimizer = tf.keras.optimizers.Adam()\n        \n        T_1 = 0.6\n        T_2 = 1.4\n        SMOOTH_FRACTION = 0.1\n        N_ITER = 5\n        \n        model.compile(optimizer=optimizer, \n                      loss=BiTemperedLogisticLoss(t1=T_1, t2=T_2, lbl_smth=SMOOTH_FRACTION, n_iter=N_ITER), \n                      metrics=['categorical_accuracy'])\n        \n        return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LR_START = 0.00001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n        \n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) \n    return lr\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ModelCheckpoint\ndef get_checkpoint(model_save_path, is_save_best = True):\n    return ModelCheckpoint(model_save_path, \n                             monitor= 'val_loss', \n                             verbose=1, \n                             save_best_only=is_save_best, \n                             mode= 'min', \n                             save_weights_only = False)\n\n# https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/EarlyStopping\ndef get_early_stopping():\n    return EarlyStopping(monitor = 'val_loss', min_delta = 0.0001, \n                           patience = PATIENCE, mode = 'min', verbose = 1,\n                           restore_best_weights = True)\n    \n# https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ReduceLROnPlateau\ndef get_learning_rate_decay():\n  return ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, \n                              patience = 1, min_delta = 0.0001, \n                              mode = 'min', verbose = 1)\n\n\ndef get_model_callback( fold_num):\n    model_save_path_best = f'{MODEL_NAME}_best_fold_{fold_num}_.h5'\n    print(\"Best model save path: \", model_save_path_best)\n\n    model_save_path_last = f'{MODEL_NAME}_last_fold_{fold_num}_.h5'\n    print(\"Last model save path: \", model_save_path_last)\n\n#saves best model\n    checkpoint_best = get_checkpoint(model_save_path_best, is_save_best = True)\n#saves the most recent model\n    checkpoint_last = get_checkpoint(model_save_path_last, is_save_best = False)\n\n    early_stopping = get_early_stopping()\n#     learning_rate_decay = get_learning_rate_decay()\n\n# https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/LearningRateScheduler\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n    \n#     if WANDB:\n#         wandb.run.name= f'{MODEL_NAME}_fold_{fold_num}'\n#         [WandbCallback(),checkpoint_best, checkpoint_last, early_stopping]\n\n    return [checkpoint_best, checkpoint_last, early_stopping, lr_callback]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\noof_pred = []; oof_labels = []; history_list = []\n\n\nfor fold , (X_train, X_valid) in enumerate(skf.split(np.arange(NUM_FILES))):\n\n    print(\"Start Fold: \",fold);\n    # print(\"X_valid: \", X_valid)\n#     MODEL_SAVE_PATH = f\"{MODEL_NAME}_{fold}.h5\"\n\n    if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n\n#     TRAIN_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/ld_train%.2i*.tfrec' % x for x in X_train])\n#     VALID_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/ld_train%.2i*.tfrec' % x for x in X_valid])\n    \n    TRAINING_FILENAMES = [DATASET_FILENAMES[x] for x in X_train]\n    VALID_FILENAMES = [DATASET_FILENAMES[x] for x in X_valid]\n    \n    \n#     np.random.shuffle(DATASET_FILENAMES)\n    \n    ct_train = count_data_items(TRAINING_FILENAMES)\n    ct_valid = count_data_items(VALID_FILENAMES)\n\n    print(\"Train File: \",ct_train)\n    print(\"Valid File: \", ct_valid)\n\n    STEPS_PER_EPOCH =  ct_train// BATCH_SIZE\n    VALIDATION_STEPS = ct_valid // BATCH_SIZE\n    print(\"Train Step: \",STEPS_PER_EPOCH)\n    print(\"Valid Step: \",VALIDATION_STEPS)\n    \n    callback_list = get_model_callback(fold)\n\n    tf.keras.backend.clear_session()\n        \n    with strategy.scope():\n        model = get_model()\n        history = model.fit( x=get_training_dataset(),\n                                steps_per_epoch = STEPS_PER_EPOCH,\n                                epochs = EPOCHS,\n                                validation_data = get_validation_dataset(), \n                                validation_steps = VALIDATION_STEPS,\n                                callbacks = callback_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()\nmodel.save('model(withBitemp).h5')\nprint(\"Done\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}