{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install --upgrade tensorflow==2.2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install --upgrade cloud-tpu-client==0.10","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os, json, cv2, math, re\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nimport random\n\n#model imports (keras/tensorflow)\nimport tensorflow as tf\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.preprocessing import image\nfrom keras import layers, models\n# from tensorflow.keras.applications import EfficientNetB0\nfrom keras.optimizers import Adam\nfrom keras.layers import Conv2D, MaxPooling2D, BatchNormalization, GlobalAveragePooling2D\nfrom keras.layers import Dense, Dropout, Flatten\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nimport tensorflow.keras.backend as K\nfrom tensorflow import keras\nfrom tensorflow.keras.optimizers import Adamax\n\nfrom kaggle_datasets import KaggleDatasets\nfrom functools import partial\n\nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.system('pip install /kaggle/input/kerasapplications -q')\nos.system('pip install /kaggle/input/efficientnet-keras-source-code/ -q --no-deps')\n\nimport efficientnet.tfkeras as efn","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Setting a Seed\n\nDoing this for reproduciblity.","metadata":{}},{"cell_type":"code","source":"SEED = 314\n\ndef seed_everything(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    random.seed(seed)    \n\nseed_everything(SEED)    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Detecting TPU\n\nThe following cell isnt necessary but is nice to double check that we are going to be using the TPU. If we have everything set up correctly the number of replicas should be 8. If we do not have the TPU turned on we will see a value of 1.\n\nADD Google Cloud Software Development Kit (SDK) to Notebook if using a private dataset (add-ons tab). Pretty much always using public data though.","metadata":{}},{"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')\n\nfrom tensorflow.python.framework.config import set_memory_growth\ngpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n    try:\n        # Currently, memory growth needs to be the same across GPUs\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n    except RuntimeError as e:\n        # Memory growth must be set before GPUs have been initialized\n        print(e) \n    # Mixed precision\n    from tensorflow.keras.mixed_precision import experimental as mixed_precision\n    policy = mixed_precision.Policy('mixed_float16')\n    mixed_precision.set_policy(policy)\n    print('Compute dtype: %s' % policy.compute_dtype)\n    print('Variable dtype: %s' % policy.variable_dtype)\n\n# XLA\ntf.config.optimizer.set_jit(True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Variables\n\n- note: actual file path for data is 'cassava-leaf-disease-tfrecords-center-512x512'","metadata":{}},{"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\n\nBATCH_SIZE = 2 * strategy.num_replicas_in_sync\nTARGET_SIZE = 512\nIMAGE_SIZE = [TARGET_SIZE, TARGET_SIZE]\n\nHEIGHT = TARGET_SIZE\nWIDTH = TARGET_SIZE\n\nCLASSES = ['0', '1', '2', '3', '4']\nEPOCHS = 40\nDROPOUT_RATE = 0.45 #would like to implement this \n\nNOTOP_DEMO  = '../input/xception/xception_weights_tf_dim_ordering_tf_kernels_notop.h5'\n\nNOISY_STUDENT_B0 = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b0_noisy-student_notop.h5'\nNOISY_STUDENT_B1 = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b1_noisy-student_notop.h5'\nNOISY_STUDENT_B2 = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b2_noisy-student_notop.h5'\nNOISY_STUDENT_B3 = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b3_noisy-student_notop.h5'\nNOISY_STUDENT_B4 = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b4_noisy-student_notop.h5'\nNOISY_STUDENT_B5 = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b5_noisy-student_notop.h5'\nNOISY_STUDENT_B6 = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b5_noisy-student_notop.h5'\nNOISY_STUDENT_B7 = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b7_noisy-student_notop.h5'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Splitting TFRecords\n\n\nThe number on the end of the tfrecord file corresponds to the number of images in that tfrecord.\n\nExample: 'gs://kds-100c2bc3bab7e1f77f19378980a417f43e62119932994bd622dc7cb4/Id_train01-1427.tfrec' (1427 imgs)\n","metadata":{}},{"cell_type":"code","source":"#this function counts number of images in all TFRecords\ndef count_data_items(filenames):\n    n = [int(re.compile(r'-([0-9]*)\\.').search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n#note: this filepath had to be manually selected\ndatabase_base_path = '../input/cassava-leaf-disease-tfrecords-center-512x512/'\n\n#reading train metadata\ntrain = pd.read_csv(f'{database_base_path}train.csv')\nprint(f'Train samples: {len(train)}')\n\n# Dataset paths\nGCS_PATH = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-tfrecords-center-{HEIGHT}x{WIDTH}') # Center croped and resized (15 TFRecord)\nGCS_PATH_CLASSES = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-tfrecords-classes-{HEIGHT}x{WIDTH}') # Center croped and resized (15 TFRecord) by classes\n\n# Dataset TFRecords\nFILENAMES_COMP = tf.io.gfile.glob(GCS_PATH + '/*.tfrec')\n\nFILENAMES_COMP_CBB = tf.io.gfile.glob(GCS_PATH_CLASSES + '/CBB*.tfrec')\nFILENAMES_COMP_CBSD = tf.io.gfile.glob(GCS_PATH_CLASSES + '/CBSD*.tfrec')\nFILENAMES_COMP_CGM = tf.io.gfile.glob(GCS_PATH_CLASSES + '/CGM*.tfrec')\nFILENAMES_COMP_CMD = tf.io.gfile.glob(GCS_PATH_CLASSES + '/CMD*.tfrec')\nFILENAMES_COMP_Healthy = tf.io.gfile.glob(GCS_PATH_CLASSES + '/Healthy*.tfrec')\n\n\nALL_TRAINING_FILENAMES = (FILENAMES_COMP + \n                      (10 * FILENAMES_COMP_CBB) + \n                      (5 * FILENAMES_COMP_CBSD) + \n                      (4 * FILENAMES_COMP_CGM))\n\n#ALL_TRAINING_FILENAMES = tf.io.gfile.glob(TRAINING_FILENAMES + '/*.tfrec')\nNUM_ALL_TRAINING_IMAGES = count_data_items(ALL_TRAINING_FILENAMES)\n\nprint(f'GCS: train images: {NUM_ALL_TRAINING_IMAGES}')\ndisplay(train.head())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train test split","metadata":{}},{"cell_type":"code","source":"TRAINING_FILENAMES, VALIDATION_FILENAMES = train_test_split(\n    ALL_TRAINING_FILENAMES,\n    train_size= 0.70, test_size=0.30,\n    random_state=SEED,\n)\n\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\n\nprint(\"Training Images: {}  Validation Image: {}\".format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))\nprint(\"Training Percent: {:.2f}  Validation Percent: {:.2f}\".format((NUM_TRAINING_IMAGES/NUM_ALL_TRAINING_IMAGES),\n                                                           (NUM_VALIDATION_IMAGES/NUM_ALL_TRAINING_IMAGES)))\n\nSTEPS_PER_EPOCH =  NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions","metadata":{}},{"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3) #decoding jpeg-encoded img to uint8 tensor\n    image = tf.cast(image, tf.float32) / 255.0 #cast int val to float so we can normalize it\n    image = tf.image.resize(image, [*IMAGE_SIZE]) #added this back seeing if it does anything\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) #resizing to proper shape\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_tfrecord(example, labeled=True):\n    \"\"\"\n        1. Parse data based on the 'TFREC_FORMAT' map.\n        2. Decode image.\n        3. If 'labeled' returns (image, label) if not (image, name).\n    \"\"\"\n    if labeled:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'target': tf.io.FixedLenFeature([], tf.int64), \n        }\n    else:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'image_name': tf.io.FixedLenFeature([], tf.string), \n        }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    if labeled:\n        label_or_name = tf.cast(example['target'], tf.int32)\n    else:\n        label_or_name =  example['image_name']\n    return image, label_or_name","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    \"\"\"\n        Create a Tensorflow dataset from TFRecords.\n    \"\"\"\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda x: read_tfrecord(x, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    return dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data augmentation\n\nI have a couple options for data augmentation here.\n\nThe first provides very simple augmentation, and the second provides a bit more variation in terms of structural manipulation.\n\nNOTE: from testing and others comments it seem less is more for augmentation with this dataset","metadata":{}},{"cell_type":"code","source":"def simple_data_augmenter(image, label):\n    # Thanks to the dataset.prefetch(AUTO) statement in the following function this happens essentially for free on TPU. \n    # Data pipeline code is executed on the \"CPU\" part of the TPU while the TPU itself is computing gradients.\n    \n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32) #random int and rotating img based on result\n    \n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n    \n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_flip_left_right(image)\n    \n    \n    return image, label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thanks to Chris Deotte for sharing this next data Augmentation Function. [link](https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96/comments)\n\n- alter the parameters in the transform function in order to tune the data augmentation parameters","metadata":{}},{"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Trying out another augmentation technique here called Coarse Dropout which randomly cuts out parts of the image. This is another technique to prevent overfitting.\n\nI perform this dropout function on the transform return statement.","metadata":{}},{"cell_type":"code","source":"def dropout(image, DIM=512, PROBABILITY = 0.75, CT = 5, SZ = 0.1):\n    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(CT==0)|(SZ==0): return image\n    \n    for k in range(CT):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        # COMPUTE SQUARE \n        WIDTH = tf.cast( SZ*DIM,tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # DROPOUT IMAGE\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3]) \n        three = image[ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM,DIM,3])\n    #imgplot = plt.imshow(image)\n    #plt.show()\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform(image,label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 45. * tf.random.normal([1],dtype='float32')\n    shr = 0. * tf.random.normal([1],dtype='float32') #need to test with and without shear\n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 20. * tf.random.normal([1],dtype='float32') \n    w_shift = 20. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n    \n    #PERFORMING DROPOUT ON RETURN STATEMENT\n        \n    return dropout(tf.reshape(d,[DIM,DIM,3])),label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading Data\n\nMake sure to have the correct augmenter selected in the next function. \n\n- transform - Chris Deottes Augmenter\n- simple_data_augmenter - quicker but more simple augmenter","metadata":{}},{"cell_type":"code","source":"def get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(transform, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048) #set higher than input?\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.map(transform, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\n#not used\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing Transformations\n\nVisualizing transformations function, comment out during training to save RAM.","metadata":{}},{"cell_type":"code","source":"def visualize_transformations():\n    row = 2; col = 4;\n    all_elements = get_training_dataset().unbatch()\n    one_element = tf.data.Dataset.from_tensors(next(iter(all_elements)) )\n    augmented_element = one_element.repeat().map(transform).batch(row*col)\n    \n\n    for (img,label) in augmented_element:\n        plt.figure(figsize=(15,int(15*row/col)))\n        for j in range(row*col):\n            plt.subplot(row,col,j+1)\n            plt.axis('off')\n            plt.imshow(img[j,])\n        plt.show()\n        break\n        \n#visualize_transformations()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building Model\n\nAlso note that we're using sparse_categorical_crossentropy as our loss function, because we did not one-hot encode our labels.\n\n--\n\nThere is a very useful repository on github where I am trying the 'noisy-student' starting weights for the EfficientNet models. Check old notebook version for direct download, but I used the same thing from a dataset so I can still load this in when internet is turned off. \n\nGithub Repo -> [Link](https://github.com/qubvel/efficientnet)","metadata":{}},{"cell_type":"markdown","source":"## Defining model","metadata":{}},{"cell_type":"code","source":"def create_model():\n    \n    model = models.Sequential()\n    \n    model.add(efn.EfficientNetB3(\n       include_top = False, \n        weights = NOISY_STUDENT_B3, \n        input_shape = (TARGET_SIZE, TARGET_SIZE, 3)\n    ))\n  \n    #model.add(keras.applications.Xception(\n    #    input_shape=(TARGET_SIZE, TARGET_SIZE, 3),\n    #    weights=NOTOP_DEMO,\n    #    include_top=False)\n    #)\n    model.add(layers.GlobalAveragePooling2D())\n    model.add(layers.Dropout(DROPOUT_RATE))\n    model.add(layers.Dense(5, activation = \"softmax\"))# 5 is the dimensionality of the output space \"5 options\"\n    \n    loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False)\n\n    model.compile(optimizer = 'adam',\n                  loss = loss,\n                  metrics = ['accuracy','sparse_categorical_accuracy']) #try sparse_categorical_accuracy here\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In order to ensure that our model is trained on the TPU, we build it using strategy.scope()","metadata":{}},{"cell_type":"code","source":"#with strategy.scope():\nmodel = create_model()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('./Untrained_TPU_model.h5')\n\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Callbacks\n\n\n#### Custom Learning Rate Scheduler\n\n","metadata":{}},{"cell_type":"code","source":"# Learning rate schedule for TPU, GPU and CPU.\n# Using an LR ramp up because fine-tuning a pre-trained model.\n# Starting with a high LR would break the pre-trained weights.\n\nLR_START = 0.00001\nLR_MAX = 0.0001\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 3\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = 0.85\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        #cosine decay\n        progress = (epoch - LR_RAMPUP_EPOCHS) / (EPOCHS - LR_RAMPUP_EPOCHS)\n        lr = LR_MAX * (0.5 * (1.0 + tf.math.cos(np.pi * ((1.0 * progress) % 1.0))))\n        \n        #exponential decay\n        #lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \n#setting verbose=True allows us to see LR in model training\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n\n#visualizing the learning rate schedule\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\n\nsns.set(style='whitegrid')\nplt.figure(figsize=(13, 5))\nplt.xlabel('Epoch')\nplt.ylabel('Learning Rate')\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Early_Stopping + ModelCheckpoint","metadata":{}},{"cell_type":"code","source":"#note if you monitor=val_loss then mode=min, if monitor=val_sparse_categorical_accuracy then mode=max\nmodel_save = ModelCheckpoint('./TPU_Model_best_weights.h5', \n                             save_best_only = True, \n                             save_weights_only = True,\n                             monitor = 'val_loss', \n                             mode = 'min',\n                             verbose = 1)\n\nmy_early_stopper = EarlyStopping(monitor = 'val_loss', min_delta = 0.001, \n                           patience = 3, mode = 'min', verbose = 1,\n                           restore_best_weights = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\njson_log = open('/kaggle/working/loss_log.json', mode='w', buffering=1)\njson_logging_callback = keras.callbacks.LambdaCallback(\n    on_train_begin=lambda logs: json_log.write('{ \"train\": [ \\n'),\n    on_epoch_end=lambda epoch, logs: json_log.write(\n        json.dumps({'epoch': epoch, 'loss': logs['loss'], 'acc': logs['accuracy']}) + ',\\n'),\n    on_train_end=lambda logs: json_log.write(']\\n}'),\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fitting the Model","metadata":{}},{"cell_type":"code","source":"print(\"Started Training\")\nhistory = model.fit(x=get_training_dataset(),\n                    epochs=EPOCHS,\n                    steps_per_epoch = STEPS_PER_EPOCH,\n                    validation_steps=VALID_STEPS,\n                    validation_data=get_validation_dataset(),\n                    callbacks = [lr_callback, model_save,json_logging_callback],\n                    verbose=1,\n                   )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_weights('./TPU_Model_final_weights.h5')\nmodel.save('./TPU_Model_Final')\nmodel.save(\"./TPU_Model_Final.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing Model History","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(13, 5))\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title(\"Model Loss\")\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend(['Train', 'Test'])\nplt.ylim(ymax = 2, ymin = 0)\nplt.grid()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(13, 5))\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('Model Accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend(['Train','Test'])\nplt.grid()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(13, 5))\nplt.plot(history.history['sparse_categorical_accuracy'])\nplt.plot(history.history['val_sparse_categorical_accuracy'])\nplt.title('Sparse Categorical Accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Sparse Categorical Accuracy')\nplt.legend(['Train','Test'])\nplt.grid()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Making predictions.","metadata":{}},{"cell_type":"code","source":"test_datagen_flow=get_training_dataset()\n\ntest_loss, test_acc = model.evaluate(test_datagen_flow, verbose=1)\n\nprint('\\nTest accuracy:', test_acc)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}