{"cells":[{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":" %config Completer.use_jedi = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U efficientnet","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nimport numpy as np\nimport pandas as pd\nfrom kaggle_datasets import KaggleDatasets\nfrom functools import partial\nimport matplotlib.pyplot as plt\nimport re\nfrom sklearn.model_selection import StratifiedKFold\nimport math\nimport tensorflow.keras.backend as K\nimport gc\nfrom keras.callbacks import Callback\nimport tensorflow_addons as tfa\nimport efficientnet.keras as efc\n\nprint(\"Tensorflow version\" + tf.__version__)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 42\ntf.random.set_seed(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Setting up TPU Strategy"},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device: ', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas: ', strategy.num_replicas_in_sync)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CONFIG"},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_PATH = KaggleDatasets().get_gcs_path(dataset_dir='cassava-leaf-disease-tfrecords-512x512')\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [512, 512]\nCLASSES = ['0','1','2','3','4']\nEPOCHS = 10\nN_FOLDS = 4","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loading\n## Helper methods"},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32)\n    image = tf.reshape(image, [512,512, 3])\n    return image\n\ndef onehot(image,label):\n    CLASSES = 5\n    return image,tf.one_hot(label,CLASSES)\n\ndef read_tfrecord(example, labeled):\n    tfrecord_format= {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'target': tf.io.FixedLenFeature([], tf.int64),\n        'image_name': tf.io.FixedLenFeature([], tf.string)\n    } if labeled else{\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'image_name': tf.io.FixedLenFeature([], tf.string)\n    }\n    \n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'],tf.int32)\n        return onehot(image,label)\n    idnum = example['image_name']\n    return image, idnum\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    return dataset\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cv = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=42)\n\nfiles = tf.io.gfile.glob(GCS_PATH + '/Id_train*.tfrec')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Augmentations\n## Perspective Warping"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ported from docs.fast.ai \n\ndef find_coeffs(orig_pts, targ_pts):\n    matrix = []\n    for p1, p2 in zip(targ_pts, orig_pts):\n        matrix.append([p1[0], p1[1], 1, 0, 0, 0, -p2[0]*p1[0], -p2[0]*p1[1]])\n        matrix.append([0, 0, 0, p1[0], p1[1], 1, -p2[1]*p1[0], -p2[1]*p1[1]])\n    A = np.array(matrix, dtype=np.float32)[None,...]\n    B = np.array(orig_pts, dtype=np.float32)\n    B = np.reshape(B, [8,1])[None,...]\n    \n    return tf.linalg.solve(A, B)[0][:,0]\n\ndef get_coords_persp_transform(magnitude=0.2,p=0.5):\n    H,W = 1,1\n    # Generate random numbers for coordinates\n    ys = np.random.uniform(0,magnitude, [4])\n    xs = np.random.uniform(0,magnitude, [4])\n    # prepare coordinates the coordinates\n    ys = ys*np.array([H,1,1,H])\n    ys[1:3] = 1 - (1 - ys[[0,3]])*ys[[1,2]]\n    xs = xs * np.array([W,W,1,1])\n    xs[2:] = 1 - (1 - xs[:2])*xs[2:]\n\n    return np.stack([xs,ys])\n    \n\ndef get_persp_mat(IMAGE_SIZE,magnitude=0.2):\n    H,W = IMAGE_SIZE\n    src_coords = np.transpose(get_coords_persp_transform(magnitude=0.2)*H).astype(np.int32)\n    targ_coords = np.transpose(np.array([[0,0,W,W],[0,H,H,0]])).astype(np.int32)\n    p_mat = find_coeffs(targ_coords,src_coords)\n    p_mat = tf.concat([p_mat, [1]],axis=0)\n    return tf.reshape(p_mat, [3,3])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Random Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))\n\ndef transform(image,label,name=None,p_rot=0.5,p_shr=0.5,p_h_zoom=0.5,p_w_zoom=0.5,p_h_shift=0.5,p_w_shift=0.5,p_persp_warp=0.5, mag_persp_warp=0.3):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 60. * tf.random.normal([1],dtype='float32') if np.random.uniform() < p_rot else tf.constant([0.])\n    shr = .2 * tf.random.normal([1],dtype='float32') if np.random.uniform() < p_shr else  tf.constant([0.])\n    h_zoom = 0.9 + tf.random.uniform([1],maxval=4., dtype='float32')/10. if np.random.uniform() < p_h_zoom else tf.constant([1.])\n    w_zoom = 0.9 + tf.random.uniform([1],maxval=4.,dtype='float32')/10. if np.random.uniform() < p_w_zoom else  tf.constant([1.])\n    h_shift = 10. * tf.random.uniform([1],dtype='float32') if np.random.uniform() < p_h_shift else tf.constant([0.])\n    w_shift = 10. * tf.random.uniform([1],dtype='float32') if np.random.uniform() < p_w_shift else  tf.constant([0.])\n    persp = get_persp_mat(IMAGE_SIZE,magnitude=mag_persp_warp) if np.random.uniform() < p_persp_warp else np.eye(3)\n    \n    # GET TRANSFORMATION MATRIX    \n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n    \n    m = m @ persp\n    \n    d = tfa.image.transform(image, tf.reshape(m, [9])[:-1], fill_mode='reflect')\n    d = tf.reshape(d,[512,512,3])\n    seeds = np.random.randint([4])    \n    d = tf.image.stateless_random_flip_left_right(d, [seeds[0],seeds[0]+3])\n    d = tf.image.stateless_random_flip_up_down(d, [seeds[0], seeds[0]+3])\n    if np.random.random() < 0.7:\n        d = tf.image.stateless_random_contrast(d, 0.3, 0.9, [seeds[0], seeds[0]+3])\n        d = tf.image.stateless_random_brightness(d, 0.1, [seeds[0], seeds[0]+3])\n#     offset = (512-DIM)//2\n#     d = tf.slice(d, [offset,offset,0],[DIM, DIM,3])\n    \n    if name:\n        return d,label, name\n    return d,label\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow_probability as tfp\n\ndef cutmix(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 5\n    imgs = []; labs = []\n    for j in range(BATCH_SIZE):\n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast( tf.random.uniform([],0,BATCH_SIZE),tf.int32)\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE,CLASSES))\n    return image2,label2\n\ndef mixup(image, label, PROBABILITY = 0.7, alpha=0.4):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 5\n    beta = tfp.distributions.Beta(alpha, alpha).sample(BATCH_SIZE)\n    \n    imgs = []; labs = []\n    for j in range(BATCH_SIZE):\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,BATCH_SIZE),tf.int32)\n        a = beta[j]*P # this is beta dist with alpha=1.0\n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        # MAKE CUTMIX LABEL\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n#         print(lab1, lab2)\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE,CLASSES))\n#     print(label, label2)\n\n    return image2,label2\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rescale_images(image, label):\n    return tf.multiply(image, 1/255.), label\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# validation set transforms\ndef resize(image, label, size=IMAGE_SIZE):\n    return tf.image.resize(\n        image, size, method=tf.image.ResizeMethod.NEAREST_NEIGHBOR, preserve_aspect_ratio=False,\n        antialias=False, name=None\n    ), label\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset(FILENAMES):\n    dataset = load_dataset(FILENAMES, labeled=True)\n    dataset = dataset.map(transform, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.map(rescale_images, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(512)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.map(resize, num_parallel_calls=AUTOTUNE)\n#     dataset = dataset.map(mixup, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_dataset(FILENAMES,ordered=True):\n    dataset = load_dataset(FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.map(rescale_images, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.map(resize, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_ds():\n    it = iter(cv.split(files, ['']*len(files)))\n    trn_idx, val_idx = next(it)\n\n    TRAINING_FILENAMES = np.array(files)[trn_idx].tolist()\n    VALID_FILENAMES = np.array(files)[val_idx].tolist()\n\n    return get_training_dataset(TRAINING_FILENAMES)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ds = get_train_ds()\n\n# imgs, lbls = next(iter(ds.take(1)))\n\ndef plot_images(num_images=9, ):\n    images, labels = next(iter(get_train_ds().take(1)))\n    rows = math.ceil(num_images/3)\n    fig, axes = plt.subplots(rows, 3, constrained_layout=False, figsize=(20,20))\n    for i, ax in enumerate(axes.flatten()):\n        ax.imshow(images[i])\n        ax.set_title(str(labels[i].numpy()))\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_images(15)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LR Scheduler - One Cycle Schedule"},{"metadata":{"trusted":true},"cell_type":"code","source":"def sched_lin(start, end, pos): return start + pos*(end-start)\n\ndef sched_cos(start, end, pos): return start + (1 + np.cos(np.pi*(1-pos))) * (end-start)/2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Scheduler:\n    \"\"\"\n    Scheduler used to schedule learning rate and momentum for Cyclic Learning Rate. \n    Args:\n        pct_start: percent progress when to start annealing\n        max_val: maximum value of the hyperparameter during the schedule.\n        init_div: default=100., div factor to calculate the initial value for the schedule.\n        div_fac: default=25000., div factor to calculate the final value for the schedule.\n        sched_func: default=cosine_scheduling. the type of scheduling for each part of the scheduler\n    \"\"\"\n    def __init__(self, pct_start, max_val, init_div=25., div_fact=1000000., sched_func=sched_cos):\n        self.pcts = tf.cumsum(tf.constant([0,pct_start,1-pct_start]))\n        self.max_val = max_val\n        self.init_val = max_val/init_div\n        self.final_val = max_val/div_fact\n        if isinstance(sched_func, (list,tuple)):\n            if len(sched_func)>2: raise ValueError(f\"The sched functions should be only two, received {len(sched_func)} \")\n            self.scheds = [partial(sched_func[0],start=self.init_val,end=self.max_val), partial(sched_func[1],start=self.max_val, end=self.final_val)]           \n        self.scheds = [partial(sched_func,start=self.init_val,end=self.max_val), partial(sched_func,start=self.max_val, end=self.final_val)]\n        \n    def __call__(self, pos):\n        if pos==1: return self.final_val\n        idx = tf.where(tf.not_equal(pos>=self.pcts, tf.constant(False)))[-1].numpy()[-1]\n        sched = self.scheds[idx]\n        actual_pos = (pos-self.pcts[idx])/(self.pcts[idx+1]-self.pcts[idx])\n        return sched(pos=actual_pos)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class OneCycleLR(keras.callbacks.Callback):\n    def __init__(self, \n                 num_samples,\n                 batch_size,\n                 steps_per_epoch,\n                 max_lr, \n                 init_div=25.0,\n                 div_fact=1000000.0,\n                 pct_start=0.3,\n                 maximum_momentum=0.94,\n                 minimum_momentum=0.85,\n                 sched_func=sched_cos,\n                 verbose=True):\n        super(OneCycleLR, self).__init__()\n        \n            \n        self.initial_lr = max_lr\n        self.STEPS_PER_EPOCH = steps_per_epoch\n        self.max_momentum = maximum_momentum\n        self.min_momentum = minimum_momentum\n        self.verbose = verbose\n        self.lr_schedule = Scheduler(pct_start, max_lr,init_div, div_fact, sched_func=sched_func)\n        self.mom_div = minimum_momentum/maximum_momentum\n        self.mom_schedule = Scheduler(pct_start, minimum_momentum, self.mom_div,self.mom_div, sched_func=sched_func)\n        self.lrs = []\n        self.moms = []\n        \n    def reset(self):\n        K.set_value(self.model.optimizer.lr, self.lr_schedule.init_val)\n        self.update_momentum('momentum', self.mom_schedule.init_val)\n        self.update_momentum('beta_1', self.mom_schedule.init_val)\n\n    def calculate_lr(self, pos):\n        \"\"\"\n        Calculates and returns learning rate for the next batch.\n        \"\"\"\n        return self.lr_schedule(pos)\n        \n        \n    def calculate_momentum(self, pos):\n        \"\"\"\n        Calculates and returns the momentum for the next batch.\n        \"\"\"\n        return self.mom_schedule(pos)\n    \n    def on_train_begin(self, logs=None):\n        self.reset()\n        self.history = {}\n        self.curr_batch = 0\n    \n    def update_momentum(self, param_name, value):\n        if hasattr(self.model.optimizer,param_name):\n            K.set_value(getattr(self.model.optimizer,param_name), value)\n            self.moms.append(value)\n    \n    def on_train_batch_begin(self, batch, logs=None):\n        pos = self.curr_batch/(self.params['epochs']*self.STEPS_PER_EPOCH)\n        lr = self.calculate_lr(pos)\n        #set the new learning rate\n        K.set_value(self.model.optimizer.lr, lr)\n        self.lrs.append(lr)\n        \n        #set the new momentum\n        self.update_momentum('momentum', self.calculate_momentum(pos))\n        self.update_momentum('beta_1', self.calculate_momentum(pos))\n        self.curr_batch += 1\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model - Transfer Learning"},{"metadata":{"trusted":true},"cell_type":"code","source":"# The batchnorm layers can be used while fine tuning the model to a new dataset.\ndef trainable_bn(x):\n    if isinstance(x,tf.keras.layers.BatchNormalization):\n         x.trainable=True ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaModel(keras.Model):\n    def __init__(self, model_name='eff', pretrained=True):\n        super(CassavaModel, self).__init__()\n#         prep = tf.keras.layers.Lambda(tf.keras.applications.resnet_v2.preprocess_input)\n#         base_model = tf.keras.applications.ResNet50V2(weights='imagenet',include_top=False)\n\n        eff=efc.EfficientNetB7(weights='noisy-student',include_top=False)\n        self.model=tf.keras.Sequential()\n#         self.model.add(prep)\n#         trainable_bn(base_model)\n        self.model.add(eff)\n        self.model.add(tf.keras.layers.Dropout(0.5))\n        self.model.add(tf.keras.layers.GlobalAveragePooling2D())\n        self.model.add(tf.keras.layers.Dense(5,activation='softmax'))\n\n    def call(self, inputs):\n#         x = self.encoder(inputs)\n#         x = self.pool_1(x)\n# #         x = tf.concat([self.pool_1(x),self.pool_2(x)], axis=1)\n#         x = self.head(x)\n        return self.model(inputs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def f1_metric(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    recall = true_positives / (possible_positives + K.epsilon())\n    f1_val = 2*(precision*recall)/(precision+recall+K.epsilon())\n    return f1_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(model_name='eff'):\n    with strategy.scope():        \n        model = CassavaModel(model_name)\n        model.compile(\n            optimizer=tf.keras.optimizers.Adam(lr=LR),\n            loss=tf.keras.losses.CategoricalCrossentropy(),\n            metrics=[tf.keras.metrics.CategoricalAccuracy(name='acc', dtype=None),\n                     f1_metric])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_tta(model, ds, tta_times=5):\n    yhats = []\n    for i in range(tta_times):\n        yhats.append(model.predict(ds,verbose=1).squeeze())\n    yhats = np.stack(yhats,axis=0).sum(axis=0)\n    if len(yhats.shape) < 2:\n        yhats = yhats[None,:]\n    return yhats.argmax(axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_score(model, ds, tta_times=1):\n    preds = predict_tta(model, ds, tta_times=tta_times)\n    labels = []\n    for img, label in ds:\n        labels.extend(label.numpy().argmax(axis=1).tolist())\n    return (preds == np.array(labels)).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import ReduceLROnPlateau,ModelCheckpoint,EarlyStopping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LR=1e-4*strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def checkpoint_callback(fold):\n    return ModelCheckpoint(f'best_{fold}.h5',verbose=1,monitor='val_loss',save_best_only=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"redlr=ReduceLROnPlateau(monitor='val_loss',patience=3,verbose=1)\nchkpt=checkpoint_callback\nes=EarlyStopping(patience=8,verbose=1,restore_best_weights=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit(TRAINING_FILENAMES, VALID_FILENAMES, train_ds, valid_ds, model,\n        lr,n_epochs=EPOCHS,callbacks=None):\n    NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\n    NUM_VALIDATION_IMAGES = count_data_items(VALID_FILENAMES)\n    STEPS_PER_EPOCH = NUM_TRAINING_IMAGES//BATCH_SIZE\n    VALID_STEPS = NUM_VALIDATION_IMAGES//BATCH_SIZE\n    \n    one_cycle_lr = OneCycleLR(NUM_TRAINING_IMAGES, BATCH_SIZE,STEPS_PER_EPOCH, lr)#*strategy.num_replicas_in_sync )\n#     ONE_CYCLE_LR.append(one_cycle_lr)\n    return model.fit(train_ds, \n                    steps_per_epoch=STEPS_PER_EPOCH,\n                    epochs=n_epochs,\n                    callbacks=callbacks if callbacks else [one_cycle_lr],\n                    validation_data=valid_ds,\n                    validation_steps=VALID_STEPS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run(EPOCHS, model_type='eff'):\n    model_types = {'eff': 'EfficientNetB3', 'resnet': 'Resnet50'}\n    plt.figure(figsize=(20,20))\n    for fold, (trn_idx, val_idx) in enumerate(cv.split(files, ['']*len(files))):\n        TRAINING_FILENAMES = np.array(files)[trn_idx].tolist()\n        VALID_FILENAMES = np.array(files)[val_idx].tolist()\n        chkpt=checkpoint_callback(fold)\n        callbacks=[redlr,chkpt,es]\n        train_dataset = get_training_dataset(TRAINING_FILENAMES)\n        valid_dataset = get_validation_dataset(VALID_FILENAMES)\n        model = get_model(model_type)\n#         print(model)\n#         history = fit(TRAINING_FILENAMES, VALID_FILENAMES,\n#                       train_dataset, valid_dataset, model, lr=1e-2/2, n_epochs=EPOCHS)\n#         plt.plot(np.arange(0,EPOCHS),history.history['loss'],'r-',label='frozen train loss')\n#         plt.plot(np.arange(0,EPOCHS),history.history['acc'],'r--',label='frozen train accuracy')\n\n#         plt.plot(np.arange(0,EPOCHS),history.history['val_loss'],'b-',label='frozen val loss')\n#         plt.plot(np.arange(0,EPOCHS),history.history['val_acc'],'b--',label='frozen val accuracy')\n        print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')\n        print(f'$$$$$$$$$$$$$$$ MODEL TYPE:: {model_types[model_type]}')\n        print('$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$')\n        \n        model.trainable = True\n\n        history = fit(TRAINING_FILENAMES, VALID_FILENAMES,\n                      train_dataset, valid_dataset, model, lr=1e-4, n_epochs=EPOCHS,callbacks=callbacks)\n#         plt.plot(np.arange(0,len(history.history['loss'])),history.history['loss'],'r:',label='unfrozen train loss')\n#         plt.plot(np.arange(0,len(history.history['loss'])),history.history['acc'],'r-.',label='unfrozen train accuracy')\n\n#         plt.plot(np.arange(0,len(history.history['loss'])),history.history['val_loss'],'b:',label='unfrozen val loss')\n#         plt.plot(np.arange(0,len(history.history['loss'])),history.history['val_acc'],'b-.',label='unfrozen val accuracy')\n\n\n        print(f\"Validation Accuracy {get_score(model, valid_dataset)} with {EPOCHS} Unfrozen\")\n#         model.save(f'/kaggle/working/{model_types[model_type]}-fold-{fold}-unfreeze', save_format='tf')\n        del model\n        gc.collect()\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# transforms\nrun(20, 'eff')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# 4 one cycle epochs\n# run(4, 'eff')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# # No aug\n# run(4, 'eff')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}