{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install --quiet efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install keras-swa\nfrom swa.tfkeras import SWA","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install tensorflow-addons=='0.9.1'\nimport tensorflow_addons as tfa","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install git+https://github.com/qubvel/classification_models.git","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math, os, re, warnings, random\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.utils import class_weight\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras import optimizers, applications, Sequential, losses, metrics\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler , ReduceLROnPlateau\nimport efficientnet.tfkeras as efn\nfrom sklearn.model_selection import StratifiedKFold\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16 * REPLICAS\nLEARNING_RATE = 3e-5 * REPLICAS\nEPOCHS = 40\nHEIGHT = 512\nWIDTH = 512\nCHANNELS = 3\nN_CLASSES = 5\nES_PATIENCE = 10\nN_FOLDS = 5\nMODEL_NAME = 'EfficientNet4'\nfocal_loss = 0\nSWA = False\nlabel_smoothing = 0.005\ncutmix_rate = 0.2\nmixup_rate = 0.2\ngridmask_rate = 0\nimg_size = 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\ndatabase_base_path = '/kaggle/input/cassava-leaf-disease-classification/'\ntrain = pd.read_csv(database_base_path + 'train.csv')\n\nprint('Train samples: %d' % len(train))\n\n# GCS_PATH = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification') # Original dataset\n# GCS_PATH = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-tfrecords-{HEIGHT}x{WIDTH}') # Only resized\nGCS_PATH = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-tfrecords-center-{HEIGHT}x{WIDTH}') # Center croped\nGCS_PATH_EXT = KaggleDatasets().get_gcs_path('cassava-leaf-disease-tfrecords-classes-ext-512x512') \n# TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train_tfrecords/*.tfrec') # Original TFRecords\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/*.tfrec')\nEXT_FILENAMES = tf.io.gfile.glob(GCS_PATH_EXT + '/*.tfrec')\nTRAINING_FILENAMES += EXT_FILENAMES\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nprint(f\"GCS: train images: {NUM_TRAINING_IMAGES}\")\n\ndisplay(train.head())\n\n\nCLASSES = ['Cassava Bacterial Blight', \n           'Cassava Brown Streak Disease', \n           'Cassava Green Mottle', \n           'Cassava Mosaic Disease', \n           'Healthy']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CutMix"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\ndef cutmix(image, label, PROBABILITY = cutmix_rate):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = img_size\n    \n    imgs = []; labs = []\n    for j in range(BATCH_SIZE):\n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast( tf.random.uniform([],0,BATCH_SIZE),tf.int32)\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.round(tf.math.maximum(0,y-WIDTH//2))\n        yb = tf.math.round(tf.math.minimum(DIM,y+WIDTH//2))\n        xa = tf.math.round(tf.math.maximum(0,x-WIDTH//2))\n        xb = tf.math.round(tf.math.minimum(DIM,x+WIDTH//2))\n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        # MAKE CUTMIX LABEL\n        a = tf.cast((xb-xa)*(yb-ya)/DIM/DIM,tf.float32)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],N_CLASSES)\n            lab2 = tf.one_hot(label[k],N_CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        \n        tf.print(\"{} {} {}\".format(a,lab1,lab2))\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE,N_CLASSES))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MixUp"},{"metadata":{"trusted":true},"cell_type":"code","source":"def mixup(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = img_size\n    \n    imgs = []; labs = []\n    for j in range(BATCH_SIZE):\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,BATCH_SIZE),tf.int32)\n        a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        # MAKE CUTMIX LABEL\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],N_CLASSES)\n            lab2 = tf.one_hot(label[k],N_CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE,N_CLASSES))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# GridMask"},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform(image, inv_mat, image_shape):\n    h, w, c = image_shape\n    cx, cy = w//2, h//2\n    new_xs = tf.repeat( tf.range(-cx, cx, 1), h)\n    new_ys = tf.tile( tf.range(-cy, cy, 1), [w])\n    new_zs = tf.ones([h*w], dtype=tf.int32)\n    old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))\n    old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)\n    clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)\n    clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)\n    clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)\n    old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))\n    old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))\n    new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))\n    new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))\n    old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)\n    new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)\n    rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))\n    rotated_image_channel = list()\n    for i in range(c):\n        vals = rotated_image_values[:,i]\n        sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])\n        rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))\n    return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def random_rotate(image, angle, image_shape):\n    def get_rotation_mat_inv(angle):\n        # transform to radian\n        angle = math.pi * angle / 180\n        cos_val = tf.math.cos(angle)\n        sin_val = tf.math.sin(angle)\n        one = tf.constant([1], tf.float32)\n        zero = tf.constant([0], tf.float32)\n        rot_mat_inv = tf.concat([cos_val, sin_val, zero, -sin_val, cos_val, zero, zero, zero, one], axis=0)\n        rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])\n        return rot_mat_inv\n    angle = float(angle) * tf.random.normal([1],dtype='float32')\n    rot_mat_inv = get_rotation_mat_inv(angle)\n    return transform(image, rot_mat_inv, image_shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):\n    h, w = image_height, image_width\n    hh = int(np.ceil(np.sqrt(h*h+w*w)))\n    hh = hh+1 if hh%2==1 else hh\n    d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)\n    l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)\n\n    st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n    st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n\n    y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)\n    x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)\n\n    for i in range(0, hh//d+1):\n        s1 = i * d + st_h\n        s2 = i * d + st_w\n        y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)\n        x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)\n\n    x_clip_mask = tf.logical_or(x_ranges < 0 , x_ranges > hh-1)\n    y_clip_mask = tf.logical_or(y_ranges < 0 , y_ranges > hh-1)\n    clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)\n\n    x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))\n    y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))\n\n    hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)), tf.int32)])\n    x_ranges = tf.repeat(x_ranges, hh)\n    y_ranges = tf.repeat(y_ranges, hh)\n\n    y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))\n    x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))\n\n    y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64),  tf.zeros_like(y_ranges), [hh, hh])\n    y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)\n\n    x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])\n    x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)\n\n    mask = tf.expand_dims( tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)\n\n    mask = random_rotate(mask, rotate_angle, [hh, hh, 1])\n    mask = tf.image.crop_to_bounding_box(mask, (hh-h)//2, (hh-w)//2, image_height, image_width)\n\n    return mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def apply_grid_mask(image, image_shape, PROBABILITY = gridmask_rate):\n    AugParams = {\n        'd1' : 100,\n        'd2': 160,\n        'rotate' : 45,\n        'ratio' : 0.3\n    }\n    \n        \n    mask = GridMask(image_shape[0], image_shape[1], AugParams['d1'], AugParams['d2'], AugParams['rotate'], AugParams['ratio'])\n    if image_shape[-1] == 3:\n        mask = tf.concat([mask, mask, mask], axis=-1)\n        mask = tf.cast(mask,tf.float32)\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n    if P==1:\n        return image*mask\n    else:\n        return image\n\ndef gridmask(img_batch, label_batch):\n    return apply_grid_mask(img_batch, (img_size,img_size, 3)), label_batch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# More data augs :"},{"metadata":{"trusted":true},"cell_type":"code","source":"AugParams = {\n    'scale_factor':0.5,\n    'scale_prob':0.5,\n    'rot_range':90,\n    'rot_prob':0.5,\n    'blockout_sl':0.1,\n    'blockout_sh':0.2,\n    'blockout_rl':0.4,\n    'blockout_prob':0.5,\n    'blur_ksize':3,\n    'blur_sigma':1,\n    'blur_prob':0.5\n}\nIMG_DIM = (HEIGHT,WIDTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_rotate(image, angle):\n\n    if len(image.get_shape().as_list()) != 3:\n        raise ValueError('`image_rotate` only support image with 3 dimension(h, w, c)`')\n\n    angle = tf.cast(angle, tf.float32)\n    h, w, c = HEIGHT, WIDTH, 3\n    cy, cx = h//2, w//2\n\n    ys = tf.range(h)\n    xs = tf.range(w)\n\n    ys_vec = tf.tile(ys, [w])\n    xs_vec = tf.reshape( tf.tile(xs, [h]), [h,w] )\n    xs_vec = tf.reshape( tf.transpose(xs_vec, [1,0]), [-1])\n\n    ys_vec_centered, xs_vec_centered = ys_vec - cy, xs_vec - cx\n    new_coord_centered = tf.cast(tf.stack([ys_vec_centered, xs_vec_centered]), tf.float32)\n\n    inv_rot_mat = tf.reshape( tf.dynamic_stitch([0,1,2,3], [tf.cos(angle), tf.sin(angle), -tf.sin(angle), tf.cos(angle)]), [2,2])\n    old_coord_centered = tf.matmul(inv_rot_mat, new_coord_centered)\n\n    old_ys_vec_centered, old_xs_vec_centered = old_coord_centered[0,:], old_coord_centered[1,:]\n    old_ys_vec = tf.cast( tf.round(old_ys_vec_centered+cy), tf.int32)\n    old_xs_vec = tf.cast( tf.round(old_xs_vec_centered+cx), tf.int32)\n\n    outside_ind = tf.logical_or( tf.logical_or(old_ys_vec > h-1 , old_ys_vec < 0), tf.logical_or(old_xs_vec > w-1 , old_xs_vec<0))\n\n    old_ys_vec = tf.boolean_mask(old_ys_vec, tf.logical_not(outside_ind))\n    old_xs_vec = tf.boolean_mask(old_xs_vec, tf.logical_not(outside_ind))\n\n    ys_vec = tf.boolean_mask(ys_vec, tf.logical_not(outside_ind))\n    xs_vec = tf.boolean_mask(xs_vec, tf.logical_not(outside_ind))\n\n    old_coord = tf.cast(tf.transpose(tf.stack([old_ys_vec, old_xs_vec]), [1,0]), tf.int32)\n    new_coord = tf.cast(tf.transpose(tf.stack([ys_vec, xs_vec]), [1,0]), tf.int64)\n\n    channel_vals = tf.split(image, c, axis=-1)\n    rotated_channel_vals = list()\n    for channel_val in channel_vals:\n        rotated_channel_val = tf.gather_nd(channel_val, old_coord)\n\n        sparse_rotated_channel_val = tf.SparseTensor(new_coord, tf.squeeze(rotated_channel_val,axis=-1), [h, w])\n        rotated_channel_vals.append(tf.sparse.to_dense(sparse_rotated_channel_val, default_value=0, validate_indices=False))\n\n    rotated_image = tf.transpose(tf.stack(rotated_channel_vals), [1, 2, 0])\n    return rotated_image\n    \ndef random_blockout(img, sl=0.1, sh=0.2, rl=0.4):\n\n    h, w, c = WIDTH, HEIGHT, 3\n    origin_area = tf.cast(h*w, tf.float32)\n\n    e_size_l = tf.cast(tf.round(tf.sqrt(origin_area * sl * rl)), tf.int32)\n    e_size_h = tf.cast(tf.round(tf.sqrt(origin_area * sh / rl)), tf.int32)\n\n    e_height_h = tf.minimum(e_size_h, h)\n    e_width_h = tf.minimum(e_size_h, w)\n\n    erase_height = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_height_h, dtype=tf.int32)\n    erase_width = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_width_h, dtype=tf.int32)\n\n    erase_area = tf.zeros(shape=[erase_height, erase_width, c])\n    erase_area = tf.cast(erase_area, tf.uint8)\n\n    pad_h = h - erase_height\n    pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)\n    pad_bottom = pad_h - pad_top\n\n    pad_w = w - erase_width\n    pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)\n    pad_right = pad_w - pad_left\n\n    erase_mask = tf.pad([erase_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)\n    erase_mask = tf.squeeze(erase_mask, axis=0)\n    erased_img = tf.multiply(tf.cast(img,tf.float32), tf.cast(erase_mask, tf.float32))\n\n    return tf.cast(erased_img, img.dtype)\n    \ndef zoom_out(x, scale_factor):\n\n    resize_x = tf.random.uniform(shape=[], minval=tf.cast(IMG_DIM[1]//(1/scale_factor), tf.int32), maxval=IMG_DIM[1], dtype=tf.int32)\n    resize_y = tf.random.uniform(shape=[], minval=tf.cast(IMG_DIM[0]//(1/scale_factor), tf.int32), maxval=IMG_DIM[0], dtype=tf.int32)\n    top_pad = (IMG_DIM[0] - resize_y) // 2\n    bottom_pad = IMG_DIM[0] - resize_y - top_pad\n    left_pad = (IMG_DIM[1] - resize_x ) // 2\n    right_pad = IMG_DIM[1] - resize_x - left_pad\n        \n    x = tf.image.resize(x, (resize_y, resize_x))\n    x = tf.pad([x], [[0,0], [top_pad, bottom_pad], [left_pad, right_pad], [0,0]])\n    x = tf.image.resize(x, IMG_DIM)\n    return tf.squeeze(x, axis=0)\n    \ndef zoom_in(x, scale_factor):\n\n    scales = list(np.arange(0.5, 1.0, 0.05))\n    boxes = np.zeros((len(scales),4))\n            \n    for i, scale in enumerate(scales):\n        x_min = y_min = 0.5 - (0.5*scale)\n        x_max = y_max = 0.5 + (0.5*scale)\n        boxes[i] = [x_min, y_min, x_max, y_max]\n        \n    def random_crop(x):\n        crop = tf.image.crop_and_resize([x], boxes=boxes, box_indices=np.zeros(len(boxes)), crop_size=IMG_DIM)\n        return crop[tf.random.uniform(shape=[], minval=0, maxval=len(scales), dtype=tf.int32)]\n        \n    return random_crop(x)\n\ndef gaussian_blur(img, ksize=5, sigma=1):\n    \n    def gaussian_kernel(size=3, sigma=1):\n\n        x_range = tf.range(-(size-1)//2, (size-1)//2 + 1, 1)\n        y_range = tf.range((size-1)//2, -(size-1)//2 - 1, -1)\n\n        xs, ys = tf.meshgrid(x_range, y_range)\n        kernel = tf.exp(-(xs**2 + ys**2)/(2*(sigma**2))) / (2*np.pi*(sigma**2))\n        return tf.cast( kernel / tf.reduce_sum(kernel), tf.float32)\n    \n    kernel = gaussian_kernel(ksize, sigma)\n    kernel = tf.expand_dims(tf.expand_dims(kernel, axis=-1), axis=-1)\n    \n    r, g, b = tf.split(img, [1,1,1], axis=-1)\n    r_blur = tf.nn.conv2d([r], kernel, [1,1,1,1], 'SAME')\n    g_blur = tf.nn.conv2d([g], kernel, [1,1,1,1], 'SAME')\n    b_blur = tf.nn.conv2d([b], kernel, [1,1,1,1], 'SAME')\n\n    blur_image = tf.concat([r_blur, g_blur, b_blur], axis=-1)\n    return tf.squeeze(blur_image, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def augmentation(image,label):\n    image = tf.cast(image, tf.float32)\n    #Gaussian blur\n    if tf.random.uniform(shape=[], minval=0.0, maxval=1.0) > AugParams['blur_prob']:\n        image = gaussian_blur(image, AugParams['blur_ksize'], AugParams['blur_sigma'])\n    \n    #Random block out\n    if tf.random.uniform(shape=[], minval=0.0, maxval=1.0) > AugParams['blockout_prob']:\n        image = random_blockout(image, AugParams['blockout_sl'], AugParams['blockout_sh'], AugParams['blockout_rl'])\n        \n    #Random scale\n    if tf.random.uniform(shape=[], minval=0.0, maxval=1.0) > AugParams['scale_prob']:\n        if tf.random.uniform(shape=[], minval=0.0, maxval=1.0) > 0.5:\n            image = zoom_in(image, AugParams['scale_factor'])\n        else:\n            image = zoom_out(image, AugParams['scale_factor'])\n    #Random rotate\n    if tf.random.uniform(shape=[], minval=0.0, maxval=1.0) > AugParams['rot_prob']:\n        angle = tf.random.uniform(shape=[], minval=-AugParams['rot_range'], maxval=AugParams['rot_range'], dtype=tf.int32)\n        image = image_rotate(image,angle)\n    \n    return tf.cast(image, tf.uint8),label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    # Shear\n    if p_shear > .2:\n        if p_shear > .6:\n            image = transform_shear(image, HEIGHT, shear=20.)\n        else:\n            image = transform_shear(image, HEIGHT, shear=-20.)\n    # Rotation\n    if p_rotation > .2:\n        if p_rotation > .6:\n            image = transform_rotation(image, HEIGHT, rotation=45.)\n        else:\n            image = transform_rotation(image, HEIGHT, rotation=-45.)\n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > .75:\n        image = tf.image.transpose(image)\n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n    # Pixel-level transforms\n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n    # Crops\n    if p_crop > .7:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.6)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.8)\n    elif p_crop > .4:\n        crop_size = tf.random.uniform([], int(HEIGHT*.6), HEIGHT, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n            \n    image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Datasets utility functions\ndef to_float32_2(image, label):\n    max_val = tf.reduce_max(label, axis=-1,keepdims=True)\n    cond = tf.equal(label, max_val)\n    label = tf.where(cond, tf.ones_like(label), tf.zeros_like(label))\n    return tf.cast(image, tf.float32), tf.cast(label, tf.int32)\n\ndef to_float32(image, label):\n    return tf.cast(image, tf.float32), label\n\n\ndef one_hot(image,label):\n    NUMCLASSES = len(CLASSES)\n    return image,tf.one_hot(label,NUMCLASSES)\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n                      \n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_tfrecord(example, labeled=True):\n    if labeled:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'target': tf.io.FixedLenFeature([], tf.int64), \n        }\n    else:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'image_name': tf.io.FixedLenFeature([], tf.string), \n        }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    if labeled:\n        label_or_name = tf.cast(example['target'], tf.int32)\n    else:\n        label_or_name =  example['image_name']\n    return image, label_or_name\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda x: read_tfrecord(x, labeled=labeled), num_parallel_calls=AUTO)\n    return dataset\n\ndef get_dataset(FILENAMES, labeled=True, ordered=False, repeated=False, augment=False,valid=False):\n    dataset = load_dataset(FILENAMES, labeled=labeled, ordered=ordered)\n    if augment:\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n        #dataset = dataset.map(augmentation, num_parallel_calls=AUTO)\n    if repeated:\n        dataset = dataset.repeat()\n    if not ordered:\n        dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    if valid :\n        dataset = dataset.map(one_hot,num_parallel_calls=AUTO)\n    else :\n        if cutmix_rate :\n            dataset = dataset.map(cutmix, num_parallel_calls=AUTO)\n        if mixup_rate :\n            dataset = dataset.map(mixup, num_parallel_calls=AUTO)\n        if gridmask_rate :\n            dataset = dataset.map(gridmask, num_parallel_calls=AUTO)\n\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualization utility functions\nnp.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n\ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n    \n# Visualize model predictions\ndef dataset_to_numpy_util(dataset, N):\n    dataset = dataset.unbatch().batch(N)\n    for images, labels in dataset:\n        numpy_images = images.numpy()\n        numpy_labels = labels.numpy()\n        break;  \n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    label = np.argmax(label, axis=-1)\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(label, str(correct), ', shoud be ' if not correct else '',\n                                correct_label if not correct else ''), correct\n\ndef display_one_flower_eval(image, title, subplot, red=False):\n    plt.subplot(subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    plt.title(title, fontsize=14, color='red' if red else 'black')\n    return subplot+1\n\ndef display_9_images_with_predictions(images, predictions, labels):\n    subplot=331\n    plt.figure(figsize=(13,13))\n    for i, image in enumerate(images):\n        title, correct = title_from_label_and_target(predictions[i], labels[i])\n        subplot = display_one_flower_eval(image, title, subplot, not correct)\n        if i >= 8:\n            break;\n              \n    plt.tight_layout()\n    plt.subplots_adjust(wspace=0.1, hspace=0.1)\n    plt.show()\n    \n    \n# Model evaluation\ndef plot_metrics(history):\n    metric_list = [m for m in list(history.keys()) if m is not 'lr']\n    size = len(metric_list)//2\n    fig, axes = plt.subplots(size, 1, sharex='col', figsize=(20, size * 4))\n    if size > 1:\n        axes = axes.flatten()\n    else:\n        axes = [axes]\n    \n    for index in range(len(metric_list)//2):\n        metric_name = metric_list[index]\n        val_metric_name = metric_list[index+size]\n        axes[index].plot(history[metric_name], label='Train %s' % metric_name)\n        axes[index].plot(history[val_metric_name], label='Validation %s' % metric_name)\n        axes[index].legend(loc='best', fontsize=16)\n        axes[index].set_title(metric_name)\n        if 'loss' in metric_name:\n            axes[index].axvline(np.argmin(history[metric_name]), linestyle='dashed')\n            axes[index].axvline(np.argmin(history[val_metric_name]), linestyle='dashed', color='orange')\n        else:\n            axes[index].axvline(np.argmax(history[metric_name]), linestyle='dashed')\n            axes[index].axvline(np.argmax(history[val_metric_name]), linestyle='dashed', color='orange')\n\n    plt.xlabel('Epochs', fontsize=16)\n    sns.despine()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_dataset(TRAINING_FILENAMES, ordered=True)\ntrain_iter = iter(train_dataset.unbatch().batch(20))\n\n'''display_batch_of_images(next(train_iter))\ndisplay_batch_of_images(next(train_iter))'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_count = train.groupby('label', as_index=False).count()\nlabel_count.rename(columns={'image_id': 'Count', 'label': 'Label'}, inplace=True)\nlabel_count['Label'] = label_count['Label'].apply(lambda x: CLASSES[x])\n\nfig, ax = plt.subplots(1, 1, figsize=(18, 10))\nax = sns.barplot(x=label_count['Count'], y=label_count['Label'], palette='viridis')\nax.tick_params(labelsize=16)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LR_START = 1e-8\nLR_MIN = 1e-8\nLR_MAX = LEARNING_RATE\nLR_RAMPUP_EPOCHS = 3\nLR_SUSTAIN_EPOCHS = 0\nN_CYCLES = .5\n\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        progress = (epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) / (EPOCHS - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)\n        lr = LR_MAX * (0.5 * (1.0 + tf.math.cos(math.pi * N_CYCLES * 2.0 * progress)))\n        if LR_MIN is not None:\n            lr = tf.math.maximum(LR_MIN, lr)\n            \n    return lr\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\n\nsns.set(style='whitegrid')\nfig, ax = plt.subplots(figsize=(20, 6))\nplt.plot(rng, y)\n\nprint(f'{EPOCHS} total epochs and {NUM_TRAINING_IMAGES//BATCH_SIZE} steps per epoch')\nprint(f'Learning rate schedule: {y[0]:.3g} to {max(y):.3g} to {y[-1]:.3g}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_fn(input_shape, N_CLASSES):\n    input_image = L.Input(shape=input_shape, name='input_image')\n    base_model = efn.EfficientNetB4(input_tensor=input_image, \n                                    include_top=False, \n                                    weights='noisy-student', \n                                    pooling='avg')\n\n    model = tf.keras.Sequential([\n        base_model,\n        L.Dropout(.25),\n        L.Dense(N_CLASSES, activation='softmax', name='output')\n    ])\n\n    optimizer = optimizers.Adam(lr=LEARNING_RATE)\n    model.compile(optimizer=optimizer, \n                  loss=losses.SparseCategoricalCrossentropy(), \n                  metrics=['sparse_categorical_accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Focal Loss "},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend as K\n\nimport dill\n\n\ndef binary_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n\n        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\n    return binary_focal_loss_fixed\n\n\ndef categorical_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Softmax version of focal loss.\n           m\n      FL = ∑  -alpha * (1 - p_o,c)^gamma * y_o,c * log(p_o,c)\n          c=1\n      where m = number of classes, c = class and o = observation\n    Parameters:\n      alpha -- the same as weighing factor in balanced cross entropy\n      gamma -- focusing parameter for modulating factor (1-p)\n    Default value:\n      gamma -- 2.0 as mentioned in the paper\n      alpha -- 0.25 as mentioned in the paper\n    References:\n        Official paper: https://arxiv.org/pdf/1708.02002.pdf\n        https://www.tensorflow.org/api_docs/python/tf/keras/backend/categorical_crossentropy\n    Usage:\n     model.compile(loss=[categorical_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def categorical_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred: A tensor resulting from a softmax\n        :return: Output tensor.\n        \"\"\"\n\n        # Scale predictions so that the class probas of each sample sum to 1\n        y_pred /= K.sum(y_pred, axis=-1, keepdims=True)\n\n        # Clip the prediction value to prevent NaN's and Inf's\n        epsilon = K.epsilon()\n        y_pred = K.clip(y_pred, epsilon, 1. - epsilon)\n\n        # Calculate Cross Entropy\n        cross_entropy = -y_true * K.log(y_pred)\n\n        # Calculate Focal Loss\n        loss = alpha * K.pow(1 - y_pred, gamma) * cross_entropy\n\n        # Sum the losses in mini_batch\n        return K.sum(loss, axis=1)\n\n    return categorical_focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.applications import DenseNet121, DenseNet201 , DenseNet169\nfrom tensorflow.keras.applications import vgg16\nfrom tensorflow.keras.applications.inception_v3 import InceptionV3\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.applications import ResNet101 , ResNet101V2 , ResNet152 , ResNet152V2 , ResNet50V2 , ResNet50\nfrom tensorflow.keras.applications import MobileNet , MobileNetV2\nfrom tensorflow.keras.applications import InceptionResNetV2\nimport tensorflow.keras.layers as L\nfrom tensorflow.keras.models import Model,Sequential\nfrom tensorflow.keras.losses import CategoricalCrossentropy\nfrom classification_models.keras import Classifiers\n\ndef get_model_generalized(name,input_shape,N_CLASSES):\n    \n    if name == 'EfficientNet0' :\n        base_model = efn.EfficientNetB0(weights='noisy-student',\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n    elif name == 'EfficientNet1' :\n        base_model = efn.EfficientNetB1(weights='noisy-student',\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n    elif name == 'EfficientNet2' :\n        base_model = efn.EfficientNetB2(weights='noisy-student',\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n    elif name == 'EfficientNet4' :\n        base_model = efn.EfficientNetB4(weights='noisy-student',\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n    elif name == 'EfficientNet5' :\n        base_model = efn.EfficientNetB5(weights='noisy-student',\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n        #for layer in base_model.layers :\n        #    layer.trainable = False\n    elif name == 'EfficientNet6' :\n        base_model = efn.EfficientNetB6(weights='noisy-student',\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n    elif name == 'EfficientNet7' :\n        base_model = efn.EfficientNetB7(weights='noisy-student',\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n        #base_model.trainable = True\n        #for layer in base_model.layers[:-20] :\n        #    layer.trainable = True\n    elif name == 'EfficientNet3' :\n        base_model = efn.EfficientNetB3(weights='noisy-student',\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n        #base_model.trainable = True\n        #for layer in base_model.layers[:-25] :\n        #    layer.trainable = True        \n            \n    elif name == 'DenseNet201' :\n        base_model = DenseNet201(weights='imagenet',include_top=False,input_shape=input_shape)\n    elif name == 'DenseNet169' :\n        base_model = DenseNet169(weights='imagenet',include_top=False,input_shape=input_shape)\n        #base_model.trainable = True\n    elif name == 'DenseNet121' :\n        base_model = DenseNet121(weights='imagenet',include_top=False,input_shape=input_shape)\n        #for layer in base_model.layers[:-25] :\n        #    layer.trainable = True\n    elif name == 'MobileNet' :\n        base_model = MobileNet(weights = 'imagenet', include_top=False,input_shape=input_shape)\n    elif name == 'Inception' :\n        base_model = InceptionV3(weights = 'imagenet',include_top=False,input_shape=input_shape)\n    elif name == 'ResNet50' :\n        base_model = ResNet50(weights = 'imagenet',include_top=False,input_shape=input_shape)\n    elif name == 'ResNet50V2' :\n        base_model = ResNet50V2(weights = 'imagenet',include_top=False,input_shape=input_shape)\n    elif name == 'ResNet101' :\n        base_model = ResNet101(weights = 'imagenet',include_top=False,input_shape=input_shape)\n    elif name == 'ResNet101V2' :\n        base_model = ResNet101V2(weights = 'imagenet',include_top=False,input_shape=input_shape)\n    elif name == 'ResNet152' :\n        base_model = ResNet152(weights = 'imagenet',include_top=False,input_shape=input_shape)\n    elif name == 'ResNet152V2' :\n        base_model = ResNet152V2(weights = 'imagenet',include_top=False,input_shape=input_shape)\n    elif name == 'Incepresnet' :\n        base_model = InceptionResNetV2(weights = 'imagenet',include_top=False,input_shape=input_shape) \n        #base_model.trainable = True\n        #for layer in base_model.layers[:-25] :\n        #    layer.trainable = True\n    elif name == 'seresnet18' : \n        model, preprocess_input = Classifiers.get('seresnet18')\n        base_model = model(input_shape=input_shape, weights='imagenet', include_top=False)\n    elif name == 'seresnet34' : \n        model, preprocess_input = Classifiers.get('seresnet34')\n        base_model = model(input_shape=input_shape, weights='imagenet', include_top=False)\n    elif name == 'seresnet50' : \n        model, preprocess_input = Classifiers.get('seresnet50')\n        base_model = model(input_shape=input_shape, weights='imagenet', include_top=False)\n    elif name == 'seresnet101' : \n        model, preprocess_input = Classifiers.get('seresnet101')\n        base_model = model(input_shape=input_shape, weights='imagenet', include_top=False)\n    elif name == 'seresnet152' : \n        model, preprocess_input = Classifiers.get('seresnet152')\n        base_model = model(input_shape=input_shape, weights='imagenet', include_top=False)\n    elif name == 'seresnext50' : \n        model, preprocess_input = Classifiers.get('seresnext50')\n        base_model = model(input_shape=input_shape, weights='imagenet', include_top=False)\n    elif name == 'seresnext101' : \n        model, preprocess_input = Classifiers.get('seresnext101')\n        base_model = model(input_shape=input_shape, weights='imagenet', include_top=False)\n    x = base_model.output\n    x = L.GlobalAveragePooling2D()(x)\n    x = L.Dense(512,activation='relu')(x)\n    x = L.Dropout(0.3)(x)\n    x = L.Dense(256,activation='relu')(x)\n    x = L.Dropout(0.2)(x)\n    predictions = L.Dense(N_CLASSES,activation='softmax')(x)\n    model = Model(inputs = base_model.input, outputs=predictions) \n    \n    metric = ['categorical_accuracy']\n    if focal_loss : \n        loss=categorical_focal_loss(gamma=2., alpha=.25) #tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO)\n        metric = [tfa.metrics.F1Score(num_classes=5, average=\"macro\"),'categorical_accuracy']\n    elif label_smoothing :\n        loss=CategoricalCrossentropy(label_smoothing=label_smoothing)\n    else :\n        loss = 'categorical_crossentropy'\n    if SWA :\n        opt = tf.keras.optimizers.Adam(lr=1e-5) # roll back\n        opt = tfa.optimizers.SWA(opt)\n    else :\n        opt = 'adam'\n        \n    optimizer = optimizers.Adam(lr=1e-4)                                      #optimizers.Adam(lr=LEARNING_RATE)\n    model.compile(optimizer=optimizer, \n                  loss=loss, \n                  metrics=metric)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#skf = StratifiedKFold(n_splits=N_FOLDS, random_state=seed, shuffle=True)\nskf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)\noof_pred = []; oof_labels = []; history_list = []\noof_pred2 = []; oof_labels2 = [] \next_train = ['/CBB0-330.tfrec',\n             '/CBB1-75.tfrec',\n             '/CBSD0-330.tfrec',\n             '/CBSD1-330.tfrec',\n             '/CBSD2-330.tfrec',\n             '/CBSD3-330.tfrec',\n             '/CGM0-330.tfrec',\n             '/CGM1-330.tfrec',\n             '/CMD0-330.tfrec',\n            '/CMD1-330.tfrec',\n            '/CMD2-330.tfrec',\n            '/CMD3-330.tfrec',\n            '/CMD4-330.tfrec',\n            '/CMD5-330.tfrec',\n            '/CMD6-330.tfrec',\n            '/Healthy0-313.tfrec']\next_valid = []\n\nfor fold,(idxT, idxV) in enumerate(skf.split(np.arange(15))):\n    if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print(f'\\nFOLD: {fold+1}')\n    print(f'TRAIN: {idxT} VALID: {idxV}')\n\n    # Create train and validation sets\n    TRAIN_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/Id_train%.2i*.tfrec' % x for x in idxT])\n    print(len(TRAIN_FILENAMES))\n    VALID_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/Id_train%.2i*.tfrec' % x for x in idxV])\n    \n    EXT_DATA_TRAIN = tf.io.gfile.glob([GCS_PATH_EXT + x for x in ext_train])\n    TRAIN_FILENAMES += EXT_DATA_TRAIN\n    print(len(TRAIN_FILENAMES))\n    \n    np.random.shuffle(TRAINING_FILENAMES)\n\n    ct_train = count_data_items(TRAIN_FILENAMES)\n    STEPS_PER_EPOCH = ct_train // BATCH_SIZE\n    \n    ## MODEL\n    K.clear_session()\n    with strategy.scope():\n        #model = model_fn((None, None, CHANNELS), N_CLASSES)\n        model = get_model_generalized(MODEL_NAME,(None, None, CHANNELS), N_CLASSES)\n        \n    model_path = f'model_{fold}.h5'\n    es = EarlyStopping(monitor='val_categorical_accuracy', mode='max', \n                       patience=ES_PATIENCE, restore_best_weights=True, verbose=1)\n    mc = ModelCheckpoint(f'best_model{fold}.h5', monitor = 'val_categorical_accuracy' , mode = 'max', verbose = 1 , save_best_only = True)\n    \n\n    reduce_lr =  ReduceLROnPlateau(monitor = \"val_categorical_accuracy\", factor = 0.5, patience = 4,\n      verbose = 1, mode = \"auto\", epsilon = 1e-04, cooldown = 0,\n      min_lr = 1e-8)\n\n\n    ## TRAIN\n    history = model.fit(x=get_dataset(TRAIN_FILENAMES, labeled=True, ordered=False, repeated=True, augment=True,valid=False).map(to_float32_2), \n                        validation_data=get_dataset(VALID_FILENAMES, labeled=True, ordered=True, repeated=False, augment=False,valid=True).map(to_float32_2), \n                        steps_per_epoch=STEPS_PER_EPOCH, \n                        callbacks=[es, mc ,reduce_lr],  #LearningRateScheduler(lrfn, verbose=0)\n                        epochs=EPOCHS,  \n                        verbose=2).history\n      \n    history_list.append(history)\n    # Save last model weights\n    model.save_weights(model_path)\n\n    # OOF predictions\n    ds_valid = get_dataset(VALID_FILENAMES, labeled=True, ordered=True, repeated=False,valid=True, augment=False).map(to_float32_2)\n    oof_labels.append([target.numpy() for img, target in iter(ds_valid.unbatch())])\n    x_oof = ds_valid.map(lambda image, image_name: image)\n    oof_pred.append(np.argmax(model.predict(x_oof), axis=-1))\n    model.load_weights(f'best_model{fold}.h5')\n    oof_pred2.append(np.argmax(model.predict(x_oof),axis=-1))\n    \n    ## RESULTS\n    print(f\"#### FOLD {fold+1} OOF Accuracy = {np.max(history['val_categorical_accuracy']):.3f}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold, history in enumerate(history_list):\n    print(f'\\nFOLD: {fold+1}')\n    plot_metrics(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_true = np.argmax(np.concatenate(oof_labels),axis=1)\ny_preds = np.concatenate(oof_pred)\n\nprint(classification_report(y_true, y_preds, target_names=CLASSES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_preds2 = np.concatenate(oof_pred2)\n\nprint(classification_report(y_true, y_preds2, target_names=CLASSES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score\n\nprint(accuracy_score(y_true,y_preds2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score\n\nprint(accuracy_score(y_true,y_preds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(20, 12))\ntrain_cfn_matrix = confusion_matrix(y_true, y_preds, labels=range(len(CLASSES)))\ntrain_cfn_matrix = (train_cfn_matrix.T / train_cfn_matrix.sum(axis=1)).T\ntrain_df_cm = pd.DataFrame(train_cfn_matrix, index=CLASSES, columns=CLASSES)\nax = sns.heatmap(train_df_cm, cmap='Blues', annot=True, fmt='.2f', linewidths=.5).set_title('Train', fontsize=30)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''x_samp, y_samp = dataset_to_numpy_util(train_dataset, 18)\n\nx_samp_1, y_samp_1 = x_samp[:9,:,:,:], y_samp[:9]\nsamp_preds_1 = model.predict(x_samp_1, batch_size=9)\ndisplay_9_images_with_predictions(x_samp_1, samp_preds_1, y_samp_1)\n\nx_samp_2, y_samp_2 = x_samp[9:,:,:,:], y_samp[9:]\nsamp_preds_2 = model.predict(x_samp_2, batch_size=9)\ndisplay_9_images_with_predictions(x_samp_2, samp_preds_2, y_samp_2)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}