{"cells":[{"metadata":{},"cell_type":"markdown","source":"using this and ensembling effcient nets and densenet and incepresnet will get you a bronze medal sadly i didn't choose this submission :( ","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install image-classifiers==1.0.0b1\nfrom classification_models.tfkeras import Classifiers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install keras-swa\nfrom swa.tfkeras import SWA","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os , math , re , random\nimport cv2\nimport matplotlib.pyplot as plt\n\nfrom kaggle_datasets import KaggleDatasets\n\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.metrics import classification_report\nfrom sklearn.utils import shuffle\n\n\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\n\nfrom tensorflow.keras.callbacks import EarlyStopping,LearningRateScheduler, ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras.models import Model,Sequential\n\n\nimport efficientnet.tfkeras as efn\nfrom tensorflow.keras.applications import DenseNet121, DenseNet201\nfrom tensorflow.keras.applications import vgg16\nfrom tensorflow.keras.applications.inception_v3 import InceptionV3\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.applications import MobileNet , MobileNetV2\nfrom tensorflow.keras.applications import InceptionResNetV2\nfrom tensorflow.keras import optimizers\n\nfrom tensorflow.keras import backend as K\n\n!pip install tensorflow-addons=='0.9.1'\nimport tensorflow_addons as tfa","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for reproducible results :\ndef seed_everything(seed=13):\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n    os.environ['TF_KERAS'] = '1'\n    random.seed(seed)\n    \nseed_everything(1234)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try :\n    tpu=tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on :',tpu.master())\nexcept ValueError :\n    tpu = None\n\nif tpu :    \n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse :\n    strategy = tf.distribute.get_strategy()\n    \nprint('Replicas :',strategy.num_replicas_in_sync)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO  = tf.data.experimental.AUTOTUNE\n\nGCS_DS_PATH2 = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')\nGCS_DS_PATH1 = KaggleDatasets().get_gcs_path('512x512-melanoma-tfrecords-70k-images')\nEPOCHS = 20\nBATCH_SIZE = 8 * strategy.num_replicas_in_sync \nimg_size = 468 # 468 for effnet b5 , b1 , b3\nSEED =  1234\nnb_classes = 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Using Images : ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntest = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\nsub = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\n\ntrain_paths = train.image_name.apply(lambda x : GCS_DS_PATH1 + '/train/' +x + '.jpg').values\ntest_paths = test.image_name.apply(lambda x : GCS_DS_PATH1 + '/test/' +x + '.jpg').values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train['sex'].value_counts())\nprint('*'*100)\nprint(train['diagnosis'].value_counts())\nprint('*'*100)\nprint(train['anatom_site_general_challenge'].value_counts())\nprint(train['benign_malignant'].value_counts())\nprint(train['target'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.patient_id.duplicated().any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dup_patients_test = test[test.patient_id.duplicated() == True]\nunique_patient_test_ids = list(set(dup_patients_test['patient_id']))\npatient = test[test['patient_id'] == unique_patient_test_ids[2]] \npatient = patient.reset_index(drop=True)\nimages = []\n\nfor i in range(len(patient)) :\n    img = '../input/siim-isic-melanoma-classification/jpeg/test/'+patient['image_name'][i]+'.jpg'\n    n = cv2.imread(img)\n    n = cv2.cvtColor(n, cv2.COLOR_BGR2GRAY)\n    images.append(n)\n\n    \nplt.figure(figsize=(10,10))\nfor i in range(len(images)) :\n    plt.subplot(8,8,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(images[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dup_patients = train[train.patient_id.duplicated() == True ]\ndup_patients","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_patient_ids = set(dup_patients['patient_id'])\nlen(unique_patient_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mislabeled_patient_ids = []\nfor patient_id in unique_patient_ids :    \n    class_count = len(dup_patients[dup_patients['patient_id' ] == patient_id].benign_malignant.value_counts())\n    if class_count > 1 :\n        mislabeled_patient_ids.append(patient_id)\n        print(patient_id ,' is mislabeled :')\n        print(dup_patients[dup_patients['patient_id' ] == patient_id].benign_malignant.value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(mislabeled_patient_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dup_patients.loc[dup_patients['patient_id'].isin(mislabeled_patient_ids)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dup_patients.loc[dup_patients['patient_id'].isin(mislabeled_patient_ids)].target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pos = train[train['target'] == 1]\ntrain_neg_ids = []\nfor i in range(1200) :\n    if train['target'][i] == 0 :\n        train_neg_ids.append(i)\ntrain_neg = train.iloc[train_neg_ids]   \ntrain_balanced = pd.concat([train_pos,train_neg])\ntrain_balanced = train_balanced.sample(frac=1).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_balanced_paths = train_balanced.image_name.apply(lambda x : GCS_DS_PATH + '/jpeg/train/' + x + '.jpg').values\ntrain_balanced_labels = train_balanced.loc[:,'target'].values\n\nfrom sklearn.model_selection import train_test_split\nX_train_paths, X_valid_paths, Y_train, Y_valid = train_test_split(train_balanced_paths,train_balanced_labels, test_size=0.15, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bool_random_brightness = False # 0.902 no improvement\nbool_random_contrast =False  # doesn't improve\nbool_random_hue = False\nbool_random_saturation = False\n\ngridmask_rate = 0.4 #improve 0.4\ncutmix_rate =  0.4   #improve 0.4\nmixup_rate = 0 #doesn't improve\nrotation = False\nrandom_blackout = False\ncrop_size = 0\ntransforms = False\nmicro_aug = False\nhair_aug = True # hair aug can be good","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations :","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Cutmix : ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# batch\ndef cutmix(image, label, PROBABILITY = cutmix_rate):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    \n    DIM = img_size    \n    imgs = []; labs = []\n    \n    for j in range(BATCH_SIZE):\n        \n        #random_uniform( shape, minval=0, maxval=None)        \n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n        \n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast(tf.random.uniform([], 0, BATCH_SIZE), tf.int32)\n        \n        # CHOOSE RANDOM LOCATION\n        x = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)\n        y = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)\n        \n        # Beta(1, 1)\n        b = tf.random.uniform([], 0, 1) # this is beta dist with alpha=1.0\n        \n\n        WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        \n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]        \n        #ya:yb\n        middle = tf.concat([one,two,three],axis=1)\n\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        \n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        lab1 = label[j,]\n        lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n\n    image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE, nb_classes))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## MixUp :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def mixup(image, label, PROBABILITY = mixup_rate):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = img_size\n    \n    imgs = []; labs = []\n    for j in range(BATCH_SIZE):\n        \n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,BATCH_SIZE),tf.int32)\n        a = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n\n        #mixup\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n        if P==1:\n            a=0.\n        \n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        \n        # MAKE CUTMIX LABEL\n        lab1 = label[j,]\n        lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE,nb_classes))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## GridMask :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform(image, inv_mat, image_shape):\n    h, w, c = image_shape\n    cx, cy = w//2, h//2\n    new_xs = tf.repeat( tf.range(-cx, cx, 1), h)\n    new_ys = tf.tile( tf.range(-cy, cy, 1), [w])\n    new_zs = tf.ones([h*w], dtype=tf.int32)\n    old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))\n    old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)\n    clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)\n    clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)\n    clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)\n    old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))\n    old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))\n    new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))\n    new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))\n    old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)\n    new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)\n    rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))\n    rotated_image_channel = list()\n    for i in range(c):\n        vals = rotated_image_values[:,i]\n        sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])\n        rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))\n    return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def random_rotate(image, angle, image_shape):\n    def get_rotation_mat_inv(angle):\n        # transform to radian\n        angle = math.pi * angle / 180\n        cos_val = tf.math.cos(angle)\n        sin_val = tf.math.sin(angle)\n        one = tf.constant([1], tf.float32)\n        zero = tf.constant([0], tf.float32)\n        rot_mat_inv = tf.concat([cos_val, sin_val, zero, -sin_val, cos_val, zero, zero, zero, one], axis=0)\n        rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])\n        return rot_mat_inv\n    angle = float(angle) * tf.random.normal([1],dtype='float32')\n    rot_mat_inv = get_rotation_mat_inv(angle)\n    return transform(image, rot_mat_inv, image_shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):\n    h, w = image_height, image_width\n    hh = int(np.ceil(np.sqrt(h*h+w*w)))\n    hh = hh+1 if hh%2==1 else hh\n    d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)\n    l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)\n\n    st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n    st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n\n    y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)\n    x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)\n\n    for i in range(0, hh//d+1):\n        s1 = i * d + st_h\n        s2 = i * d + st_w\n        y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)\n        x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)\n\n    x_clip_mask = tf.logical_or(x_ranges < 0 , x_ranges > hh-1)\n    y_clip_mask = tf.logical_or(y_ranges < 0 , y_ranges > hh-1)\n    clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)\n\n    x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))\n    y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))\n\n    hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)), tf.int32)])\n    x_ranges = tf.repeat(x_ranges, hh)\n    y_ranges = tf.repeat(y_ranges, hh)\n\n    y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))\n    x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))\n\n    y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64),  tf.zeros_like(y_ranges), [hh, hh])\n    y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)\n\n    x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])\n    x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)\n\n    mask = tf.expand_dims( tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)\n\n    mask = random_rotate(mask, rotate_angle, [hh, hh, 1])\n    mask = tf.image.crop_to_bounding_box(mask, (hh-h)//2, (hh-w)//2, image_height, image_width)\n\n    return mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def apply_grid_mask(image, image_shape, PROBABILITY = gridmask_rate):\n    AugParams = {\n        'd1' : 100,\n        'd2': 160,\n        'rotate' : 45,\n        'ratio' : 0.3\n    }\n    \n        \n    mask = GridMask(image_shape[0], image_shape[1], AugParams['d1'], AugParams['d2'], AugParams['rotate'], AugParams['ratio'])\n    if image_shape[-1] == 3:\n        mask = tf.concat([mask, mask, mask], axis=-1)\n        mask = tf.cast(mask,tf.float32)\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n    if P==1:\n        return image*mask\n    else:\n        return image\n\ndef gridmask(img_batch, label_batch):\n    return apply_grid_mask(img_batch, (img_size,img_size, 3)), label_batch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Shearing Zooming Shifting and Rotating :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))\n\ncfg = dict(\ntransform_prob    =  1.0,\nrot               = 180.0,\nshr               =   2.0,\nhzoom             =   8.0,\nwzoom             =   8.0,\nhshift            =   8.0,\nwshift            =   8.0,\n)\n\ndef transform_shear_rot(image,cfg):\n    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = img_size\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = cfg['rot'] * tf.random.normal([1],dtype='float32')\n    shr = cfg['shr'] * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/cfg['hzoom']\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/cfg['wzoom']\n    h_shift = cfg['hshift'] * tf.random.normal([1],dtype='float32') \n    w_shift = cfg['wshift'] * tf.random.normal([1],dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils for data :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    image = tf.image.resize(image, [img_size,img_size])\n    return image\n\ndef decode_image2(filename,label=None, image_size=(img_size,img_size)) :\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits,channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image,image_size)\n    if label == None :\n        return image\n    else :\n        return image, label\n\n\ndef data_augment(image, label=None,seed = 2020):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image,seed=seed)\n    image = tf.image.random_flip_up_down(image,seed=seed)\n    #image = tf.keras.preprocessing.image.random_rotation(image,45) \n    #if transforms :\n    #    image = transform_shear_rot(image,cfg)\n    if crop_size :   \n        image = tf.image.random_crop(image, size=[crop_size, crop_size, 3], seed=seed)\n    if bool_random_brightness:\n        image = tf.image.random_brightness(image,0.2)\n    if bool_random_contrast:\n        image = tf.image.random_contrast(image,0.6,1.4)\n    if bool_random_hue:\n        image = tf.image.random_hue(image,0.07)\n    if bool_random_saturation:\n        image = tf.image.random_saturation(image,0.5,1.5)\n    if random_blackout :\n        image= transform_random_blackout(image)\n        \n    if label == None :\n        return image\n    else :\n        return image,label ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Using tf.data.Dataset :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_train_data(train_paths,train_labels) :\n    \n    \n    train_dataset=(tf.data.Dataset\n    .from_tensor_slices((train_paths,train_labels.astype(np.float32)))\n    .map(decode_image2,num_parallel_calls = AUTO)\n    .map(data_augment,num_parallel_calls = AUTO)\n    .repeat()\n    .shuffle(512)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO))\n    \n    if cutmix_rate :\n        train_dataset = train_dataset.map(cutmix,num_parallel_calls = AUTO) \n    if mixup_rate : \n        train_dataset = train_dataset.map(mixup, num_parallel_calls = AUTO)\n    if gridmask_rate:\n        train_dataset =train_dataset.map(gridmask, num_parallel_calls=AUTO)    \n     \n    return train_dataset \n\ndef create_validation_data(valid_paths,valid_labels) :\n    valid_data = (\n        tf.data.Dataset\n        .from_tensor_slices((valid_paths,valid_labels))\n        .map(decode_image2, num_parallel_calls = AUTO)\n        #.map(data_augment, num_parallel_calls= AUTO)\n        .batch(BATCH_SIZE)\n        .prefetch(AUTO)\n        \n    ) \n    return valid_data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data using TF records :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#TRAINING_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH2 + '/tfrecords/train*')\n#TEST_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH2 + '/tfrecords/test*')\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH1 + '/train*')\n#TEST_FILENAMES = tf.io.gfile.glob('../input/siim-isic-melanoma-classification/tfrecords/test*')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH1 + '/test*')\n\n#TRAINING_FILENAMES = tf.io.gfile.glob('../input/512x512-melanoma-tfrecords-70k-images/train*')\n#TEST_FILENAMES = tf.io.gfile.glob('../input/512x512-melanoma-tfrecords-70k-images/test*')\n\nIMAGE_SIZE = [512, 512] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Read train tf Records :\ndef read_labeled_tfrecord(example):\n    \n    \n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n \n    \n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.float32)\n    return image, label\n\n#Read test tf Records :\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['image_name']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\n   \n    \ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)# prefetch next batch while training (autotune prefetch buffer size)\n    \n    if cutmix_rate :\n        dataset = dataset.map(cutmix,num_parallel_calls = AUTO) \n    if mixup_rate : \n        dataset = dataset.map(mixup, num_parallel_calls = AUTO)\n    if rotation :\n        dataset = dataset.map(transform_rotation, num_parallel_calls = AUTO)\n   # if blackout :\n   #     dataset = dataset.map(transform_random_blackout, num_parallel_calls = AUTO)\n    if gridmask_rate:\n        dataset = dataset.map(gridmask, num_parallel_calls=AUTO) \n    \n    \n    return dataset   \n\ndef get_test_dataset(ordered=False,aug=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    \n    if aug == True :        \n        \n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n        \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint('Dataset: {} training images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Callbacks :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_start = 0.00001\n\nlr_max = 0.00001 * strategy.num_replicas_in_sync\nlr_min = 0.00001 \nlr_rampup_epochs = 8\nlr_sustain_epochs = 3\nlr_exp_decay = .6\n\n\ndef lrfn(epoch) :\n    if epoch < lr_rampup_epochs :\n        lr = lr_start + (lr_max-lr_min) / lr_rampup_epochs * epoch\n    elif epoch < lr_rampup_epochs + lr_sustain_epochs :\n        lr = lr_max\n    else :\n        lr = lr_min + (lr_max - lr_min) * lr_exp_decay**(epoch - lr_sustain_epochs - lr_rampup_epochs)\n    return lr\n\nlr_callback = LearningRateScheduler(lrfn, verbose = True)\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\n\nfrom matplotlib import pyplot as plt\n\nplt.plot(rng,y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reduce_lr =  ReduceLROnPlateau(monitor = \"val_loss\", factor = 0.5, patience = 10,\n  verbose = 0, mode = \"auto\", epsilon = 1e-04, cooldown = 0,\n  min_lr = 1e-5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"es = EarlyStopping(monitor = \"val_loss\" , verbose = 1 , mode = 'min' , patience = 50 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mc = ModelCheckpoint('best_model.h5', monitor = 'loss' , mode = 'min', verbose = 1 , save_best_only = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint_path = \"best_model.h5\"\n\nswa_mc = tfa.callbacks.AverageModelCheckpoint(filepath=checkpoint_path, save_weights_only=False,\n                                                   update_weights=True)\n\n\n\"\"\"start_epoch = 10\nswa_mc = SWA(start_epoch=start_epoch, \n          lr_schedule='cyclic', \n          swa_lr=1e-6,\n          swa_lr2=1e-5,\n          swa_freq=3,\n        #batch_size = BATCH_SIZE,\n          verbose=1)\"\"\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modeling :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"focal_loss = True\nlabel_smoothing = 0\nSWA = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model_generalized(name,trainable_layers=20,opt='adam',lr=0.001):\n    if name == 'EfficientNetB7' :\n        base_model = efn.EfficientNetB7(weights='imagenet',\n                                        include_top = False,\n                                        input_shape=(img_size,img_size,3)\n                                       )\n    elif name == 'EfficientNetB5' :\n        base_model = efn.EfficientNetB5(weights='imagenet',\n                                        include_top = False,\n                                        input_shape=(img_size,img_size,3)\n                                       )\n    elif name == 'EfficientNetB3' :\n        base_model = efn.EfficientNetB3(weights='imagenet',\n                                        include_top = False,\n                                        input_shape=(img_size,img_size,3)\n                                       )\n    elif name == 'EfficientNetB1' :\n        base_model = efn.EfficientNetB1(weights='imagenet',\n                                        include_top = False,\n                                        input_shape=(img_size,img_size,3)\n                                       )    \n    elif name == 'DenseNet' :\n        base_model = DenseNet201(weights='imagenet',include_top=False,input_shape=(img_size,img_size,3))\n    elif name == 'MobileNet' :\n        base_model = MobileNet(weights = 'imagenet', include_top=False,input_shape=(img_size,img_size,3))\n    elif name == 'Inception' :\n        base_model = InceptionV3(weights = 'imagenet',include_top=False,input_shape=(img_size,img_size,3))\n    elif name == 'ResNet' :\n        base_model = ResNet50(weights = 'imagenet',include_top=False,input_shape=(img_size,img_size,3))\n    elif name == 'Incepresnet' :\n        base_model = InceptionResNetV2(weights = 'imagenet',include_top=False,input_shape=(img_size,img_size,3))\n    elif name == 'SEResNet50' :\n        seresnet50, _ = Classifiers.get('seresnet50')\n        base_model =  seresnet50(weights = 'imagenet', include_top = False, input_shape = (img_size,img_size,3))\n    elif name == 'SEResNext50' :\n        seresnext50 , _ = Classifiers.get('seresnext50')\n        base_model = seresnext50(weights = 'imagenet', include_top = False,input_shape = (img_size,img_size,3))\n    elif name == 'NasNetLarge' :\n        nasnet , _ = Classifiers.get('nasnetlarge')\n        base_model = nasnet(weights= 'imagenet', include_top = False , input_shape = (img_size,img_size,3))    \n            \n    base_model.trainable = True\n    for layer in base_model.layers[:-trainable_layers] :\n        layer.trainable = True\n    layer = base_model.output\n    layer = L.GlobalAveragePooling2D()(layer)\n    layer = L.Dense(1024,activation='relu')(layer)\n    layer = L.Dropout(0.4)(layer)\n    predictions = L.Dense(nb_classes,activation='sigmoid')(layer)\n   # predictions = tf.cast(predictions,tf.float32)\n    model = Model(inputs = base_model.input, outputs=predictions)\n    \n    if focal_loss : \n        loss= tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO)\n    if label_smoothing :\n        loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing)\n    else :\n        loss = 'binary_crossentropy'\n    \n    \n    if opt == 'RMSprop' :\n        opt = optimizers.RMSprop(learning_rate = lr)\n    elif SWA == True :\n        opt = tf.keras.optimizers.Adam(lr=1e-5) # roll back\n        opt = tfa.optimizers.SWA(opt)\n    else :\n        opt =  tf.keras.optimizers.Adam(lr=1e-5) # roll back\n\n        \n    \n    model.compile(optimizer=opt,loss=loss,metrics=['accuracy',tf.keras.metrics.AUC()])  \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fixing imbalanced :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils import class_weight \nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(train.target),\n                                                 train.target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weights = { 0 :  0.50897302 , 1 : 28.36130137 }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classweights =[item for k in class_weights for item in (k, class_weights[k])]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EfficientNetB1 :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    model = get_model_generalized('EfficientNetB1')\n\nhistory = model.fit(\n    get_training_dataset(),\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS,\n    callbacks = [],\n    #class_weight = classweights,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    model = get_model_generalized('EfficientNetB7')\n\nhistory = model.fit(\n    train_balanced_data,\n    steps_per_epoch= len(Y_train) // BATCH_SIZE,\n    epochs=50,\n    callbacks = [],\n    #class_weight = classweights,\n    validation_data = valid_balanced_data\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EfficientNetB3 : ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    model = get_model_generalized('EfficientNetB3')\n\nhistory = model.fit(\n    get_training_dataset(),\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS,\n    callbacks = [],\n    #class_weight = classweights\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EfficientNetB5 :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    model = get_model_generalized('EfficientNetB5')\n\nhistory = model.fit(\n    get_training_dataset(),\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS,\n    #callbacks = [swa_mc],\n   # batch_size = BATCH_SIZE\n    #class_weight = classweights\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    model = get_model_generalized('EfficientNetB5')\n\nhistory = model.fit(\n    train_balanced_data,\n    steps_per_epoch= len(Y_train) // BATCH_SIZE,\n    epochs=50,\n    callbacks = [],\n    #class_weight = classweights,\n    validation_data = valid_balanced_data\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Resnet :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    model = get_model_generalized('ResNet')\n\nhistory = model.fit(\n    get_training_dataset(),\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS-10,\n    callbacks = [],\n    #class_weight = classweights\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SEResNet 50 :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    model = get_model_generalized('SEResNet50')\n\nhistory = model.fit(\n    get_training_dataset(),\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS-10,\n    callbacks = [],\n    #class_weight = classweights\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SEResNext 50 : ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    model = get_model_generalized('SEResNext50')\n\nhistory = model.fit(\n    get_training_dataset(),\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS-5,\n    callbacks = [],\n    #class_weight = classweights\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DenseNet :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    model = get_model_generalized('DenseNet')\n\nhistory = model.fit(\n    get_training_dataset(),\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=12,\n    callbacks = [],\n    #class_weight = classweights\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MobileNet :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    model = get_model_generalized('MobileNet')\n\nhistory = model.fit(\n    get_training_dataset(),\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=EPOCHS,\n    callbacks = [],\n    #class_weight = classweights\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Nasnet : ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    model = get_model_generalized('NasNetLarge')\n\nhistory = model.fit(\n    get_training_dataset(),\n    steps_per_epoch=STEPS_PER_EPOCH,\n    epochs=15,\n    callbacks = [],\n    #class_weight = classweights\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del model\nimport gc \ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction :","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Label Smoothing :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"label_smoothing = 0\ndef LabelSmoothing(encodings , alpha):\n    K = encodings.shape[1]\n    y_ls = (1 - alpha) * encodings + alpha / K\n    return y_ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_smoothing = 0\ntest_ds = get_test_dataset(ordered=True)\n#model.load_weights('best_model.h5')\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\nif label_smoothing :\n    probabilities = LabelSmoothing(probabilities,label_smoothing)\nprobabilities =probabilities.flatten()\nprint(probabilities)\nprint('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\npred_df = pd.DataFrame({'image_name': test_ids, 'target': probabilities})\ndel sub['target']\nsub = sub.merge(pred_df, on='image_name')\nsub.to_csv('densenet_468img_size_12epochs_fl_0.4cutmix_0.4gridmask_20trainable.csv', index=False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[sub['target'] > 0.5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ndel model\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pseudo Labeling :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[sub['target']  >  0.8] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[sub['target'] < 0.01]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test Time Augmentation :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#model.load_weights('best_model.h5')\nTTA_NUM = 3\nprobabilities = []\nfor i in range(TTA_NUM):\n    print(f'TTA Number: {i}\\n')\n    test_ds = get_test_dataset(ordered=True,aug=True)\n    test_images_ds = test_ds.map(lambda image, idnum: image)\n    probabilities.append(model.predict(test_images_ds).flatten())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tab = np.zeros((len(probabilities[1]),1))\nfor i in range(0,len(probabilities[1])) :\n    for j in range(0,TTA_NUM) :\n        tab[i] = tab[i] + probabilities[j][i]\ntab = tab / TTA_NUM              \ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\npred_df = pd.DataFrame({'image_name': test_ids, 'target': np.squeeze(tab)})\ndel sub['target']\nsub = sub.merge(pred_df, on='image_name')\nsub.to_csv('e3_468_20epochs_fl_20trainable_3tta_heavy_augs.csv', index=False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Both Image and meta-data :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# batch\ndef cutmix_v2(data, label, PROBABILITY = cutmix_rate):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    \n    DIM = img_size    \n    imgs = []; labs = []\n    \n    for j in range(BATCH_SIZE):\n        \n        #random_uniform( shape, minval=0, maxval=None)        \n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n        \n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast(tf.random.uniform([], 0, BATCH_SIZE), tf.int32)\n        \n        # CHOOSE RANDOM LOCATION\n        x = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)\n        y = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)\n        \n        # Beta(1, 1)\n        b = tf.random.uniform([], 0, 1) # this is beta dist with alpha=1.0\n        \n\n        WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        \n        # MAKE CUTMIX IMAGE\n        one = data['image_data'][j,ya:yb,0:xa,:]\n        two = data['image_data'][k,ya:yb,xa:xb,:]\n        three = data['image_data'][j,ya:yb,xb:DIM,:]        \n        #ya:yb\n        middle = tf.concat([one,two,three],axis=1)\n\n        img = tf.concat([data['image_data'][j,0:ya,:,:],middle,data['image_data'][j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        \n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        lab1 = label[j,]\n        lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n\n    image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE, nb_classes))\n    \n    data['image_data'] = image2\n    \n    return data,label2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def gridmask_v2(data, label_batch):\n    return apply_grid_mask_v2(data, (img_size,img_size, 3)), label_batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def apply_grid_mask_v2(data, image_shape, PROBABILITY = gridmask_rate):\n    AugParams = {\n        'd1' : 100,\n        'd2': 160,\n        'rotate' : 45,\n        'ratio' : 0.3\n    }\n    \n        \n    mask = GridMask(image_shape[0], image_shape[1], AugParams['d1'], AugParams['d2'], AugParams['rotate'], AugParams['ratio'])\n    if image_shape[-1] == 3:\n        mask = tf.concat([mask, mask, mask], axis=-1)\n        mask = tf.cast(mask,tf.float32)\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n    if P==1:\n        data['image_data'] = data['image_data']  * mask\n        return data\n    else:\n        return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Read train tf Records :\ndef read_labeled_tfrecord(example):\n    \n    \n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"patient_id\" : tf.io.FixedLenFeature([],tf.int64) ,\n         \"sex\": tf.io.FixedLenFeature([],tf.int64),\n      \"age_approx\": tf.io.FixedLenFeature([],tf.int64),\n      \"anatom_site_general_challenge\": tf.io.FixedLenFeature([],tf.int64),\n     # \"source\": tf.io.FixedLenFeature([],tf.int64),\n        \"target\": tf.io.FixedLenFeature([], tf.int64), # shape [] means single element  \n    }\n \n    \n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    \n    data = {}\n    \n    data['patient_id'] = tf.cast(example['patient_id'],tf.float32)\n    data['sex'] = tf.cast(example['patient_id'],tf.float32)\n    data['age_approx'] = tf.cast(example['age_approx'],tf.float32)\n    #data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)\n    \n    data['anatom_site_general_challenge'] = tf.cast(example['anatom_site_general_challenge'],tf.float32)\n    #data['source'] = tf.cast(example['source'],tf.float32)\n    \n    \n    label = tf.cast(example['target'], tf.float32)\n    return image, label , data\n\n#Read test tf Records :\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n      \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n      \"image_name\": tf.io.FixedLenFeature([], tf.string), # shape [] means single element\n      \"patient_id\" : tf.io.FixedLenFeature([],tf.int64) ,\n      \"sex\": tf.io.FixedLenFeature([],tf.int64),\n      \"age_approx\": tf.io.FixedLenFeature([],tf.int64),\n      \"anatom_site_general_challenge\": tf.io.FixedLenFeature([],tf.int64),\n      #\"source\": tf.io.FixedLenFeature([],tf.int64),\n\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    img_name = example['image_name']\n    \n    data = {}\n    \n    data['patient_id'] = tf.cast(example['patient_id'],tf.float32)\n    data['sex'] = tf.cast(example['patient_id'],tf.float32)\n    data['age_approx'] = tf.cast(example['age_approx'],tf.float32)\n    \n    #data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)\n    \n    \n    \n    data['anatom_site_general_challenge'] = tf.cast(example['anatom_site_general_challenge'],tf.float32)\n    #data['source'] = tf.cast(example['source'],tf.float32)\n    \n    return image, img_name , data # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    \n    return dataset\n\n\ndef train_setup(image,label,data) :\n    #anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32) for i in range(7)]\n    tab_data=[tf.cast(data[tfeat], dtype=tf.float32) for tfeat in ['sex','age_approx','patient_id','anatom_site_general_challenge']]#,'source','anatom_site_general_challenge',\n    tabular=tf.stack(tab_data) #+anatom\n    \n    return {'image_data' : image , 'meta_data' : tabular} , label\n\ndef test_setup(image,image_name,data) :\n    #anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32) for i in range(7)]\n    tab_data=[tf.cast(data[tfeat], dtype=tf.float32) for tfeat in ['sex','age_approx','patient_id','anatom_site_general_challenge']] #,'source','anatom_site_general_challenge',\n    tabular=tf.stack(tab_data) #+anatom\n    return {'image_data' : image , 'meta_data' : tabular } , image_name\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(data, label=None,seed = 2020):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    \n    data['image_data'] = tf.image.random_flip_left_right(data['image_data'],seed=seed)\n    data['image_data'] = tf.image.random_flip_up_down(data['image_data'],seed=seed)\n   \n    #data['image_data'] = tf.keras.preprocessing.image.random_rotation(data['image_data'],45) \n    #if transforms :\n    #    data['image_data'] = transform_shear_rot(data['image_data'],cfg)\n    \n  \n     \n    if crop_size :   \n        data['image_data'] = tf.image.random_crop(data['image_data'], size=[crop_size, crop_size, 3], seed=seed)\n    if bool_random_brightness:\n        data['image_data'] = tf.image.random_brightness(data['image_data'],0.2)\n    if bool_random_contrast:\n        data['image_data'] = tf.image.random_contrast(data['image_data'],0.6,1.4)\n    if bool_random_hue:\n        data['image_data'] = tf.image.random_hue(data['image_data'],0.07)\n    if bool_random_saturation:\n        data['image_data'] = tf.image.random_saturation(data['image_data'],0.5,1.5)\n    if random_blackout :\n        data['image_data'] = transform_random_blackout(data['image_data'])\n        \n    if label == None :\n        return data\n    else :\n        return data,label ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Crop MicroScope circle : ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"'''def crop_microscope(img_to_crop):\n    pad_y = img_to_crop.shape[0]//200 \n    pad_x = img_to_crop.shape[1]//200\n    img = img_to_crop[pad_y:-pad_y, pad_y:-pad_y,:]\n    \n#cropping 0.5% from every side, because some microscope images\n#have frames along the edges so cv2.boundingRect crops by frame, \n#but not by needed part of the image.\n    \n    \n    gray = cv2.cvtColor(np.float32(img),cv2.COLOR_BGR2GRAY)\n    _,thresh = cv2.threshold(gray,50,255,cv2.THRESH_BINARY) \n    x,y,w,h = cv2.boundingRect(thresh) #getting crop points\n    \n#since we cropped borders we need to uncrop it back \n    if y!=0: \n        y = y+pad_y\n    if h == thresh.shape[0]:\n        h = h+pad_y\n    if x !=0:\n        x = x +pad_x\n    if w == thresh.shape[1]:\n        w = w + pad_x\n    h = h+pad_y\n    w = w + pad_x\n    img = img_to_crop[y:y+h,x:x+w]\n    return img\n\n\ndef crop_circle(image) :\n    out = tf.py_function(crop_microscope, [image], tf.float32)\n    return  out'''\n    \n    \nclass Microscope:\n    def __init__(self, p: float = 0.5):\n        self.p = p\n\n    def __call__(self, data , label):\n        for k in range(BATCH_SIZE) :\n            if random.random() < self.p:\n                circle = cv2.circle((np.ones(data['image_data'][k].shape) * 255).astype(np.uint8),\n                            (data['image_data'][k].shape[0]//2, data['image_data'][k][1]//2),\n                            random.randint(data['image_data'][k].shape[0]//2 - 3, data['image_data'][k].shape[0]//2 + 15),\n                            (0, 0, 0),\n                            -1)\n\n                mask = circle - 255\n                data['image_data'][k] = np.multiply(data['image_data'][k], mask)\n\n        return data , label\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(p={self.p})'\n    \nmicro = Microscope()   \n\nclass Microscope_test:\n    def __init__(self, p: float = 0.5):\n        self.p = p\n\n    def __call__(self, data , image_name):\n        if random.random() < self.p:\n            circle = cv2.circle((np.ones(data['image_data'].shape) * 255).astype(np.uint8),\n                        (data['image_data'].shape[0]//2, data['image_data'][1]//2),\n                        random.randint(data['image_data'].shape[0]//2 - 3, data['image_data'].shape[0]//2 + 15),\n                        (0, 0, 0),\n                        -1)\n\n            mask = circle - 255\n            data['image_data'] = np.multiply(data['image_data'], mask)\n\n        return data , image_name\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(p={self.p})'\n\n\n\nclass AdvancedHairAugmentation:\n    def __init__(self, hairs: int = 4, hairs_folder: str = \"\"):\n        self.hairs = hairs\n        self.hairs_folder = hairs_folder\n\n    def __call__(self, data,label):\n        for k in range(BATCH_SIZE) :\n            n_hairs = random.randint(0, self.hairs)\n\n            if not n_hairs:\n                return data , label\n\n            height, width, _ = data['image_data'][k].shape  # target image width and height\n            hair_images = [im for im in os.listdir(self.hairs_folder) if 'png' in im]\n\n            for _ in range(n_hairs):\n                hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))\n                hair = cv2.flip(hair, random.choice([-1, 0, 1]))\n                hair = cv2.rotate(hair, random.choice([0, 1, 2]))\n\n                h_height, h_width, _ = hair.shape  # hair image width and height\n                roi_ho = random.randint(0, data['image_data'][k].shape[0] - hair.shape[0])\n                roi_wo = random.randint(0, data['image_data'][k].shape[1] - hair.shape[1])\n                roi = data['image_data'][k][roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]\n\n                img2gray = cv2.cvtColor(np.float32(hair), cv2.COLOR_BGR2GRAY)\n                ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)\n                mask_inv = cv2.bitwise_not(mask)\n                img_bg = cv2.bitwise_and(np.float32(roi),np.float32(roi) , mask=mask_inv)#\n                hair_fg = cv2.bitwise_and(np.float32(hair),np.float32(hair), mask=mask) #\n\n                dst = cv2.add(img_bg, hair_fg)\n                data['image_data'][k][roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst\n\n        return data , label\n    \nhair = AdvancedHairAugmentation(hairs_folder = '../input/melanoma-hairs')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(train_setup , num_parallel_calls = AUTO)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)# prefetch next batch while training (autotune prefetch buffer size)\n    \n    if hair_aug :\n        dataset = dataset.map(hair, num_parallel_calls = AUTO)\n    if micro_aug :\n        dataset = dataset.map(micro,num_parallel_calls = AUTO)\n    if cutmix_rate :\n        dataset = dataset.map(cutmix_v2,num_parallel_calls = AUTO) \n    if mixup_rate : \n        dataset = dataset.map(mixup, num_parallel_calls = AUTO)\n    if rotation :\n        dataset = dataset.map(transform_rotation, num_parallel_calls = AUTO)\n   # if blackout :\n   #     dataset = dataset.map(transform_random_blackout, num_parallel_calls = AUTO)\n    if gridmask_rate:\n        dataset = dataset.map(gridmask_v2, num_parallel_calls=AUTO)  \n    \n    return dataset   \n\ndef get_test_dataset(ordered=False,aug=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.map(test_setup , num_parallel_calls = AUTO)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    \n    if aug == True :        \n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n        #dataset = dataset.map(micro, num_parallel_calls = AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modeling with both Image and Meta Data : ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"focal_loss = True\nlabel_smoothing = 0\nSWA = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def roc_auc_loss(\n    labels,\n    logits,\n    weights=1.0,\n    surrogate_type='xent',\n    scope=None):\n  \"\"\"Computes ROC AUC loss.\n  The area under the ROC curve is the probability p that a randomly chosen\n  positive example will be scored higher than a randomly chosen negative\n  example. This loss approximates 1-p by using a surrogate (either hinge loss or\n  cross entropy) for the indicator function. Specifically, the loss is:\n    sum_i sum_j w_i*w_j*loss(logit_i - logit_j)\n  where i ranges over the positive datapoints, j ranges over the negative\n  datapoints, logit_k denotes the logit (or score) of the k-th datapoint, and\n  loss is either the hinge or log loss given a positive label.\n  Args:\n    labels: A `Tensor` of shape [batch_size] or [batch_size, num_labels].\n    logits: A `Tensor` with the same shape and dtype as `labels`.\n    weights: Coefficients for the loss. Must be a scalar or `Tensor` of shape\n      [batch_size] or [batch_size, num_labels].\n    surrogate_type: Either 'xent' or 'hinge', specifying which upper bound\n      should be used for the indicator function.\n    scope: Optional scope for `name_scope`.\n  Returns:\n    loss: A `Tensor` of the same shape as `logits` with the component-wise loss.\n    other_outputs: An empty dictionary, for consistency.\n  Raises:\n    ValueError: If `surrogate_type` is not `xent` or `hinge`.\n  \"\"\"\n  with tf.name_scope(scope, 'roc_auc', [labels, logits, weights]):\n    # Convert inputs to tensors and standardize dtypes.\n    labels, logits, weights, original_shape = _prepare_labels_logits_weights(\n        labels, logits, weights)\n\n    # Create tensors of pairwise differences for logits and labels, and\n    # pairwise products of weights. These have shape\n    # [batch_size, batch_size, num_labels].\n    logits_difference = tf.expand_dims(logits, 0) - tf.expand_dims(logits, 1)\n    labels_difference = tf.expand_dims(labels, 0) - tf.expand_dims(labels, 1)\n    weights_product = tf.expand_dims(weights, 0) * tf.expand_dims(weights, 1)\n\n    signed_logits_difference = labels_difference * logits_difference\n    raw_loss = util.weighted_surrogate_loss(\n        labels=tf.ones_like(signed_logits_difference),\n        logits=signed_logits_difference,\n        surrogate_type=surrogate_type)\n    weighted_loss = weights_product * raw_loss\n\n    # Zero out entries of the loss where labels_difference zero (so loss is only\n    # computed on pairs with different labels).\n    loss = tf.reduce_mean(tf.abs(labels_difference) * weighted_loss, 0) * 0.5\n    loss = tf.reshape(loss, original_shape)\n    return loss, {}\n\n\ndef custom_loss_wrapper() :\n    def roc_auc_loss(\n    labels,\n    logits,\n    weights=1.0,\n    surrogate_type='xent',\n    scope=None):\n        \n      \"\"\"Computes ROC AUC loss.\n      The area under the ROC curve is the probability p that a randomly chosen\n      positive example will be scored higher than a randomly chosen negative\n      example. This loss approximates 1-p by using a surrogate (either hinge loss or\n      cross entropy) for the indicator function. Specifically, the loss is:\n        sum_i sum_j w_i*w_j*loss(logit_i - logit_j)\n      where i ranges over the positive datapoints, j ranges over the negative\n      datapoints, logit_k denotes the logit (or score) of the k-th datapoint, and\n      loss is either the hinge or log loss given a positive label.\n      Args:\n        labels: A `Tensor` of shape [batch_size] or [batch_size, num_labels].\n        logits: A `Tensor` with the same shape and dtype as `labels`.\n        weights: Coefficients for the loss. Must be a scalar or `Tensor` of shape\n          [batch_size] or [batch_size, num_labels].\n        surrogate_type: Either 'xent' or 'hinge', specifying which upper bound\n          should be used for the indicator function.\n        scope: Optional scope for `name_scope`.\n      Returns:\n        loss: A `Tensor` of the same shape as `logits` with the component-wise loss.\n        other_outputs: An empty dictionary, for consistency.\n      Raises:\n        ValueError: If `surrogate_type` is not `xent` or `hinge`.\n      \"\"\"\n      with tf.name_scope(scope, 'roc_auc', [labels, logits, weights]):\n        # Convert inputs to tensors and standardize dtypes.\n        labels, logits, weights, original_shape = _prepare_labels_logits_weights(\n            labels, logits, weights)\n\n        # Create tensors of pairwise differences for logits and labels, and\n        # pairwise products of weights. These have shape\n        # [batch_size, batch_size, num_labels].\n        logits_difference = tf.expand_dims(logits, 0) - tf.expand_dims(logits, 1)\n        labels_difference = tf.expand_dims(labels, 0) - tf.expand_dims(labels, 1)\n        weights_product = tf.expand_dims(weights, 0) * tf.expand_dims(weights, 1)\n\n        signed_logits_difference = labels_difference * logits_difference\n        raw_loss = util.weighted_surrogate_loss(\n            labels=tf.ones_like(signed_logits_difference),\n            logits=signed_logits_difference,\n            surrogate_type=surrogate_type)\n        weighted_loss = weights_product * raw_loss\n\n        # Zero out entries of the loss where labels_difference zero (so loss is only\n        # computed on pairs with different labels).\n        loss = tf.reduce_mean(tf.abs(labels_difference) * weighted_loss, 0) * 0.5\n        loss = tf.reshape(loss, original_shape)\n        return loss, {}\n    return roc_auc_loss    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" def get_model_generalized_v2(name,trainable_layers=20,opt='adam',lr=0.001):\n    if name == 'EfficientNetB2' :\n        base_model = efn.EfficientNetB2(weights='imagenet',\n                                        include_top = False,\n                                       )\n    elif name == 'EfficientNetB5' :\n        base_model = efn.EfficientNetB5(weights='imagenet',\n                                        include_top = False,\n                                       )\n    elif name == 'EfficientNetB3' :\n        base_model = efn.EfficientNetB3(weights='imagenet',\n                                        include_top = False,\n                                       )\n    elif name == 'EfficientNetB1' :\n        base_model = efn.EfficientNetB1(weights='imagenet',\n                                        include_top = False,\n                                       )    \n    elif name == 'DenseNet' :\n        base_model = DenseNet201(weights='imagenet',include_top=False)\n    elif name == 'MobileNet' :\n        base_model = MobileNet(weights = 'imagenet', include_top=False)\n    elif name == 'Inception' :\n        base_model = InceptionV3(weights = 'imagenet',include_top=False)\n    elif name == 'ResNet' :\n        base_model = ResNet50(weights = 'imagenet',include_top=False)\n    elif name == 'Incepresnet' :\n        base_model = InceptionResNetV2(weights = 'imagenet',include_top=False)\n    elif name == 'SEResNet50' :\n        seresnet50, _ = Classifiers.get('seresnet50')\n        base_model =  seresnet50(weights = 'imagenet', include_top = False)\n    elif name == 'SEResNext50' :\n        seresnext50 , _ = Classifiers.get('seresnext50')\n        base_model = seresnext50(weights = 'imagenet', include_top = False)\n    elif name == 'NasNetLarge' :\n        nasnet , _ = Classifiers.get('nasnetlarge')\n        base_model = nasnet(weights= 'imagenet', include_top = False )    \n            \n    base_model.trainable = True\n    for layer in base_model.layers[:-trainable_layers] :\n        layer.trainable = True\n        \n    inp1 = L.Input(shape = (img_size,img_size, 3), name = 'image_data')\n    inp2 = L.Input(shape = (4), name = 'meta_data')  \n    \n    \n    layer1 = base_model(inp1)\n    layer1 = L.GlobalAveragePooling2D()(layer1)\n    layer1 = L.Dense(1024,activation='relu')(layer1)\n    layer1 = L.Dropout(0.4)(layer1)\n    layer1 = L.Dense(512,activation='relu')(layer1)\n    layer1 = L.Dropout(0.4)(layer1)#0.4\n    layer1 = L.Dense(256,activation='relu')(layer1)\n    layer1 = L.Dropout(0.4)(layer1) #0.4\n    \n    #\n    layer2 = L.Dense(128,activation = 'relu')(inp2)\n    layer2 = L.Dropout(0.4)(layer2)\n    layer2 = L.Dense(64, activation = 'relu')(layer2)\n    layer2 = L.BatchNormalization()(layer2)\n    layer2 = L.Dropout(0.3)(layer2) #0.3\n    layer2 = L.Dense(32, activation = 'relu')(layer2)\n    layer2 = L.BatchNormalization()(layer2)\n    layer2 = L.Dropout(0.3)(layer2) #0.3\n    #layer2 = L.Dense(16, activation = 'relu')(layer2)\n    #layer2 = L.BatchNormalization()(layer2)\n    #layer2 = L.Dropout(0.2)(layer2)\n    \n    concat = L.concatenate([layer1, layer2])\n    \n    concat = L.Dense(512,activation = 'relu')(concat) #512 is best 0.933\n    concat = L.Dropout(0.4)(concat)\n    \n    #concat = L.Dense(256,activation = 'relu')(concat) #testing\n    #concat = L.Dropout(0.3)(concat)    \n    \n    predictions = L.Dense(nb_classes,activation='sigmoid')(concat)\n   # predictions = tf.cast(predictions,tf.float32)\n    model = Model(inputs = [inp1,inp2], outputs=[predictions])\n    \n    if focal_loss : \n        loss= tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO)\n    if label_smoothing :\n        loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing)\n    else :\n        loss = 'binary_crossentropy'\n    \n    \n    if opt == 'RMSprop' :\n        opt = optimizers.RMSprop(learning_rate = lr)\n    elif SWA == True :\n        opt = tf.keras.optimizers.Adam(lr=1e-5) # roll back\n        opt = tfa.optimizers.SWA(opt)\n    else :\n        opt =  tf.keras.optimizers.Adam(lr=1e-5) # roll back\n\n        \n    model.compile(optimizer=opt,loss=loss,metrics=['accuracy',tf.keras.metrics.AUC()])  \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    model = get_model_generalized_v2('EfficientNetB2')\n\n    history = model.fit(\n        get_training_dataset(),\n        steps_per_epoch=STEPS_PER_EPOCH,\n        epochs=EPOCHS,\n        callbacks = [],\n        #class_weight = classweights\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"label_smoothing = 0\ntest_ds = get_test_dataset(ordered=True)\n#model.load_weights('best_model.h5')\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)\nif label_smoothing :\n    probabilities = LabelSmoothing(probabilities,label_smoothing)\nprobabilities =probabilities.flatten()\nprint(probabilities)\nprint('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\npred_df = pd.DataFrame({'image_name': test_ids, 'target': probabilities})\ndel sub['target']\nsub = sub.merge(pred_df, on='image_name')\nsub.to_csv('E4_468img_size_20epochs_fl_v2_1024.4_512.4_256.4cnn128.4_64.3_32.3meta_cutmix_grid_512both_hair.csv', index=False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test Time Augs :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#model.load_weights('best_model.h5')\nTTA_NUM = 5\nprobabilities = []\nfor i in range(TTA_NUM):\n    print(f'TTA Number: {i}\\n')\n    test_ds = get_test_dataset(ordered=True,aug=True)\n    test_images_ds = test_ds.map(lambda image, idnum: image)\n    probabilities.append(model.predict(test_images_ds).flatten())\ntab = np.zeros((len(probabilities[1]),1))\nfor i in range(0,len(probabilities[1])) :\n    for j in range(0,TTA_NUM) :\n        tab[i] = tab[i] + probabilities[j][i]\ntab = tab / TTA_NUM              \ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\npred_df = pd.DataFrame({'image_name': test_ids, 'target': np.squeeze(tab)})\ndel sub['target']\nsub = sub.merge(pred_df, on='image_name')\nsub.to_csv('e4_468_20epochs_fl_20trainable_5ttav2.csv', index=False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}