{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# first load libraries\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nimport os , math , re , random\nimport glob \n\nimport pydicom\nimport cv2\nimport time\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport albumentations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\n\nfrom tensorflow.keras.callbacks import EarlyStopping,LearningRateScheduler, ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras.models import Model,Sequential\n\n\nimport efficientnet.tfkeras as efn\nfrom tensorflow.keras.applications import DenseNet121, DenseNet201\nfrom tensorflow.keras.applications import vgg16\nfrom tensorflow.keras.applications.inception_v3 import InceptionV3\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.applications import MobileNet , MobileNetV2\nfrom tensorflow.keras.applications import InceptionResNetV2\nfrom tensorflow.keras import optimizers\n\n!pip install tensorflow-addons=='0.9.1'\nimport tensorflow_addons as tfa","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\n\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.metrics import classification_report\nfrom sklearn.utils import shuffle\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !git clone https://github.com/chentinghao/download_google_drive.git\n    \n# !python ./download_google_drive/download_gdrive.py 1HoHknBnHFhdKkQkeOkEyrRz_Ete47WMY best_model_b0_vloss=0.0162_vauc=0.9325.h5\n# !python ./download_google_drive/download_gdrive.py 1dNqwBxPVVDc2KQVtCunuRWYqYTrR48Hg best_model_b3_vacc=0.9159_vloss=0.0182_vauc=0.9332.h5\n# !python ./download_google_drive/download_gdrive.py 1h8T4M9gdxJLSbED2yMfUUFHdtC3-TwXL best_model_b5_vacc=0.9239_vloss=0.0165_vauc=0.9347.h5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"OVERVIEW THE DATA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# for reproducible results :\ndef seed_everything(seed=13):\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n    os.environ['TF_KERAS'] = '1'\n    random.seed(seed)\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try :\n    tpu=tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on :',tpu.master())\nexcept ValueError :\n    tpu = None\n\nif tpu :    \n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse :\n    strategy = tf.distribute.get_strategy()\n    \nprint('Replicas :',strategy.num_replicas_in_sync)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO  = tf.data.experimental.AUTOTUNE\n\nGCS_DS_PATH2 = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')\nGCS_DS_PATH1 = KaggleDatasets().get_gcs_path('512x512-melanoma-tfrecords-70k-images')\nEPOCHS = 10\nBATCH_SIZE = 8 * strategy.num_replicas_in_sync \nimg_size = 468 # 468 for effnet b5\nSEED =  42\nnb_classes = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntest = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\nsub = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\n\ntrain_paths = train.image_name.apply(lambda x : GCS_DS_PATH1 + '/train/' +x + '.jpg').values\ntest_paths = test.image_name.apply(lambda x : GCS_DS_PATH1 + '/test/' +x + '.jpg').values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['age_approx'].fillna(train['age_approx'].mean(),inplace = True)\ntrain['sex'].fillna('unknown_sex',inplace = True)\ntrain['anatom_site_general_challenge'].fillna('unknown_anatom',inplace = True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train['sex'].value_counts())\nprint('*'*100)\nprint(train['diagnosis'].value_counts())\nprint('*'*100)\nprint(train['anatom_site_general_challenge'].value_counts())\nprint(train['benign_malignant'].value_counts())\nprint(train['target'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"one_hot_sex = pd.get_dummies(train.sex, prefix='sex')\none_hot_diagnosis = pd.get_dummies(train.diagnosis , prefix = 'disgnosis')\none_hot_anatom = pd.get_dummies(train.anatom_site_general_challenge ,prefix = 'anatom')\n\ntrain = train.join(one_hot_sex)\ntrain = train.join(one_hot_diagnosis)\ntrain = train.join(one_hot_anatom)\n\ntrain = train.drop(['sex','diagnosis','anatom_site_general_challenge','benign_malignant'],axis = 1)\ntrain['id'] = train['patient_id'].map(lambda x : int(x[3:]))\ntrain.drop('patient_id',axis = 1,inplace = True)\n\n\none_hot_sex = pd.get_dummies(test.sex, prefix='sex')\none_hot_anatom = pd.get_dummies(test.anatom_site_general_challenge ,prefix = 'anatom')\n\ntest = test.join(one_hot_sex)\ntest = test.join(one_hot_diagnosis)\ntest = test.join(one_hot_anatom)\n\ntest = test.drop(['sex','anatom_site_general_challenge'],axis = 1)\ntest['id'] = test['patient_id'].map(lambda x : int(x[3:]))\ntest.drop('patient_id',axis = 1,inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\ndup_patients_test = test[test.id.duplicated() == True]\nunique_patient_test_ids = list(set(dup_patients_test['id']))\npatient = test[test['id'] == unique_patient_test_ids[2]] \npatient = patient.reset_index(drop=True)\nimages = []\nfor i in range(len(patient)) :\n    img = '../input/siim-isic-melanoma-classification/jpeg/test/'+patient['image_name'][i]+'.jpg'\n    n = cv2.imread(img)\n    n = cv2.cvtColor(n, cv2.COLOR_BGR2GRAY)\n    images.append(n)\n\nimport matplotlib.pyplot as plt\nplt.figure(figsize=(10,10))\nfor i in range(len(images)) :\n    plt.subplot(8,8,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(images[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bool_random_brightness = False # 0.902 no improvement\nbool_random_contrast =False  # doesn't improve\nbool_random_hue = False\nbool_random_saturation = False\n\ngridmask_rate = 0.4 #improve \ncutmix_rate = 0.4 #improve\nmixup_rate = 0 #doesn't improve\nrotation = False\nrandom_blackout = False\ncrop_size = 0\ntransforms = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# batch\ndef cutmix_b0(image, label, PROBABILITY = cutmix_rate, img_size=224):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    \n    DIM = img_size    \n    imgs = []; labs = []\n    \n    for j in range(BATCH_SIZE):\n        \n        #random_uniform( shape, minval=0, maxval=None)        \n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n        \n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast(tf.random.uniform([], 0, BATCH_SIZE), tf.int32)\n        \n        # CHOOSE RANDOM LOCATION\n        x = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)\n        y = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)\n        \n        # Beta(1, 1)\n        b = tf.random.uniform([], 0, 1) # this is beta dist with alpha=1.0\n        \n\n        WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        \n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]        \n        #ya:yb\n        middle = tf.concat([one,two,three],axis=1)\n\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        \n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        lab1 = label[j,]\n        lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n\n    image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE, nb_classes))\n    return image2, label2\n\n\n# batch\ndef cutmix_b3(image, label, PROBABILITY = cutmix_rate, img_size=320):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    \n    DIM = img_size    \n    imgs = []; labs = []\n    \n    for j in range(BATCH_SIZE):\n        \n        #random_uniform( shape, minval=0, maxval=None)        \n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n        \n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast(tf.random.uniform([], 0, BATCH_SIZE), tf.int32)\n        \n        # CHOOSE RANDOM LOCATION\n        x = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)\n        y = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)\n        \n        # Beta(1, 1)\n        b = tf.random.uniform([], 0, 1) # this is beta dist with alpha=1.0\n        \n\n        WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        \n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]        \n        #ya:yb\n        middle = tf.concat([one,two,three],axis=1)\n\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        \n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        lab1 = label[j,]\n        lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n\n    image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE, nb_classes))\n    return image2, label2\n\n\n# batch\ndef cutmix_b5(image, label, PROBABILITY = cutmix_rate, img_size=468):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    \n    DIM = img_size    \n    imgs = []; labs = []\n    \n    for j in range(BATCH_SIZE):\n        \n        #random_uniform( shape, minval=0, maxval=None)        \n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n        \n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast(tf.random.uniform([], 0, BATCH_SIZE), tf.int32)\n        \n        # CHOOSE RANDOM LOCATION\n        x = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)\n        y = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)\n        \n        # Beta(1, 1)\n        b = tf.random.uniform([], 0, 1) # this is beta dist with alpha=1.0\n        \n\n        WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        \n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]        \n        #ya:yb\n        middle = tf.concat([one,two,three],axis=1)\n\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        \n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        lab1 = label[j,]\n        lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n\n    image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE, nb_classes))\n    return image2, label2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mixup(image, label, PROBABILITY = mixup_rate, img_size=224):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = img_size\n    \n    imgs = []; labs = []\n    for j in range(BATCH_SIZE):\n        \n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,BATCH_SIZE),tf.int32)\n        a = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n\n        #mixup\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n        if P==1:\n            a=0.\n        \n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        \n        # MAKE CUTMIX LABEL\n        lab1 = label[j,]\n        lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE,nb_classes))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform(image, inv_mat, image_shape):\n    h, w, c = image_shape\n    cx, cy = w//2, h//2\n    new_xs = tf.repeat( tf.range(-cx, cx, 1), h)\n    new_ys = tf.tile( tf.range(-cy, cy, 1), [w])\n    new_zs = tf.ones([h*w], dtype=tf.int32)\n    old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))\n    old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)\n    clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)\n    clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)\n    clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)\n    old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))\n    old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))\n    new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))\n    new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))\n    old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)\n    new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)\n    rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))\n    rotated_image_channel = list()\n    for i in range(c):\n        vals = rotated_image_values[:,i]\n        sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])\n        rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))\n    return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])\n\ndef GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):\n    h, w = image_height, image_width\n    hh = int(np.ceil(np.sqrt(h*h+w*w)))\n    hh = hh+1 if hh%2==1 else hh\n    d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)\n    l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)\n\n    st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n    st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n\n    y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)\n    x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)\n\n    for i in range(0, hh//d+1):\n        s1 = i * d + st_h\n        s2 = i * d + st_w\n        y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)\n        x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)\n\n    x_clip_mask = tf.logical_or(x_ranges < 0 , x_ranges > hh-1)\n    y_clip_mask = tf.logical_or(y_ranges < 0 , y_ranges > hh-1)\n    clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)\n\n    x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))\n    y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))\n\n    hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)), tf.int32)])\n    x_ranges = tf.repeat(x_ranges, hh)\n    y_ranges = tf.repeat(y_ranges, hh)\n\n    y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))\n    x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))\n\n    y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64),  tf.zeros_like(y_ranges), [hh, hh])\n    y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)\n\n    x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])\n    x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)\n\n    mask = tf.expand_dims( tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)\n\n    mask = random_rotate(mask, rotate_angle, [hh, hh, 1])\n    mask = tf.image.crop_to_bounding_box(mask, (hh-h)//2, (hh-w)//2, image_height, image_width)\n\n    return mask\n\ndef apply_grid_mask(image, image_shape, PROBABILITY = gridmask_rate):\n    AugParams = {\n        'd1' : 100,\n        'd2': 160,\n        'rotate' : 45,\n        'ratio' : 0.3\n    }\n    \n        \n    mask = GridMask(image_shape[0], image_shape[1], AugParams['d1'], AugParams['d2'], AugParams['rotate'], AugParams['ratio'])\n    if image_shape[-1] == 3:\n        mask = tf.concat([mask, mask, mask], axis=-1)\n        mask = tf.cast(mask,tf.float32)\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n    if P==1:\n        return image*mask\n    else:\n        return image\n\ndef gridmask_b0(img_batch, label_batch, img_size=224):\n    return apply_grid_mask(img_batch, (img_size, img_size, 3)), label_batch\n\ndef gridmask_b3(img_batch, label_batch, img_size=320):\n    return apply_grid_mask(img_batch, (img_size, img_size, 3)), label_batch\n\ndef gridmask_b5(img_batch, label_batch, img_size=468):\n    return apply_grid_mask(img_batch, (img_size, img_size, 3)), label_batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def random_rotate(image, angle, image_shape):\n    def get_rotation_mat_inv(angle):\n        # transform to radian\n        angle = math.pi * angle / 180\n        cos_val = tf.math.cos(angle)\n        sin_val = tf.math.sin(angle)\n        one = tf.constant([1], tf.float32)\n        zero = tf.constant([0], tf.float32)\n        rot_mat_inv = tf.concat([cos_val, sin_val, zero, -sin_val, cos_val, zero, zero, zero, one], axis=0)\n        rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])\n        return rot_mat_inv\n    angle = float(angle) * tf.random.normal([1],dtype='float32')\n    rot_mat_inv = get_rotation_mat_inv(angle)\n    return transform(image, rot_mat_inv, image_shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data, img_size=468):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    image = tf.image.resize(image, [img_size,img_size])\n    return image\n\n\ndef data_augment(image, label=None, seed = 2020):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image,seed=seed)\n    image = tf.image.random_flip_up_down(image,seed=seed)\n    image = tf.image.random_brightness(image,0.2)\n    image = tf.image.random_contrast(image,0.6,1.4)\n    image = tf.image.random_hue(image,0.07)\n    image = tf.image.random_saturation(image,0.5,1.5)\n        \n    if label == None :\n        return image\n    else :\n        return image, label ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#TRAINING_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH2 + '/tfrecords/train*')\n#TEST_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH2 + '/tfrecords/test*')\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH1 + '/train*')\n#TEST_FILENAMES = tf.io.gfile.glob('../input/siim-isic-melanoma-classification/tfrecords/test*')\n# VAL_FILENAMES = TRAINING_FILENAMES[int(0.9*len(TRAINING_FILENAMES)):]\n# TRAINING_FILENAMES = TRAINING_FILENAMES[:int(0.9*len(TRAINING_FILENAMES))]\nTEST_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH1 + '/test*')\n\n#TRAINING_FILENAMES = tf.io.gfile.glob('../input/512x512-melanoma-tfrecords-70k-images/train*')\n#TEST_FILENAMES = tf.io.gfile.glob('../input/512x512-melanoma-tfrecords-70k-images/test*')\n\nIMAGE_SIZE = [512, 512] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Read train tf Records :\ndef read_labeled_tfrecord(example, img_size=468):\n    \n    \n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n \n    \n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'], img_size)\n    idnum = example['image_name']\n    label = tf.cast(example['target'], tf.float32)\n    return image, label\n\n#Read test tf Records :\ndef read_unlabeled_tfrecord(example, img_size=468):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'], img_size)\n    idnum = example['image_name']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False, img_size=468):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(lambda x: read_labeled_tfrecord(x, img_size) if labeled else read_unlabeled_tfrecord(x, img_size), num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\n   \n    \ndef get_training_dataset(img_size=468, name='b0'):\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True, img_size=img_size)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048, seed=img_size)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)# prefetch next batch while training (autotune prefetch buffer size)\n#     if name == 'b0':\n#         dataset = dataset.map(cutmix_b0,   num_parallel_calls = AUTO) \n#         dataset = dataset.map(gridmask_b0, num_parallel_calls = AUTO) \n#     if name == 'b3':\n#         dataset = dataset.map(cutmix_b3,   num_parallel_calls = AUTO) \n#         dataset = dataset.map(gridmask_b3, num_parallel_calls = AUTO) \n#     if name == 'b5':\n#         dataset = dataset.map(cutmix_b5,   num_parallel_calls = AUTO) \n#         dataset = dataset.map(gridmask_b5, num_parallel_calls = AUTO) \n    return dataset   \n\ndef get_val_dataset(img_size=468):\n    dataset = load_dataset(VAL_FILENAMES, labeled=True, img_size=img_size)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048, seed=img_size)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n        \n    return dataset\n\ndef get_test_dataset(ordered=False,img_size=468,aug=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, img_size=img_size, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n        \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint('Dataset: {} training images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reduce_lr =  ReduceLROnPlateau(monitor = \"val_loss\", factor = 0.5, patience = 3,\n  verbose = 0, mode = \"auto\", epsilon = 1e-04, cooldown = 0,\n  min_lr = 1e-6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"es = EarlyStopping(monitor = \"val_los\" , verbose = 1 , mode = 'min' , patience = 10 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mc_b5 = ModelCheckpoint('best_model_b5.h5', monitor = 'loss' , mode = 'min', verbose = 1 , save_best_only = True)\nmc_b3 = ModelCheckpoint('best_model_b3.h5', monitor = 'loss' , mode = 'min', verbose = 1 , save_best_only = True)\nmc_b0 = ModelCheckpoint('best_model_b0.h5', monitor = 'loss' , mode = 'min', verbose = 1 , save_best_only = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"focal_loss = True\nlabel_smoothing = 0\nSWA = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model_generalized(name,trainable_layers=25,opt='adam',lr=0.001,img_size=468, auc=None):\n    if name == 'EfficientNetB7' :\n        base_model = efn.EfficientNetB7(weights='imagenet',\n                                        include_top = False,\n                                        input_shape=(img_size,img_size,3)\n                                       )\n    elif name == 'EfficientNetB5' :\n        base_model = efn.EfficientNetB5(weights='imagenet',\n                                        include_top = False,\n                                        input_shape=(img_size,img_size,3)\n                                       )\n    elif name == 'EfficientNetB3' :\n        base_model = efn.EfficientNetB3(weights='imagenet',\n                                        include_top = False,\n                                        input_shape=(img_size,img_size,3)\n                                       )\n    elif name == 'EfficientNetB0' :\n        base_model = efn.EfficientNetB0(weights='imagenet',\n                                        include_top = False,\n                                        input_shape=(img_size,img_size,3)\n                                       )    \n\n    base_model.trainable = True\n    for layer in base_model.layers[:-trainable_layers] :\n        layer.trainable = True\n    layer = base_model.output\n    layer = L.GlobalAveragePooling2D()(layer)\n    layer = L.Dense(img_size*2,activation=tf.keras.activations.softplus)(layer)\n    layer = L.Dropout(0.5)(layer)\n    layer = L.BatchNormalization()(layer)\n    predictions = L.Dense(nb_classes,activation='sigmoid')(layer)\n   # predictions = tf.cast(predictions,tf.float32)\n    model = Model(inputs = base_model.input, outputs=predictions)\n    \n    if focal_loss : \n        loss= tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO)\n    elif label_smoothing :\n        loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing)\n    else :\n        loss = 'binary_crossentropy'\n    \n    \n    if opt == 'RMSprop' :\n        opt = optimizers.RMSprop(learning_rate = lr)\n    elif SWA == True :\n        opt = tf.keras.optimizers.Adam(lr=1e-4) # roll back\n        opt = tfa.optimizers.SWA(opt)\n    else :\n        opt =  tf.keras.optimizers.Adam(lr=1e-4) # roll back\n\n    model.compile(optimizer=opt,loss=loss,metrics=['accuracy', auc])  \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    auc = tf.keras.metrics.AUC(name='auc')\n    model_b0 = get_model_generalized('EfficientNetB0', img_size=224, auc=auc)\n\nhistory = model_b0.fit(\n    get_training_dataset(img_size=224, name='b0'),\n    steps_per_epoch=STEPS_PER_EPOCH,\n#     validation_data=get_val_dataset(img_size=224),\n#     validation_steps=STEPS_PER_EPOCH // 10,\n    epochs=20,\n    callbacks = [reduce_lr, es, mc_b0],\n    #class_weight = classweights,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    auc = tf.keras.metrics.AUC(name='auc')\n    model_b3 = get_model_generalized('EfficientNetB3', img_size=320, auc=auc)\n\nhistory = model_b3.fit(\n    get_training_dataset(img_size=320, name='b3'),\n    steps_per_epoch=STEPS_PER_EPOCH,\n#     validation_data=get_val_dataset(img_size=320),\n#     validation_steps=STEPS_PER_EPOCH // 10,\n    epochs=20,\n    callbacks = [reduce_lr, es, mc_b3],\n    #class_weight = classweights,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    auc = tf.keras.metrics.AUC(name='auc')\n    model_b5 = get_model_generalized('EfficientNetB5', img_size=468, auc=auc)\n\nhistory = model_b5.fit(\n    get_training_dataset(img_size=468, name='b5'),\n    steps_per_epoch=STEPS_PER_EPOCH,\n#     validation_data=get_val_dataset(img_size=468),\n#     validation_steps=STEPS_PER_EPOCH // 10,\n    epochs=20,\n    callbacks = [reduce_lr, es, mc_b5],\n    #class_weight = classweights,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !git clone https://github.com/chentinghao/download_google_drive.git\n    \n# !python ./download_google_drive/download_gdrive.py 1HoHknBnHFhdKkQkeOkEyrRz_Ete47WMY best_model_b0_vloss=0.0162_vauc=0.9325.h5\n# !python ./download_google_drive/download_gdrive.py 1dNqwBxPVVDc2KQVtCunuRWYqYTrR48Hg best_model_b3_vacc=0.9159_vloss=0.0182_vauc=0.9332.h5\n# !python ./download_google_drive/download_gdrive.py 1h8T4M9gdxJLSbED2yMfUUFHdtC3-TwXL best_model_b5_vacc=0.9239_vloss=0.0165_vauc=0.9347.h5\n# !python ./download_google_drive/download_gdrive.py 1h8T4M9gdxJLSbED2yMfUUFHdtC3-TwXL best_model_b5_vacc=0.9239_vloss=0.0165_vauc=0.9347.h5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds_b5 = get_test_dataset(ordered=True, img_size=468)\ntest_ds_b3 = get_test_dataset(ordered=True, img_size=320)\ntest_ds_b0 = get_test_dataset(ordered=True, img_size=224)\n\n# models_name = ['best_model_b0_vloss=0.0162_vauc=0.9325.h5', \n#                'best_model_b3_vacc=0.9159_vloss=0.0182_vauc=0.9332.h5', \n#                'best_model_b5_vacc=0.9239_vloss=0.0165_vauc=0.9347.h5']\n\nmodels_name = ['best_model_b0.h5', \n               'best_model_b3.h5', \n               'best_model_b5.h5']\n\nfor name in models_name:\n    if name[12] == '5':\n        model_b5.load_weights(name)\n    if name[12] == '3':\n        model_b3.load_weights(name)\n    if name[12] == '0':\n        model_b0.load_weights(name)\n\nprint('Computing predictions...')\ntest_images_ds_b5 = test_ds_b5.map(lambda image, idnum: image)\ntest_images_ds_b3 = test_ds_b3.map(lambda image, idnum: image)\ntest_images_ds_b0 = test_ds_b0.map(lambda image, idnum: image)\n\nprobabilities_b5 = model_b5.predict(test_images_ds_b5)\nprobabilities_b3 = model_b3.predict(test_images_ds_b3)\nprobabilities_b0 = model_b0.predict(test_images_ds_b0)\n\nprobabilities = (probabilities_b0 + probabilities_b3 + probabilities_b5) / 3\n\nif label_smoothing :\n    probabilities = LabelSmoothing(probabilities,label_smoothing)\n    \nprobabilities_b5 =probabilities_b5.flatten()\nprobabilities_b3 =probabilities_b3.flatten()\nprobabilities_b0 =probabilities_b0.flatten()\nprobabilities =probabilities.flatten()\n\nprobabilities_list = [probabilities_b0, probabilities_b3, probabilities_b5]\n\nprint('Generating submission.csv file...')\ntest_ids_ds = test_ds_b0.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\npred_df_b5 = pd.DataFrame({'image_name': test_ids, 'target': probabilities_b5})\npred_df_b3 = pd.DataFrame({'image_name': test_ids, 'target': probabilities_b3})\npred_df_b0 = pd.DataFrame({'image_name': test_ids, 'target': probabilities_b0})\npred_df_mean = pd.DataFrame({'image_name': test_ids, 'target': probabilities})\n\nfor name in models_name:\n    if name[12] == '5':\n        pred_df_b5.to_csv('pred_df_b5_submission.csv', index=False)\n    if name[12] == '3':\n        pred_df_b3.to_csv('pred_df_b3_submission.csv', index=False)\n    if name[12] == '0':\n        pred_df_b0.to_csv('pred_df_b0_submission.csv', index=False)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df_mean.to_csv('pred_df_mean_submission_gamma-0.50.csv', index=False)\npred_df_mean.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df_mean.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df_mean[pred_df_mean['target'] > 0.5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ndel model_b5\ndel model_b3\ndel model_b0\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # GAMMA SEARCH\n# for gamma in range(40, 61, 3):\n#     gamma = gamma / 100\n    \n#     probabilities = probabilities - 0.5 + gamma\n#     pred_df_gamma = pd.DataFrame({'image_name': test_ids, 'target': probabilities})\n# #     sub_gamma = sub.merge(pred_df_gamma, on='image_name')\n#     pred_df_gamma.to_csv('pred_df_mean_submission_gamma-{}.csv'.format(round(gamma, 2)), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df_b5_0936 = pd.read_csv('../input/melanoma-dif-sub/pl_0.936.csv')\n\n# lets rank each prediction and then divide it by its max value to we have our predictions between 0 and 1\ndef rank_data(sub):\n    sub['target'] = sub['target'].rank() / sub['target'].rank().max()\n    return sub\n\npred_df_b5 = rank_data(pred_df_b5)\npred_df_b0 = rank_data(pred_df_b0)\npred_df_b3 = rank_data(pred_df_b3)\npred_df_b5_0936 = rank_data(pred_df_b5_0936)\npred_df_mean = rank_data(pred_df_mean)\n\npred_df_b5_0936.columns = ['image_name', 'target1']\npred_df_b5.columns = ['image_name', 'target2']\npred_df_mean.columns = ['image_name', 'target3']\npred_df_b3.columns = ['image_name', 'target4']\npred_df_b0.columns = ['image_name', 'target5']\n\nf_sub = pred_df_mean.merge(pred_df_b5_0936, on = 'image_name').merge(pred_df_b3, on = 'image_name').merge(pred_df_b0, on = 'image_name').merge(pred_df_b5, on = 'image_name')\nf_sub['target'] = f_sub['target1'] * 0.3 + f_sub['target2'] * 0.3 + f_sub['target3'] * 0.05 + f_sub['target4'] * 0.3 + f_sub['target5'] * 0.05\nf_sub = f_sub[['image_name', 'target']]\nf_sub.to_csv('blend_sub.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}