{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# General libraries\nimport os\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport random\nimport cv2\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom IPython.display import clear_output\nfrom math import pi\nfrom math import cos\nfrom math import floor\nfrom sklearn.model_selection import KFold\nfrom kaggle_datasets import KaggleDatasets\nimport albumentations as alb\nfrom sklearn.metrics import roc_auc_score\n\n# Deep learning libraries\nfrom tensorflow.keras import layers\nimport tensorflow.keras.backend as K\nfrom keras import models\nfrom keras import backend\nfrom keras.callbacks import Callback\nfrom keras.models import Model, Sequential\nfrom keras.layers import Input, Dense, Flatten, Dropout, BatchNormalization\nfrom keras.layers import Conv2D, SeparableConv2D, MaxPool2D, LeakyReLU, Activation\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\nimport tensorflow as tf, re, math\nfrom keras import optimizers\n\n!pip install -q efficientnet >> /dev/null\nimport efficientnet.tfkeras as efn\n\nDATA_SETS_2020 = {'train' : '../input/siim-isic-melanoma-classification/train.csv',\n                  'test' : '../input/siim-isic-melanoma-classification/test.csv',\n                  'sub' : '../input/siim-isic-melanoma-classification/sample_submission.csv'}\n\n# USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\nSEED = 13\n\n# NUMBER OF FOLDS. USE 3, 5, OR 15 \nFOLDS = 5\n\n# WHICH IMAGE SIZES TO LOAD EACH FOLD\n# CHOOSE 128, 256, 384, 512, 768\nIMG_SIZES = [384] * FOLDS\n\n# INCLUDE OLD COMP DATA?\nINC2019 = [True] * FOLDS\nINC2018_2017 = [True] * FOLDS\n\n# SEGMENTED DATASET?\nSEGMENTED = False\n\n# BATCH SIZE AND EPOCHS\nBATCH_SIZES = [32] * FOLDS\nEPOCHS = [15] * FOLDS\n\n# WHICH EFFICIENTNET\nEFF_NETS = [6] * FOLDS\n\n# MIN LEARNING RATE\nLR_MIN = 1e-6\n\n# TRAINING VERBOSE\nVERBOSE = False\n\n# WEIGHTS FOR FOLD MODELS WHEN PREDICTING TEST\nWGTS = [1/FOLDS] * FOLDS\n\n# AUGMENTATION PARAMETERS\nROT_ = 180.0\nSHR_ = 8.0\nHZOOM_ = 14.0\nWZOOM_ = 14.0\nHSHIFT_ = 14.0\nWSHIFT_ = 14.0\n\nADD_HAIR_PROB = 0.5\nADD_HAIR = True\nN_HAIRS = 10\n\nCUTOUT_PROB = 0.25\nCUTOUT = True\n\nSOLARIZE_PROB = 0.25\nEQUALIZE_PROB = 0.25\n\n# TEST TIME AUGMENTATION STEPS\nTTA = 11\n\n# TPU CONFIGURATION\nDEVICE = 'TPU'\nprint(\"connecting to TPU...\")\nif DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"_uuid":"ad191228-fab3-4f24-a14c-5ee2da828e99","_cell_guid":"0a1a8dce-4d35-4f1a-859c-028055e75134","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TFRecords augmentation","metadata":{}},{"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\ndef transform(image, DIM=256):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"https://www.kaggle.com/cdeotte/tfrecord-experiments-upsample-and-coarse-dropout","metadata":{}},{"cell_type":"code","source":"def dropout(image, DIM=256, PROBABILITY = 0.75, CT = 4, SZ = 0.2):\n    # input - one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n\n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1) < PROBABILITY, tf.int32)\n    if (P == 0)|(CT == 0)|(SZ == 0): return image\n\n    for k in range( CT ):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        # COMPUTE SQUARE \n        WIDTH = tf.cast( SZ*DIM,tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # DROPOUT IMAGE\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3]) \n        three = image[ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM,DIM,3])\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"https://www.kaggle.com/szacho/augmix-data-augmentation-on-tpu","metadata":{}},{"cell_type":"code","source":"def float_parameter(level, maxval):\n    return tf.cast((level) * maxval / 10., tf.float32)\n\ndef sample_level(n):\n    return tf.random.uniform(shape=[1], minval=0.1, maxval=n, dtype=tf.float32)\n\ndef solarize_add(image, level, probability=1.0):\n    P = tf.cast( tf.random.uniform([],0,1) < probability, tf.int32)\n    if P == 0: \n        return image\n    # For each pixel in the image less than threshold\n    # we add 'addition' amount to it and then clip the\n    # pixel value to be between 0 and 255. The value\n    # of 'addition' is between -128 and 128.\n    threshold = float_parameter(sample_level(level), 1)\n    addition = float_parameter(sample_level(level), 0.5)\n    rand_var = tf.random.uniform(shape=[], dtype=tf.float32)\n    addition = tf.cond(rand_var > 0.5, lambda: addition, lambda: -addition)\n\n    added_image = tf.cast(image, tf.float32) + addition\n    added_image = tf.cast(tf.clip_by_value(added_image, 0, 1), tf.float32)\n    return tf.where(image < threshold, added_image, image)\n\ndef equalize(image, probability=1.0):\n    P = tf.cast( tf.random.uniform([],0,1) < probability, tf.int32)\n    if P == 0: \n        return image\n    image = tf.cast(tf.math.scalar_mul(255, image), tf.uint8)\n\n    def scale_channel(im, c):\n        im = tf.cast(im[:, :, c], tf.int32)\n        # Compute the histogram of the image channel.\n        histo = tf.histogram_fixed_width(im, [0, 255], nbins=256)\n        # For the purposes of computing the step, filter out the nonzeros.\n        nonzero = tf.where(tf.not_equal(histo, 0))\n        nonzero_histo = tf.reshape(tf.gather(histo, nonzero), [-1])\n        step = (tf.reduce_sum(nonzero_histo) - nonzero_histo[-1]) // 255\n\n        def build_lut(histo, step):\n            # Compute the cumulative sum, shifting by step // 2\n            # and then normalization by step.\n            lut = (tf.cumsum(histo) + (step // 2)) // step\n            # Shift lut, prepending with 0.\n            lut = tf.concat([[0], lut[:-1]], 0)\n            # Clip the counts to be in range.  This is done\n            # in the C code for image.point.\n            return tf.clip_by_value(lut, 0, 255)\n\n        # If step is zero, return the original image.  Otherwise, build\n        # lut from the full histogram and step and then index from it.\n        result = tf.cond(tf.equal(step, 0),\n                        lambda: im,\n                        lambda: tf.gather(build_lut(histo, step), im))\n\n        return tf.cast(result, tf.uint8)\n\n    # Assumes RGB for now.  Scales each channel independently\n    # and then stacks the result.\n    s1 = scale_channel(image, 0)\n    s2 = scale_channel(image, 1)\n    s3 = scale_channel(image, 2)\n    image = tf.stack([s1, s2, s3], 2)\n\n    return tf.cast(tf.clip_by_value(tf.math.divide(image, 255), 0, 1), tf.float32)\n\ndef autocontrast(image, probability=1.0):\n    P = tf.cast( tf.random.uniform([],0,1) < probability, tf.int32)\n    if P == 0: \n        return image\n    image = tf.cast(tf.math.scalar_mul(255, image), tf.uint8)\n\n    def scale_channel(image):\n        # A possibly cheaper version can be done using cumsum/unique_with_counts\n        # over the histogram values, rather than iterating over the entire image.\n        # to compute mins and maxes.\n        lo = tf.cast(tf.reduce_min(image), tf.float32)\n        hi = tf.cast(tf.reduce_max(image), tf.float32)\n\n        # Scale the image, making the lowest value 0 and the highest value 255.\n        def scale_values(im):\n            scale = 255.0 / (hi - lo)\n            offset = -lo * scale\n            im = tf.cast(im, tf.float32) * scale + offset\n            im = tf.clip_by_value(im, 0.0, 255.0)\n            return tf.cast(im, tf.uint8)\n\n        result = tf.cond(hi > lo, lambda: scale_values(image), lambda: image)\n        return result\n\n    # Assumes RGB for now.  Scales each channel independently\n    # and then stacks the result.\n    s1 = scale_channel(image[:, :, 0])\n    s2 = scale_channel(image[:, :, 1])\n    s3 = scale_channel(image[:, :, 2])\n    image = tf.stack([s1, s2, s3], 2)\n    return tf.cast(tf.clip_by_value(tf.math.divide(image, 255), 0, 1), tf.float32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hair adding\n* Source: https://www.kaggle.com/graf10a/siim-data-augmentation-in-tf-hair-batch-affine","metadata":{}},{"cell_type":"code","source":"GCS_PATH={}\nGCS_PATH['hairs']=KaggleDatasets().get_gcs_path('melanoma-hairs')\nhair_images=tf.io.gfile.glob(GCS_PATH['hairs'] + '/*.png')\nhair_images_tf=tf.convert_to_tensor(hair_images)\n\ndef hair_aug(image, dim=384, n_max=6, probability=1.0):\n    P = tf.cast( tf.random.uniform([],0,1) < probability, tf.int32)\n    if P == 0: \n        return image\n    # Copy the input image, so it won't be changed\n    img=tf.identity(image) \n    # Randomly choose the number of hairs to augment (up to n_max)\n    n_hairs = tf.random.uniform(shape=[], maxval=tf.constant(n_max)+1, dtype=tf.int32)\n    \n    im_height=tf.shape(img)[1]\n    im_width=tf.shape(img)[0]\n    \n    if n_hairs == 0:\n        return img\n\n    for _ in tf.range(n_hairs):\n        # Read a random hair image\n        i=tf.random.uniform(shape=[], maxval=tf.shape(hair_images_tf)[0], \n                            dtype=tf.int32)\n        fname=hair_images_tf[i]\n\n        bits = tf.io.read_file(fname)\n        hair = tf.image.decode_jpeg(bits)\n        \n        scale=tf.cast(dim/256, dtype=tf.int32)\n        \n        # Rescale the hair image to the right size (256 -- original size)\n        new_width=scale*tf.shape(hair)[1]\n        new_height=scale*tf.shape(hair)[0]\n        hair = tf.image.resize(hair, [new_height, new_width])\n\n        \n        # Random flips of the hair image\n        hair = tf.image.random_flip_left_right(hair)\n        hair = tf.image.random_flip_up_down(hair)\n        # Random number of 90 degree rotations\n        n_rot=tf.random.uniform(shape=[], maxval=4,\n                                dtype=tf.int32)\n        hair = tf.image.rot90(hair, k=n_rot)\n        \n        h_height=tf.shape(hair)[0]\n        h_width=tf.shape(hair)[1]\n        \n        roi_h0 = tf.random.uniform(shape=[], maxval=im_height - h_height + 1, \n                                    dtype=tf.int32)\n        roi_w0 = tf.random.uniform(shape=[], maxval=im_width - h_width + 1, \n                                    dtype=tf.int32)\n\n\n        roi = img[roi_h0:(roi_h0 + h_height), roi_w0:(roi_w0 + h_width)]  \n\n        # Convert the hair image to grayscale \n        # (slice to remove the trainsparency channel)\n        hair2gray = tf.image.rgb_to_grayscale(hair[:, :, :3])\n\n        mask=hair2gray>10\n\n        img_bg = tf.multiply(roi, tf.cast(tf.image.grayscale_to_rgb(~mask),\n                                          dtype=tf.float32))\n        hair_fg = tf.multiply(tf.cast(hair[:, :, :3], dtype=tf.int32),\n                              tf.cast(tf.image.grayscale_to_rgb(mask), \n                                      dtype=tf.int32\n                                      )\n                             )\n\n        dst = tf.add(img_bg, tf.cast(hair_fg, dtype=tf.float32)/255)\n\n        paddings = tf.stack([\n            [roi_h0, im_height-(roi_h0 + h_height)], \n            [roi_w0, im_width-(roi_w0 + h_width)],\n            [0, 0]\n        ])\n\n        # Pad dst with zeros to make it the same shape as image.\n        dst_padded=tf.pad(dst, paddings, \"CONSTANT\")\n        # Create a boolean mask with zeros at the pixels of\n        # the augmentation segment and ones everywhere else\n        mask_img=tf.pad(tf.ones_like(dst), paddings, \"CONSTANT\")\n        mask_img=~tf.cast(mask_img, dtype=tf.bool)\n        # Make a hole in the original image at the location\n        # of the augmentation segment\n        img_hole=tf.multiply(img, tf.cast(mask_img, dtype=tf.float32))\n        # Inserting the augmentation segment in place of the hole\n        img=tf.add(img_hole, dst_padded)\n        \n    return img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TFRecords handling","metadata":{}},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n        'patient_id'                   : tf.io.FixedLenFeature([], tf.int64),\n        'sex'                          : tf.io.FixedLenFeature([], tf.int64),\n        'age_approx'                   : tf.io.FixedLenFeature([], tf.int64),\n        'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n        'diagnosis'                    : tf.io.FixedLenFeature([], tf.int64),\n        'target'                       : tf.io.FixedLenFeature([], tf.int64)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['target']\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['image_name'] if return_image_name else 0\n \ndef prepare_image(img, augment=True, dim=256):    \n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.cast(img, tf.float32) / 255.0\n    \n    if augment:\n        if ADD_HAIR == True: \n            img = hair_aug(img, n_max=N_HAIRS, probability=ADD_HAIR_PROB)\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_saturation(img, 0.7, 1.3)\n        img = tf.image.random_contrast(img, 0.8, 1.2)\n        img = tf.image.random_brightness(img, 0.2)\n        img = transform(img, DIM=dim)\n        if CUTOUT == True:\n            img = dropout(img, DIM=dim, PROBABILITY = CUTOUT_PROB, CT = 1, SZ = 0.25)\n        \n        img = solarize_add(img, level=1, probability=SOLARIZE_PROB)\n        img = equalize(img, probability=EQUALIZE_PROB)\n        img = autocontrast(img, probability=0.5)\n        \n    img = tf.reshape(img, [dim,dim, 3])\n    return img\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\ndef get_dataset(files, augment = False, shuffle = False, repeat = False, labeled=True, return_image_names=True, batch_size=16, dim=256):\n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), num_parallel_calls=AUTO)      \n    \n    ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment, dim=dim), imgname_or_label), num_parallel_calls=AUTO)\n    ds = ds.batch(batch_size * REPLICAS)\n    ds = ds.prefetch(AUTO)\n    return ds\n\ndef prepare_k_datasets():\n    GCS_PATH = [None]*FOLDS\n    \n    for i,k in enumerate(IMG_SIZES):\n        if SEGMENTED == False:\n            GCS_PATH[i] = KaggleDatasets().get_gcs_path('isic2020and2019-%ix%i-tfrec'%(k,k))\n        else:\n            GCS_PATH[i] = KaggleDatasets().get_gcs_path('isic2020and2019-%ix%i-tfrec-segmented'%(k,k))\n    \n    skf = KFold(n_splits=FOLDS, shuffle=True, random_state=SEED)\n    train_folds = []; valid_folds = []; test_folds = []\n    files_train = []; files_valid = []; files_test = []\n    \n    for k, (idxT, idxV) in enumerate(skf.split(np.arange(15))):\n        # CREATE TRAIN AND VALIDATION SUBSETS\n        files_train.append(tf.io.gfile.glob([GCS_PATH[k] + '/2020_train%.2i*.tfrec'%x for x in idxT]))\n        files_valid.append(tf.io.gfile.glob([GCS_PATH[k] + '/2020_train%.2i*.tfrec'%x for x in idxV]))\n        files_test.append(np.sort(np.array(tf.io.gfile.glob(GCS_PATH[k] + '/2020_test*.tfrec'))))\n        \n        if INC2019[k]:\n            files_train[k] += tf.io.gfile.glob([GCS_PATH[k] + '/2019_train%.2i*.tfrec'%x for x in idxT*2+1])\n            \n        if INC2018_2017[k]:\n            files_train[k] += tf.io.gfile.glob([GCS_PATH[k] + '/2019_train%.2i*.tfrec'%x for x in idxT*2])\n            \n        train_ds = get_dataset(files_train, augment=True, shuffle=True, repeat=True, dim=IMG_SIZES[k], batch_size = BATCH_SIZES[k])    \n        valid_ds = get_dataset(files_valid, augment=False, shuffle=False, repeat=False, dim=IMG_SIZES[k])\n        test_ds  = get_dataset(files_test, labeled=False, return_image_names=False, augment=False, shuffle=False, repeat=False, dim=IMG_SIZES[k])\n        \n        train_folds.append(train_ds)\n        valid_folds.append(valid_ds)\n        test_folds.append(test_ds)\n        \n    return train_folds, valid_folds, test_folds, files_train, files_valid, files_test\n    \ndef plot_batch(fold):\n    batch = fold.unbatch().batch(BATCH_SIZES[0])\n    batch = iter(batch)\n    batch = next(batch)\n    images = batch[0]\n    batch_size = images.shape[0]\n    \n    plt.figure(figsize=(30,10))\n    for i in range(0, batch_size):\n        image = images[i]\n        plt.subplot((int)(batch_size/8), 8, (int)(i+1))\n        plt.imshow(image)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_folds, valid_folds, test_folds, files_train, files_valid, files_test = prepare_k_datasets()\nprint('TRAIN BATCH')\nplot_batch(train_folds[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('VALID BATCH')\nplot_batch(valid_folds[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('TEST BATCH')\nplot_batch(test_folds[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model define and training","metadata":{}},{"cell_type":"code","source":"# Download models\nEFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\n        \ndef get_lr_callback_CA(batch_size=32, epochs=12, cycles=2, lr_max=1e-4, fold_number=0):   \n    def lrfn(epoch):\n        epochs_per_cycle = floor(epochs/cycles)\n        cos_inner = (pi * (epoch % epochs_per_cycle)) / (epochs_per_cycle)\n        lr = lr_max/2 * (cos(cos_inner) + 1)\n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    \n    rng = [i for i in range(epochs if EPOCHS[fold_number]<epochs else EPOCHS[fold_number])]\n    y = [lrfn(x) for x in rng]\n    plt.plot(rng, y)\n    plt.title(\"Learning rate schedule: {:.3g} to {:.3g}\".format(y[0], y[-1]))\n    plt.xlabel('Epoch')\n    plt.ylabel('Learning rate')\n    plt.tight_layout()\n    plt.show()\n    \n    return lr_callback        \n\ndef get_lr_callback(batch_size=8, epochs=12, fold_number=0):\n    lr_start   = LR_MIN\n    lr_max     = 0.00000125 * REPLICAS * batch_size\n    lr_min     = LR_MIN\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    \n    rng = [i for i in range(epochs if EPOCHS[fold_number]<epochs else EPOCHS[fold_number])]\n    y = [lrfn(x) for x in rng]\n    plt.plot(rng, y)\n    plt.title(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))\n    plt.xlabel('Epoch')\n    plt.ylabel('Learning rate')\n    plt.tight_layout()\n    plt.show()\n    \n    return lr_callback\n\ndef build_model(img_dims, ef):\n    inp = tf.keras.layers.Input(shape=(img_dims, img_dims, 3))\n    base = EFNS[ef](input_shape=(img_dims, img_dims, 3), weights='imagenet', include_top=False)\n    x = base(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = Dropout(0.3)(x)\n    x = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n    model = tf.keras.Model(inputs=inp, outputs=x)\n\n    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001), \n                  loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05), \n                  metrics=['AUC', 'binary_accuracy'])\n    return model\n\ndef train_model(epochs_number, train_gen, valid_gen, steps_per_epoch, fold_number):\n    early_stopping = tf.keras.callbacks.EarlyStopping(patience=6, restore_best_weights=True)\n    checkpoint = tf.keras.callbacks.ModelCheckpoint('./FOLD' + str(fold_number+1) + '_model.h5', monitor='val_loss', verbose=False, \n                                                    save_best_only=True, save_weights_only=True, mode='min', save_freq='epoch')\n\n    history = model.fit(train_gen,\n                        epochs=epochs_number,\n                        steps_per_epoch=steps_per_epoch,\n                        validation_data=valid_gen,\n                        callbacks=[get_lr_callback(BATCH_SIZES[fold_number], epochs=EPOCHS[fold_number], fold_number=fold_number), checkpoint, early_stopping],\n                        #callbacks=[get_lr_callback_CA(BATCH_SIZES[fold_number], epochs=EPOCHS[fold_number], cycles=2, lr_max=3e-4, fold_number=fold_number), checkpoint, early_stopping],\n                        verbose=VERBOSE)\n    return history","metadata":{"_uuid":"fdb6603a-abdd-4b0d-8bf5-9e25564f8427","_cell_guid":"aa9bae95-afe1-44a0-b3ec-89f31aa9b654","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = [] \npreds = np.zeros((count_data_items(files_test[0]), 1))\n\nfor k in range(0, FOLDS):\n    # DISPLAY FOLD INFO\n    if DEVICE=='TPU':\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print('FOLD', k+1)\n    print('Image Size %i with EfficientNet B%i and batch_size %i, INC2019=%i, INC(2018,2017)=%i'\n          % (IMG_SIZES[k],EFF_NETS[k],BATCH_SIZES[k]*REPLICAS, INC2019[k], INC2018_2017[k]))\n    print('Train/valid/test images count: ', count_data_items(files_train[k]), '/', count_data_items(files_valid[k]), '/', count_data_items(files_test[k]))\n\n    print('Building...')\n    K.clear_session()\n    with strategy.scope():\n        model = build_model(IMG_SIZES[k], EFF_NETS[k])\n\n    print('Training...')\n    history = train_model(epochs_number=EPOCHS[k], \n                          train_gen=train_folds[k], \n                          valid_gen=valid_folds[k], \n                          steps_per_epoch=count_data_items(files_train[k])/BATCH_SIZES[k]//REPLICAS, \n                          fold_number=k)\n\n    print('Best weights loading...')\n    model.load_weights('./FOLD' + str(k+1) + '_model.h5')\n    \n    # PREDICT OOF USING TTA\n    print('Predicting OOF with TTA...')\n    ds_valid = get_dataset(files_valid[k], labeled=False, return_image_names=False, augment=True, repeat=True, shuffle=False, dim=IMG_SIZES[k], batch_size=BATCH_SIZES[k] * 4)\n    ct_valid = count_data_items(files_valid[k]) \n    STEPS = TTA * ct_valid / BATCH_SIZES[k] / 4 / REPLICAS\n    pred = model.predict(ds_valid, steps=STEPS, verbose=VERBOSE)[:TTA * ct_valid,] \n    oof_pred.append( np.mean(pred.reshape((ct_valid, TTA), order='F'), axis=1) )\n    \n    # GET OOF TARGETS AND NAMES\n    ds_valid = get_dataset(files_valid[k], augment=False, repeat=False, dim=IMG_SIZES[k], labeled=True, return_image_names=True)\n    oof_tar.append( np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) )\n    oof_folds.append( np.ones_like(oof_tar[-1], dtype='int8') * k )\n    ds = get_dataset(files_valid[k], augment=False, repeat=False, dim=IMG_SIZES[k], labeled=False, return_image_names=True)\n    oof_names.append( np.array([img_name.numpy().decode(\"utf8\") for img, img_name in iter(ds.unbatch())]))\n    \n    # PREDICT TEST USING TTA\n    print('Predicting Test with TTA...')\n    ds_test = get_dataset(files_test[k], labeled=False, return_image_names=False, augment=True, repeat=True, shuffle=False, dim=IMG_SIZES[k], batch_size=BATCH_SIZES[k] * 4)\n    ct_test = count_data_items(files_test[k])\n    STEPS = TTA * ct_test / BATCH_SIZES[k] / 4 / REPLICAS\n    pred = model.predict(ds_test, steps=STEPS, verbose=VERBOSE)[:TTA * ct_test,] \n    preds[:,0] += np.mean(pred.reshape((ct_test, TTA), order='F'), axis=1) * WGTS[k]\n    \n    # REPORT RESULTS\n    auc = roc_auc_score(oof_tar[-1], oof_pred[-1])\n    oof_val.append(np.max( history.history['val_auc'] ))\n    print('OOF AUC without TTA = %.4f, with TTA = %.4f' % (oof_val[-1], auc))\n\n    print('History ploting...')\n    plt.figure(figsize=(15,5))\n    plt.plot(history.history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n    plt.plot(history.history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n    x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n    xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n    plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.4f'%y,size=14)\n    plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n    plt.legend(loc=2)\n    plt2 = plt.gca().twinx()\n    plt2.plot(history.history['loss'],'-o',label='Train Loss',color='#2ca02c') \n    plt2.plot(history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n    x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n    ydist = plt.ylim()[1] - plt.ylim()[0]\n    plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss\\n%.4f'%y,size=14)\n    plt.ylabel('Loss',size=14)\n    plt.title('FOLD %i - Image Size %i, EfficientNet B%i'%(k+1,IMG_SIZES[k],EFF_NETS[k]),size=18)\n    plt.legend(loc=3)\n    plt.show() ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculate OOF AUC","metadata":{}},{"cell_type":"code","source":"# COMPUTE OVERALL OOF AUC\noof = np.concatenate(oof_pred)\ntrue = np.concatenate(oof_tar)\nnames = np.concatenate(oof_names) \nfolds = np.concatenate(oof_folds)\nauc = roc_auc_score(true, oof)\nprint('Overall OOF AUC with TTA = %.4f' % auc)\n\n# SAVE OOF TO DISK\ndf_oof = pd.DataFrame(dict(image_name=names, target=true, pred=oof, fold=folds))\ndf_oof.to_csv('oof.csv', index=False)\ndf_oof.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission to Competition","metadata":{"_uuid":"bbd83280-2ec9-4381-876c-fa9e039b867f","_cell_guid":"e42c1b26-93d6-4bfd-ad74-6bc748899e1d","trusted":true}},{"cell_type":"code","source":"ds = get_dataset(files_test[0], augment=False, repeat=False, dim=IMG_SIZES[0], labeled=False, return_image_names=True)\nimage_names = np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(ds.unbatch())])\n\nsubmission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))\nsubmission = submission.sort_values('image_name') \nsubmission.to_csv('./submission.csv', index=False)\n\nprint('Max value: ', np.amax(preds[:,0]))\nprint('Min value: ', np.amin(preds[:,0]))\nprint('Mean: ', np.mean(preds[:,0]))\n\nplt.hist(submission.target,bins=100)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}