{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet >> /dev/null\n# !pip install -q squeezenet >> /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\n# from squeezenet import squeezenet\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = \"TPU\" #or \"GPU\"\n\n# USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\nSEED = 9226 #42\n\n# NUMBER OF FOLDS. USE 3, 5, OR 15 \nFOLDS = 5\n\n# WHICH IMAGE SIZES TO LOAD EACH FOLD\n# CHOOSE 128, 192, 256, 384, 512, 768 \n# IMG_SIZES = [128]*FOLDS\n# IMG_SIZES = [192]*FOLDS\n# IMG_SIZES = [256]*FOLDS\nIMG_SIZES = [384]*FOLDS\n# IMG_SIZES = [512]*FOLDS\n# IMG_SIZES = [768]*FOLDS\n\n# INCLUDE OLD COMP DATA? YES=1 NO=0\nINC2019 = [1]*FOLDS\nINC2018 = [1]*FOLDS\n\n# BATCH SIZE AND EPOCHS\nBATCH_SIZES = [32]*FOLDS\nEPOCHS = [5]*FOLDS\n\n# WHICH EFFICIENTNET B? TO USE\n# EFF_NETS = [6,6,6,6,6]\n\n# WEIGHTS FOR FOLD MODELS WHEN PREDICTING TEST\nWGTS = [1/FOLDS]*FOLDS\n\n# TEST TIME AUGMENTATION STEPS\nTTA = 1 #11","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS\nfor i,k in enumerate(IMG_SIZES):\n    GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))\n    GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))\nprint(GCS_PATH)\nprint(GCS_PATH2)\n# files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))\n# files_test  = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')))\n# print(files_train)\n# print(files_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROT_ = 180.0 #180.0\nSHR_ = 2.0\nHZOOM_ = 8.0\nWZOOM_ = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, DIM=256):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n        'patient_id'                   : tf.io.FixedLenFeature([], tf.int64),\n        'sex'                          : tf.io.FixedLenFeature([], tf.int64),\n        'age_approx'                   : tf.io.FixedLenFeature([], tf.int64),\n        'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n        'diagnosis'                    : tf.io.FixedLenFeature([], tf.int64),\n        'target'                       : tf.io.FixedLenFeature([], tf.int64)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['target']\n\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['image_name'] if return_image_name else 0\n\n \ndef prepare_image(img, augment=True, dim=256):    \n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.cast(img, tf.float32) / 255.0\n    \n    if augment:\n        img = transform(img,DIM=dim)\n        img = tf.image.random_flip_left_right(img)\n        #img = tf.image.random_hue(img, 0.01)\n#         img = tf.image.random_saturation(img, 0.7, 1.3)\n#         img = tf.image.random_contrast(img, 0.8, 1.2)\n        img = tf.image.random_brightness(img, 0.15)\n                      \n    img = tf.reshape(img, [dim,dim, 3])\n            \n    return img\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(files, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_names=True, batch_size=16, dim=256):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n                    num_parallel_calls=AUTO)      \n    \n    ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment, dim=dim), \n                                               imgname_or_label), \n                num_parallel_calls=AUTO)\n    \n    ds = ds.batch(batch_size * REPLICAS)\n    ds = ds.prefetch(AUTO)\n    return ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n# #         efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]\n\n# #*************Imaportamt Note*****************\n# #For 128: mp11-(7,7), 3; mp12-(5,5), 3--> 6x6\n# #For 192: mp11-(6,6), 3; mp12-(4,4), 3--> 10x10\n# #For 256: mp11-(4,4), 4; mp12-(6,6), 2-->14x14\n# #For 384: mp11-(4,4), 4; mp12-(6,6), 2-->22x22\n# #For 512: mp11-(4,4), 4; mp12-(6,6), 2-->30x30\n# #For 768: mp11-(4,4), 4; mp12-(6,6), 2-->46x46\n\n# fNoList = [32, 32, 32, 32, 32]\n# def eswa_cnn(dim=128, alphaVal=0.1):\n#     inp = tf.keras.layers.Input(shape=(dim,dim,3))\n#     c1 = tf.keras.layers.Conv2D(fNoList[0], kernel_size=(2,2), strides=2, padding='valid', name='Conv1')(inp)  #32x75x75\n#     c1 = tf.keras.layers.LeakyReLU(alpha=alphaVal, name='Conv1_LR')(c1)\n#     mp11 = tf.keras.layers.AveragePooling2D(pool_size=(4,4), strides=4, padding='valid', name='AP1_conv1')(c1)  #32x24x24...\n#     mp12 = tf.keras.layers.AveragePooling2D(pool_size=(6,6), strides=2, padding='valid', name='AP2_conv1')(mp11)  #32x8x8\n\n#     mpIn = tf.keras.layers.AveragePooling2D(pool_size=(2,2), strides=2, padding='valid', name='AP_ForCCW_Conv1')(inp)  #3x75x75\n#     mpIn2 = tf.keras.layers.AveragePooling2D(pool_size=(3,3), strides=1, padding='valid', name='AP_ForCCW_Conv3')(mpIn)  #3x73x73\n#     mpIn3 = tf.keras.layers.AveragePooling2D(pool_size=(3,3), strides=1, padding='valid', name='AP_ForCCW_Conv4')(mpIn2)  #3x71x71\n#     c1InCC = tf.keras.layers.concatenate([mpIn, c1], name='CCW_Conv1', axis=-1) #35x75x75\n\n#     c2 = tf.keras.layers.Conv2D(fNoList[1], kernel_size=(2,2), strides=1, padding='valid', name='Conv2')(c1InCC)  #32x74x74\n#     c2 = tf.keras.layers.LeakyReLU(alpha=alphaVal, name='Conv2_LR')(c2)\n#     c3 = tf.keras.layers.Conv2D(fNoList[2], kernel_size=(2,2), strides=1, padding='valid', name='Conv3')(c2)   #32x73x73\n#     c3 = tf.keras.layers.LeakyReLU(alpha=alphaVal, name='Conv3_LR')(c3)\n#     c3InCC = tf.keras.layers.concatenate([mpIn2, c3], name='CCW_Conv3', axis=-1) #35x73x73\n#     c4 = tf.keras.layers.Conv2D(fNoList[3], kernel_size=(3,3), strides=1, padding='valid', name='Conv4')(c3InCC)   #32x71x71\n#     c4 = tf.keras.layers.LeakyReLU(alpha=alphaVal, name='Conv4_LR')(c4)\n#     c4InCC = tf.keras.layers.concatenate([mpIn3, c4], name='CCW_Conv4', axis=-1) #35x71x71\n#     c5 = tf.keras.layers.Conv2D(fNoList[4], kernel_size=(6,6), strides=2, padding='valid', name='Conv5')(c4InCC)   #32x32x32\n#     c5 = tf.keras.layers.LeakyReLU(alpha=alphaVal, name='Conv5_LR')(c5)\n#     mp5 = tf.keras.layers.AveragePooling2D(pool_size=(8,8), strides=4, padding='valid', name='AP_Conv5')(c5)  #32x7x7\n\n#     merge_layer = tf.keras.layers.concatenate([mp12, mp5], name='CC_AP12wAP5', axis=-1)\n#     flat = tf.keras.layers.Flatten(name='Flat_For_HLIn')(merge_layer)\n#     fcl1 = tf.keras.layers.Dense(128, name='HL1')(flat)\n#     fcl1 = tf.keras.layers.LeakyReLU(alpha=alphaVal, name='HL1_Act_LR')(fcl1)\n#     output_layer = tf.keras.layers.Dense(1, activation='sigmoid', name='Output_Layer')(fcl1)\n    \n#     model = tf.keras.Model(inputs=inp, outputs=output_layer)\n#     return model\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def LeNet (dim=128):\n#     inp = tf.keras.layers.Input(shape=(dim,dim,3))\n#     x = tf.keras.layers.Conv2D(6, kernel_size=(5,5), strides=1, padding='valid', activation='tanh')(inp)\n#     x = tf.keras.layers.AveragePooling2D(pool_size=(2,2), strides=2, padding='valid')(x)\n#     x = tf.keras.layers.Conv2D(16, kernel_size=(5,5), strides=1, padding='valid',activation='tanh')(x)\n#     x = tf.keras.layers.AveragePooling2D(pool_size=(2,2), strides=2, padding='valid')(x)\n#     x = tf.keras.layers.Conv2D(120, kernel_size=(5,5), strides=1, padding='valid',activation='tanh')(x)\n#     x = tf.keras.layers.Flatten()(x)\n#     x = tf.keras.layers.Dense(84, activation='tanh')(x)\n#     x = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n#     model = tf.keras.Model(inputs=inp, outputs=x)\n#     return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def LeNet_mod (dim=128):\n#     inp = tf.keras.layers.Input(shape=(dim,dim,3))\n#     x = tf.keras.layers.Conv2D(32, kernel_size=(5,5), strides=1, padding='valid', activation='relu')(inp)\n#     x = tf.keras.layers.AveragePooling2D(pool_size=(2,2), strides=2, padding='valid')(x)\n#     x = tf.keras.layers.Conv2D(64, kernel_size=(5,5), strides=1, padding='valid',activation='relu')(x)\n#     x = tf.keras.layers.AveragePooling2D(pool_size=(2,2), strides=2, padding='valid')(x)\n#     x = tf.keras.layers.Conv2D(128, kernel_size=(3,3), strides=1, padding='valid',activation='relu')(x)\n#     x = tf.keras.layers.AveragePooling2D(pool_size=(3,3), strides=3, padding='valid')(x)\n#     x = tf.keras.layers.Conv2D(256, kernel_size=(3,3), strides=1, padding='valid',activation='relu')(x)\n#     x = tf.keras.layers.Conv2D(256, kernel_size=(3,3), strides=1, padding='valid',activation='relu')(x)\n#     x = tf.keras.layers.GlobalAveragePooling2D()(x)\n# #     x = tf.keras.layers.Flatten()(x)\n#     x = tf.keras.layers.Dense(128, activation='relu')(x)\n#     x = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n#     model = tf.keras.Model(inputs=inp, outputs=x)\n#     return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def fireMod(inputs, squeeze_depth, expand_depth):\n#     sqz1x1 = tf.keras.layers.Conv2D(filters=squeeze_depth, kernel_size=(1,1), strides=1, activation='relu')(inputs)\n#     expnd1x1 = tf.keras.layers.Conv2D(filters=expand_depth, kernel_size=(1,1), strides=1, activation='relu')(sqz1x1)\n#     expnd3x3 = tf.keras.layers.Conv2D(filters=expand_depth, kernel_size=(3,3), strides=1, padding='same', activation='relu')(sqz1x1)\n#     fireModOut = tf.keras.layers.concatenate([expnd1x1, expnd3x3], axis=-1)\n#     return fireModOut\n\n# def squeezeNet_v0(inputs):\n#     x = tf.keras.layers.Conv2D(filters=96, kernel_size=(7,7), strides=(2, 2), padding='valid', activation='relu')(inputs)\n#     x = tf.keras.layers.MaxPool2D(pool_size=(3, 3), strides=(2, 2), padding='valid')(x)\n#     fire2 = fireMod(x, 16, 64)\n#     fire3 = fireMod(fire2, 16, 64)\n#     fire4 = fireMod(fire3, 32, 128)\n#     x = tf.keras.layers.MaxPool2D(pool_size=(3, 3), strides=(2, 2), padding='valid')(fire4)\n#     fire5 = fireMod(x, 32, 128)\n#     fire6 = fireMod(fire5, 48, 192)\n#     fire7 = fireMod(fire6, 48, 192)\n#     fire8 = fireMod(fire7, 64, 256)\n#     x = tf.keras.layers.MaxPool2D(pool_size=(3, 3), strides=(2, 2), padding='valid')(fire8)\n#     fire9 = fireMod(x, 64, 256)\n#     x = tf.keras.layers.Conv2D(filters=1000, kernel_size=(1,1), strides=(1, 1), padding='valid', activation='relu')(fire9)\n#     return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def conv_block(ip, filters_num, bottleneck=True):\n#     x = tf.keras.layers.Activation('relu')(ip)\n#     if bottleneck:\n#         inter_channel = filters_num * 4\n#         x = tf.keras.layers.Conv2D(filters=inter_channel, kernel_size=(1, 1), padding='same')(x)\n#         x = tf.keras.layers.Activation('relu')(x)\n#         x = tf.keras.layers.Conv2D(filters=filters_num, kernel_size=(3, 3), padding='same')(x)\n#     return x\n\n# def denseBlock(inputs, layers_num, filters_num, growth_rate, bottleneck=True, grow_filters_num=True):\n#     x = inputs\n#     for i in range(layers_num):\n#         cb = conv_block(x, growth_rate, bottleneck)\n#         x = tf.keras.layers.concatenate([x, cb], axis=-1)\n#         if grow_filters_num:\n#                 filters_num += growth_rate\n#     return x, filters_num\n\n# def transBlock(ip, filters_num, compression=1.0, transition_pooling='max'):\n#     x = tf.keras.layers.Activation('relu')(ip)\n#     x = tf.keras.layers.Conv2D(filters=int(filters_num * compression), kernel_size=(1, 1), padding='same')(x)\n#     if transition_pooling == 'avg':\n#         x = tf.keras.layers.AveragePooling2D((2, 2), strides=(2, 2))(x)\n#     elif transition_pooling == 'max':\n#         x = tf.keras.layers.MaxPool2D((2, 2), strides=(2, 2))(x)\n#     return x\n\n# def denseNet(inputs, depth=40, nb_dense_block=3, growth_rate=12, nb_filter=-1,\n#                        nb_layers_per_block=-1, bottleneck=True, subsample_initial_block=True,\n#                        reduction=0.0, transition_pooling='avg'):\n#     # layers in each dense block\n#     if type(nb_layers_per_block) is list or type(nb_layers_per_block) is tuple:\n#         nb_layers = list(nb_layers_per_block)  # Convert tuple to list\n#         if len(nb_layers) != (nb_dense_block):\n#             raise ValueError('If `nb_dense_block` is a list, its length must match '\n#                             'the number of layers provided by `nb_layers`.')\n#         final_nb_layer = nb_layers[-1]\n#         nb_layers = nb_layers[:-1]\n#     else:\n#         if nb_layers_per_block == -1:\n#             assert (depth - 4) % 3 == 0, 'Depth must be 3 N + 4 if nb_layers_per_block == -1'\n#             count = int((depth - 4) / 3)\n#             if bottleneck:\n#                 count = count // 2\n#             nb_layers = [count for _ in range(nb_dense_block)]\n#             final_nb_layer = count\n#         else:\n#             final_nb_layer = nb_layers_per_block\n#             nb_layers = [nb_layers_per_block] * nb_dense_block\n    \n#     # compute initial nb_filter if -1, else accept users initial nb_filter\n#     if nb_filter <= 0:\n#         nb_filter = 2 * growth_rate\n\n#     # compute compression factor\n#     compression = 1.0 - reduction\n\n#     # Initial convolution\n#     if subsample_initial_block:\n#         initial_kernel = (7, 7)\n#         initial_strides = (2, 2)\n#     else:\n#         initial_kernel = (3, 3)\n#         initial_strides = (1, 1)\n\n#     x = tf.keras.layers.Conv2D(filters=nb_filter, kernel_size=initial_kernel, padding='same', strides=initial_strides)(inputs)\n\n#     if subsample_initial_block:\n#         x = tf.keras.layers.Activation('relu')(x)\n#         x = tf.keras.layers.MaxPool2D((3, 3), strides=(2, 2), padding='same')(x)\n\n#     # Add dense blocks\n#     for block_idx in range(nb_dense_block - 1):\n#         x, nb_filter = denseBlock(x, nb_layers[block_idx], nb_filter, growth_rate, bottleneck=bottleneck)\n#         # add transition_block\n#         x = transBlock(x, nb_filter, compression=compression, transition_pooling=transition_pooling)\n#         nb_filter = int(nb_filter * compression)\n\n#     # The last dense_block does not have a transition_block\n#     x, nb_filter = denseBlock(x, final_nb_layer, nb_filter, growth_rate, bottleneck=bottleneck)\n#     x = tf.keras.layers.Activation('relu')(x)\n#     return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #************** For FusedCNN **************\n# # import tensorflow as tf\n\n# if IMG_SIZES[0]==128:\n#     poolSize1, stride1 = (5,5), 5\n#     poolSize2, stride2 = (5,5), 2\n# elif IMG_SIZES[0]==192:\n#     poolSize1, stride1 = (6,6), 4\n#     poolSize2, stride2 = (3,3), 2\n# elif IMG_SIZES[0]==256:\n#     poolSize1, stride1 = (7,7), 4\n#     poolSize2, stride2 = (3,3), 2\n# elif IMG_SIZES[0]==384:\n#     poolSize1, stride1 = (9,9), 6\n#     poolSize2, stride2 = (4,4), 3\n# elif IMG_SIZES[0]==512:\n#     poolSize1, stride1 = (9,9), 8\n#     poolSize2, stride2 = (4,4), 4\n# elif IMG_SIZES[0]==768:\n#     poolSize1, stride1 = (10,10), 6\n#     poolSize2, stride2 = (5,5), 3\n    \n# def fusedCNN(inputs):\n#     c1 = tf.keras.layers.Conv2D(96, kernel_size=(11,11), strides=2, padding='valid', activation='relu', name='Conv1')(inputs)  #96x55x55\n#     mp1 = tf.keras.layers.MaxPool2D(pool_size=(3,3), strides=2, padding='valid', name='MaxPool1_conv1')(c1)  #96x27x27\n#     ap1 = tf.keras.layers.AveragePooling2D(pool_size=poolSize1, strides=stride1, padding='valid', name='AvgPool1_conv1')(mp1)  #96x6x6\n    \n# #     BN1 = BatchNormalization()(mp1) #96x27x27\n#     c2 = tf.keras.layers.Conv2D(256, kernel_size=(5,5), strides=1, padding='same', activation='relu', name='Conv2')(mp1)  #96x27x27\n#     mp2 = tf.keras.layers.MaxPool2D(pool_size=(3,3), strides=2, padding='valid', name='MaxPool1_conv2')(c2)  #256x13x13\n#     ap2 = tf.keras.layers.AveragePooling2D(pool_size=poolSize2, strides=stride2, padding='valid', name='AvgPool1_conv2')(mp2)  #256x6x6\n    \n# #     BN2 = BatchNormalization()(mp2) #256x13x13\n#     c3 = tf.keras.layers.Conv2D(384, kernel_size=(3,3), strides=1, padding='same', activation='relu', name='Conv3')(mp2)   #256x13x13\n#     c4 = tf.keras.layers.Conv2D(384, kernel_size=(3,3), strides=1, padding='same', activation='relu', name='Conv4')(c3)   #384x13x13\n#     c5 = tf.keras.layers.Conv2D(256, kernel_size=(3,3), strides=1, padding='same', activation='relu', name='Conv5')(c4)   #256x13x13\n#     mp5 = tf.keras.layers.MaxPool2D(pool_size=poolSize2, strides=stride2, padding='valid', name='MaxPool_Conv5')(c5)  #256x6x6\n    \n#     merge_layer = tf.keras.layers.concatenate([ap1, ap2, mp5], name='CC_AP1AP2mp5', axis=-1)\n#     return merge_layer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def identity_block(input_tensor, kernel_size, filters, stage, block):\n#     \"\"\"The identity block is the block that has no conv layer at shortcut.\n\n#     # Arguments\n#         input_tensor: input tensor\n#         kernel_size: defualt 3, the kernel size of middle conv layer at main path\n#         filters: list of integers, the filterss of 3 conv layer at main path\n#         stage: integer, current stage label, used for generating layer names\n#         block: 'a','b'..., current block label, used for generating layer names\n\n#     # Returns\n#         Output tensor for the block.\n#     \"\"\"\n#     filters1, filters2, filters3 = filters\n#     if K.image_data_format() == 'channels_last':\n#         bn_axis = 3\n#     else:\n#         bn_axis = 1\n#     conv_name_base = 'res' + str(stage) + block + '_branch'\n#     bn_name_base = 'bn' + str(stage) + block + '_branch'\n#     x = tf.keras.layers.Conv2D(filters1, (1, 1), name=conv_name_base + '2a')(input_tensor)\n# #     x = tf.keras.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)\n#     x = tf.keras.layers.Activation('relu')(x)\n#     x = tf.keras.layers.Conv2D(filters2, kernel_size, padding='same', name=conv_name_base + '2b')(x)\n# #     x = tf.keras.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)\n#     x = tf.keras.layers.Activation('relu')(x)\n#     x = tf.keras.layers.Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x)\n# #     x = tf.keras.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)\n#     x = tf.keras.layers.add([x, input_tensor])\n#     x = tf.keras.layers.Activation('relu')(x)\n#     return x\n\n# def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)):\n#     \"\"\"conv_block is the block that has a conv layer at shortcut\n\n#     # Arguments\n#         input_tensor: input tensor\n#         kernel_size: defualt 3, the kernel size of middle conv layer at main path\n#         filters: list of integers, the filterss of 3 conv layer at main path\n#         stage: integer, current stage label, used for generating layer names\n#         block: 'a','b'..., current block label, used for generating layer names\n\n#     # Returns\n#         Output tensor for the block.\n\n#     Note that from stage 3, the first conv layer at main path is with strides=(2,2)\n#     And the shortcut should have strides=(2,2) as well\n#     \"\"\"\n#     filters1, filters2, filters3 = filters\n#     if K.image_data_format() == 'channels_last':\n#         bn_axis = 3\n#     else:\n#         bn_axis = 1\n#     conv_name_base = 'res' + str(stage) + block + '_branch'\n#     bn_name_base = 'bn' + str(stage) + block + '_branch'\n#     x = tf.keras.layers.Conv2D(filters1, (1, 1), strides=strides, name=conv_name_base + '2a')(input_tensor)\n# #     x = tf.keras.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)\n#     x = tf.keras.layers.Activation('relu')(x)\n#     x = tf.keras.layers.Conv2D(filters2, kernel_size, padding='same', name=conv_name_base + '2b')(x)\n# #     x = tf.keras.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)\n#     x = tf.keras.layers.Activation('relu')(x)\n#     x = tf.keras.layers.Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x)\n# #     x = tf.keras.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)\n#     shortcut = tf.keras.layers.Conv2D(filters3, (1, 1), strides=strides, name=conv_name_base + '1')(input_tensor)\n# #     shortcut = tf.keras.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut)\n#     x = tf.keras.layers.add([x, shortcut])\n#     x = tf.keras.layers.Activation('relu')(x)\n#     return x\n\n# def ResNet32(input_img):\n#     if K.image_data_format() == 'channels_last':\n#         bn_axis = 3\n#     else:\n\n#         bn_axis = 1\n#     x = tf.keras.layers.ZeroPadding2D((3, 3))(input_img)\n#     x = tf.keras.layers.Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(x)\n# #     x = tf.keras.layers.BatchNormalization(axis=bn_axis, name='bn_conv1')(x)\n#     x = tf.keras.layers.Activation('relu')(x)\n#     x = tf.keras.layers.MaxPooling2D((3, 3), strides=(2, 2))(x)\n#     x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))\n#     x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')\n#     x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')\n#     x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')\n#     x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')\n#     x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')\n#     x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')\n#     x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')\n#     x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')\n#     x = tf.keras.layers.AveragePooling2D((4, 4), name='avg_pool')(x)\n#     return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def inceptionNet_v3(input_img, activation='relu'):\n#     if activation == 'relu':\n#         actFunc = tf.nn.relu\n#     elif activation == 'swish':\n#         actFunc = tf.nn.swish\n#     elif activation == 'sigmoid':\n#         actFunc = tf.nn.sigmoid\n#     elif activation == 'tanh':\n#         actFunc = tf.nn.tanh\n  \n#     x = tf.keras.layers.Conv2D(32, (3, 3), strides=(2, 2), use_bias=False, name='block1_conv1')(input_img)\n#     x = tf.keras.layers.BatchNormalization(name='block1_conv1_bn')(x)\n#     x = tf.keras.layers.Activation(actFunc, name='block1_conv1_act_' + activation)(x)\n#     x = tf.keras.layers.Conv2D(64, (3, 3), use_bias=False, name='block1_conv2')(x)\n#     x = tf.keras.layers.BatchNormalization(name='block1_conv2_bn')(x)\n#     x = tf.keras.layers.Activation(actFunc, name='block1_conv2_act_' + activation)(x)\n\n#     residual = tf.keras.layers.Conv2D(128, (1, 1), strides=(2, 2), padding='same', use_bias=False)(x)\n#     residual = tf.keras.layers.BatchNormalization()(residual)\n\n#     x = tf.keras.layers.SeparableConv2D(128, (3, 3), padding='same', use_bias=False, name='block2_sepconv1')(x)\n#     x = tf.keras.layers.BatchNormalization(name='block2_sepconv1_bn')(x)\n#     x = tf.keras.layers.Activation(actFunc, name='block2_sepconv2_act_' + activation)(x)\n#     x = tf.keras.layers.SeparableConv2D(128, (3, 3), padding='same', use_bias=False, name='block2_sepconv2')(x)\n#     x = tf.keras.layers.BatchNormalization(name='block2_sepconv2_bn')(x)\n#     x = tf.keras.layers.MaxPool2D((3, 3), strides=(2, 2), padding='same', name='block2_pool')(x)\n#     x = tf.keras.layers.add([x, residual])\n\n#     residual = tf.keras.layers.Conv2D(256, (1, 1), strides=(2, 2), padding='same', use_bias=False)(x)\n#     residual = tf.keras.layers.BatchNormalization()(residual)\n\n#     x = tf.keras.layers.Activation(actFunc, name='block3_sepconv1_act_' + activation)(x)\n#     x = tf.keras.layers.SeparableConv2D(256, (3, 3), padding='same', use_bias=False, name='block3_sepconv1')(x)\n#     x = tf.keras.layers.BatchNormalization(name='block3_sepconv1_bn')(x)\n#     x = tf.keras.layers.Activation(actFunc, name='block3_sepconv2_act_' + activation)(x)\n#     x = tf.keras.layers.SeparableConv2D(256, (3, 3), padding='same', use_bias=False, name='block3_sepconv2')(x)\n#     x = tf.keras.layers.BatchNormalization(name='block3_sepconv2_bn')(x)\n#     x = tf.keras.layers.MaxPool2D((3, 3), strides=(2, 2), padding='same', name='block3_pool')(x)\n#     x = tf.keras.layers.add([x, residual])\n\n#     residual = tf.keras.layers.Conv2D(728, (1, 1), strides=(2, 2), padding='same', use_bias=False)(x)\n#     residual = tf.keras.layers.BatchNormalization()(residual)\n\n#     x = tf.keras.layers.Activation(actFunc, name='block4_sepconv1_act_' + activation)(x)\n#     x = tf.keras.layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name='block4_sepconv1')(x)\n#     x = tf.keras.layers.BatchNormalization(name='block4_sepconv1_bn')(x)\n#     x = tf.keras.layers.Activation(actFunc, name='block4_sepconv2_act_' + activation)(x)\n#     x = tf.keras.layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name='block4_sepconv2')(x)\n#     x = tf.keras.layers.BatchNormalization(name='block4_sepconv2_bn')(x)\n#     x = tf.keras.layers.MaxPool2D((3, 3), strides=(2, 2), padding='same', name='block4_pool')(x)\n#     x = tf.keras.layers.add([x, residual])\n  \n#     for i in range(8):\n#         residual = x\n#         prefix = 'block' + str(i + 5)\n#         x = tf.keras.layers.Activation(actFunc, name=prefix + '_sepconv1_act_' + activation)(x)\n#         x = tf.keras.layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv1')(x)\n#         x = tf.keras.layers.BatchNormalization(name=prefix + '_sepconv1_bn')(x)\n#         x = tf.keras.layers.Activation(actFunc, name=prefix + '_sepconv2_act_' + activation)(x)\n#         x = tf.keras.layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv2')(x)\n#         x = tf.keras.layers.BatchNormalization(name=prefix + '_sepconv2_bn')(x)\n#         x = tf.keras.layers.Activation(actFunc, name=prefix + '_sepconv3_act_' + activation)(x)\n#         x = tf.keras.layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv3')(x)\n#         x = tf.keras.layers.BatchNormalization(name=prefix + '_sepconv3_bn')(x)\n#         x = tf.keras.layers.add([x, residual])\n\n#     residual = tf.keras.layers.Conv2D(1024, (1, 1), strides=(2, 2), padding='same', use_bias=False)(x)\n#     residual = tf.keras.layers.BatchNormalization()(residual)\n\n#     x = tf.keras.layers.Activation(actFunc, name='block13_sepconv1_act_' + activation)(x)\n#     x = tf.keras.layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name='block13_sepconv1')(x)\n#     x = tf.keras.layers.BatchNormalization(name='block13_sepconv1_bn')(x)\n#     x = tf.keras.layers.Activation(actFunc, name='block13_sepconv2_act_' + activation)(x)\n#     x = tf.keras.layers.SeparableConv2D(1024, (3, 3), padding='same', use_bias=False, name='block13_sepconv2')(x)\n#     x = tf.keras.layers.BatchNormalization(name='block13_sepconv2_bn')(x)\n#     x = tf.keras.layers.MaxPool2D((3, 3), strides=(2, 2), padding='same', name='block13_pool')(x)\n#     x = tf.keras.layers.add([x, residual])\n\n#     x = tf.keras.layers.SeparableConv2D(1536, (3, 3), padding='same', use_bias=False, name='block14_sepconv1')(x)\n#     x = tf.keras.layers.BatchNormalization(name='block14_sepconv1_bn')(x)\n#     x = tf.keras.layers.Activation(actFunc, name='block14_sepconv1_act_' + activation)(x)\n#     x = tf.keras.layers.SeparableConv2D(2048, (3, 3), padding='same', use_bias=False, name='block14_sepconv2')(x)\n#     x = tf.keras.layers.BatchNormalization(name='block14_sepconv2_bn')(x)\n#     x = tf.keras.layers.Activation(actFunc, name='block14_sepconv2_act_' + activation)(x)\n#     return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\n\ndef build_model(dim=128, ef=7):\n    inp = tf.keras.layers.Input(shape=(dim,dim,3))\n    base = EFNS[ef](input_shape=(dim,dim,3),weights='imagenet',include_top=False)\n    x = base(inp)\n#     x = tf.keras.layers.GlobalAveragePooling2D()(x)\n#     x = tf.keras.layers.Dense(1,activation='sigmoid')(x)\n\n#     model = eswa_cnn(dim)\n#     x = squeezeNet_v0(inp)\n#     x = denseNet(inp)\n#     x = fusedCNN(inp)\n#     x = ResNet32(inp)\n#     x = inceptionNet_v3(inp, 'swish')\n\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(1,activation='sigmoid')(x)\n    model = tf.keras.Model(inputs=inp, outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.0001)\n    loss = tf.keras.losses.BinaryCrossentropy(from_logits=False, label_smoothing=0) \n    model.compile(optimizer=opt,loss=loss,metrics=['accuracy'])\n    return model\n\nmodel = build_model(dim=IMG_SIZES[0])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model_name = 'DenseNet40'\n# model_name = 'SqueezeNet_v0'\n# model_name = 'FusedCNN'\n# model_name = 'ResNet32'\nmodel_name = 'EfficientNetB7'\n# model_name = 'InceptionNet_v3_swish'\n\ntraining_plat = 'Kaggle_' + DEVICE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 1\nDISPLAY_PLOT = False #True\n\nskf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)\noof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = [] \n\nfiles_test  = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')))\npreds = np.zeros((count_data_items(files_test),1))\n\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(15))):\n    \n    # DISPLAY FOLD INFO\n    if DEVICE=='TPU':\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print('#'*25); print('#### FOLD',fold+1)\n#     print('#### Image Size %i with EfficientNet B%i and batch_size %i'%\n#           (IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))\n    print('#### Image Size %i with %s and batch_size %i'%\n          (IMG_SIZES[fold], model_name, BATCH_SIZES[fold]*REPLICAS))\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])\n    if INC2019[fold]:\n        files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])\n        print('#### Using 2019 external data')\n    if INC2018[fold]:\n        files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])\n        print('#### Using 2018+2017 external data')\n    np.random.shuffle(files_train); print('#'*25)\n    files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])\n    files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))\n    \n#     print('Train Files: ', files_train)\n#     print('Valid Files: ', files_valid)\n#     print('Test Files: ', files_test)\n#     print('Index Train: ', idxT)\n#     print('Index Valid: ', idxV)\n    \n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():\n        model = build_model(dim=IMG_SIZES[fold]) #,ef=EFF_NETS[fold])\n        \n    # SAVE BEST MODEL EACH FOLD\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i-%i-bs%i-ep%i-%s-%s-weight.h5'%(fold,IMG_SIZES[fold],BATCH_SIZES[fold],EPOCHS[fold],model_name,training_plat), monitor='val_loss', verbose=1, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n   \n    # TRAIN\n    print('Training...')\n    earlyStop = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", verbose=1, mode=\"min\", patience=5) #************ Check\n    history = model.fit(\n        get_dataset(files_train, augment=True, shuffle=True, repeat=True,\n                dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]), \n        epochs=EPOCHS[fold], callbacks = [sv, earlyStop],#get_lr_callback(BATCH_SIZES[fold])], \n        steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,\n        validation_data=get_dataset(files_valid,augment=False,shuffle=False,\n                repeat=False,dim=IMG_SIZES[fold]), #class_weight = {0:1,1:2},\n        verbose=VERBOSE\n    )\n    \n    print('Loading best model...')\n    model.load_weights('fold-%i-%i-bs%i-ep%i-%s-%s-weight.h5'%(fold,IMG_SIZES[fold],BATCH_SIZES[fold],EPOCHS[fold],model_name,training_plat))\n    \n#     # PREDICT OOF USING TTA\n#     print('Predicting OOF with TTA...')\n#     ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,\n#             repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n#     ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS\n#     pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,] \n#     oof_pred.append( np.mean(pred.reshape((ct_valid,TTA),order='F'),axis=1) )                 \n#     #oof_pred.append(model.predict(get_dataset(files_valid,dim=IMG_SIZES[fold]),verbose=1))\n    \n#     # GET OOF TARGETS AND NAMES\n#     ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n#             labeled=True, return_image_names=True)\n#     oof_tar.append( np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) )\n#     oof_folds.append( np.ones_like(oof_tar[-1],dtype='int8')*fold )\n#     ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n#                 labeled=False, return_image_names=True)\n#     oof_names.append( np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(ds.unbatch())]))\n    \n    # PREDICT TEST USING TTA\n    print('Predicting Test with TTA...')\n    ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n            repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n    ct_test = count_data_items(files_test);\n    STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS\n    pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,] \n    preds[:,0] += np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1) * WGTS[fold]\n    print(preds)\n    \n    \n    # REPORT RESULTS\n#     auc = roc_auc_score(oof_tar[-1],oof_pred[-1])\n#     oof_val.append(np.max( history.history['val_auc'] ))\n#     print('#### FOLD %i OOF AUC without TTA = %.3f, with TTA = %.3f'%(fold+1,oof_val[-1],auc))\n    \n    # PLOT TRAINING\n    if DISPLAY_PLOT:\n        plt.figure(figsize=(15,5))\n        plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n        plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n        x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n        plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        plt2 = plt.gca().twinx()\n        plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        plt.ylabel('Loss',size=14)\n        plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'%\n                (fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)\n        plt.legend(loc=3)\n        plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #************** Only For Test Data ***************#\n\n# VERBOSE = 1\n# files_test  = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')))\n# preds = np.zeros((count_data_items(files_test),1))\n\n# for pti in range(FOLDS):\n#     # DISPLAY FOLD INFO\n#     if DEVICE=='TPU':\n#         if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n#     print('#'*25); print('#### FOLD',pti+1)\n# #     print('#### Image Size %i with EfficientNet B%i and batch_size %i'%\n# #           (IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))\n#     print('#### Image Size %i with %s and batch_size %i'%\n#           (IMG_SIZES[pti], model_name, BATCH_SIZES[pti]*REPLICAS))\n    \n# #     files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[pti] + '/test*.tfrec')))\n    \n#     # BUILD MODEL\n#     K.clear_session()\n#     with strategy.scope():\n#         model = build_model(dim=IMG_SIZES[pti]) #,ef=EFF_NETS[fold])\n    \n#     print('Loading best model...')\n#     try:\n#         model.load_weights('fold-%i-%i-bs%i-ep%i-%s-%s-weight.h5'%(pti,IMG_SIZES[pti],BATCH_SIZES[pti],EPOCHS[pti],model_name,training_plat))\n#     except:\n#         print(\"Something went wrong when writing to the file\")\n#     finally:\n#         dirName = '/kaggle/input/weights-densenet40-768-5fold/'\n#         model.load_weights(dirName + 'fold-%i-%i-bs%i-ep%i-%s-%s-weight.h5'%(pti,IMG_SIZES[pti],BATCH_SIZES[pti],EPOCHS[pti],model_name,training_plat))\n        \n\n#     # PREDICT TEST USING TTA\n#     print('Predicting Test with TTA...')\n#     ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n#             repeat=True,shuffle=False,dim=IMG_SIZES[pti],batch_size=BATCH_SIZES[pti]*4)\n#     ct_test = count_data_items(files_test);\n#     STEPS = TTA * ct_test/BATCH_SIZES[pti]/4/REPLICAS\n#     pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,] \n#     preds[:,0] += np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1) * WGTS[pti]\n#     print(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[0],\n                 labeled=False, return_image_names=True)\n\nimage_names = np.array([img_name.numpy().decode(\"utf-8\") \n                        for img, img_name in iter(ds.unbatch())])\nprint(image_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))\nsubmission = submission.sort_values('image_name') \nprint(submission)\n# submission.to_csv('/kaggle/working/test_out_%s_%s_%i_%i_171820_GPU.csv'%(training_plat, model_name, IMG_SIZES[fold], FOLDS), index=False)\nsubmission.to_csv('/kaggle/working/test_out_%s_%s_%i_bs%i_ep%i_%i_17181920.csv'%(training_plat, model_name, IMG_SIZES[0], BATCH_SIZES[0], EPOCHS[0], FOLDS), index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(submission.target,bins=100)\nplt.show()\n\nprint('Possible Melanomas: ', len(submission.target[submission.target>0.5]))\nprint('Melanoma Probability:\\n', submission.target[submission.target>0.5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}