{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport os\nimport numpy as np\nimport pandas as pd \nimport pydicom\nimport PIL\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom tqdm import tqdm\nfrom kaggle_datasets import KaggleDatasets\n\nimport tensorflow as tf\nprint(\"Tensorflow version \" + tf.__version__)\nfrom tensorflow.keras.applications import ResNet50,MobileNetV2\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport tensorflow.keras.backend as K\nimport tensorflow_addons as tfa\n\n! pip install -q efficientnet\nimport efficientnet.tfkeras as efn\n\n# Input data files are available in the read-only \"../input/\" directory\n\n\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = 'TPU'\nif DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint('REPLICAS: {}'.format(REPLICAS))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Global variables\n\nDATA_PATH = '../input/siim-isic-melanoma-classification/'\nIMAGE_PATH= DATA_PATH+'jpeg/train/'\n\nDIM = 128\nBATCH_SIZE = 16 * REPLICAS\nEPOCHS = 5\nVERBOSE =1\nLR = 1e-3\n\n#For using TPU's, you must read data from a GCS bucket. For using GPU's you can skip the GCS path and use ../input/{data folder}\nGCS_PATH = KaggleDatasets().get_gcs_path(\"jpeg-melanoma-{}x{}\".format(DIM,DIM)) # for resized JPEG images\n\nGCS_PATH_2019 = KaggleDatasets().get_gcs_path(\"jpeg-isic2019-{}x{}\".format(DIM,DIM))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(DATA_PATH+'train.csv')\ntest = pd.read_csv(DATA_PATH+'test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train.benign_malignant.value_counts().plot.bar();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# number of unique patient id's\ntrain.patient_id.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby('sex').size()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_mel = train[train.target==1]\ntrain_no_mel = train[train.target==0]\n\n#img2arr = np.array(image)\nimage_names = np.random.choice(train_mel.image_name,size=6, replace=False)\nplt.figure(figsize=(12,10))\nfor idx,image_name in enumerate(image_names):\n    full_image_path = DATA_PATH+'jpeg/train/'+image_name+'.jpg'\n    image = PIL.Image.open(full_image_path)\n    plt.subplot(3,2,idx+1)\n    plt.imshow(image);\nplt.suptitle('Examples of Melanoma');\n\nimage_names = np.random.choice(train_no_mel.image_name,size=6, replace=False)\nplt.figure(figsize=(12,10))\nfor idx,image_name in enumerate(image_names):\n    full_image_path = DATA_PATH+'jpeg/train/'+image_name+'.jpg'\n    image = PIL.Image.open(full_image_path)\n    plt.subplot(3,2,idx+1)\n    plt.imshow(image);\nplt.suptitle('Examples of no Melanoma');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Observations:\n- To me, it's very difficult to tell a Melanoma image from a non-Melanoma image. Both the image sets look similar to the untrained eye. This will be challenging!\n- Notice that the images are of different sizes, hence we'll have to resize them. This can be done as a pre-processing step or on-the-fly while iterating the data loader. Loading the resized images directly from the disk will be faster, so I'll use pre-resized JPEG images from @cdeotte [here](https://www.kaggle.com/cdeotte/jpeg-melanoma-192x192).","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load the train and test files from the resized folder because it has information about duplicate images\ntrain =pd.read_csv(GCS_PATH+'/train.csv')\ntest =pd.read_csv(GCS_PATH+'/test.csv')\n\n# remove the 434 duplicate images\ntrain = train.loc[~(train.tfrecord == -1), :].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Augmentation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_images(image_name,size=(128,128)):\n    '''\n    Function to resize images using Pillow\n    '''\n    image = PIL.Image.open(os.path.join(IMAGE_PATH,image_name+'.jpg'))\n    image = image.resize(size, resample = PIL.Image.LANCZOS)\n    return image\n\ntrain_datagen = ImageDataGenerator(\n            rescale=1./255,\n            rotation_range=90,\n            width_shift_range=0.2,\n            height_shift_range=0.2,\n            shear_range=0.2,\n            zoom_range=0.3,\n            horizontal_flip=True,\n            fill_mode='nearest')\n\nidx = np.random.randint(train.shape[0])\nsample_image_resized = resize_images(train.image_name[idx])\nit = train_datagen.flow(np.expand_dims(sample_image_resized,axis=0), batch_size=1)\n\nfig = plt.figure(figsize=(12,10))\nfor i in range(9):\n    plt.subplot(330 + 1 + i)\n    plt.imshow(it.next()[0,:,:,:])\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROT_ = 180.0\nSHR_ = 2.0\nHZOOM_ = 8.0\nWZOOM_ = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0\n\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformation matrix which transforms indices\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = np.pi * rotation / 180.\n    shear    = np.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, DIM=192):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Loader","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def parse_function(filename, label=None):\n    image_string = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(image_string, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    # uncomment the line below if you wan't to resize on the fly, I'm using resized images already\n    #image = tf.image.resize(image, [64, 64])\n    if DEVICE== 'TPU':\n        image = tf.reshape(image, [DIM,DIM, 3])  # explicit size needed for TPU\n    return image, label\n\n    \ndef augment_image(image, label):\n#    image = transform(image, DIM=DIM)\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_brightness(image, max_delta=32.0 / 255.0)\n    image = tf.image.random_saturation(image, lower=0.5, upper=1.5)\n    image = tf.image.random_hue(image, 0.01)\n    image = tf.image.random_contrast(image, 0.8, 1.2)\n    #Make sure the image is still in [0, 1]\n    image = tf.clip_by_value(image, 0.0, 1.0)\n\n    return image, label\n\n\ndef data_loader(filenames, labels, augment=False, repeat=True, shuffle=True):\n    \"\"\"\n    Create tf Dataset for training and validation sets\n    \"\"\"\n    dataset = tf.data.Dataset.from_tensor_slices((filenames, labels))\n    \n    dataset = dataset.map(parse_function, num_parallel_calls=AUTO)\n    dataset = dataset.cache()\n    if shuffle:\n        dataset = dataset.shuffle(2048)\n    if repeat:\n        dataset = dataset.repeat()\n    if augment:\n        dataset = dataset.map(augment_image, num_parallel_calls=AUTO)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(buffer_size = AUTO)\n    return dataset\n\n\n#for i, label in data_loader(filenames, labels, augment=True).take(1):\n#    print(i.shape)\n\ndef data_loader_unlabelled(filenames, augment=False, repeat=True):\n    dataset = tf.data.Dataset.from_tensor_slices(filenames)\n    dataset = dataset.map(parse_function, num_parallel_calls=AUTO)\n    dataset = dataset.cache()\n    if augment:\n        dataset = dataset.map(augment_image, num_parallel_calls=AUTO)\n    if repeat:\n        dataset = dataset.repeat()\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(buffer_size = AUTO)\n    return dataset\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_files = get_filenames(train['image_name'], GCS_PATH)\n\n#for i, label in data_loader(train_files, labels=np.ones(len(train_files)), repeat=False, augment=True,shuffle=False).take(10):\n#    print(i.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(input_shape = (192,192,3), pretrained_model= ResNet50): \n    inp = tf.keras.layers.Input(shape=input_shape)\n    base_model = pretrained_model(include_top=False, weights='imagenet', input_shape=input_shape)\n    print(\"Using {} as the base model\".format(base_model.name))\n# weight freezing for Resnet50  \n    if base_model.name == \"resnet50\":\n        for layer in base_model.layers[:143]:\n            layer.trainable = False\n        for layer in base_model.layers[143:]:\n            layer.trainable = True\n    x = base_model(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(1, activation = 'sigmoid')(x)\n    model = tf.keras.Model(inputs= inp, outputs = x)\n    optimizer = tf.keras.optimizers.Adam(learning_rate= LR)\n    model.compile(optimizer= optimizer,\n              loss= tfa.losses.SigmoidFocalCrossEntropy(gamma = 2.0, alpha = 0.80), #tf.keras.losses.BinaryCrossentropy(from_logits=True),\n              metrics= ['AUC'])\n    return model\n\n\n\ndef get_lr_callback(batch_size=BATCH_SIZE):\n    lr_start   = 0.000005\n    lr_max     = 0.00000125  * batch_size\n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_model(input_shape = (DIM,DIM,3))\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef get_filenames(df, path, train_or_test = 'train'):\n    if train_or_test =='train':\n        fnames_list = df.apply(lambda img_name: path+\"/train/\"+str(img_name)+'.jpg').values.tolist()\n    elif train_or_test =='test':\n        fnames_list = df.apply(lambda img_name: path+\"/test/\"+str(img_name)+'.jpg').values.tolist()\n    else:\n        print('Invalid argument')\n        return None\n    return fnames_list\n\n## test file order\n\ndef _data_loader_test(filenames):\n    dataset = tf.data.Dataset.from_tensor_slices(filenames)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset\n\ntest_files = get_filenames(test['image_name'], path = GCS_PATH, train_or_test='test')\ntest_data = _data_loader_test(test_files)\n#list(test_data.as_numpy_iterator()\ndload_files = []\nfor img in iter(test_data.unbatch()):\n    dload_files.append(img.numpy().decode(\"utf-8\"))\n    \nassert test_files[:10] == dload_files[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## test input pipeline\n\ndef show_dataset(thumb_size, cols, rows, ds):\n    mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols + (cols-1), \n                                             thumb_size*rows + (rows-1)))\n   \n    for idx, data in enumerate(iter(ds)):\n        img, _ = data[0], data[1]\n        ix  = idx % cols\n        iy  = idx // cols\n        img = np.clip(img.numpy() * 255, 0, 255).astype(np.uint8)\n        img = PIL.Image.fromarray(img)\n        img = img.resize((thumb_size, thumb_size), resample=PIL.Image.BILINEAR)\n        mosaic.paste(img, (ix*thumb_size + ix, \n                           iy*thumb_size + iy))\n\n    display(mosaic)\n    \ntrain_files = get_filenames(train.loc[np.arange(75), 'image_name'], GCS_PATH)\nds = data_loader(train_files, labels = None).unbatch().take(10*5)   \nshow_dataset(64, 10, 5, ds)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"   \ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kfold_n_splits = 5\nvalid_set_aug=True \ntest_set_aug=True\npreds = []\nskf = StratifiedKFold(n_splits=kfold_n_splits,shuffle=True,random_state=123123)\n\nfor fold,(idxT,idxV) in enumerate(skf.split(train.image_name,train.target.values)):\n\n    print('#'*25)\n    print('### FOLD {}'.format(fold+1))\n    print('#'*25)\n\n    if DEVICE=='TPU':\n        if tpu: \n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            print('using TPU')\n    elif DEVICE=='GPU': \n        print(\"Using GPU\")\n    K.clear_session()\n    with strategy.scope():\n        model = build_model(input_shape = (DIM,DIM,3), pretrained_model = efn.EfficientNetB1)\n        \n    train_files, valid_files = get_filenames(train.loc[idxT, 'image_name'], GCS_PATH), get_filenames(train.loc[idxV, 'image_name'], GCS_PATH) \n    train_labels, valid_labels = train.loc[idxT, 'target'].values.tolist(), train.loc[idxV]['target'].values.tolist()\n    \n    train_data = data_loader(train_files, labels = train_labels, augment=True, repeat=True, shuffle=False)\n    valid_data = data_loader(valid_files, labels = valid_labels, augment=False, repeat=False, shuffle=False)\n\n    sv_best_epoch = tf.keras.callbacks.ModelCheckpoint(\n        \"fold{}.h5\".format(fold+1), monitor='val_loss', verbose=1, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n\n    history = model.fit(train_data,epochs = EPOCHS,  class_weight = {0: 1, 1: 1.5},\n                        steps_per_epoch= np.ceil(len(train_files)/BATCH_SIZE),\n                        verbose= 1, callbacks=[sv_best_epoch,get_lr_callback(BATCH_SIZE)],validation_data = valid_data)\n       \n    print(\"#\"*5+\" Loading model weights from best epoch \"+\"#\"*5)\n    model.load_weights(\"fold{}.h5\".format(fold+1))\n\n    TTA = 5\n    if valid_set_aug:\n        print('#'*5+\" With validation set augmentation size {}\".format(TTA)+'#'*5)\n        valid_data_tta = data_loader_unlabelled(valid_files, augment=True, repeat=True)\n        ypred_valid = model.predict(valid_data_tta, steps = np.ceil(TTA*len(valid_files)/BATCH_SIZE), verbose=1)\n        ypred_valid = ypred_valid[:len(valid_files)*TTA].reshape((len(valid_files),TTA), order = 'F') # Fortran like indexing, augmentations in columns\n        ypred_valid = ypred_valid.mean(axis = 1) # take the average across number of augmentations\n    else:\n        print('#'*5+\" Without validation set augmentation \"+'#'*5)\n        valid_data_no_tta = data_loader_unlabelled(valid_files, augment=False, repeat=False)\n        ypred_valid = model.predict(valid_data_no_tta, verbose =1)\n\n    auc_valid = roc_auc_score(valid_labels, ypred_valid)        \n    print('AUC of validation fold {} = {}'.format(fold+1, auc_valid))\n\n    print('Predicting Test image class')\n    test_files = get_filenames(test['image_name'], path = GCS_PATH, train_or_test='test')\n\n    if test_set_aug:\n        print(\"Using test set augmentation\")\n        test_data = data_loader_unlabelled(test_files, augment=True, repeat=True)\n        ypred_test = model.predict(test_data, steps = np.ceil(TTA*len(test_files)/BATCH_SIZE), verbose=1)\n        ypred_test = ypred_test[:len(test_files)*TTA].reshape((len(test_files),TTA), order = 'F')\n        ypred_test = ypred_test.mean(axis = 1)\n    else:\n        print(\"Without test set augmentation\")\n        test_data  = data_loader_unlabelled(test_files, augment=False, repeat=False)\n        ypred_test = model.predict(test_data, verbose=1)\n    preds += ypred_test/kfold_n_splits\n    \n    display_training_curves(\n    history.history['loss'],\n    history.history['val_loss'],\n    'loss',\n    211,\n    )\n    display_training_curves(\n    history.history['auc'],\n    history.history['val_auc'],\n    'AUC',\n    212,\n    )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#filenames = train['image_name'].apply(lambda img_name: GCS_PATH+\"train/\"+str(img_name)+'.jpg').values.tolist()\n#labels = train['target'].values.tolist()\n\n\n\ndef train_model(augment=True, kfold_n_splits = 5, valid_set_aug=True, test_set_aug=True):\n    preds = []\n    skf = StratifiedKFold(n_splits=kfold_n_splits,shuffle=True,random_state=1212)\n    \n    for fold,(idxT,idxV) in enumerate(skf.split(train.image_name,train.target.values)):\n\n        print('#'*25)\n        print('### FOLD {}'.format(fold+1))\n        print('#'*25)\n    \n        K.clear_session()\n        with strategy.scope():\n            model = build_model()\n        \n        train_files, valid_files = get_filenames(train.loc[idxT, 'image_name'], GCS_PATH), get_filenames(train.loc[idxV, 'image_name'], GCS_PATH) \n        train_labels, valid_labels = train.loc[idxT, 'target'].values.tolist(), train.loc[idxV]['target'].values.tolist()\n\n        train_data = data_loader(train_files, labels = train_labels, augment=True, repeat=True, shuffle=True)\n        valid_data = data_loader(valid_files, labels = valid_labels, augment=False, repeat=False, shuffle=False)\n        \n        sv_best_epoch = tf.keras.callbacks.ModelCheckpoint(\n            \"fold{}.h5\".format(fold+1), monitor='val_loss', verbose=1, save_best_only=True,\n            save_weights_only=True, mode='min', save_freq='epoch')\n        \n        history = model.fit(train_data,epochs = EPOCHS,  class_weight = {0: 1, 1: 1.5},\n                            steps_per_epoch= np.ceil(len(train_files)/BATCH_SIZE),\n                            verbose= VERBOSE, callbacks=[sv_best_epoch,get_lr_callback(BATCH_SIZE)],validation_data = valid_data)\n        \n        print(\"#\"*5+\" Loading model weights from best epoch \"+\"#\"*5)\n        model.load_weights(\"fold{}.h5\".format(fold+1))\n        \n        TTA = 5\n        if valid_set_aug:\n            print('#'*5+\" With validation set augmentation size {}\".format(TTA)+'#'*5)\n            valid_data_tta = data_loader_unlabelled(valid_files, augment=True, repeat=True)\n            ypred_valid = model.predict(valid_data_tta, steps = np.ceil(TTA*len(valid_files)/BATCH_SIZE), verbose=1)\n            ypred_valid = ypred_valid[:len(valid_files)*TTA].reshape((len(valid_files),TTA), order = 'F') # Fortran like indexing, augmentations in columns\n            ypred_valid = ypred_valid.mean(axis = 1) # take the average across number of augmentations\n        else:\n            print('#'*5+\" Without validation set augmentation \"+'#'*5)\n            valid_data_no_tta = data_loader_unlabelled(valid_files, augment=False, repeat=False)\n            ypred_valid = model.predict(valid_data_no_tta, verbose =1)\n            \n        auc_valid = roc_auc_score(valid_labels, ypred_valid)        \n        print('AUC of validation fold {} = {}'.format(fold+1, auc_valid))\n        \n        print('Predicting Test image class')\n        test_files = get_filenames(test['image_name'], path = GCS_PATH, train_or_test='test')\n        \n        if test_set_aug:\n            print(\"Using test set augmentation\")\n            test_data = data_loader_unlabelled(test_files, augment=True, repeat=True)\n            ypred_test = model.predict(test_data, steps = np.ceil(TTA*len(test_files)/BATCH_SIZE), verbose=1)\n            ypred_test = ypred_test[:len(test_files)*TTA].reshape((len(test_files),TTA), order = 'F')\n            ypred_test = ypred_test.mean(axis = 1)\n        else:\n            print(\"Without test set augmentation\")\n            test_data  = data_loader_unlabelled(test_files, augment=False, repeat=False)\n            ypred_test = model.predict(test_data, verbose=1)\n        preds += ypred_test/kfold_n_splits\n    return preds\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_model(augment=True, kfold_n_splits = 5)\nsub_df = pd.read_csv(DATA_PATH+\"sample_submission.csv\")\nsub_df.loc[:,'target'] = preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}