{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Libraries and Configurations","metadata":{}},{"cell_type":"code","source":"import random, re, math\nimport numpy as np, pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nimport tensorflow as tf, tensorflow.keras.backend as K\nfrom kaggle_datasets import KaggleDatasets\nprint('Tensorflow version ' + tf.__version__)\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-05T08:40:14.59417Z","iopub.execute_input":"2021-12-05T08:40:14.594541Z","iopub.status.idle":"2021-12-05T08:40:20.138221Z","shell.execute_reply.started":"2021-12-05T08:40:14.594445Z","shell.execute_reply":"2021-12-05T08:40:20.137405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:40:20.139837Z","iopub.execute_input":"2021-12-05T08:40:20.140066Z","iopub.status.idle":"2021-12-05T08:40:25.917164Z","shell.execute_reply.started":"2021-12-05T08:40:20.14004Z","shell.execute_reply":"2021-12-05T08:40:25.916648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Configuration\nIMAGE_SIZE = [512, 512]\nEPOCHS = 25\nFOLDS = 5\nSEED = 777\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nAUG_BATCH = BATCH_SIZE\nFIRST_FOLD_ONLY = False","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:40:25.918228Z","iopub.execute_input":"2021-12-05T08:40:25.918968Z","iopub.status.idle":"2021-12-05T08:40:25.923592Z","shell.execute_reply.started":"2021-12-05T08:40:25.918934Z","shell.execute_reply":"2021-12-05T08:40:25.922845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mixed Precision and/or XLA\nMixed Precision and XLA are not being used in this notebook but you can experiment using them. Change the following booleans to enable mixed precision and/or XLA on GPU/TPU. By default TPU already uses some mixed precision but we can add more (and it already uses XLA). These allow the GPU/TPU memory to handle larger batch sizes and can speed up the training process. The Nvidia V100 GPU has special Tensor Cores which get utilized when mixed precision is enabled. Unfortunately Kaggle's Nvidia P100 GPU does not have Tensor Cores to receive speed up.","metadata":{}},{"cell_type":"code","source":"MIXED_PRECISION = False\nXLA_ACCELERATE = False\n\nif MIXED_PRECISION:\n    from tensorflow.keras.mixed_precision import experimental as mixed_precision\n    if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')\n    else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n    mixed_precision.set_policy(policy)\n    print('Mixed precision enabled')\n\nif XLA_ACCELERATE:\n    tf.config.optimizer.set_jit(True)\n    print('Accelerated Linear Algebra enabled')","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:40:25.924765Z","iopub.execute_input":"2021-12-05T08:40:25.925493Z","iopub.status.idle":"2021-12-05T08:40:25.934023Z","shell.execute_reply.started":"2021-12-05T08:40:25.925451Z","shell.execute_reply":"2021-12-05T08:40:25.933425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Directories and Classes","metadata":{}},{"cell_type":"code","source":"GCS_PATH    = KaggleDatasets().get_gcs_path('melanoma-512x512')\nGCS_PATH1=KaggleDatasets().get_gcs_path('malignant-v2-512x512')\n#GCS_PATH_2019    = KaggleDatasets().get_gcs_path('isic2019-384x384-cc')\n\n#files_train_2019 = np.array(tf.io.gfile.glob(GCS_PATH_2019 + '/train*.tfrec'))\n#files_train_2020 = np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec'))\n\n#files_train = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\nmalignant= tf.io.gfile.glob(GCS_PATH1 + '/train*.tfrec')\n#files_train += malignant\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\nTRAINING_FILENAMES +=malignant\n#VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec') # predictions on this dataset should be submitted for the competition","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:40:25.936043Z","iopub.execute_input":"2021-12-05T08:40:25.936229Z","iopub.status.idle":"2021-12-05T08:40:26.959697Z","shell.execute_reply.started":"2021-12-05T08:40:25.936208Z","shell.execute_reply":"2021-12-05T08:40:26.958814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASSES = ['B','M']                                                                                                                                                # 100 - 102","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:40:26.960864Z","iopub.execute_input":"2021-12-05T08:40:26.961094Z","iopub.status.idle":"2021-12-05T08:40:26.96563Z","shell.execute_reply.started":"2021-12-05T08:40:26.961067Z","shell.execute_reply":"2021-12-05T08:40:26.964847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAINING_FILENAMES,TEST_FILENAMES = train_test_split(TRAINING_FILENAMES,test_size = 0.03,random_state =42)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:40:26.967135Z","iopub.execute_input":"2021-12-05T08:40:26.967637Z","iopub.status.idle":"2021-12-05T08:40:26.979928Z","shell.execute_reply.started":"2021-12-05T08:40:26.967604Z","shell.execute_reply":"2021-12-05T08:40:26.97914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Custom LR scheduler\nFrom starter [kernel][1]\n\n[1]: https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu","metadata":{}},{"cell_type":"code","source":"# Learning rate schedule for TPU, GPU and CPU.\n# Using an LR ramp up because fine-tuning a pre-trained model.\n# Starting with a high LR would break the pre-trained weights.\n\nLR_START = 0.00001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n\nrng = [i for i in range(25 if EPOCHS<25 else EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:40:26.981077Z","iopub.execute_input":"2021-12-05T08:40:26.981849Z","iopub.status.idle":"2021-12-05T08:40:27.231054Z","shell.execute_reply.started":"2021-12-05T08:40:26.98182Z","shell.execute_reply":"2021-12-05T08:40:27.229993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset Functions\nFrom starter [kernel][1]\n\n[1]: https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu","metadata":{}},{"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['image_name']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled = True, ordered = False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # use data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO) # returns a dataset of (image, label) pairs if labeled = True or (image, id) pair if labeld = False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    return image, label   \n\ndef get_training_dataset(dataset, do_aug=True):\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.batch(AUG_BATCH)\n    if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO) # note we put AFTER batching\n    dataset = dataset.unbatch()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(dataset, do_onehot=True):\n    dataset = dataset.batch(BATCH_SIZE)\n    if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTO) # we must use one hot like augmented train data\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = int( count_data_items(TRAINING_FILENAMES) * (FOLDS-1.)/FOLDS )\nNUM_VALIDATION_IMAGES = int( count_data_items(TRAINING_FILENAMES) * (1./FOLDS) )\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:40:27.232618Z","iopub.execute_input":"2021-12-05T08:40:27.232891Z","iopub.status.idle":"2021-12-05T08:40:27.257152Z","shell.execute_reply.started":"2021-12-05T08:40:27.232855Z","shell.execute_reply":"2021-12-05T08:40:27.256322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CutMix Augmentation\nThe following code does cutmix using the GPU/TPU. Change the variables `SWITCH`, `CUTMIX_PROB` and `MIXUP_PROB` in function `transform()` to control the amount of augmentation during training. CutMix will occur `SWITCH * CUTMIX_PROB` often and MixUp will occur `(1-SWITCH) * MIXUP_PROB` often during training.","metadata":{}},{"cell_type":"code","source":"def onehot(image,label):\n    CLASSES = 2\n    return image,tf.one_hot(label,CLASSES)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:40:27.258273Z","iopub.execute_input":"2021-12-05T08:40:27.258652Z","iopub.status.idle":"2021-12-05T08:40:27.271075Z","shell.execute_reply.started":"2021-12-05T08:40:27.258617Z","shell.execute_reply":"2021-12-05T08:40:27.270501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cutmix(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 2\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:40:27.272157Z","iopub.execute_input":"2021-12-05T08:40:27.272944Z","iopub.status.idle":"2021-12-05T08:40:27.288502Z","shell.execute_reply.started":"2021-12-05T08:40:27.272915Z","shell.execute_reply":"2021-12-05T08:40:27.287873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display CutMix Augmentation","metadata":{}},{"cell_type":"code","source":"row = 6; col = 4;\nrow = min(row,AUG_BATCH//col)\nall_elements = get_training_dataset(load_dataset(TRAINING_FILENAMES),do_aug=False).unbatch()\naugmented_element = all_elements.repeat().batch(AUG_BATCH).map(cutmix)\n\nfor (img,label) in augmented_element:\n    plt.figure(figsize=(15,int(15*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:40:27.289663Z","iopub.execute_input":"2021-12-05T08:40:27.290305Z","iopub.status.idle":"2021-12-05T08:40:53.804593Z","shell.execute_reply.started":"2021-12-05T08:40:27.290277Z","shell.execute_reply":"2021-12-05T08:40:53.80248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MixUp Augmentation\nThe following code does mixup using the GPU/TPU. Change the variables `SWITCH`, `CUTMIX_PROB` and `MIXUP_PROB` in function `transform()` to control the amount of augmentation during training. CutMix will occur `SWITCH * CUTMIX_PROB` often and MixUp will occur `(1-SWITCH) * MIXUP_PROB` often during training.","metadata":{}},{"cell_type":"code","source":"def mixup(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 2\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        # MAKE CUTMIX LABEL\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:40:53.805711Z","iopub.execute_input":"2021-12-05T08:40:53.806108Z","iopub.status.idle":"2021-12-05T08:40:53.816463Z","shell.execute_reply.started":"2021-12-05T08:40:53.806081Z","shell.execute_reply":"2021-12-05T08:40:53.81567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform(image,label):\n    # THIS FUNCTION APPLIES BOTH CUTMIX AND MIXUP\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 2\n    SWITCH = 0.5\n    CUTMIX_PROB = 0.666\n    MIXUP_PROB = 0.666\n    # FOR SWITCH PERCENT OF TIME WE DO CUTMIX AND (1-SWITCH) WE DO MIXUP\n    image2, label2 = cutmix(image, label, CUTMIX_PROB)\n    image3, label3 = mixup(image, label, MIXUP_PROB)\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        P = tf.cast( tf.random.uniform([],0,1)<=SWITCH, tf.float32)\n        imgs.append(P*image2[j,]+(1-P)*image3[j,])\n        labs.append(P*label2[j,]+(1-P)*label3[j,])\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image4,label4","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:40:53.819031Z","iopub.execute_input":"2021-12-05T08:40:53.819592Z","iopub.status.idle":"2021-12-05T08:40:53.828345Z","shell.execute_reply.started":"2021-12-05T08:40:53.819563Z","shell.execute_reply":"2021-12-05T08:40:53.82754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display MixUp Augmentation","metadata":{}},{"cell_type":"code","source":"row = 6; col = 4;\nrow = min(row,AUG_BATCH//col)\nall_elements = get_training_dataset(load_dataset(TRAINING_FILENAMES),do_aug=False).unbatch()\naugmented_element = all_elements.repeat().batch(AUG_BATCH).map(mixup)\n\nfor (img,label) in augmented_element:\n    plt.figure(figsize=(15,int(15*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:40:53.829811Z","iopub.execute_input":"2021-12-05T08:40:53.830351Z","iopub.status.idle":"2021-12-05T08:41:10.613688Z","shell.execute_reply.started":"2021-12-05T08:40:53.830313Z","shell.execute_reply":"2021-12-05T08:41:10.612746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = 6; col = 4;\nrow = min(row,AUG_BATCH//col)\nall_elements = get_training_dataset(load_dataset(TRAINING_FILENAMES),do_aug=False).unbatch()\naugmented_element = all_elements.repeat().batch(AUG_BATCH).map(transform)\n\nfor (img,label) in augmented_element:\n    plt.figure(figsize=(15,int(15*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:41:10.614916Z","iopub.execute_input":"2021-12-05T08:41:10.615158Z","iopub.status.idle":"2021-12-05T08:41:44.289388Z","shell.execute_reply.started":"2021-12-05T08:41:10.615129Z","shell.execute_reply":"2021-12-05T08:41:44.288522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build, Train, Infer Model\nThis is the 5-Fold workflow copied from Ragnar's notebook [here][1]. Now we add cutmix and/or mixup augmentation to the training images on the fly! Since cutmix and/or mixup returns one hot encoded labels, we change the loss and metric to `categorical_crossentropy` instead of `sparse_categorical_crossentropy`.\n\n[1]: https://www.kaggle.com/ragnar123/5-kfold-densenet201","metadata":{}},{"cell_type":"code","source":"!pip install focal-loss\nfrom focal_loss import BinaryFocalLoss\nloss=BinaryFocalLoss(gamma=2)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:41:44.290722Z","iopub.execute_input":"2021-12-05T08:41:44.290936Z","iopub.status.idle":"2021-12-05T08:41:53.565909Z","shell.execute_reply.started":"2021-12-05T08:41:44.290911Z","shell.execute_reply":"2021-12-05T08:41:53.564968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import DenseNet201\n\ndef get_model():\n    with strategy.scope():\n        rnet = DenseNet201(\n            input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),\n            weights='imagenet',\n            include_top=False\n        )\n        # trainable rnet\n        rnet.trainable = True\n        model = tf.keras.Sequential([\n            rnet,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(len(CLASSES), activation='sigmoid',dtype='float32')\n        ])\n    model.compile(\n        optimizer='adam',\n        loss = loss,\n        metrics=[tf.keras.metrics.AUC()]\n    )\n    return model\n\ndef train_cross_validate(folds = 5):\n    histories = []\n    models = []\n    early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 3)\n    kfold = KFold(folds, shuffle = True, random_state = SEED)\n    for f, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n        print(); print('#'*25)\n        print('### FOLD',f+1)\n        print('#'*25)\n        train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)\n        val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)\n        model = get_model()\n        ch=tf.keras.callbacks.ModelCheckpoint(f'model{f}.h5',\n                                           save_best_only=True,\n                                           monitor='val_loss',\n                                           mode='min')\n        history = model.fit(\n            get_training_dataset(train_dataset), \n            steps_per_epoch = STEPS_PER_EPOCH,\n            epochs = 5,\n            callbacks = [ch,lr_callback],# early_stopping],\n            validation_data = get_validation_dataset(val_dataset),\n            verbose=1\n        )\n        models.append(model)\n        histories.append(history)\n        if FIRST_FOLD_ONLY: break\n    return histories, models\n\ndef train_and_predict(folds = 5):\n    test_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n    test_images_ds = test_ds.map(lambda image, idnum: image)\n    print('Start training %i folds'%folds)\n    histories, models = train_cross_validate(folds = folds)\n    print('Computing predictions...')\n    # get the mean probability of the folds models\n    if FIRST_FOLD_ONLY: probabilities = np.average([models[i].predict(test_images_ds) for i in range(1)], axis = 0)\n    else: probabilities = np.average([models[i].predict(test_images_ds) for i in range(folds)], axis = 0)\n    predictions = np.argmax(probabilities, axis=-1)\n    print('Generating submission.csv file...')\n    test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n    test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\n    np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\n    return histories, models\n    \n# run train and predict\nhistories, models = train_and_predict(folds = FOLDS)","metadata":{"execution":{"iopub.status.busy":"2021-12-05T08:41:53.567585Z","iopub.execute_input":"2021-12-05T08:41:53.567845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Confusion Matrix and Validation Score\nTry forking and modifying this notebook to maximize validation score below. Tune the data augmentation and/or train for more epochs to increase accuracy. Good luck! (Code below is from starter [kernel][1]).\n\n[1]: https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu","metadata":{}},{"cell_type":"code","source":"def display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nall_labels = []; all_prob = []; all_pred = []\nkfold = KFold(FOLDS, shuffle = True, random_state = SEED)\nfor j, (trn_ind, val_ind) in enumerate( kfold.split(TRAINING_FILENAMES) ):\n    print('Inferring fold',j+1,'validation images...')\n    VAL_FILES = list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES'])\n    NUM_VALIDATION_IMAGES = count_data_items(VAL_FILES)\n    cmdataset = get_validation_dataset(load_dataset(VAL_FILES, labeled = True, ordered = True), do_onehot=False)\n    images_ds = cmdataset.map(lambda image, label: image)\n    labels_ds = cmdataset.map(lambda image, label: label).unbatch()\n    all_labels.append( next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy() ) # get everything as one batch\n    prob = models[j].predict(images_ds)\n    all_prob.append( prob )\n    all_pred.append( np.argmax(prob, axis=-1) )\n    if FIRST_FOLD_ONLY: break\ncm_correct_labels = np.concatenate(all_labels)\ncm_probabilities = np.concatenate(all_prob)\ncm_predictions = np.concatenate(all_pred)","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Correct   labels: \", cm_correct_labels.shape, cm_correct_labels)\nprint(\"Predicted labels: \", cm_predictions.shape, cm_predictions); print()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))\nscore = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\nprecision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\nrecall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\ndisplay_confusion_matrix(cmat, score, precision, recall)\nprint('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall)); print()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm=cmat\nimport sklearn.metrics\nfrom sklearn.metrics import confusion_matrix","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\ngroup_names = ['True Negative','False Positive','False Negative','True Positive']\ngroup_counts = [\"{0:0.0f}\".format(value) for value in\n                cm.flatten()]\ngroup_percentages = [\"{0:.2%}\".format(value) for value in\n                     cm.flatten()/np.sum(cm)]\nlabels = [f\"{v1}\\n{v2}\\n{v3}\" for v1, v2, v3 in\n          zip(group_names,group_counts,group_percentages)]\nlabels = np.asarray(labels).reshape(2,2)\n\nsns.heatmap(cm, annot=labels, fmt='', cmap=\"YlGnBu\")\ntn, fp, fn, tp = confusion_matrix(cm_correct_labels, cm_predictions).ravel()\nspecificity = tn /(tn+fp)\nsensitivity=  tp/ (tp+fn)\nprint('specificity: {:.6f} | sensitivity: {:.6f} '.format(specificity, sensitivity))","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}