{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"code","source":"# Install efficientnet package\n!pip install -q efficientnet","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:22.235146Z","iopub.execute_input":"2021-12-09T08:42:22.235478Z","iopub.status.idle":"2021-12-09T08:42:32.573565Z","shell.execute_reply.started":"2021-12-09T08:42:22.235393Z","shell.execute_reply":"2021-12-09T08:42:32.572663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import packages\nimport re\nimport cv2\nimport math\nimport time\nimport random\nimport sklearn\nimport itertools\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom functools import partial\nimport matplotlib.pyplot as plt\nfrom matplotlib import cm, colors\nimport efficientnet.tfkeras as efn\nimport tensorflow.keras.backend as K\nfrom tqdm import tqdm_notebook as tqdm\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import precision_recall_curve, confusion_matrix, classification_report, roc_curve, auc, ConfusionMatrixDisplay","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-09T08:42:32.575704Z","iopub.execute_input":"2021-12-09T08:42:32.576011Z","iopub.status.idle":"2021-12-09T08:42:39.744674Z","shell.execute_reply.started":"2021-12-09T08:42:32.575974Z","shell.execute_reply":"2021-12-09T08:42:39.743770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Connect to TPU\ntry:\n    AUTOTUNE = tf.data.experimental.AUTOTUNE\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n    print(\"Running on TPU \", tpu.cluster_spec().as_dict()[\"worker\"])\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept ValueError:\n    print(\"Not connected to a TPU runtime. Using CPU/GPU strategy\")\n    strategy = tf.distribute.MirroredStrategy()\n    \nREPLICAS = strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:39.745869Z","iopub.execute_input":"2021-12-09T08:42:39.746106Z","iopub.status.idle":"2021-12-09T08:42:45.495361Z","shell.execute_reply.started":"2021-12-09T08:42:39.746077Z","shell.execute_reply":"2021-12-09T08:42:45.494271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parameters","metadata":{}},{"cell_type":"code","source":"# Hyperparameters\nSEED = 21\nDIM = 384\nEPOCHS = 16\nNUM_CLASSES = 1\nBATCH_SIZE = 256\nVERBOSE_LEVEL = 1\nSAVE_OUTPUT = True\nLABEL_SMOOTHING = 0.05\n\n# LR SCHEDULE \nLR_MAX = 1e-4\nLR_MIN = 1e-7\nLR_START = 1e-4\nMODE = \"triangular2\"\nSTEP_SIZE = 4\n\n# The 2019 Data may decrease the model performance. More Infos here: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/168028 \nEXCLUDE_2019 = True","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:45.496722Z","iopub.execute_input":"2021-12-09T08:42:45.496983Z","iopub.status.idle":"2021-12-09T08:42:45.503358Z","shell.execute_reply.started":"2021-12-09T08:42:45.496955Z","shell.execute_reply":"2021-12-09T08:42:45.502597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Config for image augmentation\nAUGMENTATION_CONFIG = {\n    'ROT_': 180.0,\n    'SHR_': 6,\n    'HZOOM_': 10.0,\n    'WZOOM_': 10.0,\n    'HSHIFT_': 10.0,\n    'WSHIFT_': 10.0,\n    'CROP': 0.90,\n    'PROBABILITY_DROPOUT': 0.75\n}","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:45.505791Z","iopub.execute_input":"2021-12-09T08:42:45.506064Z","iopub.status.idle":"2021-12-09T08:42:45.515728Z","shell.execute_reply.started":"2021-12-09T08:42:45.506033Z","shell.execute_reply":"2021-12-09T08:42:45.514914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Seed is used to save the state of a random function, \n# so that it can generate the same random numbers on multiple executions\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:45.517161Z","iopub.execute_input":"2021-12-09T08:42:45.517405Z","iopub.status.idle":"2021-12-09T08:42:45.528857Z","shell.execute_reply.started":"2021-12-09T08:42:45.517376Z","shell.execute_reply":"2021-12-09T08:42:45.527859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data loading & preparation & augmentation","metadata":{}},{"cell_type":"markdown","source":"## Data loading","metadata":{}},{"cell_type":"code","source":"# Load the train and test csv files\ndf_train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ndf_test = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:45.529984Z","iopub.execute_input":"2021-12-09T08:42:45.530472Z","iopub.status.idle":"2021-12-09T08:42:45.677674Z","shell.execute_reply.started":"2021-12-09T08:42:45.530440Z","shell.execute_reply":"2021-12-09T08:42:45.677003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Countes the items inside a list of tf records\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:45.678663Z","iopub.execute_input":"2021-12-09T08:42:45.679401Z","iopub.status.idle":"2021-12-09T08:42:45.684431Z","shell.execute_reply.started":"2021-12-09T08:42:45.679368Z","shell.execute_reply":"2021-12-09T08:42:45.683373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the datasets\nGCS_PATH_2020 = KaggleDatasets().get_gcs_path('melanoma-384x384')\nGCS_PATH_OLD = KaggleDatasets().get_gcs_path('isic2019-384x384')\n\nprint(\"GCS_PATH_2020\", GCS_PATH_2020)\nprint(\"GCS_PATH_OLD\", GCS_PATH_OLD)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:45.685983Z","iopub.execute_input":"2021-12-09T08:42:45.686559Z","iopub.status.idle":"2021-12-09T08:42:46.494625Z","shell.execute_reply.started":"2021-12-09T08:42:45.686525Z","shell.execute_reply":"2021-12-09T08:42:46.493680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH_2020 + '/train*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH_2020 + '/test*.tfrec')\n\nprint(\"# TRAINING_FILENAMES\", len(TRAINING_FILENAMES))\nprint(\"# TEST_FILENAMES\", len(TEST_FILENAMES))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:46.496057Z","iopub.execute_input":"2021-12-09T08:42:46.496277Z","iopub.status.idle":"2021-12-09T08:42:46.675100Z","shell.execute_reply.started":"2021-12-09T08:42:46.496251Z","shell.execute_reply":"2021-12-09T08:42:46.674229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OLD_COMP_FILENAMES = tf.io.gfile.glob(GCS_PATH_OLD + '/*.tfrec')\nprint(\"# OLD_COMP_FILENAMES\", len(OLD_COMP_FILENAMES))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:46.676419Z","iopub.execute_input":"2021-12-09T08:42:46.676637Z","iopub.status.idle":"2021-12-09T08:42:46.753626Z","shell.execute_reply.started":"2021-12-09T08:42:46.676611Z","shell.execute_reply":"2021-12-09T08:42:46.752548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Exclude the 2019 if needed\nOLD_COMP_FILENAMES_TMP = []\nif EXCLUDE_2019:\n     for i in range(0, len(OLD_COMP_FILENAMES), 2):\n         OLD_COMP_FILENAMES_TMP.append(OLD_COMP_FILENAMES[i])\n\n     OLD_COMP_FILENAMES = OLD_COMP_FILENAMES_TMP\n     print(\"# OLD_COMP_FILENAMES\", len(OLD_COMP_FILENAMES))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:46.755462Z","iopub.execute_input":"2021-12-09T08:42:46.755808Z","iopub.status.idle":"2021-12-09T08:42:46.762328Z","shell.execute_reply.started":"2021-12-09T08:42:46.755750Z","shell.execute_reply":"2021-12-09T08:42:46.761390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only use data from 2020 for validation\nTRAINING_FILENAMES, VALIDATION_FILENAMES = train_test_split(TRAINING_FILENAMES, test_size = 0.20, random_state = SEED)\nTRAINING_FILENAMES = list(TRAINING_FILENAMES) + list(OLD_COMP_FILENAMES)\n\nrandom.shuffle(TRAINING_FILENAMES)\nrandom.shuffle(VALIDATION_FILENAMES)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:46.764112Z","iopub.execute_input":"2021-12-09T08:42:46.764426Z","iopub.status.idle":"2021-12-09T08:42:46.778611Z","shell.execute_reply.started":"2021-12-09T08:42:46.764386Z","shell.execute_reply":"2021-12-09T08:42:46.777936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test if TRAINING and VALIDATION files are valid\nfor x in TRAINING_FILENAMES:\n    if x in VALIDATION_FILENAMES:\n        raise Exception(\"TRAIN AND TEST FILES ARE NOT VALID!\")","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:46.783712Z","iopub.execute_input":"2021-12-09T08:42:46.783981Z","iopub.status.idle":"2021-12-09T08:42:46.788674Z","shell.execute_reply.started":"2021-12-09T08:42:46.783953Z","shell.execute_reply":"2021-12-09T08:42:46.787721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"# TRAINING_FILENAMES\", len(TRAINING_FILENAMES))\nprint(\"# VALIDATION_FILENAMES\", len(VALIDATION_FILENAMES))\n\nTRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nVALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\n\nprint(\"# TRAINING_IMAGES\", TRAINING_IMAGES)\nprint(\"# VALIDATION_IMAGES\", VALIDATION_IMAGES)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:46.790097Z","iopub.execute_input":"2021-12-09T08:42:46.790848Z","iopub.status.idle":"2021-12-09T08:42:46.803650Z","shell.execute_reply.started":"2021-12-09T08:42:46.790815Z","shell.execute_reply":"2021-12-09T08:42:46.802742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate an initial bias for the output layer based on the number\n# of malignant and benign cases of the 2020 train data\nbenign_cases = df_train['benign_malignant'].value_counts().benign\nmalignant_cases = df_train['benign_malignant'].value_counts().malignant\n\ninitial_bias = np.log([malignant_cases/benign_cases])\n\nprint(\"initial_bias\", initial_bias)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:46.804990Z","iopub.execute_input":"2021-12-09T08:42:46.805500Z","iopub.status.idle":"2021-12-09T08:42:46.836484Z","shell.execute_reply.started":"2021-12-09T08:42:46.805459Z","shell.execute_reply":"2021-12-09T08:42:46.835824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Augmentation","metadata":{}},{"cell_type":"code","source":"# Here we apply some manual augmentations that cannot be done with tf.image, \n# such as shearing, zooming and translation. Rotation can be done in tf.image but only in factors of 90 degrees, \n# so we do it manually instead.\n# Source: https://www.kaggle.com/teyang/melanoma-detection-using-effnet-and-meta-data#5.-Train-and-Evaluate-Model\n\nROT_ = AUGMENTATION_CONFIG['ROT_']\nSHR_ = AUGMENTATION_CONFIG['SHR_']\nHZOOM_ = AUGMENTATION_CONFIG['HZOOM_']\nWZOOM_ = AUGMENTATION_CONFIG['WZOOM_']\nHSHIFT_ = AUGMENTATION_CONFIG['HSHIFT_']\nWSHIFT_ = AUGMENTATION_CONFIG['WSHIFT_']\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, DIM=DIM):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:46.837725Z","iopub.execute_input":"2021-12-09T08:42:46.838378Z","iopub.status.idle":"2021-12-09T08:42:46.860617Z","shell.execute_reply.started":"2021-12-09T08:42:46.838341Z","shell.execute_reply":"2021-12-09T08:42:46.859560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dropout(image, PROBABILITY = AUGMENTATION_CONFIG['PROBABILITY_DROPOUT'], CT = 6, SZ = 0.20):\n    # Source: https://www.kaggle.com/cdeotte/tfrecord-experiments-upsample-and-coarse-dropout\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(CT==0)|(SZ==0): return image\n    \n    for k in range(CT):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        # COMPUTE SQUARE \n        WIDTH = tf.cast( SZ*DIM,tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # DROPOUT IMAGE\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3]) \n        three = image[ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM,DIM,3])\n    return image","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:46.862126Z","iopub.execute_input":"2021-12-09T08:42:46.863091Z","iopub.status.idle":"2021-12-09T08:42:46.878227Z","shell.execute_reply.started":"2021-12-09T08:42:46.863039Z","shell.execute_reply":"2021-12-09T08:42:46.877346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def color(x):\n    \"\"\"Color augmentation\n    Args:\n        x: Image\n\n    Returns:\n        Augmented image\n    \"\"\"        \n    x = tf.image.random_saturation(x, 0.7, 1.3, seed=SEED)\n    x = tf.image.random_contrast(x, 0.7, 1.3, seed=SEED)\n    x = tf.image.random_brightness(x, 0.1, seed=SEED)\n    return x\n\ndef flip(x):\n    \"\"\"Flip augmentation\n    Args:\n        x: Image to flip\n\n    Returns:\n        Augmented image\n    \"\"\"\n    x = tf.image.random_flip_left_right(x, seed=SEED)\n    x = tf.image.random_flip_up_down(x, seed=SEED)\n    return x\n\ndef rotate(x):\n    \"\"\"Rotate augmentation\n    Args:\n        x: Image to flip\n\n    Returns:\n        Augmented image\n    \"\"\"\n    x = tf.image.rot90(x,k=np.random.randint(4))\n    return x\n\ndef random_crop(x):\n    \"\"\"Random crop augmentation\n    Args:\n        x: Image to flip\n\n    Returns:\n        Augmented image\n    \"\"\"\n    x = tf.image.random_crop(x, size=[round(DIM*AUGMENTATION_CONFIG['CROP']), round(DIM*AUGMENTATION_CONFIG['CROP']), 3], seed=SEED)\n    x = tf.image.resize(x, [DIM, DIM])\n    x = tf.reshape(x, [DIM, DIM, 3]) \n    return x\n\ndef central_crop(x):\n    \"\"\"Central crop augmentation\n    Args:\n        x: Image to flip\n\n    Returns:\n        Augmented image\n    \"\"\"\n    x = tf.image.central_crop(x, DIM*AUGMENTATION_CONFIG['CROP'] / DIM)\n    x = tf.image.resize(x, [DIM, DIM])\n    x = tf.reshape(x, [DIM, DIM, 3]) \n    return x","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:46.879272Z","iopub.execute_input":"2021-12-09T08:42:46.880099Z","iopub.status.idle":"2021-12-09T08:42:46.897193Z","shell.execute_reply.started":"2021-12-09T08:42:46.880059Z","shell.execute_reply":"2021-12-09T08:42:46.896199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply every augmentation function inside the augmentations list to the image\ndef augment_image(image, augment=True):  \n    augmentations = [color, flip, rotate, random_crop, transform, dropout] \n    if augment:\n        for f in augmentations:\n            image = f(image)\n        \n    return image","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:46.898538Z","iopub.execute_input":"2021-12-09T08:42:46.898900Z","iopub.status.idle":"2021-12-09T08:42:46.916051Z","shell.execute_reply.started":"2021-12-09T08:42:46.898872Z","shell.execute_reply":"2021-12-09T08:42:46.915036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparation","metadata":{}},{"cell_type":"code","source":"# Normalize image pixels to values between 0 and 1\nnormalization_layer = tf.keras.layers.experimental.preprocessing.Rescaling(1./255)\n\n# Resize image to given dimensions\nresizing_layer = tf.keras.layers.experimental.preprocessing.Resizing(DIM, DIM)\n\n# Decode the image so we can use it for training\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32)\n    return image\n\n\ndef read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum\n\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.cache() # cache ds for performance gains\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n\n    # normalize the image so the values are between 0 and 1\n    dataset = dataset.map(lambda x, y: (normalization_layer(x), y), num_parallel_calls=AUTOTUNE) \n    \n    # resize the images to the same height and width\n    dataset = dataset.map(lambda x, y: (resizing_layer(x), y), num_parallel_calls=AUTOTUNE) \n\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\n\ndef get_training_dataset(files=TRAINING_FILENAMES, augment=True, shuffle=True):\n    dataset = load_dataset(files, labeled=True)\n    if augment:\n        dataset = dataset.map(lambda x, y: (augment_image(x, augment=augment), y), num_parallel_calls=AUTOTUNE)\n    \n    if shuffle: \n        dataset = dataset.shuffle(1024 * REPLICAS, reshuffle_each_iteration=True)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        dataset = dataset.with_options(opt)\n    \n    dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\n\ndef get_validation_dataset(files=VALIDATION_FILENAMES, ordered=False):\n    dataset = load_dataset(files, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\n\ndef get_test_dataset(ordered=True):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:46.918005Z","iopub.execute_input":"2021-12-09T08:42:46.918296Z","iopub.status.idle":"2021-12-09T08:42:46.950709Z","shell.execute_reply.started":"2021-12-09T08:42:46.918261Z","shell.execute_reply":"2021-12-09T08:42:46.949863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Validation\nLet's look at the data and check if we did everything correct so far","metadata":{}},{"cell_type":"code","source":"# Helper function to display a certain number of images from a list of images\ndef plot_transform(num_images, images, augment = False):\n    plt.figure(figsize=(30,10))\n    for i in range(1, num_images+1):\n        image = images[0 if augment else i]\n        plt.subplot(1 ,num_images + 1, i)\n        plt.axis('off')\n        if augment:\n            img = augment_image(image)\n        else:\n            img = image\n        plt.imshow(np.clip(img, 0, 1))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:46.952441Z","iopub.execute_input":"2021-12-09T08:42:46.952979Z","iopub.status.idle":"2021-12-09T08:42:46.960505Z","shell.execute_reply.started":"2021-12-09T08:42:46.952939Z","shell.execute_reply":"2021-12-09T08:42:46.959912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_dataset = get_training_dataset(files=TRAINING_FILENAMES, augment=False)\nexample_dataset = example_dataset.unbatch().batch(15)\nexample_batch = iter(example_dataset) \nimage_batch, label_batch = next(example_batch)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:46.962017Z","iopub.execute_input":"2021-12-09T08:42:46.962552Z","iopub.status.idle":"2021-12-09T08:42:51.946917Z","shell.execute_reply.started":"2021-12-09T08:42:46.962511Z","shell.execute_reply":"2021-12-09T08:42:51.946113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = [(x) for x in image_batch]","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:51.948373Z","iopub.execute_input":"2021-12-09T08:42:51.948735Z","iopub.status.idle":"2021-12-09T08:42:51.962568Z","shell.execute_reply.started":"2021-12-09T08:42:51.948702Z","shell.execute_reply":"2021-12-09T08:42:51.961651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_transform(7, images, augment=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:51.963842Z","iopub.execute_input":"2021-12-09T08:42:51.965127Z","iopub.status.idle":"2021-12-09T08:42:52.975328Z","shell.execute_reply.started":"2021-12-09T08:42:51.965083Z","shell.execute_reply":"2021-12-09T08:42:52.974431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_transform(7, images, augment=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:52.977027Z","iopub.execute_input":"2021-12-09T08:42:52.977638Z","iopub.status.idle":"2021-12-09T08:42:54.662433Z","shell.execute_reply.started":"2021-12-09T08:42:52.977592Z","shell.execute_reply":"2021-12-09T08:42:54.661452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print the pixel values and the data type\nfor i in range(10):\n    image = image_batch[i]\n    print(\"min:\", np.min(image), \" -  max:\", np.max(image))\n\nprint(image.dtype)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:54.663727Z","iopub.execute_input":"2021-12-09T08:42:54.664578Z","iopub.status.idle":"2021-12-09T08:42:54.740898Z","shell.execute_reply.started":"2021-12-09T08:42:54.664533Z","shell.execute_reply":"2021-12-09T08:42:54.739910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"training_dataset = get_training_dataset(augment=True)\nvalidation_dataset = get_validation_dataset()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:54.742053Z","iopub.execute_input":"2021-12-09T08:42:54.742282Z","iopub.status.idle":"2021-12-09T08:42:56.482867Z","shell.execute_reply.started":"2021-12-09T08:42:54.742255Z","shell.execute_reply":"2021-12-09T08:42:56.482136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Helper function to get the model parameters and functions\ndef get_model_parameters(lr, epochs):\n    optimizer = tf.keras.optimizers.Adam(learning_rate=lr)\n    loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=LABEL_SMOOTHING)\n    metrics = [\n        tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n        tf.keras.metrics.AUC(name='auc'),\n    ]\n\n    return loss, metrics, optimizer\n\n# Helper function to compile the model\ndef compile_model(model):\n    loss, metrics, optimizer = get_model_parameters(LR_START, EPOCHS)\n    model.compile(\n        loss=loss,\n        metrics=metrics,\n        optimizer=optimizer,\n    )\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:56.483827Z","iopub.execute_input":"2021-12-09T08:42:56.484514Z","iopub.status.idle":"2021-12-09T08:42:56.491686Z","shell.execute_reply.started":"2021-12-09T08:42:56.484480Z","shell.execute_reply":"2021-12-09T08:42:56.490578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clear the session - this helps when we are creating multiple models\nK.clear_session()\n\n# Creating the model in the strategy scope places the model on the TPU\nwith strategy.scope():\n    output_bias = None\n    if initial_bias is not None:\n        output_bias = tf.keras.initializers.Constant(initial_bias)\n\n    base_model = efn.EfficientNetB5(\n        include_top=False, \n        weights='noisy-student', \n        input_shape=[DIM,DIM,3]\n    )\n\n    base_model.trainable = False\n    \n    model = tf.keras.models.Sequential([\n        base_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(8, activation='relu'),\n        tf.keras.layers.Dense(NUM_CLASSES, activation='sigmoid', bias_initializer=output_bias)\n    ])\n    model = compile_model(model)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:42:56.492806Z","iopub.execute_input":"2021-12-09T08:42:56.493025Z","iopub.status.idle":"2021-12-09T08:43:38.051853Z","shell.execute_reply.started":"2021-12-09T08:42:56.493001Z","shell.execute_reply":"2021-12-09T08:43:38.051025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:43:38.052924Z","iopub.execute_input":"2021-12-09T08:43:38.053146Z","iopub.status.idle":"2021-12-09T08:43:38.096434Z","shell.execute_reply.started":"2021-12-09T08:43:38.053119Z","shell.execute_reply":"2021-12-09T08:43:38.095477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Initial Training","metadata":{}},{"cell_type":"code","source":"history = model.fit(\n    training_dataset,\n    epochs=3,\n    validation_data=validation_dataset,\n    verbose=VERBOSE_LEVEL\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:43:38.097714Z","iopub.execute_input":"2021-12-09T08:43:38.098069Z","iopub.status.idle":"2021-12-09T08:47:13.056505Z","shell.execute_reply.started":"2021-12-09T08:43:38.098036Z","shell.execute_reply":"2021-12-09T08:47:13.055489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Full Training","metadata":{}},{"cell_type":"markdown","source":"Train for 16 Epochs with a cyclical learning rate schedule","metadata":{}},{"cell_type":"code","source":"K.clear_session()\nwith strategy.scope():\n    # Make the whole model trainable\n    base_model.trainable = True\n    # We need to compile the model again after changing the layers\n    model = compile_model(model)\n    \nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:47:13.057906Z","iopub.execute_input":"2021-12-09T08:47:13.058142Z","iopub.status.idle":"2021-12-09T08:47:13.271830Z","shell.execute_reply.started":"2021-12-09T08:47:13.058114Z","shell.execute_reply":"2021-12-09T08:47:13.270659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the cyclical learning rate scheduler\n\nfrom tensorflow.python.framework import ops\nfrom tensorflow.python.ops import math_ops\nfrom tensorflow.python.eager import context\n\ndef cyclic_learning_rate(global_step,\n                         learning_rate=0.01,\n                         max_lr=0.1,\n                         step_size=20.,\n                         gamma=0.99994,\n                         mode='triangular',\n                         name=None):\n    \"\"\"\n    Function to define the learning rate schedule\n    Source: https://www.pyimagesearch.com/2019/07/29/cyclical-learning-rates-with-keras-and-deep-learning/\n    \"\"\"\n    if global_step is None:\n        raise ValueError(\"global_step is required for cyclic_learning_rate.\")\n\n    learning_rate = ops.convert_to_tensor(\n        learning_rate, name=\"learning_rate\")\n\n    dtype = learning_rate.dtype\n    global_step = math_ops.cast(global_step, dtype)\n    step_size = math_ops.cast(step_size, dtype)\n\n    def cyclic_lr():\n        \"\"\"Helper to recompute learning rate; most helpful in eager-mode.\"\"\"\n        # computing: cycle = floor( 1 + global_step / ( 2 * step_size ) )\n        double_step = math_ops.multiply(2., step_size)\n        global_div_double_step = math_ops.divide(global_step, double_step)\n        cycle = math_ops.floor(math_ops.add(1., global_div_double_step))\n        # computing: x = abs( global_step / step_size – 2 * cycle + 1 )\n        double_cycle = math_ops.multiply(2., cycle)\n        global_div_step = math_ops.divide(global_step, step_size)\n        tmp = math_ops.subtract(global_div_step, double_cycle)\n        x = math_ops.abs(math_ops.add(1., tmp))\n        # computing: clr = learning_rate + ( max_lr – learning_rate ) * max( 0, 1 - x )\n        a1 = math_ops.maximum(0., math_ops.subtract(1., x))\n        a2 = math_ops.subtract(max_lr, learning_rate)\n        clr = math_ops.multiply(a1, a2)\n        if mode == 'triangular2':\n            clr = math_ops.divide(clr, math_ops.cast(math_ops.pow(2, math_ops.cast(\n                cycle-1, tf.int32)), tf.float32))\n        if mode == 'exp_range':\n            clr = math_ops.multiply(math_ops.pow(gamma, global_step), clr)\n        return math_ops.add(clr, learning_rate, name=name)\n\n    if not context.executing_eagerly():\n        cyclic_lr = cyclic_lr()\n\n    return cyclic_lr\n\n\n# Helper function to create a tf callback from the learning rate scheduler\ndef get_lr_callback(mode, learning_rate, max_lr, step_size):\n    \"\"\"\n    Returns the LearningRateScheduler function for the clr\n    \"\"\"\n    def lrfn(epoch):\n        return float(\n            cyclic_learning_rate(\n                epoch,\n                mode=mode,\n                learning_rate=learning_rate,\n                max_lr=max_lr,\n                step_size=step_size,\n            )().numpy()\n        )\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=2)\n    return lr_callback\n\n\n# Helper function to plot the cyclical learning rate\ndef plot_clr(mode, learning_rate, max_lr, step_size, epochs):\n    \"\"\"\n    Plots the learning rate for each epoch\n    \"\"\"\n    rates = []\n    for i in range(0, epochs):\n        x = cyclic_learning_rate(\n            i,\n            mode=mode,\n            learning_rate=learning_rate,\n            max_lr=max_lr,\n            step_size=step_size,\n        )().numpy()\n        rates.append(x)\n\n    plt.xlabel('Iterations (epochs)')\n    plt.ylabel('Learning rate')\n    plt.plot(range(epochs), rates)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:47:13.273422Z","iopub.execute_input":"2021-12-09T08:47:13.273660Z","iopub.status.idle":"2021-12-09T08:47:13.297625Z","shell.execute_reply.started":"2021-12-09T08:47:13.273633Z","shell.execute_reply":"2021-12-09T08:47:13.296578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Helper function to display the training plots\ndef display_training_curves(training, validation, title, subplot):\n    \"\"\"\n    Plots the training process\n    Source: https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu\n    \"\"\"\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(20,15), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:47:13.298974Z","iopub.execute_input":"2021-12-09T08:47:13.299310Z","iopub.status.idle":"2021-12-09T08:47:13.316196Z","shell.execute_reply.started":"2021-12-09T08:47:13.299275Z","shell.execute_reply":"2021-12-09T08:47:13.314873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create and plot the cyclical learning rate\nlr_callback = get_lr_callback(mode=MODE, learning_rate=LR_MIN, max_lr=LR_MAX, step_size=STEP_SIZE)\nplot_clr(MODE, LR_MIN, LR_MAX, STEP_SIZE, EPOCHS)\n\ncallbacks = [lr_callback]","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:47:13.317989Z","iopub.execute_input":"2021-12-09T08:47:13.318244Z","iopub.status.idle":"2021-12-09T08:47:13.652023Z","shell.execute_reply.started":"2021-12-09T08:47:13.318216Z","shell.execute_reply":"2021-12-09T08:47:13.651055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    training_dataset,\n    epochs=EPOCHS,\n    callbacks=callbacks,\n    validation_data=validation_dataset,\n    verbose=VERBOSE_LEVEL\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T08:47:13.653579Z","iopub.execute_input":"2021-12-09T08:47:13.653860Z","iopub.status.idle":"2021-12-09T09:14:36.730990Z","shell.execute_reply.started":"2021-12-09T08:47:13.653829Z","shell.execute_reply":"2021-12-09T09:14:36.729892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's look a the training process so far\ndisplay_training_curves(\n    history.history['loss'], \n    history.history['val_loss'], \n    'loss', \n    311\n)\ndisplay_training_curves(\n    history.history['auc'], \n    history.history['val_auc'], \n    'auc', \n    312\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:14:36.732930Z","iopub.execute_input":"2021-12-09T09:14:36.733260Z","iopub.status.idle":"2021-12-09T09:14:37.286634Z","shell.execute_reply.started":"2021-12-09T09:14:36.733218Z","shell.execute_reply":"2021-12-09T09:14:37.285671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:14:37.287738Z","iopub.execute_input":"2021-12-09T09:14:37.288019Z","iopub.status.idle":"2021-12-09T09:14:44.660289Z","shell.execute_reply.started":"2021-12-09T09:14:37.287990Z","shell.execute_reply":"2021-12-09T09:14:44.659364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train with ReduceLROnPlateau schedule for max. 16 Epochs or until model does not further improve.","metadata":{}},{"cell_type":"code","source":"earlyStopping = tf.keras.callbacks.EarlyStopping(\n    monitor='val_loss', \n    patience=5, \n    verbose=VERBOSE_LEVEL,\n    restore_best_weights=True\n)\n\nlr_callback = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=1, verbose=VERBOSE_LEVEL)\n\ncallbacks = [lr_callback, earlyStopping]","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:14:44.663372Z","iopub.execute_input":"2021-12-09T09:14:44.663615Z","iopub.status.idle":"2021-12-09T09:14:44.669698Z","shell.execute_reply.started":"2021-12-09T09:14:44.663588Z","shell.execute_reply":"2021-12-09T09:14:44.668616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    training_dataset,\n    epochs=EPOCHS,\n    callbacks=callbacks,\n    validation_data=validation_dataset,\n    verbose=VERBOSE_LEVEL\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:14:44.675289Z","iopub.execute_input":"2021-12-09T09:14:44.675678Z","iopub.status.idle":"2021-12-09T09:26:02.445034Z","shell.execute_reply.started":"2021-12-09T09:14:44.675638Z","shell.execute_reply":"2021-12-09T09:26:02.443993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:02.450077Z","iopub.execute_input":"2021-12-09T09:26:02.450747Z","iopub.status.idle":"2021-12-09T09:26:11.668807Z","shell.execute_reply.started":"2021-12-09T09:26:02.450702Z","shell.execute_reply":"2021-12-09T09:26:11.667412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation","metadata":{}},{"cell_type":"code","source":"val_dataset = get_validation_dataset(ordered=True)\nnum_images = count_data_items(VALIDATION_FILENAMES)\n\n# Get the images from the dataset \nval_dataset_images = val_dataset.map(lambda image, image_name: image)\n\n# Get the labels from the dataset\nval_dataset_image_name = val_dataset.map(lambda image, image_name: image_name).unbatch()\nlabels = next(iter(val_dataset_image_name.batch(num_images))).numpy().astype('U')\nlabels = [int(x) for x in labels]","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:11.671177Z","iopub.execute_input":"2021-12-09T09:26:11.671423Z","iopub.status.idle":"2021-12-09T09:26:13.053076Z","shell.execute_reply.started":"2021-12-09T09:26:11.671395Z","shell.execute_reply":"2021-12-09T09:26:13.050812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict on the images\npredictions = model.predict(val_dataset_images, verbose=1, steps=math.ceil(len(labels) / BATCH_SIZE))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:13.055050Z","iopub.execute_input":"2021-12-09T09:26:13.055933Z","iopub.status.idle":"2021-12-09T09:26:38.282510Z","shell.execute_reply.started":"2021-12-09T09:26:13.055891Z","shell.execute_reply":"2021-12-09T09:26:38.281276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Helper function to calculate the F1 Score\ndef calc_f1(prec, recall):\n    return 2*(prec*recall)/(prec+recall) if recall and prec else 0","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:38.284669Z","iopub.execute_input":"2021-12-09T09:26:38.285076Z","iopub.status.idle":"2021-12-09T09:26:38.291836Z","shell.execute_reply.started":"2021-12-09T09:26:38.285044Z","shell.execute_reply":"2021-12-09T09:26:38.290784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the precision, recall and the thresholds from the labels and predictions\nprecision, recall, thresholds = precision_recall_curve(labels, predictions)\n\n# Calculate the f1 score for each threshold\nf1score = [calc_f1(precision[i], recall[i]) for i in range(len(thresholds))]\n\n# Get the highest f1score\nidx = np.argmax(f1score)\n\n# Get the highest precision, recall, threshold and f1score\nprecision = round(precision[idx], 4)\nrecall = round(recall[idx], 4)\nthreshold = round(thresholds[idx], 4)\nf1score = round(f1score[idx], 4)\n\nprint('Precision:', precision)\nprint('Recall:', recall)\nprint('Threshold:', threshold)\nprint('F1 Score:', f1score)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:38.292971Z","iopub.execute_input":"2021-12-09T09:26:38.293206Z","iopub.status.idle":"2021-12-09T09:26:38.331797Z","shell.execute_reply.started":"2021-12-09T09:26:38.293178Z","shell.execute_reply":"2021-12-09T09:26:38.330720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the ROC/AUC\nfpr, tpr, thresholds = roc_curve(labels, predictions, pos_label=1)\nfig, c_ax = plt.subplots(1, 1, figsize=(8, 8))\nc_ax.plot(fpr, tpr, label='%s (AUC:%0.2f)' % ('Target', auc(fpr, tpr)))\nc_ax.plot([0, 1], [0, 1], color='navy', lw=1, linestyle='--')\nc_ax.legend()\nc_ax.set_xlabel('False Positive Rate')\nc_ax.set_ylabel('True Positive Rate')","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:38.333744Z","iopub.execute_input":"2021-12-09T09:26:38.334083Z","iopub.status.idle":"2021-12-09T09:26:38.588776Z","shell.execute_reply.started":"2021-12-09T09:26:38.334042Z","shell.execute_reply":"2021-12-09T09:26:38.585794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot a confusion matrix\nbinary_preds = [0 if x < threshold else 1 for x in predictions]\ncm = confusion_matrix(labels, binary_preds)\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm)\ndisp.plot()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:38.590115Z","iopub.execute_input":"2021-12-09T09:26:38.590357Z","iopub.status.idle":"2021-12-09T09:26:38.890715Z","shell.execute_reply.started":"2021-12-09T09:26:38.590329Z","shell.execute_reply":"2021-12-09T09:26:38.890088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Grad-Cam\n\nWe can visually confirm where our network is looking with Grad-CAM, ensuring that it looks at the right patterns in the image and activating around them.\nSource: https://keras.io/examples/vision/grad_cam & https://arxiv.org/abs/1610.02391","metadata":{}},{"cell_type":"code","source":"%%capture\n# Download melanoma image from wikipedia\n!wget https://upload.wikimedia.org/wikipedia/commons/thumb/6/6c/Melanoma.jpg/512px-Melanoma.jpg","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:38.891985Z","iopub.execute_input":"2021-12-09T09:26:38.892451Z","iopub.status.idle":"2021-12-09T09:26:40.137364Z","shell.execute_reply.started":"2021-12-09T09:26:38.892400Z","shell.execute_reply":"2021-12-09T09:26:40.136300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the CNNs output layer\nefficientnet_model = False\nfor layer in model.layers:\n    if layer.name == \"efficientnet-b5\":\n        efficientnet_model = layer","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:40.138980Z","iopub.execute_input":"2021-12-09T09:26:40.139856Z","iopub.status.idle":"2021-12-09T09:26:40.144778Z","shell.execute_reply.started":"2021-12-09T09:26:40.139812Z","shell.execute_reply":"2021-12-09T09:26:40.144084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_PATH = \"./512px-Melanoma.jpg\"\nimg = tf.keras.preprocessing.image.load_img(IMAGE_PATH, target_size=(DIM, DIM))\nplt.imshow(img)\norigin_img = img","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:40.146399Z","iopub.execute_input":"2021-12-09T09:26:40.146937Z","iopub.status.idle":"2021-12-09T09:26:40.441485Z","shell.execute_reply.started":"2021-12-09T09:26:40.146895Z","shell.execute_reply":"2021-12-09T09:26:40.440373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the prediction for the image\nprediction = model.predict(np.expand_dims(img, axis=0))\nbinary_prediction = [0 if x < 0.5 else 1 for x in prediction]\nprint(\"Prediction: \" + (\"Benign\" if binary_prediction == 0 else \"Malignant\"))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:40.442961Z","iopub.execute_input":"2021-12-09T09:26:40.443314Z","iopub.status.idle":"2021-12-09T09:26:53.403995Z","shell.execute_reply.started":"2021-12-09T09:26:40.443267Z","shell.execute_reply":"2021-12-09T09:26:53.402860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# First, we create a model that maps the input image to the activations\n# of the last conv layer as well as the output predictions\ngrad_model = tf.keras.models.Model(\n    [efficientnet_model.inputs], [efficientnet_model.get_layer('top_conv').output, efficientnet_model.output]\n)\n\n# Then, we compute the gradient of the top predicted class for our input image\n# with respect to the activations of the last conv layer\nwith tf.GradientTape() as tape:\n    last_conv_layer_output, preds = grad_model(np.expand_dims(img, axis=0))\n    class_channel = preds[:, round(np.mean(tf.argmax(preds[0]).numpy()))]\n\n# This is the gradient of the output neuron (top predicted or chosen)\n# with regard to the output feature map of the last conv layer\ngrads = tape.gradient(class_channel, last_conv_layer_output)\n\n# This is a vector where each entry is the mean intensity of the gradient\n# over a specific feature map channel\npooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))\n\n# We multiply each channel in the feature map array\n# by \"how important this channel is\" with regard to the top predicted class\n# then sum all the channels to obtain the heatmap class activation\nlast_conv_layer_output = last_conv_layer_output[0]\nheatmap = last_conv_layer_output @ pooled_grads[..., tf.newaxis]\nheatmap = tf.squeeze(heatmap)\n\n# For visualization purpose, we will also normalize the heatmap between 0 & 1\nheatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)\nheatmap = heatmap.numpy()\nheatmap_array = heatmap","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:53.405750Z","iopub.execute_input":"2021-12-09T09:26:53.406094Z","iopub.status.idle":"2021-12-09T09:26:58.134536Z","shell.execute_reply.started":"2021-12-09T09:26:53.406052Z","shell.execute_reply":"2021-12-09T09:26:58.133522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display heatmap\nplt.matshow(heatmap)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:58.135826Z","iopub.execute_input":"2021-12-09T09:26:58.136075Z","iopub.status.idle":"2021-12-09T09:26:58.403158Z","shell.execute_reply.started":"2021-12-09T09:26:58.136046Z","shell.execute_reply":"2021-12-09T09:26:58.401863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.cm as cm\n\n# Load the original image\no_img = tf.keras.preprocessing.image.load_img(IMAGE_PATH, target_size=(DIM, DIM))\n\n# Rescale heatmap to a range 0-255\nheatmap = np.uint8(255 * heatmap)\n\n# Use jet colormap to colorize heatmap\njet = cm.get_cmap(\"jet\")\n\n# Use RGB values of the colormap\njet_colors = jet(np.arange(256))[:, :3]\njet_heatmap = jet_colors[heatmap]\n\n# Create an image with RGB colorized heatmap\njet_heatmap = tf.keras.preprocessing.image.array_to_img(jet_heatmap)\njet_heatmap = jet_heatmap.resize((DIM,DIM))\njet_heatmap = tf.keras.preprocessing.image.img_to_array(jet_heatmap)\n\n# Superimpose the heatmap on original image\nsuperimposed_img = jet_heatmap * 0.4 + o_img\nsuperimposed_img = tf.keras.preprocessing.image.array_to_img(superimposed_img)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:58.405193Z","iopub.execute_input":"2021-12-09T09:26:58.405643Z","iopub.status.idle":"2021-12-09T09:26:58.441738Z","shell.execute_reply.started":"2021-12-09T09:26:58.405589Z","shell.execute_reply":"2021-12-09T09:26:58.440395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot both images\nfig = plt.figure(figsize = (12, 8))\n\n# Create subplot with the grad cam image\nax1 = fig.add_subplot(1, 2, 1)\nax1 = ax1.imshow(superimposed_img)\n\n# Create subplot with the origin image\nax2 = fig.add_subplot(1, 2, 2)\nax2.imshow(origin_img)\n\n# Add a colorbar\ncmap = cm.jet\nnorm = colors.Normalize(vmin=np.min(heatmap_array), vmax=np.max(heatmap_array))\nfig.colorbar(cm.ScalarMappable(norm=norm, cmap=cmap), orientation='horizontal', label='Output layer activation')","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:58.443649Z","iopub.execute_input":"2021-12-09T09:26:58.444182Z","iopub.status.idle":"2021-12-09T09:26:59.198858Z","shell.execute_reply.started":"2021-12-09T09:26:58.444129Z","shell.execute_reply":"2021-12-09T09:26:59.197861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"test_dataset = get_test_dataset(ordered=True)\nnum_test_images = count_data_items(TEST_FILENAMES)\n\ntest_dataset_images = test_dataset.map(lambda image, image_name: image)\ntest_dataset_image_name = test_dataset.map(lambda image, image_name: image_name).unbatch()\ntest_ids = next(iter(test_dataset_image_name.batch(num_test_images))).numpy().astype('U')","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:26:59.200219Z","iopub.execute_input":"2021-12-09T09:26:59.200455Z","iopub.status.idle":"2021-12-09T09:27:04.143561Z","shell.execute_reply.started":"2021-12-09T09:26:59.200425Z","shell.execute_reply":"2021-12-09T09:27:04.142663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(test_dataset_images, verbose=1, steps=math.ceil(len(test_ids) / BATCH_SIZE))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:27:04.145237Z","iopub.execute_input":"2021-12-09T09:27:04.145585Z","iopub.status.idle":"2021-12-09T09:27:18.265108Z","shell.execute_reply.started":"2021-12-09T09:27:04.145545Z","shell.execute_reply":"2021-12-09T09:27:18.264018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(predictions)})\npred_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:27:18.267060Z","iopub.execute_input":"2021-12-09T09:27:18.267415Z","iopub.status.idle":"2021-12-09T09:27:18.307366Z","shell.execute_reply.started":"2021-12-09T09:27:18.267371Z","shell.execute_reply":"2021-12-09T09:27:18.306740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(np.round(pred_df['target'].values)).value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:27:18.308377Z","iopub.execute_input":"2021-12-09T09:27:18.308962Z","iopub.status.idle":"2021-12-09T09:27:18.322629Z","shell.execute_reply.started":"2021-12-09T09:27:18.308920Z","shell.execute_reply":"2021-12-09T09:27:18.321907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T09:27:18.323840Z","iopub.execute_input":"2021-12-09T09:27:18.324595Z","iopub.status.idle":"2021-12-09T09:27:18.379864Z","shell.execute_reply.started":"2021-12-09T09:27:18.324551Z","shell.execute_reply":"2021-12-09T09:27:18.379050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}