{"cells":[{"metadata":{},"cell_type":"markdown","source":"[Inference Notebook](https://www.kaggle.com/venkat555/ranzcr-clip-tpu-densenet-with-kfold-inference/)\n\n**Credits** \n* Flowers TPU Notebook \n* Fellow Kagglers - All the amazing posts and kernels to learn from \n* Using various image size which are already stratified by InstanceUID from https://www.kaggle.com/prateek0x/creating-stratified-groupkfold-tfrecords-256x256\n* https://www.kaggle.com/prateek0x/stratified-groupkfold-with-efn-tfrecords"},{"metadata":{},"cell_type":"markdown","source":"## Header Imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport math, re, os\nimport tensorflow as tf\nimport numpy as np\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nimport warnings, gc, random, math, os, re\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE\nfrom sklearn.model_selection import train_test_split \nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow.keras.backend as K\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Bootstrap "},{"metadata":{"trusted":true},"cell_type":"code","source":"FFO = False \nif (FFO==False):\n    FOLDS = 3\n    EPOCHS = 10\n    FIRST_FOLD_ONLY = False\n    EFN=3\nelse:\n    FOLDS = 3\n    EPOCHS = 10\n    FIRST_FOLD_ONLY = True\n    EFN=1\n\nIMG_SIZE_WIDTH = HEIGHT = WIDTH = 512 \n\nprint(\" folds={} epochs={} ffo={} efn={} save={}\".format(FOLDS,EPOCHS,FIRST_FOLD_ONLY, EFN, FFO))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\nSEED =555 \nseed = SEED\nseed_everything(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TPU Setup"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"try: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept ValueError: # no TPU found, detect GPUs\n    strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    #strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setup to read data"},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = [WIDTH, HEIGHT] # At this size, a GPU will run out of memory. Use the TPU.\n\n# For GPU training, please select 224 x 224 px image size.\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(\"ranzcr-{0}x{1}\".format(IMG_SIZE_WIDTH,IMG_SIZE_WIDTH)) # you can list the bucket with \"!gsutil ls $GCS_DS_PATH\"\nprint( \"GCS_DS_PATH :{} \".format(GCS_DS_PATH))\n   \nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nAUG_BATCH = BATCH_SIZE\nCHANNELS=3\n\ntraining_filenames = []\ntraining_filenames.append(GCS_DS_PATH + '/*.tfrec')\nTRAINING_FILENAMES = tf.io.gfile.glob(training_filenames)\nGCS_TEST_DS_PATH = KaggleDatasets().get_gcs_path(\"ranzcr-clip-catheter-line-classification\".format(IMG_SIZE_WIDTH,IMG_SIZE_WIDTH))\nTEST_FILENAMES = tf.io.gfile.glob(GCS_TEST_DS_PATH + '/test_tfrecords/*.tfrec') \nprint(\" [file names] train {} , test : {}\".format(TRAINING_FILENAMES,TEST_FILENAMES))\n# inferred from \n# label_num_to_disease_map.json\nCLASSES = [    \n    'ETT - Abnormal',\n    'ETT - Borderline',\n    'ETT - Normal',\n    'NGT - Abnormal',\n    'NGT - Borderline',\n    'NGT - Incompletely Imaged',\n    'NGT - Normal',\n    'CVC - Abnormal',\n    'CVC - Borderlinedex',\n    'CVC - Normal',\n    'Swan Ganz Catheter Present']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setup a learning rate scheduler"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Using an LR ramp up because fine-tuning a pre-trained model.\n# Starting with a high LR would break the pre-trained weights.\n\nLR_START = 0.00001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Augmentation \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    # Shear\n    if p_shear > .2:\n        if p_shear > .6:\n            image = transform_shear(image, HEIGHT, shear=20.)\n        else:\n            image = transform_shear(image, HEIGHT, shear=-20.)\n            \n    # Rotation\n    if p_rotation > .2:\n        if p_rotation > .6:\n            image = transform_rotation(image, HEIGHT, rotation=45.)\n        else:\n            image = transform_rotation(image, HEIGHT, rotation=-45.)\n    \n    \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > .75:\n        image = tf.image.transpose(image)\n        \n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n        \n    # Pixel-level transforms\n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n        \n    # Crops\n    if p_crop > .7:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.6)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.8)\n    elif p_crop > .4:\n        crop_size = tf.random.uniform([], int(HEIGHT*.6), HEIGHT, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n            \n    image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset Transformation utilities"},{"metadata":{"trusted":true},"cell_type":"code","source":"def onehot(image,label):\n    NUMCLASSES = len(CLASSES)\n    return image,tf.one_hot(label,NUMCLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_float32_2(image, label):\n    max_val = tf.reduce_max(label, axis=-1,keepdims=True)\n    cond = tf.equal(label, max_val)\n    label = tf.where(cond, tf.ones_like(label), tf.zeros_like(label))\n    return tf.cast(image, tf.float32), tf.cast(label, tf.int32)\n\ndef to_float32(image, label):\n    return tf.cast(image, tf.float32), label\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [IMG_SIZE_WIDTH,IMG_SIZE_WIDTH, 3]) # explicit size needed for TPU\n    return image\n\ndef decode_test_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [IMG_SIZE_WIDTH*2,IMG_SIZE_WIDTH*2, 3]) # explicit size needed for TPU\n    return image\n\n# Create a dictionary describing the features.\n\n\ndef read_labeled_tfrecord(example):\n    # Create a dictionary describing the features.\n    LABELED_TFREC_FORMAT = {\n        \"StudyInstanceUID\"           : tf.io.FixedLenFeature([], tf.string),\n        \"image\"                      : tf.io.FixedLenFeature([], tf.string),\n        \"ETT - Abnormal\"             : tf.io.FixedLenFeature([], tf.int64), \n        \"ETT - Borderline\"           : tf.io.FixedLenFeature([], tf.int64), \n        \"ETT - Normal\"               : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Abnormal\"             : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Borderline\"           : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Incompletely Imaged\"  : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Normal\"               : tf.io.FixedLenFeature([], tf.int64), \n        \"CVC - Abnormal\"             : tf.io.FixedLenFeature([], tf.int64), \n        \"CVC - Borderline\"           : tf.io.FixedLenFeature([], tf.int64), \n        \"CVC - Normal\"               : tf.io.FixedLenFeature([], tf.int64), \n        \"Swan Ganz Catheter Present\" : tf.io.FixedLenFeature([], tf.int64),\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image']) \n    image= tf.image.resize(image, [IMAGE_SIZE[0],IMAGE_SIZE[0]])\n    uid= example[\"StudyInstanceUID\"]\n    cvca = example[\"CVC - Abnormal\"]\n    cvcb = example[\"CVC - Borderline\"]\n    cvcn = example[\"CVC - Normal\"]\n    etta = example[\"ETT - Abnormal\"]\n    ettb = example[\"ETT - Borderline\"]\n    ettn = example[\"ETT - Normal\"]\n    ngta = example[\"NGT - Abnormal\"]\n    ngtb = example[\"NGT - Borderline\"]\n    ngti = example[\"NGT - Incompletely Imaged\"]\n    ngtn = example[\"NGT - Normal\"]\n    sgcp = example[\"Swan Ganz Catheter Present\"]\n\n    label  = [  etta, ettb, ettn, ngta, ngtb, ngti, ngtn,cvca, cvcb, cvcn , sgcp]\n#     label = tf.cast(0, tf.int32)\n#     for i in range(len(values)):\n#         if ( values[i]==1):\n#             label = tf.cast(i, tf.int32)\n    label=[tf.cast(i,tf.float32) for i in label]\n    return image,label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT  = {\n    \"StudyInstanceUID\" : tf.io.FixedLenFeature([], tf.string),\n    \"image\" : tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_test_image(example['image'])\n    image= tf.image.resize(image, [IMAGE_SIZE[0],IMAGE_SIZE[0]])\n    image_name = example['StudyInstanceUID']\n    return image, image_name # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef data_augment_old(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    # RandomCrop, VFlip, HFilp, RandomRotate\n    #image = tf.image.rot90(image,k=np.random.randint(4))\n    image = tf.image.random_flip_left_right(image , seed=SEED)\n    image= tf.image.random_flip_up_down(image, seed=SEED)\n    IMG_SIZE=IMAGE_SIZE[0]\n    # Add 6 pixels of padding\n    #image = tf.image.resize_with_crop_or_pad(image, IMG_SIZE + 6, IMG_SIZE + 6) \n    # Random crop back to the original size\n    #image = tf.image.random_crop(image, size=[IMG_SIZE, IMG_SIZE, 3])\n    image = tf.image.random_brightness(image, max_delta=0.5) # Random brightness\n    image = tf.image.random_saturation(image, 0, 2, seed=SEED)\n    image = tf.image.adjust_saturation(image, 3)\n    \n    #image = tf.image.central_crop(image, central_fraction=0.5)\n    return image, label   \n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_training_dataset(dataset, do_aug=True , do_onehot=False):\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.batch(AUG_BATCH)\n    #if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO) # note we put AFTER batching\n    if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTO) \n    dataset = dataset.unbatch()\n    \n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    #the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n    #c = 0\n    #for filename in filenames:\n    #    c += sum(1 for _ in tf.data.TFRecordDataset(filename))\n    #return c\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint(\"STEPS_PER_EPOCH {}\".format(STEPS_PER_EPOCH))\n#NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\n#print('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))\nprint('Dataset: {} training images,  {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Print Data Shapes"},{"metadata":{},"cell_type":"markdown","source":"#### Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"# data dump\nprint(\"Training data shapes:\")\ntraining_raw_dataset=load_dataset(TRAINING_FILENAMES, labeled=True)\n\nfor image,label in get_training_dataset(training_raw_dataset, do_aug=False , do_onehot=False).take(3):\n    print(image.numpy().shape, label.numpy().shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Peek at training data\ntraining_dataset = get_training_dataset(training_raw_dataset , do_aug=False , do_onehot=False ).map(to_float32)\ntraining_dataset = training_dataset.unbatch().batch(20)\ntrain_batch = iter(training_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for image,label in get_training_dataset(training_raw_dataset, do_aug=False , do_onehot=False).take(1):\n    print(image.numpy().shape, label.numpy() , type(label.numpy()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualization utilities"},{"metadata":{"trusted":true},"cell_type":"code","source":"# numpy and matplotlib defaults\nnp.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data    \n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n    \ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        #print(\"label {}\".format(label))\n        title = '' if label is None else CLASSES[np.argmax(label,axis=-1)]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n\ndef display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_batch_of_images(next(train_batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# peer at test data\ntest_dataset = get_test_dataset()\ntest_dataset = test_dataset.unbatch().batch(20)\ntest_batch = iter(test_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_batch_of_images(next(test_batch), None)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Test"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Test data shapes:\")\nfor image, image_name in get_test_dataset().take(3):\n    print(image.numpy().shape )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Modelling"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import keras\nfrom tensorflow.keras import regularizers\nimport efficientnet.tfkeras as efn\nbackbones = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6\n            ]\n\ndef get_model():\n    lr_scheduler = keras.optimizers.schedules.ExponentialDecay(\n        initial_learning_rate=1e-5, \n        decay_steps=10000, \n        decay_rate=0.9)\n\n    with strategy.scope():\n\n        #pretrained_model = tf.keras.applications.EfficientNetB0(input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),weights='imagenet', include_top=False)\n        pretrained_model = backbones[EFN](input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),weights='noisy-student', include_top=False)\n        #pretrained_model.trainable = False # False = transfer learning, True = fine-tuning\n        pretrained_model.trainable = True\n        \n        set_trainable = False\n#         for layer in pretrained_model.layers:\n#             if layer.name in ['conv5_block32_1_conv', 'conv5_block32_2_conv', 'conv5_block32_1_bn' , 'conv5_block32_1_relu' ,\n#                              'conv5_block32_concat' , 'bn' , 'relu']:\n#                 set_trainable = True\n#             if set_trainable:\n#                 layer.trainable = True\n#             else:\n#                 layer.trainable = False\n        \n        layers = [(layer, layer.name, layer.trainable) for layer in pretrained_model.layers]\n        layer_frame=pd.DataFrame(layers, columns=['Layer Type', 'Layer Name', 'Layer Trainable']) \n\n        model = tf.keras.Sequential([\n            #img_adjust_layer,\n            pretrained_model,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            \n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.Dense(1024,activation='relu'  ),\n            tf.keras.layers.Dropout(0.3),\n            \n            tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.Dense(512,activation='relu'  ),\n            tf.keras.layers.Dropout(0.3),\n            \n            tf.keras.layers.Dense(len(CLASSES), activation='sigmoid',dtype='float32')\n        ])\n        \n        auc = keras.metrics.AUC(name='auc')\n \n    opt = tf.keras.optimizers.Adam(lr=0.00001) \n    model.compile(\n        optimizer=opt,\n        loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05),\n        metrics= auc\n    )\n    return model, layer_frame\n\nbase_model , layer_frame = get_model()\n# Let's take a look to see how many layers are in the base model\nprint(\"Number of layers in the base model: \", len(base_model.layers))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"layer_frame.tail(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_layers = [(layer, layer.name, layer.trainable) for layer in base_model.layers]\nbase_layer_frame=pd.DataFrame(base_layers, columns=['Layer Type', 'Layer Name', 'Layer Trainable']) \nbase_layer_frame","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setup for KFold "},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_dataset_for_kfold(dataset, do_onehot=True):\n    dataset = dataset.batch(BATCH_SIZE)\n    if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTO) # we must use one hot like augmented train data\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_callbacks(model_save_path,fold,verbose=1 ):\n    verbose = int(verbose>0)\n    checkpoint_filepath = \"{}/eff{}-cmodel-fold{}.h5\".format(model_save_path , EFN, fold)\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(\n        filepath=checkpoint_filepath,\n        save_weights_only=True,\n        monitor='val_auc',\n        mode='max',\n        save_best_only=True,\n        verbose=verbose)\n\n    reducelr = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor='val_auc',\n        mode='max',\n        factor=0.1,\n        patience=3,\n        verbose=0)\n\n    earlystop = tf.keras.callbacks.EarlyStopping(\n        monitor='val_auc',\n        mode='max',\n        patience=5, \n        verbose=verbose)\n\n    callbacks = [reducelr, earlystop,checkpoint]\n    return callbacks","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\nimport tensorflow.keras.backend as K\nVERBOSE =1\ndef train_cross_validate(folds = 5):\n    histories = []\n    models = []\n    # Define per-fold score containers\n    acc_per_fold = []\n    loss_per_fold = []\n    #early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 10)\n    kfold = KFold(folds, shuffle = True, random_state = SEED)\n    for f, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n        print(); print('#'*25)\n        print('### FOLD',f+1)\n        print('#'*25)\n        train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)\n        val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)\n        K.clear_session()\n        model = base_model\n        data_for_validation = get_validation_dataset_for_kfold(val_dataset , do_onehot=False)\n            \n        history = model.fit(\n            get_training_dataset(train_dataset, do_aug=False , do_onehot=False),\n            steps_per_epoch = STEPS_PER_EPOCH,\n            epochs = EPOCHS,\n            callbacks = create_callbacks(\".\",f, VERBOSE),\n            validation_data = data_for_validation,\n            verbose=2\n        )\n        scores = model.evaluate(data_for_validation, verbose=0)\n        #print(scores)\n        print(f'Score for fold {f+1}: {model.metrics_names[0]} of {scores[0]}')\n        acc_per_fold.append(scores[1] * 100)\n        loss_per_fold.append(scores[0])\n        model.save(\"model-fold{}.h5\".format(f+1))\n        models.append(model)\n        histories.append(history)\n        if FIRST_FOLD_ONLY: break\n    return histories, models,acc_per_fold, loss_per_fold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_and_predict(folds = 5):\n    test_ds = get_test_dataset(ordered=True) #map(data_augment, num_parallel_calls=AUTO) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n    test_images_ds = test_ds.map(lambda image, idnum: image)\n    print('Start training %i folds'%folds)\n    histories, models,acc_per_fold,loss_per_fold  = train_cross_validate(folds = folds)\n    # == Provide average scores ==\n    print('------------------------------------------------------------------------')\n    print('Score per fold')\n    for i in range(0, len(acc_per_fold)):\n        print('------------------------------------------------------------------------')\n        print(f'> Fold {i+1} - Loss: {loss_per_fold[i]} - Accuracy: {acc_per_fold[i]}%')\n    print('------------------------------------------------------------------------')\n    print('Average scores for all folds:')\n    print(f'> Accuracy: {np.mean(acc_per_fold)} (+- {np.std(acc_per_fold)})')\n    print(f'> Loss: {np.mean(loss_per_fold)}')\n    print('------------------------------------------------------------------------')\n    \n    print('Computing predictions...')\n    # get the mean probability of the folds models\n    if FIRST_FOLD_ONLY: probabilities = np.average([models[i].predict(test_images_ds) for i in range(1)], axis = 0)\n    else: probabilities = np.average([models[i].predict(test_images_ds) for i in range(folds)], axis = 0)\n    \n    return histories, models, probabilities , test_ds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train and Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nhistories, models, probabilities, test_ds = train_and_predict(folds = FOLDS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate submission file "},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids] +  [probabilities[:,i] for i in range(probabilities.shape[1])]), fmt=['%s', '%f','%f' , '%f', '%f','%f' , '%f', '%f','%f' , '%f', '%f','%f'  ], delimiter=',', header='StudyInstanceUID,ETT - Abnormal,ETT - Borderline,ETT - Normal,NGT - Abnormal,NGT - Borderline,NGT - Incompletely Imaged,NGT - Normal,CVC - Abnormal,CVC - Borderline,CVC - Normal,Swan Ganz Catheter Present', comments='')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!head submission.csv\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()\n    \ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Plot train/loss curves"},{"metadata":{"trusted":true},"cell_type":"code","source":"for history in histories : \n    #print(history)\n    display_training_curves(history.history['loss'], history.history['val_loss'], 'loss', 211)\n    display_training_curves(history.history['auc'], history.history['val_auc'], 'accuracy', 212)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n## Confusion Matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_class_name(label):\n    index = 0 \n    size = len(CLASSES)\n    name = 9\n    for i in range(size):\n        if ( label[i]==1):\n            name = i\n            \n    \n    return name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_correct_labels(cm_correct_labels):\n    labels = [] \n    for x in cm_correct_labels :\n        labels.append(get_class_name(x))\n    \n    return labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nall_labels = []; all_prob = []; all_pred = [];  raw_labels = []\nkfold = KFold(FOLDS, shuffle = True, random_state = SEED)\nfor j, (trn_ind, val_ind) in enumerate( kfold.split(TRAINING_FILENAMES) ):\n    print('Inferring fold',j+1,'validation images...')\n    VAL_FILES = list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES'])\n    NUM_VALIDATION_IMAGES = count_data_items(VAL_FILES)\n    cmdataset = get_validation_dataset_for_kfold(load_dataset(VAL_FILES, labeled = True, ordered = True), do_onehot=False)\n    images_ds = cmdataset.map(lambda image, label: image)\n    labels_ds = cmdataset.map(lambda image, label: label).unbatch()\n    labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy()\n    raw_labels.append(labels)\n    labels = get_correct_labels(labels)\n    all_labels.append(labels  ) # get everything as one batch\n    prob = models[j].predict(images_ds)\n    all_prob.append( prob )\n    all_pred.append( np.argmax(prob, axis=-1) )\n    if FIRST_FOLD_ONLY: break\ncm_correct_labels = np.concatenate(all_labels)\ncm_probabilities = np.concatenate(all_prob)\ncm_predictions = np.concatenate(all_pred)\ncm_raw_labels = np.concatenate(raw_labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Compute ROC AUC Score "},{"metadata":{"trusted":true},"cell_type":"code","source":"y_true = pd.DataFrame(columns=CLASSES)\nfor i in range(len(CLASSES)):\n    y_true[CLASSES[i]] = cm_raw_labels[:,i]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred=pd.DataFrame(columns=CLASSES)\nfor i in range(len(CLASSES)):\n    y_pred[CLASSES[i]] = cm_probabilities[:,i]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_true.head(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred.head(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\ny_true_flat = y_true.values.reshape(-1)\ny_pred_flat = y_pred.values.reshape(-1)\nroc_auc_score(y_true_flat, y_pred_flat)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roc_auc_score(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}