{"cells":[{"metadata":{},"cell_type":"markdown","source":"[Modelling Notebook](https://www.kaggle.com/venkat555/ranzcr-clip-tpu-densenet-with-kfold/)\n\n**Credits** \n* Flowers TPU Notebook \n* Fellow Kagglers - All the amazing posts and kernels to learn from"},{"metadata":{},"cell_type":"markdown","source":"## Dependencies"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import math, os, re, warnings, random, glob\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras import Sequential\nfrom kaggle_datasets import KaggleDatasets","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Hardware configuration"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16 * REPLICAS\nHEIGHT = 512\nWIDTH = 512 \nCHANNELS = 3\nN_CLASSES = 5\nTTA_STEPS = 3 # Do TTA if > 0 \nIMAGE_SIZE = [512, 512] # At this size, a GPU will run out of memory. Use the TPU.\n                        # For GPU training, please select 224 x 224 px image size.\nSEED =555    \nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nAUG_BATCH = BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    # RandomCrop, VFlip, HFilp, RandomRotate\n    image = tf.image.rot90(image,k=np.random.randint(4))\n    image = tf.image.random_flip_left_right(image , seed=SEED)\n    image=  image = tf.image.random_flip_up_down(image, seed=SEED)\n    IMG_SIZE=IMAGE_SIZE[0]\n    # Add 6 pixels of padding\n    image = tf.image.resize_with_crop_or_pad(image, IMG_SIZE + 6, IMG_SIZE + 6) \n    # Random crop back to the original size\n    image = tf.image.random_crop(image, size=[IMG_SIZE, IMG_SIZE, 3])\n    image = tf.image.random_brightness(image, max_delta=0.5) # Random brightness\n    image = tf.image.random_saturation(image, 0, 2, seed=SEED)\n    image = tf.image.adjust_saturation(image, 3)\n    \n    #image = tf.image.central_crop(image, central_fraction=0.5)\n    return image, label ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Auxilary Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_float32_2(image, label):\n    max_val = tf.reduce_max(label, axis=-1,keepdims=True)\n    cond = tf.equal(label, max_val)\n    label = tf.where(cond, tf.ones_like(label), tf.zeros_like(label))\n    return tf.cast(image, tf.float32), tf.cast(label, tf.int32)\n\ndef to_float32(image, label):\n    return tf.cast(image, tf.float32), label\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [1024,1024, 3]) # explicit size needed for TPU\n    return image\n\n\n\n# Create a dictionary describing the features.\n\n\ndef read_labeled_tfrecord(example):\n    # Create a dictionary describing the features.\n    train_feature_description = {\n        \"CVC - Abnormal\": tf.io.FixedLenFeature([], tf.int64),\n        \"CVC - Borderline\": tf.io.FixedLenFeature([], tf.int64),\n        \"CVC - Normal\": tf.io.FixedLenFeature([], tf.int64),\n        \"ETT - Abnormal\": tf.io.FixedLenFeature([], tf.int64),\n        \"ETT - Borderline\": tf.io.FixedLenFeature([], tf.int64),\n        \"ETT - Normal\": tf.io.FixedLenFeature([], tf.int64),\n        \"NGT - Abnormal\": tf.io.FixedLenFeature([], tf.int64),\n        \"NGT - Borderline\": tf.io.FixedLenFeature([], tf.int64),\n        \"NGT - Incompletely Imaged\": tf.io.FixedLenFeature([], tf.int64),\n        \"NGT - Normal\": tf.io.FixedLenFeature([], tf.int64),\n        \"StudyInstanceUID\" : tf.io.FixedLenFeature([], tf.string),\n        \"Swan Ganz Catheter Present\" : tf.io.FixedLenFeature([], tf.int64),\n        \"image\" : tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, train_feature_description)\n    image = decode_image(example['image'])    \n    uid= example[\"StudyInstanceUID\"]\n    cvca = example[\"CVC - Abnormal\"]\n    cvcb = example[\"CVC - Borderline\"]\n    cvcn = example[\"CVC - Normal\"]\n    etta = example[\"ETT - Abnormal\"]\n    ettb = example[\"ETT - Borderline\"]\n    ettn = example[\"ETT - Normal\"]\n    ngta = example[\"NGT - Abnormal\"]\n    ngtb = example[\"NGT - Borderline\"]\n    ngti = example[\"NGT - Incompletely Imaged\"]\n    ngtn = example[\"NGT - Normal\"]\n    sgcp = example[\"Swan Ganz Catheter Present\"]\n\n    values  = [  etta, ettb, ettn, ngta, ngtb, ngti, ngtn,cvca, cvcb, cvcn , sgcp]\n    label = tf.cast(0, tf.int32)\n    for i in range(len(values)):\n        if ( values[i]==1):\n            label = tf.cast(i, tf.int32)\n    return image,label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT  = {\n    \"StudyInstanceUID\" : tf.io.FixedLenFeature([], tf.string),\n    \"image\" : tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    image= tf.image.resize(image, [IMAGE_SIZE[0],IMAGE_SIZE[0]])\n    image_name = example['StudyInstanceUID']\n    return image, image_name # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    # RandomCrop, VFlip, HFilp, RandomRotate\n    image = tf.image.rot90(image,k=np.random.randint(4))\n    image = tf.image.random_flip_left_right(image , seed=SEED)\n    image= tf.image.random_flip_up_down(image, seed=SEED)\n    IMG_SIZE=IMAGE_SIZE[0]\n    # Add 6 pixels of padding\n    #image = tf.image.resize_with_crop_or_pad(image, IMG_SIZE + 6, IMG_SIZE + 6) \n    # Random crop back to the original size\n    #image = tf.image.random_crop(image, size=[IMG_SIZE, IMG_SIZE, 3])\n    image = tf.image.random_brightness(image, max_delta=0.5) # Random brightness\n    image = tf.image.random_saturation(image, 0, 2, seed=SEED)\n    image = tf.image.adjust_saturation(image, 3)\n    \n    #image = tf.image.central_crop(image, central_fraction=0.5)\n    return image, label   \n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_training_dataset(dataset, do_aug=True , do_onehot=False):\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.batch(AUG_BATCH)\n    #if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO) # note we put AFTER batching\n    if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTO) \n    dataset = dataset.unbatch()\n    \n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False , tta= False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    if tta:\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    #the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n    #c = 0\n    #for filename in filenames:\n    #    c += sum(1 for _ in tf.data.TFRecordDataset(filename))\n    #return c\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Test Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"database_base_path = '/kaggle/input/ranzcr-clip-catheter-line-classification/'\nsubmission = pd.read_csv(f'{database_base_path}sample_submission.csv')\ndisplay(submission.head())\nTEST_FILENAMES = tf.io.gfile.glob(f'{database_base_path}test_tfrecords/*.tfrec') # predic\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nprint(f'GCS: test: {NUM_TEST_IMAGES}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## List Models loaded "},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path_list = glob.glob('/kaggle/input/ranzcr-clip/model*.h5')\nmodel_path_list.sort()\n\nprint('Models to predict:')\nprint(*model_path_list, sep='\\n')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test set predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import keras\n    \nmodels = []    \ni = 0\nfor model_path in model_path_list:\n    print(model_path)\n    K.clear_session()\n    models.append(keras.models.load_model(model_path))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\" TTA_STEPS = {} \".format(TTA_STEPS))\nif TTA_STEPS > 0:\n    for step in range(TTA_STEPS):\n        test_ds = get_test_dataset(ordered=True, tta=True)\n        print(f'TTA step {step+1}/{TTA_STEPS}')\n        test_images_ds = test_ds.map(lambda image, image_name: image)\n        probabilities = np.average([models[i].predict(test_images_ds) for i in range(len(models))], axis = 0)\nelse:\n    test_ds = get_test_dataset(ordered=True, tta=True)\n    test_images_ds = test_ds.map(lambda image, image_name: image)\n    probabilities = np.average([models[i].predict(test_images_ds) for i in range(1)], axis = 0)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate submission file"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids] +  [probabilities[:,i] for i in range(probabilities.shape[1])]), fmt=['%s', '%f','%f' , '%f', '%f','%f' , '%f', '%f','%f' , '%f', '%f','%f'  ], delimiter=',', header='StudyInstanceUID,ETT - Abnormal,ETT - Borderline,ETT - Normal,NGT - Abnormal,NGT - Borderline,NGT - Incompletely Imaged,NGT - Normal,CVC - Abnormal,CVC - Borderline,CVC - Normal,Swan Ganz Catheter Present', comments='')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!head submission.csv","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}