{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"### Installing Keras Package for Vision Transformer\n!pip install vit-keras\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"### Importing Necessary Packages\nimport numpy as np\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow as tf, tensorflow.keras.backend as K\nimport re,random,os,math\nimport pandas as pd\nfrom sklearn.model_selection import KFold\nfrom vit_keras import vit, utils\n\nfrom kaggle_datasets import KaggleDatasets\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('ranzcr-clip-catheter-line-classification') \nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH +'/train_tfrecords/*.tfrec') #+ tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH+'/test_tfrecords/*.tfrec') #\n\nprint(\"#\"*5)\nprint(TRAINING_FILENAMES)\nprint(TEST_FILENAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Flexible Options to try \n\nbatch_size = 4* strategy.num_replicas_in_sync\nAUTO = tf.data.experimental.AUTOTUNE\nfolds = 5\nimage_size = 128  # We'll resize input images to this size\ninput_shape = (image_size, image_size,3)\nnum_epochs = 10\n\n\n#Seed Everything\nSEED = 42\nprint(f'setting everything to seed {SEED}')\nrandom.seed(SEED)\nos.environ['PYTHONHASHSEED'] = str(SEED)\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#### Learning Rate \n\nLR_START = 0.00001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LABELED_TFREC_FORMAT = {\n    'CVC - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n    'CVC - Borderline': tf.io.FixedLenFeature([], tf.int64),\n    'CVC - Normal': tf.io.FixedLenFeature([], tf.int64),\n    'ETT - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n    'ETT - Borderline': tf.io.FixedLenFeature([], tf.int64),\n    'ETT - Normal': tf.io.FixedLenFeature([], tf.int64),\n    'NGT - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n    'NGT - Borderline': tf.io.FixedLenFeature([], tf.int64),\n    'NGT - Incompletely Imaged': tf.io.FixedLenFeature([], tf.int64),\n    'NGT - Normal': tf.io.FixedLenFeature([], tf.int64),\n    'Swan Ganz Catheter Present': tf.io.FixedLenFeature([], tf.int64),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\n\n\nUNLABELED_TFREC_FORMAT = {\n    'image': tf.io.FixedLenFeature([], tf.string),\n    'StudyInstanceUID': tf.io.FixedLenFeature([], tf.string)\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.image.resize(image, [image_size, image_size])\n    image = tf.reshape(image, [image_size, image_size, 3])\n    return image\n\n\ndef read_labeled_tfrecord(example):\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    #     label = tf.cast(example['target'], tf.int32)\n    labels = [  # Edit this to add whatever labels you want your model to predict\n        example['ETT - Abnormal'],\n        example['ETT - Borderline'],\n        example['ETT - Normal'],\n        example['NGT - Abnormal'],\n        example['NGT - Borderline'],\n        example['NGT - Incompletely Imaged'],\n        example['NGT - Normal'],\n        example['CVC - Abnormal'],\n        example['CVC - Borderline'],\n        example['CVC - Normal'],\n        example['Swan Ganz Catheter Present'],\n    ]\n    labels = tf.dtypes.cast(labels, tf.int32)\n\n    return image, labels  # returns a dataset of (image, label) pairs\n\n\ndef read_unlabeled_tfrecord(example):\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['StudyInstanceUID']\n    return image, idnum  # returns a dataset of image(s)\n\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False  # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames,\n                                      num_parallel_reads=AUTO)  # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order)  # use data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord,\n                          num_parallel_calls=AUTO)  # returns a dataset of (image, label) pairs if labeled = True or (image, id) pair if labeld = False\n    return dataset\n\n\ndef get_training_dataset(dataset, do_aug=True):\n    dataset = dataset.repeat()  # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(batch_size)\n    dataset = dataset.prefetch(AUTO)  # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\n\ndef get_validation_dataset(dataset, do_onehot=True):\n    dataset = dataset.batch(batch_size)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    #     dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.batch(batch_size)\n    dataset = dataset.prefetch(AUTO)  # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\nNUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES) * (folds - 1.) / folds)\nNUM_VALIDATION_IMAGES = int(count_data_items(TRAINING_FILENAMES) * (1. / folds))\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // batch_size\n\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES,\n                                                                                           NUM_VALIDATION_IMAGES,\n                                                                                           NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"######Training and Saving Weights For Inference for each Fold\ntrn_dict = {}\nval_dict = {}\n\nkfold = KFold(folds, shuffle=True, random_state=42)\nfor f, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n    trn_dict[f] = trn_ind\n    val_dict[f] = val_ind\n\nprint(trn_dict)\nprint(val_dict)\n\n\ndef train(fold=0):\n    with strategy.scope():\n        transformer = vit.vit_l32(\n            image_size=image_size,\n            include_top=False,\n            pretrained_top = False,\n            weights=\"imagenet21k+imagenet2012\",\n        )\n\n        model = tf.keras.Sequential([\n            transformer,\n            tf.keras.layers.Dense(11, activation='sigmoid')\n        ])\n        model.compile(\n            optimizer=tf.keras.optimizers.Adam(),\n            loss='binary_crossentropy',\n            metrics=[tf.keras.metrics.AUC(multi_label=True)])\n    print(model.summary())\n\n    checkpoint_filepath = f'./fold{fold}vit.h5'\n    checkpoint_callback = keras.callbacks.ModelCheckpoint(\n        checkpoint_filepath,\n        monitor=\"val_auc\",\n        save_best_only=True,\n        save_weights_only=True,\n        mode='max'\n    )\n    train_dataset = load_dataset(\n        list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[trn_dict[fold]]['TRAINING_FILENAMES']),\n        labeled=True)\n    val_dataset = load_dataset(\n        list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_dict[fold]]['TRAINING_FILENAMES']),\n        labeled=True, ordered=True)\n    history = model.fit(\n                        get_training_dataset(train_dataset),\n                        steps_per_epoch = STEPS_PER_EPOCH,\n                        epochs=num_epochs,\n                        callbacks = [checkpoint_callback, lr_callback], #model_checkpoint_callback\n                        validation_data = get_validation_dataset(val_dataset),\n                        verbose=1\n                        )\n    return history\n\nhistory = train(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}