{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install efficientnet\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport math, re, os\nimport tensorflow as tf\nprint(tf.__version__)\nfrom kaggle_datasets import KaggleDatasets # Only on Kaggle\nfrom tensorflow.keras import models, layers, optimizers\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.applications import Xception, ResNet50V2, NASNetLarge, InceptionResNetV2, DenseNet121\nfrom efficientnet.keras import EfficientNetB7\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout, Flatten, Input, Conv2D, multiply, LocallyConnected2D, Lambda, BatchNormalization\nfrom tensorflow.keras.models import Model, load_model\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# detect and init the TPU\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\n\n# instantiate a distribution strategy\ntpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = tpu_strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"BASE_DIR = \"/kaggle/input/ranzcr-clip-catheter-line-classification/\"\ndf_train = pd.read_csv(os.path.join(BASE_DIR, \"train.csv\"), index_col=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET_SIZE = 768\nCHANNELS = 3\nBATCH_SIZE = 16 * REPLICAS\nSTEPS_PER_EPOCH = len(df_train) // BATCH_SIZE\nOUTPUT_SIZE = 11","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_float32_2(image, label):\n    max_val = tf.reduce_max(label, axis=-1,keepdims=True)\n    cond = tf.equal(label, max_val)\n    label = tf.where(cond, tf.ones_like(label), tf.zeros_like(label))\n    return tf.cast(image, tf.float32), tf.cast(label, tf.int32)\n\ndef to_float32(image, label):\n    return tf.cast(image, tf.float32), label\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [1024,1024, 3]) # explicit size needed for TPU\n    return image\n\n\ndef read_labeled_tfrecord(example):\n    # Create a dictionary describing the features.\n    LABELED_TFREC_FORMAT = {\n        \"StudyInstanceUID\"           : tf.io.FixedLenFeature([], tf.string),\n        \"image\"                      : tf.io.FixedLenFeature([], tf.string),\n        \"ETT - Abnormal\"             : tf.io.FixedLenFeature([], tf.int64), \n        \"ETT - Borderline\"           : tf.io.FixedLenFeature([], tf.int64), \n        \"ETT - Normal\"               : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Abnormal\"             : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Borderline\"           : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Incompletely Imaged\"  : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Normal\"               : tf.io.FixedLenFeature([], tf.int64), \n        \"CVC - Abnormal\"             : tf.io.FixedLenFeature([], tf.int64), \n        \"CVC - Borderline\"           : tf.io.FixedLenFeature([], tf.int64), \n        \"CVC - Normal\"               : tf.io.FixedLenFeature([], tf.int64), \n        \"Swan Ganz Catheter Present\" : tf.io.FixedLenFeature([], tf.int64),\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image']) \n    image= tf.image.resize(image, [IMAGE_SIZE[0],IMAGE_SIZE[0]])\n    uid= example[\"StudyInstanceUID\"]\n    cvca = example[\"CVC - Abnormal\"]\n    cvcb = example[\"CVC - Borderline\"]\n    cvcn = example[\"CVC - Normal\"]\n    etta = example[\"ETT - Abnormal\"]\n    ettb = example[\"ETT - Borderline\"]\n    ettn = example[\"ETT - Normal\"]\n    ngta = example[\"NGT - Abnormal\"]\n    ngtb = example[\"NGT - Borderline\"]\n    ngti = example[\"NGT - Incompletely Imaged\"]\n    ngtn = example[\"NGT - Normal\"]\n    sgcp = example[\"Swan Ganz Catheter Present\"]\n\n    label  = [etta, ettb, ettn, ngta, ngtb, ngti, ngtn,cvca, cvcb, cvcn , sgcp]\n    label=[tf.cast(i,tf.float32) for i in label]\n    return image,label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT  = {\n    \"StudyInstanceUID\" : tf.io.FixedLenFeature([], tf.string),\n    \"image\" : tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    image= tf.image.resize(image, [IMAGE_SIZE[0],IMAGE_SIZE[0]])\n    image_name = example['StudyInstanceUID']\n    return image, image_name # returns a dataset of image(s)\n\ndef read_labeled_tf_record(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    return dataset\n\ndef data_augment(image, label):\n    image = tf.image.random_flip_left_right(image , seed=SEED)\n    image = tf.image.random_flip_up_down(image, seed=SEED)\n    image = tf.image.random_brightness(image, max_delta=0.5) # Random brightness\n    image = tf.image.random_saturation(image, 0, 2, seed=SEED)\n    image = tf.image.adjust_saturation(image, 3)\n    \n    #image = tf.image.central_crop(image, central_fraction=0.5)\n    return image, label   \n\ndef get_training_dataset(dataset):\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    #the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\ndef load_dataset(filenames, labeled = True, ordered = False):\n    ignore_order = tf.data.Options()\n    \n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    \n    dataset = (tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO).with_options(ignore_order).\n               map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO))\n    \n    return dataset\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(\"ranzcr-clip-catheter-line-classification\") # you can list the bucket with \"!gsutil ls $GCS_DS_PATH\"\nprint( \"GCS_DS_PATH :{} \".format(GCS_DS_PATH))\n\nIMAGE_SIZE = [TARGET_SIZE, TARGET_SIZE]\nSEED = 555    \nFOLDS = 3\nEPOCHS = 30\nFIRST_FOLD_ONLY = True\n\ntraining_filenames = []\ntraining_filenames.append(GCS_DS_PATH + '/train_tfrecords/*.tfrec')\nTRAINING_FILENAMES = tf.io.gfile.glob(training_filenames)\nTEST_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + '/test_tfrecords/*.tfrec')\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nprint(\"STEPS_PER_EPOCH {}\".format(STEPS_PER_EPOCH))\nprint('Dataset: {} training images,  {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    input_shape = (TARGET_SIZE, TARGET_SIZE, CHANNELS)\n    in_lay = Input(input_shape)\n    #conv_base = InceptionResNetV2(include_top = False, weights = 'imagenet', input_shape = input_shape)\n    conv_base = Xception(include_top = False, weights = 'imagenet', input_shape = input_shape)\n    #conv_base = DenseNet121(include_top = False, weights = 'imagenet', input_shape = input_shape)\n    #conv_base = EfficientNetB7(include_top = False, weights = 'imagenet', input_shape = input_shape)\n\n    conv_base.trainable = False\n    pt_features = conv_base(in_lay)\n    bn_features = BatchNormalization()(pt_features)\n    # Credit: kmader https://www.kaggle.com/kmader/attention-on-pretrained-vgg16-for-bone-age\n    # here we do an attention mechanism to turn pixels in the GAP on an off\n    attn_layer = Conv2D(64, kernel_size = (1,1), padding = 'same', activation = 'relu')(bn_features)\n    attn_layer = Conv2D(16, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n    attn_layer = LocallyConnected2D(1, kernel_size = (1,1), padding = 'valid', activation = 'sigmoid', implementation=2)(attn_layer)\n    # fan it out to all of the channels\n    pt_depth = conv_base.get_output_shape_at(0)[-1]\n    up_c2_w = np.ones((1, 1, 1, pt_depth))\n    up_c2 = Conv2D(pt_depth, kernel_size = (1,1), padding = 'same', \n                activation = 'linear', use_bias = False, weights = [up_c2_w])\n    up_c2.trainable = False\n    attn_layer = up_c2(attn_layer)\n\n    mask_features = multiply([attn_layer, bn_features])\n    gap_features = GlobalAveragePooling2D()(mask_features)\n    gap_mask = GlobalAveragePooling2D()(attn_layer)\n    # to account for missing values from the attention model\n    gap = Lambda(lambda x: x[0]/x[1], name = 'RescaleGAP')([gap_features, gap_mask])\n    gap_dr = Dropout(0.5)(gap)\n    dr_steps = Dropout(0.25)(Dense(1024, activation = 'elu')(gap_dr))\n    out_layer = Dense(11, activation = 'sigmoid')(dr_steps)\n    model = Model(inputs = [in_lay], outputs = [out_layer])\n    model.compile(optimizer = Adam(lr = 0.005), loss = 'binary_crossentropy', metrics = [\"AUC\"])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.python.keras.callbacks import ReduceLROnPlateau\nfrom tensorflow.python.platform import tf_logging as logging\n\n# https://stackoverflow.com/questions/52227286/reducelronplateau-fallback-to-the-previous-weights-with-the-minimum-acc-loss\nclass ReduceLRBacktrack(ReduceLROnPlateau):\n    def __init__(self, best_path, *args, **kwargs):\n        super(ReduceLRBacktrack, self).__init__(*args, **kwargs)\n        self.best_path = best_path\n\n    def on_epoch_end(self, epoch, logs=None):\n        current = logs.get(self.monitor)\n        if current is None:\n            logging.warning('Reduce LR on plateau conditioned on metric `%s` '\n                            'which is not available. Available metrics are: %s',\n                             self.monitor, ','.join(list(logs.keys())))\n        if not self.monitor_op(current, self.best): # not new best\n            if not self.in_cooldown(): # and we're not in cooldown\n                if self.wait+1 >= self.patience: # going to reduce lr\n                    # load best model so far\n                    print(\"Backtracking to best model before reducing LR\")\n                    self.model.load_weights(self.best_path)\n\n        super().on_epoch_end(epoch, logs) # actually reduce LR","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_dataset_for_kfold(dataset):\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef create_callbacks(model_save_path,fold):\n    model_checkpoint_path = \"{}/cmodel-{}.h5\".format(model_save_path , fold)\n    model_save = ModelCheckpoint(model_checkpoint_path, \n                                save_best_only = True, \n                                save_weights_only = True,\n                                monitor = 'val_loss', \n                                mode = 'min', verbose = 1)\n\n    early_stop = EarlyStopping(monitor = 'val_loss', min_delta = 0.0001, \n                              patience = 5, mode = 'min', verbose = 1,\n                              restore_best_weights = True)\n    \n    reduce_lr = ReduceLRBacktrack(best_path = model_checkpoint_path,\n                                  monitor = 'val_loss', factor = 0.3, \n                                  patience = 2, min_delta = 0.0001, \n                                  mode = 'min', verbose = 1)\n    \"\"\"\n    reduce_lr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.3,\n                                  patience = 2, min_delta = 0.0001, \n                                  mode = 'min', verbose = 1)\n    \"\"\"\n    callbacks = [model_save, early_stop, reduce_lr]\n    return callbacks","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\nimport tensorflow.keras.backend as K\n\nfolds=5\n#def train_cross_validate(folds = 5):\nhistories = []\nmodels = []\n# Define per-fold score containers\nacc_per_fold = []\nloss_per_fold = []\nearly_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 10)\nkfold = KFold(folds, shuffle = True, random_state = SEED)\nfor f, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n    print(); print('#'*25)\n    print('### FOLD',f+1, \"with trn_ind\", trn_ind, \"and val_ind\", val_ind)\n    print('#'*25)\n    training_df = pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES})\n    train_dataset = load_dataset(list(training_df.loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)\n    val_dataset = load_dataset(list(training_df.loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)\n    K.clear_session()\n    with tpu_strategy.scope():\n        model = get_model()\n        #model.load_weights('./cmodel-0.h5')\n    validation_dataset = get_validation_dataset_for_kfold(val_dataset)\n\n    print(\"Getting training dataset\")\n    training_dataset = get_training_dataset(train_dataset)\n\n    model.summary()\n    \n    history = model.fit(\n        training_dataset,\n        steps_per_epoch = STEPS_PER_EPOCH,\n        epochs = EPOCHS // 3,\n        callbacks = create_callbacks(\".\", f),\n        validation_data = validation_dataset\n    )\n    \n    with tpu_strategy.scope():\n        set_trainable = True\n        for layer in model.layers: # Sets all layers to trainable except the one AFTER locally_connected2d\n            print(\"Layer\", layer.name, \"set to trainable\", set_trainable)\n            layer.trainable = set_trainable\n            set_trainable = not layer.name.startswith(\"locally\") or not set_trainable\n        model.compile(optimizer = Adam(lr = 0.0001), loss = \"binary_crossentropy\", metrics = [\"AUC\"])\n\n    model.summary()\n    history_ft = model.fit(\n        training_dataset,\n        steps_per_epoch = STEPS_PER_EPOCH,\n        epochs = EPOCHS,\n        callbacks = create_callbacks(\".\", f),\n        validation_data = validation_dataset\n    )\n\n    scores = model.evaluate(validation_dataset)\n    #print(scores)\n    print(f'Score for fold {f+1}: {model.metrics_names[0]} of {scores[0]}')\n    acc_per_fold.append(scores[1] * 100)\n    loss_per_fold.append(scores[0])\n    model.save(\"model-fold{}.h5\".format(f+1))\n    models.append(model)\n    histories.append(history)\n    if FIRST_FOLD_ONLY: break\n    #return histories, models,acc_per_fold, loss_per_fold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nfor h in [history, history_ft]:\n    auc = h.history['auc']\n    val_auc = h.history['val_auc']\n    loss = h.history['loss']\n    val_loss = h.history['val_loss']\n\n    epochs = range(1, len(auc) + 1)\n\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))\n    sns.set_style(\"white\")\n    plt.suptitle('Train history', size = 15)\n\n    ax1.plot(epochs, auc, \"bo\", label = \"Training auc\")\n    ax1.plot(epochs, val_auc, \"b\", label = \"Validation auc\")\n    ax1.set_title(\"Training and validation auc\")\n    ax1.legend()\n\n    ax2.plot(epochs, loss, \"bo\", label = \"Training loss\", color = 'red')\n    ax2.plot(epochs, val_loss, \"b\", label = \"Validation loss\", color = 'red')\n    ax2.set_title(\"Training and validation loss\")\n    ax2.legend()\n\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_and_predict(folds = 5):\n    test_ds = get_test_dataset(ordered=True) #map(data_augment, num_parallel_calls=AUTO) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n    test_images_ds = test_ds.map(lambda image, idnum: image)\n    print('Start training %i folds'%folds)\n    histories, models,acc_per_fold,loss_per_fold  = train_cross_validate(folds = folds)\n    # == Provide average scores ==\n    print('------------------------------------------------------------------------')\n    print('Score per fold')\n    for i in range(0, len(acc_per_fold)):\n        print('------------------------------------------------------------------------')\n        print(f'> Fold {i+1} - Loss: {loss_per_fold[i]} - Accuracy: {acc_per_fold[i]}%')\n    print('------------------------------------------------------------------------')\n    print('Average scores for all folds:')\n    print(f'> Accuracy: {np.mean(acc_per_fold)} (+- {np.std(acc_per_fold)})')\n    print(f'> Loss: {np.mean(loss_per_fold)}')\n    print('------------------------------------------------------------------------')\n    \n    print('Computing predictions...')\n    # get the mean probability of the folds models\n    if FIRST_FOLD_ONLY: probabilities = np.average([models[i].predict(test_images_ds) for i in range(1)], axis = 0)\n    else: probabilities = np.average([models[i].predict(test_images_ds) for i in range(folds)], axis = 0)\n    \n    return histories, models, probabilities, test_ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Uncomment the below and the \"def train_cross_validate\" and \"return\" lines to run with all folds\n#%%time\n#histories, models, probabilities, test_ds = train_and_predict(folds = FOLDS)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}