{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Imports"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import KFold\nimport efficientnet.tfkeras as efn\nimport matplotlib.pyplot as plt\nimport tensorflow_addons as tfa\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport random\nimport time\nimport os","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Hardware configuration"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPUv3-8')\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    STRATEGY = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    print('Running on GPU/CPU')\n    tf.keras.mixed_precision.set_global_policy('mixed_float16')\n    STRATEGY = tf.distribute.get_strategy()\n\nBATCH_SIZE = 16 * STRATEGY.num_replicas_in_sync\n\nprint('Number of replicas:', STRATEGY.num_replicas_in_sync)\nprint('Using tensorflow %s' % tf.__version__)\nprint('Batch size: %.i' % BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n    print('Seeding everything with seed %.i.' % seed)\n    \n    \nIMG_SIZE = 768\nSEED = 42\nEPOCHS = 24\nVERBOSE = 1\nPATIENCE = [4, 2]\nFACTOR = .1\nMIN_LR = 0.9e-8\nwrapper = '%.i - %.3is - loss - %.4f - auc - %.4f - val_loss - %.4f - val_auc - %.4f - lr - %.1e'\nUSED_FOLDS = [0]\n\ngcs_path = [\n    KaggleDatasets().get_gcs_path('ranzcr-clip-raw-kfold-tfrecords'),\n    KaggleDatasets().get_gcs_path('ranzcr-clip-augmented-kfold-tfrecords-1'),\n    KaggleDatasets().get_gcs_path('ranzcr-clip-augmented-kfold-tfrecords-2'),\n    KaggleDatasets().get_gcs_path('ranzcr-clip-augmented-kfold-tfrecords-3'),\n    KaggleDatasets().get_gcs_path('ranzcr-clip-augmented-kfold-tfrecords-4'),\n    KaggleDatasets().get_gcs_path('ranzcr-clip-augmented-kfold-tfrecords-5'),\n    KaggleDatasets().get_gcs_path('ranzcr-clip-augmented-kfold-tfrecords-6'),\n    KaggleDatasets().get_gcs_path('ranzcr-clip-augmented-kfold-tfrecords-7'),\n    KaggleDatasets().get_gcs_path('ranzcr-clip-augmented-kfold-tfrecords-8')]\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Helper functions\n1. Data workflow\n2. Model Configurations"},{"metadata":{"_kg_hide-input":true,"trusted":true,"scrolled":true},"cell_type":"code","source":"feature_map = {\n    'image': tf.io.FixedLenFeature([], tf.string),\n    'StudyInstanceUID': tf.io.FixedLenFeature([], tf.string),  \n    'ETT - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n    'ETT - Borderline': tf.io.FixedLenFeature([], tf.int64),\n    'ETT - Normal': tf.io.FixedLenFeature([], tf.int64),\n    'NGT - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n    'NGT - Borderline': tf.io.FixedLenFeature([], tf.int64),\n    'NGT - Incompletely Imaged': tf.io.FixedLenFeature([], tf.int64),\n    'NGT - Normal': tf.io.FixedLenFeature([], tf.int64),\n    'CVC - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n    'CVC - Borderline': tf.io.FixedLenFeature([], tf.int64),\n    'CVC - Normal': tf.io.FixedLenFeature([], tf.int64),\n    'Swan Ganz Catheter Present': tf.io.FixedLenFeature([], tf.int64)}\n\n\ndef count_data_items(filenames):\n    return np.sum([int(x[:-6].split('-')[-1]) for x in filenames])\n\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.reshape(image, [IMG_SIZE, IMG_SIZE, 3])\n    return image\n\n\ndef scale_image(image, target):\n    image = tf.cast(image, tf.float32) / 255.\n    return image, target\n\n\ndef read_tfrecord(example):\n    example = tf.io.parse_single_example(example, feature_map)\n    image = decode_image(example['image'])\n    target = [\n        tf.cast(example['ETT - Abnormal'], tf.float32),\n        tf.cast(example['ETT - Borderline'], tf.float32),\n        tf.cast(example['ETT - Normal'], tf.float32),\n        tf.cast(example['NGT - Abnormal'], tf.float32),\n        tf.cast(example['NGT - Borderline'], tf.float32),\n        tf.cast(example['NGT - Incompletely Imaged'], tf.float32),\n        tf.cast(example['NGT - Normal'], tf.float32),\n        tf.cast(example['CVC - Abnormal'], tf.float32),\n        tf.cast(example['CVC - Borderline'], tf.float32),\n        tf.cast(example['CVC - Normal'], tf.float32),\n        tf.cast(example['Swan Ganz Catheter Present'], tf.float32)]\n    return image, target\n\n\ndef get_dataset(filenames, ordered=True, shuffled=False, repeated=False, \n                cached=False, distributed=True):\n    auto = tf.data.experimental.AUTOTUNE\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=auto)\n    if not ordered:\n        ignore_order = tf.data.Options()\n        ignore_order.experimental_deterministic = False\n        dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=auto)\n    if shuffled:\n        dataset = dataset.shuffle(2048, seed=SEED)\n    if repeated:\n        dataset = dataset.repeat()\n    dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n    dataset = dataset.map(scale_image, num_parallel_calls=auto)\n    if cached:\n        dataset = dataset.cache()\n    dataset = dataset.prefetch(auto)\n    if distributed:\n        dataset = STRATEGY.experimental_distribute_dataset(dataset)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"scrolled":true},"cell_type":"code","source":"def get_model(pretrained=False):\n    model = tf.keras.models.Sequential()\n    \n    if pretrained == True:\n        \n        pretrained_model = tf.keras.models.load_model('../input/ranzcr-clip-pretrain-tf/RANZCR_1.h5')\n    \n        for layer in pretrained_model.layers[:-1]:\n            model.add(layer)   \n            \n    else:\n        model.add(efn.EfficientNetB6(\n            include_top=False,\n            input_shape=(IMG_SIZE, IMG_SIZE, 3),\n            weights='imagenet',\n            pooling='avg'))\n    \n    model.add(tf.keras.layers.Dense(11, \n        kernel_initializer=tf.keras.initializers.RandomUniform(seed=SEED),\n        bias_initializer=tf.keras.initializers.Zeros(), name='dense_top'))\n    model.add(tf.keras.layers.Activation('sigmoid', dtype='float32'))\n    \n    return model\n\n\ndef print_function(*args):\n    print(wrapper % args)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Look through augmented images"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"filenames = tf.io.gfile.glob(os.path.join(gcs_path[1], 'fold_0/epoch_0/*.tfrec'))\n\ndataset = get_dataset(filenames[:1], shuffled=True, distributed=False)\n\nplt.figure(figsize=[20, 6])\n\nfor i, sample in enumerate(dataset.unbatch().take(10).as_numpy_iterator()):\n    plt.subplot(2, 5, i + 1)\n    plt.imshow(sample[0])\n    plt.axis('off')\n    \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inspect model"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"model = get_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Run KFold"},{"metadata":{"_kg_hide-output":true,"trusted":true,"scrolled":false},"cell_type":"code","source":"histories = []\nscores = []\n\nkfold = KFold(n_splits=5, shuffle=True, random_state=SEED)\nfolders = ['fold_0', 'fold_1', 'fold_2', 'fold_3', 'fold_4']\nswitcher = {x:x // 3 + 1 for x in range(EPOCHS)}\n\nfor i, (train_index, val_index) in enumerate(kfold.split(folders)):\n    \n    if i in USED_FOLDS:\n        \n        if tpu: \n            tf.tpu.experimental.initialize_tpu_system(tpu)\n        tf.keras.backend.clear_session()\n\n\n        @tf.function\n        def train_steps(generator):\n            def train_steps_wrapper(x, y):\n                with tf.GradientTape() as tape:\n                    probabilities = model(x, training=True)\n                    loss = loss_fn(y, probabilities)\n                    gradients = tape.gradient(loss, model.trainable_variables)\n                optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n                train_loss.update_state(loss)\n                train_accuracy.update_state(y, probabilities)\n            for _ in tf.range(steps_per_epoch):\n                STRATEGY.run(train_steps_wrapper, next(generator))\n\n\n        @tf.function\n        def val_steps(generator):\n            def val_steps_wrapper(x, y):\n                probabilities = model(x, training=False)\n                loss = loss_fn(y, probabilities)\n                val_loss.update_state(loss)\n                val_accuracy.update_state(y, probabilities)\n            for _ in tf.range(validation_steps):\n                STRATEGY.run(val_steps_wrapper, next(generator))\n\n\n        with STRATEGY.scope():\n            model = get_model(pretrained=False)\n\n            optimizer = tf.keras.optimizers.Adam()\n            loss_fn = tfa.losses.SigmoidFocalCrossEntropy(alpha = 0.50, gamma = 2.0)\n\n            train_accuracy = tf.keras.metrics.AUC(multi_label=True)\n            val_accuracy = tf.keras.metrics.AUC(multi_label=True)\n            train_loss = tf.keras.metrics.Mean()\n            val_loss = tf.keras.metrics.Mean()\n\n        val_filenames = []\n        for j in val_index:\n            val_filenames += tf.io.gfile.glob(os.path.join(\n                gcs_path[0], folders[j], '*.tfrec'))\n\n        val_dataset = get_dataset(val_filenames, cached=True)\n        validation_steps = count_data_items(val_filenames) // BATCH_SIZE\n\n        history = {'loss': [], 'auc': [], 'val_loss': [], 'val_auc': []}\n        count = [0, 0]\n        best_score = 0\n        best_loss = 10.\n\n        for epoch in range(EPOCHS):\n\n            print(f'Epoch {epoch + 1}/{EPOCHS}')\n            tic = time.time()\n\n            train_filenames = []\n            for j in train_index:\n                train_filenames += tf.io.gfile.glob(os.path.join(\n                    gcs_path[switcher[epoch]], folders[j], f'epoch_{epoch % 3}/*.tfrec'))\n            np.random.shuffle(train_filenames)\n            train_dataset = get_dataset(train_filenames, ordered=False, shuffled=True)\n\n            steps_per_epoch = count_data_items(train_filenames) // BATCH_SIZE\n\n            train_steps(iter(train_dataset))\n            val_steps(iter(val_dataset))\n\n            history['loss'].append(train_loss.result().numpy())\n            history['auc'].append(train_accuracy.result().numpy())\n            history['val_loss'].append(val_loss.result().numpy())\n            history['val_auc'].append(val_accuracy.result().numpy())\n\n            lr = tf.keras.backend.get_value(optimizer.lr)\n\n            print_function(steps_per_epoch, time.time() - tic, \n                           history['loss'][-1], history['auc'][-1],\n                           history['val_loss'][-1], history['val_auc'][-1], lr)\n\n            if history['val_loss'][-1] < best_loss:\n                best_loss = history['val_loss'][-1]\n                count[1] += 1\n\n            if history['val_auc'][-1] > best_score:\n                best_score = history['val_auc'][-1]\n                model.save_weights(f'./RANZCR_{i}.h5', overwrite=True)\n                count = [0, 0]\n\n            if history['val_loss'][-1] != best_loss and history['val_auc'][-1] != best_score:\n                count[0] += 1\n                count[1] += 1\n\n            if count[0] >= PATIENCE[0]:\n                break\n\n            if count[1] >= PATIENCE[1]:\n                new_lr = lr * FACTOR\n                if new_lr >= MIN_LR:\n                    tf.keras.backend.set_value(optimizer.lr, new_lr)\n                count[1] = 0\n\n            train_loss.reset_states()\n            train_accuracy.reset_states()\n\n            val_loss.reset_states()\n            val_accuracy.reset_states()\n\n        histories.append(pd.DataFrame(history))\n        scores.append(best_score)\n        \n    else:\n        pass","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Results"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"scores = pd.DataFrame({\n    'Fold': np.arange(len(scores)) + 1,\n    'AUC': scores})\n\ndisplay(scores.head())\n\nprint('CV %.4f' % scores['AUC'].mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"for i in range(len(histories)):\n    histories[i].loc[:, ['auc', 'val_auc']].plot()\n    plt.title(f'Fold {i + 1}')\n    plt.xlabel('Epochs')\n    plt.ylabel('AUC')\n    plt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}