{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"! pip install -q efficientnet >> /dev/null","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Import modules"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport re\nimport math\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import classification_report, roc_auc_score, roc_curve, confusion_matrix\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\n\nimport efficientnet.tfkeras as efn\n\n# from functools import partial\n# from albumentations import (\n#     Compose, RandomBrightness, JpegCompression, HueSaturationValue, RandomContrast, HorizontalFlip,\n#     Rotate\n# )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Set parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = '/kaggle/input/ranzcr-clip-catheter-line-classification'\n\nMODEL_PATH = '/kaggle/working/models'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nDEVICE = 'TPU' # ['CPU' GPU' 'TPU']\n\nENABLE_MIXED_PRECISION = True # [True False]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"XLA_ACCELERATE = True\n\n\nif XLA_ACCELERATE:\n    tf.config.optimizer.set_jit(True)\n    print('Accelerated Linear Algebra enabled')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\n\nFOLDS = 3 \n\nIMG_SIZE = 600\n\nBATCH_SIZE = 16 # [8, 16, 32, 64, 128, 256, 512]\n\nEPOCHS = 50\n\nEFF_NET = 'B7' # ['B0',B1','B2',B3','B4',B5','B6',B7']\n\nVERBOSE = 1 # [0: silent, 1: progress bar, 2: single line]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TF_RECS = len(os.listdir(f'{DATA_PATH}/train_tfrecords'))\n\nprint(NUM_TF_RECS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setup devices and settings"},{"metadata":{"trusted":true},"cell_type":"code","source":"# For kaggle tpus\nfrom kaggle_datasets import KaggleDatasets\nif DEVICE == 'TPU':\n    print('TPU')\n    DATA_PATH = KaggleDatasets().get_gcs_path(DATA_PATH.split('/')[-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEVICE == 'CPU':\n\n    strategy = tf.distribute.get_strategy()\n    print('\\nUsing Default Distribution Strategy  for CPU')\n\n\nif DEVICE == 'GPU':\n\n    gpu_accelerarors = tf.config.list_physical_devices('GPU')\n        \n    if len(gpu_accelerarors) > 1:\n        strategy = tf.distribute.MirroredStrategy()\n        print(f'Number of GPUs available: {len(gpu_accelerarors)}')\n        print('\\n Using Mirrored Distribution Strategy')\n        \n    else:\n        strategy = tf.distribute.get_strategy()\n        if len(gpu_accelerarors) == 1:\n            print(f'Number of GPUs available: 1')\n            print('\\nUsing Default Distribution Strategy for GPU')\n        else:\n            print('ERROR: GPU not available')\n            print('\\nUsing Default Distribution Strategy  for CPU')\n        \nif DEVICE == 'TPU':\n\n    try:\n        resolver = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(resolver)\n        tf.tpu.experimental.initialize_tpu_system(resolver)\n        strategy = tf.distribute.experimental.TPUStrategy(resolver)\n        tpu_accelerarors = tf.config.list_logical_devices('TPU')\n        print(f'Number of TPU cores available: {len(tpu_accelerarors)}')\n        print(f'\\nUsing TPU Distribution Strategy')\n        \n    except:\n        print('ERROR: TPU not available')\n        print('\\nUsing Default Distribution Strategy for CPU')\n        strategy = tf.distribute.get_strategy()\n        \n        \nif ENABLE_MIXED_PRECISION:\n    \n    print('\\nMixed Precision enabled:')\n    \n    if DEVICE == 'GPU':\n        policy = mixed_precision.Policy('mixed_float16')\n        \n    if DEVICE == 'TPU':\n        policy = mixed_precision.Policy('mixed_bfloat16')\n        \n    mixed_precision.set_policy(policy)\n    \n    print('\\t...Compute dtype: %s' % policy.compute_dtype)\n    print('\\t...Variable dtype: %s' % policy.variable_dtype)\n\n\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'\\nREPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset:\n    \n    feature_description = {\n        \"StudyInstanceUID\"           : tf.io.FixedLenFeature([], tf.string),\n        \"image\"                      : tf.io.FixedLenFeature([], tf.string),\n        \"ETT - Abnormal\"             : tf.io.FixedLenFeature([], tf.int64), \n        \"ETT - Borderline\"           : tf.io.FixedLenFeature([], tf.int64), \n        \"ETT - Normal\"               : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Abnormal\"             : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Borderline\"           : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Incompletely Imaged\"  : tf.io.FixedLenFeature([], tf.int64), \n        \"NGT - Normal\"               : tf.io.FixedLenFeature([], tf.int64), \n        \"CVC - Abnormal\"             : tf.io.FixedLenFeature([], tf.int64), \n        \"CVC - Borderline\"           : tf.io.FixedLenFeature([], tf.int64), \n        \"CVC - Normal\"               : tf.io.FixedLenFeature([], tf.int64), \n        \"Swan Ganz Catheter Present\" : tf.io.FixedLenFeature([], tf.int64),\n    }\n    \n#     transforms = Compose([\n#             Rotate(limit=40),\n#             RandomBrightness(limit=0.1),\n#             JpegCompression(quality_lower=85, quality_upper=100, p=0.5),\n#             HueSaturationValue(hue_shift_limit=20, sat_shift_limit=30, val_shift_limit=20, p=0.5),\n#             RandomContrast(limit=0.2, p=0.5),\n#             HorizontalFlip(),\n#         ])\n    \n    aug = tf.keras.Sequential([\n        tf.keras.layers.Activation(None, dtype='float32'),\n        tf.keras.layers.experimental.preprocessing.RandomFlip(dtype='float32'),\n        tf.keras.layers.experimental.preprocessing.RandomRotation(0.04, fill_mode='constant',dtype='float32'),\n        tf.keras.layers.experimental.preprocessing.RandomTranslation(0.15,0.15, fill_mode='constant',dtype='float32'),\n#         tf.keras.layers.experimental.preprocessing.RandomContrast((.9,1.2),dtype='float32'),\n#         tf.keras.layers.experimental.preprocessing.RandomZoom(0.2, fill_mode='constant'),\n        tf.keras.layers.experimental.preprocessing.RandomWidth(0.35, dtype='float32'),\n        tf.keras.layers.experimental.preprocessing.RandomHeight(0.35, dtype='float32'),\n        tf.keras.layers.experimental.preprocessing.Resizing(600,600, dtype='float32')\n    ])\n    \n    def __init__(self, image_size):\n        self.image_size = image_size\n        \n    def parse_function(self, example_proto):\n        example = tf.io.parse_single_example(example_proto, self.feature_description)\n        image = tf.io.decode_image(example['image'], channels=3)\n        label = [example['ETT - Abnormal'],\n                 example['ETT - Borderline'],\n                 example['ETT - Normal'],\n                 example['NGT - Abnormal'],\n                 example['NGT - Borderline'],\n                 example['NGT - Incompletely Imaged'],\n                 example['NGT - Normal'],\n                 example['CVC - Abnormal'],\n                 example['CVC - Borderline'],\n                 example['CVC - Normal'],\n                 example['Swan Ganz Catheter Present']]\n        return image, label \n    \n    \n#     def aug_fn(image):\n#         data = {\"image\":image}\n#         aug_data = self.transforms(**data)\n#         aug_img = aug_data[\"image\"]\n# #         aug_img = tf.cast(aug_img/255.0, tf.float32)\n#         aug_img = tf.image.resize(aug_img, size=[self.image_size, self.image_size])\n#         return aug_img\n    \n#     def process_data(image, label):\n#         aug_img = tf.numpy_function(func=aug_fn, inp=[image], Tout=tf.float32)\n#         return aug_img, label\n    \n    def augment_function(self, image, label): \n#         image = tf.image.random_contrast(image, 0.8, 1.2)\n#         image = tf.image.random_brightness(image, 0.1) \n        return self.aug(image, training=True), label \n    \n    def process_function(self, image, label):\n        image.set_shape([None, self.image_size, self.image_size, 3])\n        label.set_shape([None, 11])\n        image = tf.image.resize(image, [self.image_size, self.image_size], 'bilinear')/255.\n        return image, label\n            \n    def generator(self, files, batch_size=1, repeat=False, augment=False, shuffle=True, cache=False):\n        AUTO = tf.data.experimental.AUTOTUNE\n        ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n        if shuffle: \n            opt = tf.data.Options()\n            opt.experimental_deterministic = False\n            ds = ds.with_options(opt)\n            ds = ds.shuffle(2000)\n        ds = ds.map(self.parse_function, num_parallel_calls=AUTO)\n        \n        if cache:\n            ds = ds.cache()\n        \n        if repeat:\n            ds = ds.repeat()\n            \n        ds = ds.batch(batch_size)\n        \n        ds = ds.map(self.process_function, num_parallel_calls=AUTO)\n\n        if augment:\n            ds = ds.map(self.augment_function, num_parallel_calls=AUTO)        \n        \n        ds = ds.prefetch(AUTO)\n        return ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # A custom layer\n# class SpatialAttentionModule(tf.keras.layers.Layer):\n#     def __init__(self, kernel_size=3):\n#         '''\n#         paper: https://arxiv.org/abs/1807.06521\n#         code: https://gist.github.com/innat/99888fa8065ecbf3ae2b297e5c10db70\n#         '''\n#         super(SpatialAttentionModule, self).__init__()\n#         self.conv1 = tf.keras.layers.Conv2D(16, kernel_size=kernel_size, \n#                                             use_bias=False, \n#                                             kernel_initializer='he_normal',\n#                                             strides=1, padding='same', \n#                                             activation=tf.nn.relu6)\n# #         self.conv2 = tf.keras.layers.Conv2D(32, kernel_size=kernel_size, \n# #                                             use_bias=False, \n# #                                             kernel_initializer='he_normal',\n# #                                             strides=1, padding='same', \n# #                                             activation=tf.nn.relu6)\n# #         self.conv3 = tf.keras.layers.Conv2D(16, kernel_size=kernel_size, \n# #                                             use_bias=False, \n# #                                             kernel_initializer='he_normal',\n# #                                             strides=1, padding='same', \n# #                                             activation=tf.nn.relu6)\n#         self.conv2 = tf.keras.layers.Conv2D(1, kernel_size=kernel_size,  \n#                                             use_bias=False,\n#                                             kernel_initializer='he_normal',\n#                                             strides=1, padding='same', \n#                                             activation=tf.math.sigmoid)\n\n#     def call(self, inputs):\n#         avg_out = tf.reduce_mean(inputs, axis=3)\n#         max_out = tf.reduce_max(inputs,  axis=3)\n#         x = tf.stack([avg_out, max_out], axis=3) \n#         x = self.conv1(x)\n# #         x = self.conv2(x)\n# #         x = self.conv3(x)\n#         return self.conv2(x)\n\n#     def get_config(self):\n\n#         config = super().get_config().copy()\n#         config.update({\n#             'conv1': self.conv1,\n#             'conv2': self.conv2,\n#         })\n#         return config\n    \n# # A custom layer\n# class ChannelAttentionModule(tf.keras.layers.Layer):\n#     def __init__(self, ratio=8):\n#         '''\n#         paper: https://arxiv.org/abs/1807.06521\n#         code: https://gist.github.com/innat/99888fa8065ecbf3ae2b297e5c10db70\n#         '''\n#         super(ChannelAttentionModule, self).__init__()\n#         self.ratio = ratio\n#         self.gapavg = tf.keras.layers.GlobalAveragePooling2D()\n#         self.gmpmax = tf.keras.layers.GlobalMaxPooling2D()\n        \n#     def build(self, input_shape):\n#         self.conv1 = tf.keras.layers.Conv2D(input_shape[-1]//self.ratio, \n#                                             kernel_size=1, \n#                                             strides=1, padding='same',\n#                                             use_bias=True, activation=tf.nn.relu)\n    \n#         self.conv2 = tf.keras.layers.Conv2D(input_shape[-1], \n#                                             kernel_size=1, \n#                                             strides=1, padding='same',\n#                                             use_bias=True, activation=tf.nn.relu)\n#         super(ChannelAttentionModule, self).build(input_shape)\n\n#     def call(self, inputs):\n#         # compute gap and gmp pooling \n#         gapavg = self.gapavg(inputs)\n#         gmpmax = self.gmpmax(inputs)\n#         gapavg = tf.keras.layers.Reshape((1, 1, gapavg.shape[1]))(gapavg)   \n#         gmpmax = tf.keras.layers.Reshape((1, 1, gmpmax.shape[1]))(gmpmax)   \n#         # forward passing to the respected layers\n#         gapavg_out = self.conv2(self.conv1(gapavg))\n#         gmpmax_out = self.conv2(self.conv1(gmpmax))\n#         return tf.math.sigmoid(gapavg_out + gmpmax_out)\n    \n#     def get_output_shape_for(self, input_shape):\n#         return self.compute_output_shape(input_shape)\n\n#     def compute_output_shape(self, input_shape):\n#         output_len = input_shape[3]\n#         return (input_shape[0], output_len)\n    \n#     def get_config(self):\n\n#         config = super().get_config().copy()\n#         config.update({\n#             'ratio': self.ratio,\n#             'gapavg': self.gapavg,\n#             'gmpmax': self.gmpmax,\n#         })\n#         return config","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Original Src: https://github.com/bfelbo/DeepMoji/blob/master/deepmoji/attlayer.py\n# # Adoped and Modified: https://www.kaggle.com/c/human-protein-atlas-image-classification/discussion/77269#454482\n# class AttentionWeightedAverage2D(tf.keras.layers.Layer):\n#     def __init__(self, **kwargs):\n#         self.init = tf.keras.initializers.get('uniform')\n#         super(AttentionWeightedAverage2D, self).__init__(** kwargs)\n\n#     def build(self, input_shape):\n#         self.input_spec = [tf.keras.layers.InputSpec(ndim=4)]\n#         assert len(input_shape) == 4\n#         self.W = self.add_weight(shape=(input_shape[3], 1),\n#                                  name='{}_W'.format(self.name),\n#                                  initializer=self.init)\n#         self._trainable_weights = [self.W]\n#         super(AttentionWeightedAverage2D, self).build(input_shape)\n\n#     def call(self, x):\n#         # computes a probability distribution over the timesteps\n#         # uses 'max trick' for numerical stability\n#         # reshape is done to avoid issue with Tensorflow\n#         # and 2-dimensional weights\n#         logits  = K.dot(x, self.W)\n#         x_shape = K.shape(x)\n#         logits  = K.reshape(logits, (x_shape[0], x_shape[1], x_shape[2]))\n#         ai      = K.exp(logits - K.max(logits, axis=[1,2], keepdims=True))\n        \n#         att_weights    = ai / (K.sum(ai, axis=[1,2], keepdims=True) + K.epsilon())\n#         weighted_input = x * K.expand_dims(att_weights)\n#         result         = K.sum(weighted_input, axis=[1,2])\n#         return result\n\n#     def get_output_shape_for(self, input_shape):\n#         return self.compute_output_shape(input_shape)\n\n#     def compute_output_shape(self, input_shape):\n#         output_len = input_shape[3]\n#         return (input_shape[0], output_len)\n    \n#     def get_config(self):\n\n#         config = super().get_config().copy()\n#         config.update({\n#             'init': self.init,\n#         })\n#         return config","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(name, input_shape, classes, output_bias=None):\n    \n    # Dictionary mapping name to model function\n    \n    EFFICIENT_NETS = {'B0': efn.EfficientNetB0, \n                      'B1': efn.EfficientNetB1, \n                      'B2': efn.EfficientNetB2, \n                      'B3': efn.EfficientNetB3, \n                      'B4': efn.EfficientNetB4, \n                      'B5': efn.EfficientNetB5, \n                      'B6': efn.EfficientNetB6,\n                      'B7': efn.EfficientNetB7}\n    \n    # Output layer bias initialization\n    \n    if output_bias is None:\n        output_bias = 'zeros'\n    else:\n        output_bias = tf.keras.initializers.Constant(output_bias)\n        \n    \n    # Base model\n    \n    base_model = EFFICIENT_NETS[name](include_top=False, \n                                      weights='imagenet', \n                                      input_shape=input_shape)\n    \n    # Model\n    base_model.trainable = True\n    \n    for layer in base_model.layers:\n        if isinstance(layer, tf.keras.layers.BatchNormalization):\n            # we do aggressive exponential smoothing of batch norm\n            # parameters to faster adjust to our new dataset\n            layer.momentum = 0.99\n    \n    inputs = tf.keras.Input(shape=input_shape)\n    x = base_model(inputs)\n#     cam = ChannelAttentionModule()(x)\n#     camx = cam * x\n#     camx = tf.keras.layers.BatchNormalization()(camx)\n#     sam = SpatialAttentionModule()(camx)\n#     spnx = sam * camx\n#     spnx = tf.keras.layers.BatchNormalization()(spnx)\n#     gap = tf.keras.layers.GlobalAveragePooling2D()(spnx)\n#     sam1 = SpatialAttentionModule()(camx)\n#     sam1 = tf.keras.layers.BatchNormalization()(sam1)\n#     wvgx = tf.keras.layers.GlobalAveragePooling2D()(sam1)\n#     gapavg = tf.keras.layers.Average()([gap, wvgx])\n#     gapavg = tf.keras.layers.BatchNormalization()(gapavg)\n#     awgavg = AttentionWeightedAverage2D()(x)\n    \n#     x = tf.keras.layers.Add()([gap, awgavg])\n#     x = tf.keras.layers.BatchNormalization()(x)\n#     x = tf.keras.layers.Dense(256, activation=tf.nn.relu)(x)\n#     x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n#     x = tf.keras.layers.BatchNormalization()(x)\n#     x = tf.keras.layers.LeakyReLU()(x)\n    x = tf.keras.layers.Dropout(rate=0.2)(x)\n#     x = tf.keras.layers.Dense(32)(x)\n#     x = tf.keras.layers.BatchNormalization()(x)\n#     x = tf.keras.layers.LeakyReLU()(x)\n#     x = tf.keras.layers.Dropout(rate=0.3)(x)\n#     x = tf.keras.layers.Dense(128)(x)\n#     x = tf.keras.layers.BatchNormalization()(x)\n#     x = tf.keras.layers.LeakyReLU()(x)\n#     x = tf.keras.layers.Dropout(rate=0.3)(x)   \n#     x = tf.keras.layers.Reshape((320,8))(x)\n#     x = tf.keras.layers.SeparableConv1D(8, 20, activation='relu')(x)\n#     x = tf.keras.layers.BatchNormalization()(x)\n#     x = tf.keras.layers.Dropout(rate=0.2)(x)\n#     x = tf.keras.layers.SeparableConv1D(16, 20, activation='relu')(x)\n#     x = tf.keras.layers.Dropout(rate=0.2)(x)\n#     x = tf.keras.layers.BatchNormalization()(x)\n#     x = tf.keras.layers.SeparableConv1D(32, 20, activation='relu')(x)\n#     x = tf.keras.layers.Dropout(rate=0.2)(x)\n#     x = tf.keras.layers.BatchNormalization()(x)\n#     x = tf.keras.layers.GlobalMaxPooling1D()(x)\n#     x = tf.keras.layers.Dropout(rate=0.2)(x)\n#     x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Dense(classes, bias_initializer=output_bias)(x)\n    outputs = tf.keras.layers.Activation('sigmoid', dtype='float32')(x) # Supports mixed-precision training\n    \n    model = tf.keras.Model(inputs, outputs)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def compile_model(model, lr=0.0001):\n    \n    optimizer = tf.keras.optimizers.Adam(lr=lr)\n    \n    loss = 'binary_crossentropy'\n    \n#     loss = get_weighted_loss(pos_weights, neg_weights)\n        \n    metrics = [\n        tf.keras.metrics.AUC(name='auc', multi_label=True)\n    ]\n\n    model.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = create_model(name='B7', \n                             input_shape=(600,600,3), \n                             classes=11)\nmodel = compile_model(model, lr=0.01)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.utils.plot_model(model, show_shapes=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_class_weights(labels):\n    \"\"\"\n    Compute positive and negative frequences for each class.\n\n    Args:\n        labels (np.array): matrix of labels, size (num_examples, num_classes)\n    Returns:\n        positive_frequencies (np.array): array of positive frequences for each\n                                         class, size (num_classes)\n        negative_frequencies (np.array): array of negative frequences for each\n                                         class, size (num_classes)\n    \"\"\"\n    ### START CODE HERE (REPLACE INSTANCES OF 'None' with your code) ###\n    \n    # total number of patients (rows)\n    N = labels.shape[0]\n    weights = {}\n    \n    positive_frequencies = np.mean(labels, axis=0)\n#     negative_frequencies = 1 - positive_frequencies\n\n    ### END CODE HERE ###\n    w = (1 / positive_frequencies)/11.0\n    \n    return dict(enumerate(w))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\ndf = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\n\n# Get the multi-labels\nlabel_cols = df_sub.columns[1:]\nlabels = df[label_cols].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weights = compute_class_weights(labels)\nclass_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_class_freqs(labels):\n    \"\"\"\n    Compute positive and negative frequences for each class.\n\n    Args:\n        labels (np.array): matrix of labels, size (num_examples, num_classes)\n    Returns:\n        positive_frequencies (np.array): array of positive frequences for each\n                                         class, size (num_classes)\n        negative_frequencies (np.array): array of negative frequences for each\n                                         class, size (num_classes)\n    \"\"\"\n    ### START CODE HERE (REPLACE INSTANCES OF 'None' with your code) ###\n    \n    # total number of patients (rows)\n    N = labels.shape[0]\n    \n    positive_frequencies = np.mean(labels, axis=0)\n    negative_frequencies = 1 - positive_frequencies\n\n    ### END CODE HERE ###\n    return positive_frequencies, negative_frequencies","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"freq_pos, freq_neg = compute_class_freqs(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\n\ndata = pd.DataFrame({\"Class\": label_cols, \"Label\": \"Positive\", \"Value\": freq_pos})\ndata = data.append([{\"Class\": label_cols[l], \"Label\": \"Negative\", \"Value\": v} for l,v in enumerate(freq_neg)], ignore_index=True)\nplt.xticks(rotation=90)\nf = sns.barplot(x=\"Class\", y=\"Value\", hue=\"Label\" ,data=data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_callbacks(model_save_path, verbose=1, e_s=10, e=4):\n    \n    verbose = int(verbose>0)\n    \n    if not os.path.exists(model_save_path):\n        os.makedirs(model_save_path)\n    \n    cpk_path = f'{model_save_path}/model.h5'\n\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(\n        filepath=cpk_path,\n        monitor='val_loss',\n        mode='min',\n        save_best_only=True,\n        verbose=verbose\n    )\n\n    reducelr = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor='val_loss',\n        mode='min',\n        factor=0.1,\n        patience=e,\n        verbose=0\n    )\n\n    earlystop = tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss',\n        mode='min',\n        patience=e_s, \n        verbose=verbose\n    )\n    \n    callbacks = [checkpoint, reducelr, earlystop]\n    \n    return callbacks","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import pandas as pd\n\n# train = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train = train.iloc[:,1:-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train.head(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# labels_ = train.to_numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# freq_pos, freq_neg = compute_class_freqs(labels_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# labels = np.array(train.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import seaborn as sns\n\n# data = pd.DataFrame({\"Class\": labels, \"Label\": \"Positive\", \"Value\": freq_pos})\n# data = data.append([{\"Class\": labels[l], \"Label\": \"Negative\", \"Value\": v} for l,v in enumerate(freq_neg)], ignore_index=True)\n# plt.xticks(rotation=90)\n# f = sns.barplot(x=\"Class\", y=\"Value\", hue=\"Label\" ,data=data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pos_weights = freq_neg\n# neg_weights = freq_pos\n# pos_contribution = freq_pos * pos_weights \n# neg_contribution = freq_neg * neg_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data = pd.DataFrame({\"Class\": labels, \"Label\": \"Positive\", \"Value\": pos_contribution})\n# data = data.append([{\"Class\": labels[l], \"Label\": \"Negative\", \"Value\": v} \n#                         for l,v in enumerate(neg_contribution)], ignore_index=True)\n# plt.xticks(rotation=90)\n# sns.barplot(x=\"Class\", y=\"Value\", hue=\"Label\" ,data=data);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def get_weighted_loss(pos_weights, neg_weights, epsilon=1e-7):\n#     \"\"\"\n#     Return weighted loss function given negative weights and positive weights.\n\n#     Args:\n#       pos_weights (np.array): array of positive weights for each class, size (num_classes)\n#       neg_weights (np.array): array of negative weights for each class, size (num_classes)\n    \n#     Returns:\n#       weighted_loss (function): weighted loss function\n#     \"\"\"\n#     def weighted_loss(y_true, y_pred):\n#         \"\"\"\n#         Return weighted loss value. \n\n#         Args:\n#             y_true (Tensor): Tensor of true labels, size is (num_examples, num_classes)\n#             y_pred (Tensor): Tensor of predicted labels, size is (num_examples, num_classes)\n#         Returns:\n#             loss (Tensor): overall scalar loss summed across all classes\n#         \"\"\"\n        \n#         ### START CODE HERE (REPLACE INSTANCES OF 'None' with your code) ###\n\n#         loss_pos = -1. * K.sum(K.mean(pos_weights * y_true * K.log(y_pred+epsilon), axis=0))\n#         loss_neg = -1. * K.sum(K.mean(neg_weights * (1 - y_true) * K.log(1-y_pred+epsilon), axis=0))\n#         return loss_pos+loss_neg\n    \n#         ### END CODE HERE ###\n#     return weighted_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folds_val_auc = [None] * FOLDS # Store the validation auc for each fold\n\nskf = KFold(n_splits=FOLDS, shuffle=True, random_state=SEED)\n\nDISPLAY_PLOT = True\n\nprint(f'Training...')\n\n\nfor fold, (train_idx, valid_idx) in enumerate(skf.split(np.arange(NUM_TF_RECS))):\n    \n    print(f'\\n\\n{\"*\"*100} \\nFOLD: {fold+1}')\n    \n    # Input Pipeline ******************************************************\n    \n    train_files = tf.io.gfile.glob(f'{DATA_PATH}/train_tfrecords/{idx:02}*.tfrec' for idx in train_idx)\n    valid_files = tf.io.gfile.glob(f'{DATA_PATH}/train_tfrecords/{idx:02}*.tfrec' for idx in valid_idx)\n    \n    ds = Dataset(IMG_SIZE)\n    \n    train_ds = ds.generator(train_files, \n                            BATCH_SIZE*REPLICAS, \n                            repeat=True, \n                            augment=True, \n                            shuffle=True,\n                            cache=True)\n\n    valid_ds = ds.generator(valid_files, \n                            BATCH_SIZE*REPLICAS,  \n                            repeat=False, \n                            augment=False, \n                            shuffle=False,\n                            cache=False)\n    \n    \n    # Calculate the steps_per_epoch\n    \n    steps_per_epoch = count_items(train_files)//(BATCH_SIZE*REPLICAS) * 2\n    \n    \n    # Build Model ******************************************************\n    \n    if fold==0:\n        lr=0.0001\n        e_s=10\n        e=4\n    else:\n        lr=0.00001\n        e_s=5\n        e=2\n        \n    print('Learning Rate: '+str(lr))\n    \n    tf.keras.backend.clear_session()\n        \n    with strategy.scope():\n        \n        if fold==0:\n            model = create_model(name=EFF_NET, \n                             input_shape=(IMG_SIZE,IMG_SIZE,3), \n                             classes=11)\n        \n        else:\n            model.load_weights('/kaggle/working/models/model.h5')\n\n        \n        \n        model = compile_model(model, lr=lr)\n        \n    print(f'\\nModel initialized and compiled: EfficientNet-{EFF_NET}')\n    \n        \n    # Train ******************************************************\n   \n    callbacks = create_callbacks(MODEL_PATH, verbose=VERBOSE, e_s=e_s, e=e)\n\n    print(f'\\nModel training...\\n')\n    \n    history = model.fit(train_ds, \n                        epochs=EPOCHS, \n                        steps_per_epoch=steps_per_epoch,\n                        validation_data=valid_ds, \n                        callbacks=callbacks, \n                        verbose=VERBOSE,\n                       class_weight=class_weights)\n    \n    # Save acc for each fold in a list\n    folds_val_auc[fold] = max(history.history['val_auc'])\n    \n    print(f'\\nModel trained \\n\\nFOLD-{fold+1} Validation AUC = {folds_val_auc[fold]}')\n    \n    n_epochs = len(history.history['loss'])\n    \n    # PLOT TRAINING\n    # https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n    if DISPLAY_PLOT:        \n        plt.figure(figsize=(15,5))\n        plt.plot(np.arange(n_epochs),history.history['auc'],'-o',label='auc',color='#ff7f0e')\n        plt.plot(np.arange(n_epochs),history.history['val_auc'],'-o',label='Val auc',color='#1f77b4')\n        \n        x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n        \n        plt.ylabel('auc',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        \n        plt2 = plt.gca().twinx()\n        \n        plt2.plot(np.arange(n_epochs),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(np.arange(n_epochs),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        \n        x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        \n        plt.ylabel('Loss',size=14)\n        plt.legend(loc=3)\n        plt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}