{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is the fourth part of [RANZCR 1st Place Solution by TF](https://www.kaggle.com/tt195361/ranzcr-1st-place-solution-by-tf-1-make-masks). This notebook is based on [RANZCR 1st Place Soluiton Cls Model (small ver.)](https://www.kaggle.com/haqishen/ranzcr-1st-place-soluiton-cls-model-small-ver).\n\nThe fourth step is to train a classification model by using the [dataset](https://www.kaggle.com/tt195361/ranzcr-1st-place-solution-by-tf-train-data) made by [the previous step](https://www.kaggle.com/tt195361/ranzcr-1st-place-solution-by-tf-3-gen-masks).\n\nInput to the classification model is 5 channels, 3 for images and 2 for masks. The original pytorch model just adds the weight for the additional channels on the first convolutional layer to support 5 channel input.\n\nFor Keras, I did the followings:\n1. Make a model with pretrained weight as usual.\n1. Clone layers that need to change the weight for 5 channel input. For EfficientNet in Keras, the layers are normalization at 2nd and stem_conv at 4th.\n1. Make a 5 channel input layer, then call each clone and existing layers from the input to output in order.\n1. Set weight to the clone layers for 5 input channels.\n\nThe outputs of the classification model are as follows:\n* ETT: including no_ETT. Activation is softmax, loss is cross entropy.\n* Other: NGT, CVC, and Swan Ganz Catheter Present. Activation is sigmoid, loss is binary cross entropy.\n* Pred: ETT excluding no_ETT, and Other. For the AUC metrics and final prediction.\n\nFor the data augmentation, the original notebook uses [Albumentations](https://github.com/albumentations-team/albumentations). I made similar one by Tensorflow.\n\nTo run 1 fold, it took 3.5 hours. So, I used the following 3 sessions to run 5 folds.\n\n* [Version 14](https://www.kaggle.com/tt195361/ranzcr-1st-place-solution-by-tf-4-cls-model?scriptVersionId=61560523) -- fold 0\n* [Version 15](https://www.kaggle.com/tt195361/ranzcr-1st-place-solution-by-tf-4-cls-model?scriptVersionId=61573005) -- fold 1, 2\n* [Version 16](https://www.kaggle.com/tt195361/ranzcr-1st-place-solution-by-tf-4-cls-model?scriptVersionId=61603268) -- fold 3, 4","metadata":{}},{"cell_type":"markdown","source":"## Config and Libraries","metadata":{}},{"cell_type":"code","source":"DEBUG = False","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# libraries\nimport os\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport math\nimport tensorflow as tf\nfrom tensorflow import keras\nimport tensorflow.keras.layers as L\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow_addons as tfa\n\nprint(tf.__version__)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enet_type = tf.keras.applications.EfficientNetB1\nnum_classes = 12\nnum_ett_classes = 4\nnum_other_classes = num_classes - num_ett_classes\nn_ch = 5\nimage_size = 512\nbatch_size = 128 # original is 32\ninit_lr = 3e-4\nwarmup_epo = 1\n# If DEBUG == True, only run 3 epochs per fold\ncosine_epo = 29 if not DEBUG else 2\nn_epochs = warmup_epo + cosine_epo\nloss_weights = [1., 9.]\n\nVID = \"V19\"\nFOLD_I_LIST=[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ranzcr_train_data_name = 'ranzcr-1st-place-solution-by-tf-train-data'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TPU","metadata":{}},{"cell_type":"code","source":"try: # detect TPUs\n    # NEW: in Tensorflow 2.4\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() \n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError: # otherwise detect GPUs\n    strategy = tf.distribute.MirroredStrategy() # single-GPU or multi-GPU\n    \nprint(f\"Running on {strategy.num_replicas_in_sync} replicas\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path(ranzcr_train_data_name)\n\nGCS_DS_PATH","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"def decode_image(image_bytes):\n    image = tf.image.decode_jpeg(image_bytes, channels=3)\n    image = tf.reshape(image, (image_size, image_size, 3))\n    return image\n\ndef decode_mask(mask_bytes):\n    mask = tf.io.decode_png(mask_bytes, channels=3)\n    mask = tf.reshape(mask, (image_size, image_size, 3))\n    return mask\n\ndef read_tfrecord(example):\n    TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'mask': tf.io.FixedLenFeature([], tf.string),\n        'ETT - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n        'ETT - Borderline': tf.io.FixedLenFeature([], tf.int64),\n        'ETT - Normal': tf.io.FixedLenFeature([], tf.int64),\n        'NGT - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n        'NGT - Borderline': tf.io.FixedLenFeature([], tf.int64),\n        'NGT - Incompletely Imaged': tf.io.FixedLenFeature([], tf.int64),\n        'NGT - Normal': tf.io.FixedLenFeature([], tf.int64),\n        'CVC - Abnormal': tf.io.FixedLenFeature([], tf.int64),\n        'CVC - Borderline': tf.io.FixedLenFeature([], tf.int64),\n        'CVC - Normal': tf.io.FixedLenFeature([], tf.int64),\n        'Swan Ganz Catheter Present': tf.io.FixedLenFeature([], tf.int64),\n        'fold': tf.io.FixedLenFeature([], tf.int64),\n    }\n    \n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    mask = decode_mask(example['mask'])\n    ett_abnormal = example['ETT - Abnormal']\n    ett_borderline = example['ETT - Borderline']\n    ett_normal = example['ETT - Normal']\n    # Add no_ett, 1 when all ett_xxx are 0\n    no_ett = 1 - tf.math.reduce_max([\n        ett_abnormal, ett_borderline, ett_normal])\n    ngt_abnormal = example['NGT - Abnormal']\n    ngt_borderline = example['NGT - Borderline']\n    ngt_inc_imaged = example['NGT - Incompletely Imaged']\n    ngt_normal = example['NGT - Normal']\n    cvc_abnormal = example['CVC - Abnormal']\n    cvc_borderline = example['CVC - Borderline']\n    cvc_normal = example['CVC - Normal']\n    swan_ganz_cat_present = example['Swan Ganz Catheter Present']\n    fold = example['fold']\n\n    labels_ett = [\n        ett_abnormal, ett_borderline, ett_normal, no_ett ]\n    labels_other = [\n        ngt_abnormal, ngt_borderline, ngt_inc_imaged, ngt_normal,\n        cvc_abnormal, cvc_borderline, cvc_normal,\n        swan_ganz_cat_present ]\n    labels_pred = [\n        ett_abnormal, ett_borderline, ett_normal, # no no_ett\n        ngt_abnormal, ngt_borderline, ngt_inc_imaged, ngt_normal,\n        cvc_abnormal, cvc_borderline, cvc_normal,\n        swan_ganz_cat_present ]\n    labels = (labels_ett, labels_other, labels_pred)\n    \n    return image, mask, labels, fold\n\ndef load_dataset(filenames):\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=None)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=None)\n    return dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfrec_file_names = tf.io.gfile.glob(GCS_DS_PATH + '/*.tfrec')\ntfrec_file_names = \\\n    [ tfrec_file_names[0] ] if DEBUG else tfrec_file_names\nraw_ds = load_dataset(tfrec_file_names)\n\nprint(raw_ds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folds_list = []\nfor _, _, _, fold_batch in raw_ds.batch(256):\n    print('.', end='', flush=True)\n    folds_list.append(fold_batch)\n\nfolds = np.concatenate(folds_list)\nfold, counts = np.unique(folds, return_counts=True)\nfold_count_dict = dict(zip(fold, counts))\n\nfold_count_dict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fold_train_count(fold_i):\n    counts = [ \n        count for fold, count in fold_count_dict.items() \\\n        if fold != fold_i ]\n    return sum(counts)\n\ndef fold_val_count(fold_i):\n    return fold_count_dict[fold_i]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Augmentations","metadata":{}},{"cell_type":"markdown","source":"### Utilities","metadata":{}},{"cell_type":"code","source":"def image_mask_to_float_0_1(image, mask):\n    image = tf.cast(image, dtype=tf.float32) / 255.0\n    mask = tf.cast(mask, dtype=tf.float32) / 255.0\n    return image, mask\n\ndef image_mask_to_uint8(image, mask):\n    image = tf.cast(image * 255.0, dtype=tf.uint8)\n    mask = tf.cast(mask * 255.0, dtype=tf.uint8)\n    return image, mask","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_aug(aug_fun, with_mask):\n    image, mask, _, _ = next(iter(raw_ds.take(1)))\n    image, mask = image_mask_to_float_0_1(image, mask)\n    \n    plt.figure(figsize=(12, 4))\n    rows = 2\n    cols = 5\n    aug_masks = []\n    for p in range(rows*cols):\n        aug_image, aug_mask = aug_fun(image, mask)\n        aug_image, aug_mask = image_mask_to_uint8(aug_image, aug_mask)\n        aug_masks.append(aug_mask)\n        \n        plt.subplot(rows, cols, p+1)\n        plt.imshow(aug_image)\n        plt.axis(\"off\")\n    plt.tight_layout()\n    plt.show()        \n    \n    if with_mask:\n        plt.figure(figsize=(12, 4))\n        for p, aug_mask in enumerate(aug_masks):\n            plt.subplot(rows, cols, p+1)\n            plt.imshow(aug_mask)\n            plt.axis(\"off\")\n        plt.tight_layout()\n        plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def random_float(minval=0.0, maxval=1.0):\n    rnd = tf.random.uniform(\n        [], minval=minval, maxval=maxval, dtype=tf.float32)\n    return rnd\n\ndef choice(p, image1, mask1, image2, mask2):\n    rnd = random_float()\n    image = tf.where(rnd <= p, image1, image2)\n    mask = tf.where(rnd <= p, mask1, mask2)\n    return image, mask","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" def mirror_boundary(v, max_v):\n    # v % (max_v*2.0-2.0) ==> v % (512*2-2) ==> [0..1022]\n    # [0..1022] - (max_v-1.0) ==> [0..1022] - 511 ==> [-511..511]\n    # -1.0 * abs([-511..511]) ==> [-511..0]\n    # [-511..0] + max_v - 1.0 ==> [-511..0] + 511 ==> [0..511]\n    mirror_v = -1.0 * tf.math.abs(\n        v % (max_v*2.0-2.0) - (max_v-1.0)) + max_v-1.0\n    return mirror_v\n\ndef clip_boundary(v, max_v):\n    clip_v = tf.clip_by_value(v, 0.0, max_v-1.0)\n    return clip_v\n\ndef interpolate_bilinear(image, map_x, map_y):\n    def _gather(image, map_x, map_y):\n        map_stack = tf.stack([map_x, map_y]) # [ 2, height, width ]\n        map_indices = tf.transpose(\n            map_stack, perm=[1, 2, 0])       # [ height, width, 2 ]\n        map_indices = tf.cast(map_indices, dtype=tf.int32)\n        gather_image = tf.gather_nd(image, map_indices)\n        return gather_image\n    \n    ll = _gather(image, tf.math.floor(map_x), tf.math.floor(map_y))\n    lr = _gather(image, tf.math.ceil(map_x), tf.math.floor(map_y))\n    ul = _gather(image, tf.math.floor(map_x), tf.math.ceil(map_y))\n    ur = _gather(image, tf.math.ceil(map_x), tf.math.ceil(map_y))\n    \n    fraction_x = tf.expand_dims(map_x % 1.0, axis=-1) # [h, w, 1]\n    int_l = (lr - ll) * fraction_x + ll\n    int_u = (ur - ul) * fraction_x + ul\n    \n    fraction_y = tf.expand_dims(map_y % 1.0, axis=-1) # [h, w, 1]\n    interpolate_image = (int_u - int_l) * fraction_y + int_l\n    return interpolate_image\n\ndef remap(image, height, width, map_x, map_y, mode):\n    assert \\\n        mode in ('mirror', 'constant'), \\\n        \"mode is neither 'mirror' nor 'constant'\"\n\n    height_f = tf.cast(height, dtype=tf.float32)\n    width_f = tf.cast(width, dtype=tf.float32)\n    map_x = tf.reshape(map_x, shape=[height, width])\n    map_y = tf.reshape(map_y, shape=[height, width])\n    if mode == 'mirror':\n        b_map_x = mirror_boundary(map_x, width_f)\n        b_map_y = mirror_boundary(map_y, height_f)\n    else:\n        b_map_x = clip_boundary(map_x, width_f)\n        b_map_y = clip_boundary(map_y, height_f)\n        \n    image_remap = interpolate_bilinear(image, b_map_x, b_map_y)\n    \n    if mode == 'constant':\n        map_stack = tf.stack([map_x, map_y])\n        map_indices = tf.transpose(map_stack, perm=[1, 2, 0])\n        x_ge_0 = (0.0 <= map_indices[ : , : , 0])    # [h, w]\n        x_lt_w = (map_indices[ : , : , 0] < width_f)\n        y_ge_0 = (0.0 <= map_indices[ : , : , 1])\n        y_lt_h = (map_indices[ : , : , 1] < height_f)\n        inside_boundary = tf.math.reduce_all(\n            tf.stack([x_ge_0, x_lt_w, y_ge_0, y_lt_h]), axis=0) # [h, w]\n        inside_boundary = inside_boundary[ : , : , tf.newaxis]  # [h, w, 1]\n        image_remap = tf.where(inside_boundary, image_remap, 0.0)\n\n    return image_remap","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### HorizontalFlip","metadata":{}},{"cell_type":"code","source":"def HorizontalFlip(p):\n    def _do_horizontal_flip(image, mask):\n        aug_image = tf.image.flip_left_right(image)\n        aug_mask = tf.image.flip_left_right(mask)\n        return choice(p, aug_image, aug_mask, image, mask)\n    return _do_horizontal_flip","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"horizontal_flip = HorizontalFlip(p=0.5)\ncheck_aug(horizontal_flip, with_mask=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### RandomBrightness","metadata":{}},{"cell_type":"code","source":"def RandomBrightness(max_delta, p):\n    def _do_random_brightness(image, mask):\n        aug_image = tf.image.random_brightness(image, max_delta)\n        return choice(p, aug_image, mask, image, mask)\n    return _do_random_brightness","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_brightness = RandomBrightness(max_delta=0.2, p=0.75)\ncheck_aug(random_brightness, with_mask=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### RandomContrast","metadata":{}},{"cell_type":"code","source":"def RandomContrast(lower, upper, p):\n    def _do_random_contrast(image, mask):\n        aug_image = tf.image.random_contrast(image, lower, upper)\n        return choice(p, aug_image, mask, image, mask)\n    return _do_random_contrast","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_contrast = RandomContrast(lower=0.2, upper=0.8, p=0.75)\ncheck_aug(random_contrast, with_mask=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### OpticalDistortion","metadata":{}},{"cell_type":"markdown","source":"initUndistortRectifyMap -- https://docs.opencv.org/3.4/da/d54/group__imgproc__transform.html#ga7dfb72c9cf9780a347fbe3d1c47e5d5a\n","metadata":{}},{"cell_type":"code","source":"def initUndistortRectifyMap(height, width, k, dx, dy):\n    height = tf.cast(height, dtype=tf.float32)\n    width = tf.cast(width, dtype=tf.float32)\n    \n    f_x = width\n    f_y = height\n    c_x = width * 0.5 + dx\n    c_y = height * 0.5 + dy\n    \n    f_dash_x = f_x\n    c_dash_x = (width - 1.0) * 0.5\n    f_dash_y = f_y\n    c_dash_y = (height - 1.0) * 0.5\n\n    h_rng = tf.range(height, dtype=tf.float32)\n    w_rng = tf.range(width, dtype=tf.float32)\n    v, u = tf.meshgrid(h_rng, w_rng)\n    \n    x = (u - c_dash_x) / f_dash_x\n    y = (v - c_dash_y) / f_dash_y\n    x_dash = x\n    y_dash = y\n    \n    r_2 = x_dash * x_dash + y_dash * y_dash\n    r_4 = r_2 * r_2\n    x_dash_dash = x_dash * (1 + k*r_2 + k*r_4)\n    y_dash_dash = y_dash * (1 + k*r_2 + k*r_4)\n\n    map_x = x_dash_dash * f_x + c_x\n    map_y = y_dash_dash * f_y + c_y\n    return map_x, map_y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def OpticalDistortion(distort_limit, shift_limit, p=1.0):\n    def _do_optical_distortion(image, mask):\n        k = random_float(-distort_limit, distort_limit)\n        dx = random_float(-shift_limit, shift_limit)\n        dy = random_float(-shift_limit, shift_limit)\n        image_shape = tf.shape(image)\n        height = image_shape[0]\n        width = image_shape[1]\n        map_x, map_y = initUndistortRectifyMap(\n            height, width, k, dx, dy)\n        aug_image = remap(\n            image, height, width, map_x, map_y, mode='mirror')\n        aug_mask = remap(\n            mask, height, width, map_x, map_y, mode='mirror')\n        return choice(p, aug_image, aug_mask, image, mask)\n    return _do_optical_distortion","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optical_distortion = OpticalDistortion(\n    distort_limit=1.0, shift_limit=0.05, p=0.75)\ncheck_aug(optical_distortion, with_mask=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### GridDistortion","metadata":{}},{"cell_type":"code","source":"def make_grid_distorted_maps(height, width, num_steps, xsteps, ysteps):\n    def _make_maps_before_last(size, step, steps): # size=512, step=102,\n                                                   # steps.shape=[num_steps]\n        step_rep = tf.repeat(step, num_steps)  # [102, 102, 102, 102, 102]\n        step_rep_f = tf.cast(step_rep, dtype=tf.float32)\n        step_inc = step_rep_f * steps          # [102*s_0, ..., 102*s_4]\n        cur = tf.math.cumsum(step_inc)         # [si_0, si_0 + si_1, ... ]\n        zero = tf.zeros([1], dtype=tf.float32)\n        prev = tf.concat([ zero, cur[ :-1] ], axis=0) # [0, c_0, ..., c_3]\n        prev_cur = tf.stack([prev, cur])       # [[p_0, p_1, ...], [c_0, c_1, ...]]\n        ranges = tf.transpose(prev_cur)        # [[p_0, c_0], [p_1, c_1], ... ]\n\n        def _linspace_range(rng):\n            return tf.linspace(rng[0], rng[1], step)\n \n        maps_stack = tf.map_fn(_linspace_range, ranges)\n        maps = tf.reshape(maps_stack, [-1])    # [-1] flatten into 1-D\n        return maps\n    \n    def _make_last_map(size, step, last_start):\n        last_step = size - step * num_steps  # 512 - 102*5 = 2 \n        size_f = tf.cast(size, dtype=tf.float32)\n        last_map = tf.linspace(last_start, size_f-1.0, last_step)\n        return last_map\n    \n    def _make_distorted_map(size, steps):\n        step = size // num_steps               # step=102 \n        maps_before_last = _make_maps_before_last(size, step, steps[ :-1 ])\n        last_map = _make_last_map(size, step, maps_before_last[-1])\n        distorted_map = tf.concat([maps_before_last, last_map], axis=0)\n        return distorted_map\n\n    xx = _make_distorted_map(width, xsteps)\n    yy = _make_distorted_map(height, ysteps)\n    map_y, map_x = tf.meshgrid(xx, yy)\n    return map_x, map_y\n\ndef GridDistortion(num_steps, distort_limit, p=1.0):\n    def _do_grid_distortion(image, mask):\n        xsteps = tf.random.uniform(\n            [num_steps + 1],\n            minval=1.0 - distort_limit,\n            maxval=1.0 + distort_limit)\n        ysteps = tf.random.uniform(\n            [num_steps + 1],\n            minval=1.0 - distort_limit,\n            maxval=1.0 + distort_limit)\n\n        image_shape = tf.shape(image)\n        height = image_shape[0]\n        width = image_shape[1]\n        map_x, map_y = make_grid_distorted_maps(\n            height, width, num_steps, xsteps, ysteps)\n        aug_image = remap(\n            image, height, width, map_x, map_y, mode='mirror')\n        aug_mask = remap(\n            mask, height, width, map_x, map_y, mode='mirror')\n        return choice(p, aug_image, aug_mask, image, mask)\n    return _do_grid_distortion","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_distortion = GridDistortion(\n    num_steps=5, distort_limit=1.0, p=0.75)\ncheck_aug(grid_distortion, with_mask=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### OneOf","metadata":{}},{"cell_type":"code","source":"def OneOf(trans1, trans2, p):\n    def _do_one_of(image, mask):\n        image1, mask1 = trans1(image, mask)\n        image2, mask2 = trans2(image, mask)\n        aug_image, aug_mask = choice(\n            0.5, image1, mask1, image2, mask2)\n        return choice(p, aug_image, aug_mask, image, mask)\n    return _do_one_of","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_of_opt_grid_distortion = OneOf(\n    optical_distortion, grid_distortion, p=0.75)\ncheck_aug(one_of_opt_grid_distortion, with_mask=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### HeuSaturationValue","metadata":{}},{"cell_type":"code","source":"def HueSaturationValue(\n        hue_shift_limit, sat_shift_limit, val_shift_limit, p):\n    def _do_hue_saturation_value(image, mask):\n        hsv_image = tf.image.rgb_to_hsv(image)\n        hue_shift = random_float(-hue_shift_limit, hue_shift_limit)\n        sat_shift = random_float(-sat_shift_limit, sat_shift_limit)\n        val_shift = random_float(-val_shift_limit, val_shift_limit)\n\n        hue_values = (hsv_image[ ... , :1 ] + hue_shift) % 1.0\n        sat_values = tf.clip_by_value(\n            hsv_image[ ... , 1:2 ] + sat_shift, 0.0, 1.0)\n        val_values = tf.clip_by_value(\n            hsv_image[ ... , 2: ] + val_shift, 0.0, 1.0)\n        hsv_image = tf.concat(\n            [hue_values, sat_values, val_values], axis=-1)\n        aug_image = tf.image.hsv_to_rgb(hsv_image)\n        return choice(p, aug_image, mask, image, mask)\n    return _do_hue_saturation_value","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hue_saturation_value = HueSaturationValue(\n    hue_shift_limit=0.2, sat_shift_limit=0.3,\n    val_shift_limit=0.2, p=0.75)\ncheck_aug(hue_saturation_value, with_mask=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ShiftScaleRotate","metadata":{}},{"cell_type":"code","source":"def affine_transform(height, width, tx, ty, z, theta):\n    cx = (width - 1.0) * 0.5\n    cy = (height - 1.0) * 0.5\n    \n    center_shift_mat = tf.convert_to_tensor([\n        [1.0, 0.0, -cx],\n        [0.0, 1.0, -cy],\n        [0.0, 0.0, 1.0]], dtype=tf.float32)\n    trans_mat = center_shift_mat\n    \n    rot_rad = -2.0 * math.pi * theta / 360.0\n    roration_mat = tf.convert_to_tensor([\n        [tf.math.cos(rot_rad), tf.math.sin(rot_rad), 0.0],\n        [-tf.math.sin(rot_rad), tf.math.cos(rot_rad), 0.0],\n        [0.0, 0.0, 1.0]], dtype=tf.float32)\n    trans_mat = tf.linalg.matmul(roration_mat, trans_mat)\n    \n    shift_mat = tf.convert_to_tensor([\n        [1.0, 0.0, cx - tx],\n        [0.0, 1.0, cy - ty],\n        [0.0, 0.0, 1.0]], dtype=tf.float32)\n    trans_mat = tf.linalg.matmul(shift_mat, trans_mat)\n\n    zoom_mat = tf.convert_to_tensor([\n        [1.0 / z, 0.0, 0.0],\n        [0.0, 1.0 / z, 0.0],\n        [0.0, 0.0, 1.0]], dtype=tf.float32)\n    trans_mat = tf.linalg.matmul(zoom_mat, trans_mat)\n    \n    h_rng = tf.range(height, dtype=tf.float32)\n    w_rng = tf.range(width, dtype=tf.float32)\n    y, x = tf.meshgrid(h_rng, w_rng)\n    x = tf.reshape(x, [-1])\n    y = tf.reshape(y, [-1])\n    ones = tf.ones_like(x)\n    coord_mat = tf.stack([x, y, ones])\n    \n    res_mat = tf.linalg.matmul(trans_mat, coord_mat)\n    map_x = res_mat[0]\n    map_y = res_mat[1]\n    return map_x, map_y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ShiftScaleRotate(\n        shift_limit, scale_limit, rotate_limit, p):\n    def _do_shift_scale_rotate(image, mask):\n        image_shape = tf.shape(image)\n        height_i = image_shape[0]\n        width_i = image_shape[1]\n        height_f = tf.cast(height_i, dtype=tf.float32)\n        width_f = tf.cast(width_i, dtype=tf.float32)\n        tx = width_f * random_float(-shift_limit, shift_limit)\n        ty = height_f * random_float(-shift_limit, shift_limit)\n        z = random_float(1.0 - scale_limit, 1.0 + scale_limit)\n        theta = random_float(-rotate_limit, rotate_limit)\n\n        map_x, map_y = affine_transform(\n            height_f, width_f, tx, ty, z, theta)\n        aug_image = remap(\n            image, height_i, width_i, map_x, map_y, mode='constant')\n        aug_mask = remap(\n            mask, height_i, width_i, map_x, map_y, mode='constant')\n        return choice(p, aug_image, aug_mask, image, mask)\n    return _do_shift_scale_rotate","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shift_scale_rotate = ShiftScaleRotate(\n    shift_limit=0.2, scale_limit=0.3, rotate_limit=30, p=0.75)\ncheck_aug(shift_scale_rotate, with_mask=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cutout","metadata":{}},{"cell_type":"code","source":"def randints(shape, minval, maxval):\n    # maxval+1 to include maxval for the result.\n    # generated range is [minval, maxval) (maxval is not included)\n    return tf.random.uniform(\n        shape=shape, minval=minval, maxval=maxval+1, dtype=tf.int32)\n\ndef make_range_masks(size, starts, ends):\n    indice = tf.range(size, dtype=tf.int32)\n    start_masks = (\n        starts[ : , tf.newaxis] <= indice[  tf.newaxis, : ])\n    end_masks = (\n        indice[ tf.newaxis, : ] <= ends[ : , tf.newaxis])\n    range_masks = start_masks & end_masks\n    return range_masks\n\ndef make_region_mask(tops, lefts, bottoms, rights):\n    row_masks = make_range_masks(image_size, tops, bottoms)\n    col_masks = make_range_masks(image_size, lefts, rights)\n    region_masks = \\\n        row_masks[ : , : , tf.newaxis ] & \\\n        col_masks[ : , tf.newaxis, : ]\n    region_mask = tf.math.reduce_any(region_masks, axis=0)\n    region_mask = region_mask[ : , : , tf.newaxis]\n    return region_mask\n\ndef Cutout(num_cuts, mask_factor, p):\n    def _do_cutout(image, mask):\n        image_shape = tf.shape(image)\n        height_i = image_shape[0]\n        width_i = image_shape[1]\n        height_f = tf.cast(height_i, dtype=tf.float32)\n        width_f = tf.cast(width_i, dtype=tf.float32)\n        cut_h = tf.cast(height_f * mask_factor, dtype=tf.int32)\n        cut_w = tf.cast(width_f * mask_factor, dtype=tf.int32)\n\n        y_centers = randints([num_cuts], 0, image_size - 1)\n        x_centers = randints([num_cuts], 0, image_size - 1)\n        tops = tf.math.maximum(y_centers - cut_h//2, 0)\n        lefts = tf.math.maximum(x_centers - cut_w//2, 0)\n        bottoms = tf.math.minimum(tops + cut_h, height_i - 1)\n        rights = tf.math.minimum(lefts + cut_w, width_i - 1)\n\n        cut_region = make_region_mask(tops, lefts, bottoms, rights)\n        mask_value = tf.constant(0.0, dtype=tf.float32)\n        aug_image = tf.where(cut_region, mask_value, image)\n        return choice(p, aug_image, mask, image, mask)\n    return _do_cutout","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cut_out = Cutout(num_cuts=1, mask_factor=0.4, p=0.75)\ncheck_aug(cut_out, with_mask=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Series of Augmentations","metadata":{}},{"cell_type":"code","source":"def do_augment(image, mask, labels):\n    # some tensorflow augmentation expects float\n    image, mask = image_mask_to_float_0_1(image, mask)\n    \n    image, mask = horizontal_flip(image, mask)\n    image, mask = random_brightness(image, mask)\n    image, mask = random_contrast(image, mask)\n    image, mask = one_of_opt_grid_distortion(image, mask)\n    image, mask = hue_saturation_value(image, mask)\n    image, mask = shift_scale_rotate(image, mask)\n    image, mask = cut_out(image, mask)\n\n    # back to uint8\n    image, mask = image_mask_to_uint8(image, mask)\n    return image, mask, labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset 2","metadata":{}},{"cell_type":"code","source":"def select_train(ds, fold):\n    ds = ds.filter(lambda image, mask, labels, f: f != fold)\n    return ds\n    \ndef select_val(ds, fold):\n    ds = ds.filter(lambda image, mask, labels, f: f == fold)\n    return ds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drop_fold(image, mask, labels, fold):\n    return image, mask, labels\n\ndef process_image_mask(image, mask, labels):\n    mask = mask[ : , : , :-1 ]\n    combined = tf.concat([image, mask], axis=-1)\n    return combined, labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_dataset(\n        dset, augment=True, repeat=True, shuffle=1024):\n    dset = dset.map(drop_fold, num_parallel_calls=AUTOTUNE)\n    dset = dset.repeat() if repeat else dset\n    dset = dset.map(\n        do_augment, num_parallel_calls=AUTOTUNE) if augment else dset\n    dset = dset.map(process_image_mask, num_parallel_calls=AUTOTUNE)\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(batch_size)\n    dset = dset.prefetch(AUTOTUNE)\n    return dset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_datasets(fold_i):\n    train_ds = select_train(raw_ds, fold_i)\n    train_ds = build_dataset(\n        train_ds, augment=True, repeat=True, shuffle=1024)\n\n    val_ds = select_val(raw_ds, fold_i)\n    val_ds = build_dataset(\n        val_ds, augment=False, repeat=False, shuffle=None)\n\n    train_steps = fold_train_count(fold_i) // batch_size\n    val_steps = fold_val_count(fold_i) // batch_size\n\n    return train_ds, val_ds, train_steps, val_steps","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization","metadata":{}},{"cell_type":"code","source":"train_ds, val_ds, train_steps, val_steps = make_datasets(0)\n\nprint(train_ds)\nprint(val_ds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pylab import rcParams\n\ndef show_images(ds):\n    rcParams['figure.figsize'] = 20,10\n\n    f, axarr = plt.subplots(1,5)\n    imgs = []\n    ds_iter = iter(ds.unbatch())\n    for p in range(5):\n        img, labels = next(ds_iter)\n        axarr[p].imshow(img[ : , : , :3 ])\n        imgs.append(img)\n\n    f, axarr = plt.subplots(1,5)\n    for p in range(5):\n        axarr[p].imshow(imgs[p][ : , : , 3])\n\n    f, axarr = plt.subplots(1,5)\n    for p in range(5):\n        axarr[p].imshow(imgs[p][ : , : , 4])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_images(train_ds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_images(val_ds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"markdown","source":"Reference: _clone_layer() function in https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/keras/models.py","metadata":{}},{"cell_type":"code","source":"def clone_layer(layer):\n    config = layer.get_config()\n    return layer.__class__.from_config(config)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reference: [How to replace (or insert) intermediate layer in Keras model?](https://stackoverflow.com/questions/49492255/how-to-replace-or-insert-intermediate-layer-in-keras-model)","metadata":{}},{"cell_type":"code","source":"def make_input_layers_dict(enet_3_ch, stem_conv_index):\n    input_layers_dict = {}\n    for layer in enet_3_ch.layers[ stem_conv_index: ]:\n        for node in layer._outbound_nodes:\n            layer_name = node.outbound_layer.name\n            if layer_name not in input_layers_dict:\n                input_layers_dict[layer_name] = [layer.name]\n            else:\n                input_layers_dict[layer_name].append(layer.name)\n    return input_layers_dict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def call_layers(enet_3_ch, stem_conv_index, input_layers_dict, x):\n    new_output_tensor_dict = {\n        \"stem_conv\": x\n    }\n    model_outputs = []\n    for layer in enet_3_ch.layers[ stem_conv_index + 1: ]:\n        # Determine input tensors\n        layer_input = [\n            new_output_tensor_dict[layer_aux] \n            for layer_aux in input_layers_dict[layer.name] ]\n        if len(layer_input) == 1:\n            layer_input = layer_input[0]\n\n        x = layer(layer_input)\n        new_output_tensor_dict[layer.name] = x\n\n        if layer.name in enet_3_ch.output_names:\n            model_outputs.append(x)\n\n    return model_outputs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_normalization_n_ch_weights(weights_3_ch):\n    # [ mean, stdev, bias ]\n    weight_3_0 = weights_3_ch[0]\n    weight_n_0 = np.concatenate(\n        [ weight_3_0, weight_3_0[ :(n_ch - 3)] ])\n    \n    weight_3_1 = weights_3_ch[1]\n    weight_n_1 = np.concatenate(\n        [ weight_3_1, weight_3_1[ :(n_ch - 3)] ])\n    \n    weight_n_2 = weights_3_ch[2]\n    \n    weights_n_ch = [weight_n_0, weight_n_1, weight_n_2]\n    return weights_n_ch","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_stem_conv_n_ch_weights(weights_3_ch):\n    # [ coefficient, bias ]\n    # stem_conv does not use bias, so only has one weight.\n    weight_3_0 = weights_3_ch[0]\n    weight_n_0 = np.concatenate(\n        # [ filter_height, filter_width, in_channel, out_channel ]\n        [ weight_3_0, weight_3_0[ : , : , :(n_ch - 3), : ]], axis=2)\n    return [ weight_n_0 ]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_n_ch_enet():\n    # Make a model with pretrained weight as usual.\n    enet_3_ch = enet_type(\n        include_top=False, weights='imagenet',\n        input_shape=(image_size, image_size, 3),\n        pooling='avg')\n    \n    stem_conv_index = 4\n    assert \\\n        enet_3_ch.layers[stem_conv_index].name == \"stem_conv\", \\\n        \"Efn layer 4 is not 'stem_conv'\"\n    \n    input_layers_dict = \\\n        make_input_layers_dict(enet_3_ch, stem_conv_index)\n    \n    # Clone layers that need to change the weight for 5 channel input.\n    rescaling = enet_3_ch.layers[1]\n    normalization_3_ch = enet_3_ch.layers[2]\n    normalization_n_ch = clone_layer(normalization_3_ch)\n    stem_conv_pad = enet_3_ch.layers[3]\n    stem_conv_3_ch = enet_3_ch.layers[stem_conv_index]\n    stem_conv_n_ch = clone_layer(stem_conv_3_ch)\n\n    # Make a 5 channel input layer, then call.\n    n_ch_inputs = tf.keras.Input(\n        shape=(image_size, image_size, n_ch), name=\"enet_n_ch_inputs\")\n    x = n_ch_inputs\n    x = rescaling(x)\n    x = normalization_n_ch(x)\n    x = stem_conv_pad(x)\n    x = stem_conv_n_ch(x)\n    n_ch_outputs = call_layers(\n        enet_3_ch, stem_conv_index, input_layers_dict, x)\n    \n    enet_n_ch = tf.keras.Model(\n        inputs=n_ch_inputs, outputs=n_ch_outputs, name=\"enet_n_ch\")\n    \n    # Set weight to the clone layers for 5 input channels.\n    normalization_weights_3_ch = normalization_3_ch.get_weights()\n    normalization_weights_n_ch = \\\n        make_normalization_n_ch_weights(normalization_weights_3_ch)\n    normalization_n_ch.set_weights(normalization_weights_n_ch)\n    \n    stem_conv_weights_3_ch = stem_conv_3_ch.get_weights()\n    stem_conv_weights_n_ch = \\\n        make_stem_conv_n_ch_weights(stem_conv_weights_3_ch)\n    stem_conv_n_ch.set_weights(stem_conv_weights_n_ch)\n    \n    return enet_n_ch","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_pred_output(x):\n    ett = x[0]\n    other = x[1]\n    ett_wo_no_ett = ett[ : , :3 ]\n    pred = tf.concat([ett_wo_no_ett, other], axis=-1)\n    return pred\n\ndef make_model():\n    with strategy.scope(): \n        enet = make_n_ch_enet()\n        \n        inputs = tf.keras.Input(\n            shape=(image_size, image_size, n_ch), name=\"inputs\")\n        x = enet(inputs)\n        x = L.Dropout(0.5, name='dropout')(x)\n        output_ett = L.Dense(\n            num_ett_classes, activation='softmax', name='ett')(x)\n        output_other = L.Dense(\n            num_other_classes, activation='sigmoid', name='other')(x)\n        output_pred = L.Lambda(\n            make_pred_output, name='pred')([output_ett, output_other])\n\n        model = tf.keras.Model(\n            inputs=inputs,\n            outputs=[output_ett, output_other, output_pred],\n            name=\"cls_model\")\n\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(),\n        loss={\n            \"ett\": \"categorical_crossentropy\",\n            \"other\": \"binary_crossentropy\" },\n        loss_weights={\n            \"ett\": loss_weights[0] / sum(loss_weights),\n            \"other\": loss_weights[1] / sum(loss_weights) },\n        metrics={\n            \"pred\": tf.keras.metrics.AUC(multi_label=True, name=\"auc\") },\n        # overheads and allows the XLA compiler to unroll the loop on TPU\n        # and optimize hardware utilization.\n        # needs to be commented out for Tensorflow 2.3\n        steps_per_execution=8)\n    model.summary()\n\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = make_model()\ninitial_weights = model.get_weights()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"LR_START = init_lr\nLR_MAX = 1e-3\nLR_MIN = 1e-5\nLR_RAMPUP_EPOCHS = warmup_epo\nLR_SUSTAIN_EPOCHS = 0\nEPOCHS = n_epochs\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        decay_total_epochs = EPOCHS - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\n        decay_epoch_index = epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n        phase = math.pi * decay_epoch_index / decay_total_epochs\n        cosine_decay = 0.5 * (1 + math.cos(phase))\n        lr = (LR_MAX - LR_MIN) * cosine_decay + LR_MIN\n    return lr\n\nrng = [i for i in range(EPOCHS)]\nlr_y = [lrfn(x) for x in rng]\nplt.figure(figsize=(10, 4))\nplt.plot(rng, lr_y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\". \\\n      format(lr_y[0], max(lr_y), lr_y[-1]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cb_monitor = 'val_pred_auc'\n\nclass RestoreBestWeights(tf.keras.callbacks.Callback):\n    def __init__(self):\n        super(RestoreBestWeights, self).__init__()\n        self.best_monitor = -np.Inf\n        self.best_weights = None\n        self.best_epoch = None\n        \n    def on_epoch_end(self, epoch, logs=None):\n        current_monitor = logs.get(cb_monitor)\n        if current_monitor > self.best_monitor:\n            self.best_monitor = current_monitor\n            self.best_weights = self.model.get_weights()\n            self.best_epoch = epoch\n            \n    def on_train_end(self, logs=None):\n        print(\"Restoring best weights on epoch {0}, {1} was {2:.5f}\".format(\n            self.best_epoch + 1, cb_monitor, self.best_monitor))\n        self.model.set_weights(self.best_weights)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_callbacks(fold_i):\n    best_model_file_name = \"cls_model_{0}_{1}.hdf5\".format(VID, fold_i)\n    cb_mode = 'max'\n    cb_min_delta = 1e-4\n    cb_verbose = 1\n\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(\n        best_model_file_name, save_best_only=True,\n        save_weights_only=False, monitor=cb_monitor, mode=cb_mode,\n        verbose=cb_verbose)\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = False)\n    restore_best_weights = RestoreBestWeights()\n    \n    return checkpoint, lr_callback, restore_best_weights","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_one_fold(fold_i):\n    train_dataset, val_dataset, train_steps, val_steps = \\\n        make_datasets(fold_i)\n    checkpoint, lr_callback, restore_best_weights = \\\n        make_callbacks(fold_i)\n    history = model.fit(\n        train_dataset, \n        epochs=EPOCHS,\n        verbose=1,\n        callbacks=[checkpoint, lr_callback, restore_best_weights],\n        steps_per_epoch=train_steps,\n        validation_data=val_dataset,\n        validation_steps=val_steps)\n    return history, val_dataset, val_steps","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(history, title, labels, subplot):\n    plt.subplot(*subplot)\n    plt.title(title)\n    for label in labels:\n        plt.plot(history.history[label], label=label)\n    plt.legend()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_fit_result(history):\n    plt.figure(figsize=(12, 4))\n    plot_history(\n        history,\n        \"Loss\", ['loss', 'val_loss'], (1, 2, 1))\n    plot_history(\n        history,\n        \"AUC\", ['pred_auc', 'val_pred_auc'], (1, 2, 2))\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_one_fold(model, val_dataset, val_steps):\n    val_true_list = [] \n    for _, (_, _, labels_pred) in val_dataset:\n        val_true_list.append(labels_pred)\n    val_true = np.concatenate(val_true_list, axis=0)\n   \n    # Causes the error :ValueError: The two structures don't have\n    # the same nested structure.\n    # val_preds = model.predict(val_dataset, steps=val_steps)\n    # val_pred = val_preds[2]\n    val_pred_list = []\n    for val_data in val_dataset:\n        val_preds = model(val_data[0], training=False)\n        val_pred_list.append(val_preds[2])\n    val_pred = np.concatenate(val_pred_list, axis=0)\n    \n    val_true = val_true[ : len(val_pred) ]\n    return val_true, val_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_columns = [\n    'ETT - Abnormal',\n    'ETT - Borderline',\n    'ETT - Normal',\n    'NGT - Abnormal',\n    'NGT - Borderline',\n    'NGT - Incompletely Imaged',\n    'NGT - Normal',\n    'CVC - Abnormal',\n    'CVC - Borderline',\n    'CVC - Normal',\n    'Swan Ganz Catheter Present',\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\ndef show_roc_auc_score(val_true, val_pred):\n    auc_list = []\n    for i in range(val_true.shape[-1]):\n        auc = roc_auc_score(val_true[ : , i], val_pred[ : , i])\n        print(\"{0:30s}: {1:.4f}\".format(target_columns[i], auc))\n        auc_list.append(auc)\n\n    mean_auc = np.mean(auc_list)\n    print(\"{0:30s}: {1:.4f}\".format(\"Mean\", mean_auc))\n    \n    plt.figure(figsize=(8, 4))\n    plt.plot(auc_list)\n    ticks = np.arange(len(target_columns))\n    plt.xticks(ticks=ticks, labels=target_columns, rotation=90)\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold_i in FOLD_I_LIST:\n    print(\"####################\")\n    print(\"# Fold {0}\".format(fold_i))\n    model.set_weights(initial_weights)\n    history, val_dataset, val_steps = fit_one_fold(fold_i)\n    plot_fit_result(history)\n    val_true, val_pred = predict_one_fold(model, val_dataset, val_steps)\n    show_roc_auc_score(val_true, val_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}