{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook tries to run the aux loss model explained in the discussion [Easy Trick to Add Aux Loss](https://www.kaggle.com/c/siim-covid19-detection/discussion/263676).","metadata":{}},{"cell_type":"code","source":"%env SM_FRAMEWORK=tf.keras\n!pip install -U segmentation-models","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:18.53615Z","iopub.execute_input":"2021-08-11T20:58:18.536677Z","iopub.status.idle":"2021-08-11T20:58:28.593155Z","shell.execute_reply.started":"2021-08-11T20:58:18.536565Z","shell.execute_reply":"2021-08-11T20:58:28.591598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:28.595319Z","iopub.execute_input":"2021-08-11T20:58:28.595641Z","iopub.status.idle":"2021-08-11T20:58:28.601525Z","shell.execute_reply.started":"2021-08-11T20:58:28.595606Z","shell.execute_reply":"2021-08-11T20:58:28.600202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nimport math\nimport pandas as pd\nimport tensorflow_addons as tfa\nimport segmentation_models as sm\n\nprint(tf.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-11T20:58:28.604391Z","iopub.execute_input":"2021-08-11T20:58:28.604781Z","iopub.status.idle":"2021-08-11T20:58:36.175846Z","shell.execute_reply.started":"2021-08-11T20:58:28.604747Z","shell.execute_reply":"2021-08-11T20:58:36.17482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEG_MODEL = sm.FPN\nBACKBONE = 'efficientnetb4'\nIMAGE_SIZE = 512\nBATCH_SIZE = 128\nINIT_LR = 1e-4\nWARMUP_EPO = 2\nCOSINE_EPO = 28 if not DEBUG else 2\nN_EPOCHS = WARMUP_EPO + COSINE_EPO\nN_FOLDS = 5\n\nVID = 'V03'\n# FOLD_I_LIST = [0, 1, 2, 3, 4]\nFOLD_I_LIST = [0]\nFOLD_I_LIST = FOLD_I_LIST[ :1 ] if DEBUG else FOLD_I_LIST","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:36.177129Z","iopub.execute_input":"2021-08-11T20:58:36.177421Z","iopub.status.idle":"2021-08-11T20:58:36.183048Z","shell.execute_reply.started":"2021-08-11T20:58:36.177393Z","shell.execute_reply":"2021-08-11T20:58:36.182212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_name = 'siim-covid19-tfrecord-for-training'\nMAX_BBOXES = 8\nN_STUDY_LABELS = 4","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:36.184047Z","iopub.execute_input":"2021-08-11T20:58:36.184449Z","iopub.status.idle":"2021-08-11T20:58:36.198778Z","shell.execute_reply.started":"2021-08-11T20:58:36.184419Z","shell.execute_reply":"2021-08-11T20:58:36.197965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TPU","metadata":{}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() \n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError: # otherwise detect GPUs\n    strategy = tf.distribute.MirroredStrategy() # single-GPU or multi-GPU\n    \nprint(f\"Running on {strategy.num_replicas_in_sync} replicas\")","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:36.199799Z","iopub.execute_input":"2021-08-11T20:58:36.200229Z","iopub.status.idle":"2021-08-11T20:58:41.968146Z","shell.execute_reply.started":"2021-08-11T20:58:36.200199Z","shell.execute_reply":"2021-08-11T20:58:41.966469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path(train_data_name)\n\nGCS_DS_PATH","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:41.969712Z","iopub.execute_input":"2021-08-11T20:58:41.970098Z","iopub.status.idle":"2021-08-11T20:58:42.433212Z","shell.execute_reply.started":"2021-08-11T20:58:41.970061Z","shell.execute_reply":"2021-08-11T20:58:42.431974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"def decode_image(image_bytes):\n    image = tf.image.decode_jpeg(image_bytes, channels=3)\n    image = tf.reshape(image, [IMAGE_SIZE, IMAGE_SIZE, 3])\n    return image\n\ndef read_tfrecord(example):\n    TFREC_FORMAT = {\n        'image_id': tf.io.FixedLenFeature([], tf.string),\n        'study_id': tf.io.FixedLenFeature([], tf.string),\n        'fold': tf.io.FixedLenFeature([], tf.int64),\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'xray_height': tf.io.FixedLenFeature([], tf.int64),\n        'xray_width': tf.io.FixedLenFeature([], tf.int64),\n        'bbox_label': tf.io.FixedLenFeature([MAX_BBOXES], tf.string),\n        'confidence': tf.io.FixedLenFeature([MAX_BBOXES], tf.float32),\n        'left': tf.io.FixedLenFeature([MAX_BBOXES], tf.float32),\n        'top': tf.io.FixedLenFeature([MAX_BBOXES], tf.float32),\n        'right': tf.io.FixedLenFeature([MAX_BBOXES], tf.float32),\n        'bottom': tf.io.FixedLenFeature([MAX_BBOXES], tf.float32),\n        'Negative for Pneumonia': tf.io.FixedLenFeature([], tf.int64),\n        'Typical Appearance': tf.io.FixedLenFeature([], tf.int64),\n        'Indeterminate Appearance': tf.io.FixedLenFeature([], tf.int64),\n        'Atypical Appearance': tf.io.FixedLenFeature([], tf.int64),\n    }\n    \n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    ids = [example['image_id'], example['study_id']]\n    fold = example['fold']\n    image = decode_image(example['image'])\n    bbox_label = example['bbox_label']\n\n    xray_height = tf.cast(example['xray_height'], dtype=tf.float32)\n    xray_width = tf.cast(example['xray_width'], dtype=tf.float32)\n    def _calc_bbox_pos(example_pos, xray_size):\n        pos = example_pos * IMAGE_SIZE / xray_size\n        pos = tf.cast(tf.math.round(pos), dtype=tf.int64)\n        pos = tf.clip_by_value(pos, 0, IMAGE_SIZE - 1)\n        return pos\n    left = _calc_bbox_pos(example['left'], xray_width)\n    top = _calc_bbox_pos(example['top'], xray_height)\n    right = _calc_bbox_pos(example['right'], xray_width)\n    bottom = _calc_bbox_pos(example['bottom'], xray_height)\n    bbox_pos = [left, top, right, bottom]\n\n    study_label = [\n        example['Negative for Pneumonia'], example['Typical Appearance'],\n        example['Indeterminate Appearance'], example['Atypical Appearance']]\n    return ids, fold, image, bbox_label, bbox_pos, study_label","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:42.437248Z","iopub.execute_input":"2021-08-11T20:58:42.437581Z","iopub.status.idle":"2021-08-11T20:58:42.45505Z","shell.execute_reply.started":"2021-08-11T20:58:42.437548Z","shell.execute_reply":"2021-08-11T20:58:42.453789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opacity_label = tf.constant(\"opacity\".encode('utf-8'))\n\ndef process_bbox(ids, fold, image, bbox_label, bbox_pos, study_label):\n    # opacity => 1.0, none => 0.0\n    bbox_label = tf.cast(\n        bbox_label == opacity_label, dtype=tf.float32)\n    return ids, fold, image, bbox_label, bbox_pos, study_label","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:42.458262Z","iopub.execute_input":"2021-08-11T20:58:42.458909Z","iopub.status.idle":"2021-08-11T20:58:42.4776Z","shell.execute_reply.started":"2021-08-11T20:58:42.458838Z","shell.execute_reply":"2021-08-11T20:58:42.475922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_mask(ids, fold, image, bbox_label, bbox_pos, study_label):\n    lefts   = bbox_pos[0]\n    tops    = bbox_pos[1]\n    rights  = bbox_pos[2]\n    bottoms = bbox_pos[3]\n\n    def _make_one_mask(i):\n        mask_height = bottoms[i] - tops[i] + 1\n        mask_width = rights[i] - lefts[i] + 1\n        mask_shape = [mask_height, mask_width]\n        mask = bbox_label[i] * tf.ones(mask_shape, dtype=tf.float32)\n\n        paddings = [\n            [tops[i], IMAGE_SIZE - bottoms[i] - 1],\n            [lefts[i], IMAGE_SIZE - rights[i] - 1]]\n        mask = tf.pad(mask, paddings, mode='CONSTANT')\n        return mask\n\n    num_masks_rng = tf.range(MAX_BBOXES, dtype=tf.int64)\n    masks = tf.map_fn(\n        _make_one_mask, num_masks_rng,\n        fn_output_signature=tf.float32)\n    mask = tf.math.reduce_sum(masks, axis=0)\n    mask = tf.math.minimum(1.0, mask)\n    mask = tf.reshape(mask, [IMAGE_SIZE, IMAGE_SIZE, 1])\n    return ids, fold, image, mask, study_label","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:42.479354Z","iopub.execute_input":"2021-08-11T20:58:42.479705Z","iopub.status.idle":"2021-08-11T20:58:42.492215Z","shell.execute_reply.started":"2021-08-11T20:58:42.47967Z","shell.execute_reply":"2021-08-11T20:58:42.490783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dataset(filenames):\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=None)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=None)\n    dataset = dataset.map(process_bbox, num_parallel_calls=None)\n    dataset = dataset.map(make_mask, num_parallel_calls=None)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:42.494087Z","iopub.execute_input":"2021-08-11T20:58:42.4946Z","iopub.status.idle":"2021-08-11T20:58:42.510158Z","shell.execute_reply.started":"2021-08-11T20:58:42.494561Z","shell.execute_reply":"2021-08-11T20:58:42.50916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfrec_file_names = sorted(tf.io.gfile.glob(GCS_DS_PATH + '/*.tfrec'))\ntfrec_file_names = \\\n    [ tfrec_file_names[ :4 ] ] if DEBUG else tfrec_file_names\nraw_ds = load_dataset(tfrec_file_names)\n\nprint(raw_ds)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:42.51136Z","iopub.execute_input":"2021-08-11T20:58:42.511676Z","iopub.status.idle":"2021-08-11T20:58:43.699775Z","shell.execute_reply.started":"2021-08-11T20:58:42.511643Z","shell.execute_reply":"2021-08-11T20:58:43.698706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fold Information","metadata":{}},{"cell_type":"code","source":"study_id_list = []\nfold_batch_list = []\nfor ids_batch, fold_batch, _, _, _ in raw_ds.batch(256):\n    print('.', end='', flush=True)\n    study_id_bin_array = ids_batch[ :, 1].numpy()\n    for study_id_bin in study_id_bin_array:\n        study_id_str = study_id_bin.decode('utf-8')\n        study_id_list.append(study_id_str)\n    fold_batch_list.append(fold_batch.numpy())\n\nprint()\nfold_array = np.concatenate(fold_batch_list, axis=0)\nfold_info_df = pd.DataFrame({\n    'study_id': study_id_list,\n    'fold': fold_array })\n\nfold_info_df","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:43.700979Z","iopub.execute_input":"2021-08-11T20:58:43.701277Z","iopub.status.idle":"2021-08-11T20:58:54.652655Z","shell.execute_reply.started":"2021-08-11T20:58:43.701247Z","shell.execute_reply":"2021-08-11T20:58:54.651248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold_info_df['fold'].value_counts().sort_index()","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:54.654311Z","iopub.execute_input":"2021-08-11T20:58:54.654798Z","iopub.status.idle":"2021-08-11T20:58:54.673234Z","shell.execute_reply.started":"2021-08-11T20:58:54.65475Z","shell.execute_reply":"2021-08-11T20:58:54.671813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_count(fold_i):\n    return sum(fold_info_df['fold'] != fold_i)\n\ndef get_val_count(fold_i):\n    return sum(fold_info_df['fold'] == fold_i)\n\ndef get_val_study_ids(fold_i):\n    fold_mask = (fold_info_df['fold'] == fold_i)\n    val_study_ids = fold_info_df.loc[fold_mask, 'study_id']\n    return val_study_ids.values","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:54.67523Z","iopub.execute_input":"2021-08-11T20:58:54.676005Z","iopub.status.idle":"2021-08-11T20:58:54.684238Z","shell.execute_reply.started":"2021-08-11T20:58:54.675954Z","shell.execute_reply":"2021-08-11T20:58:54.682976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Augmentation","metadata":{}},{"cell_type":"code","source":"def image_to_float_0_1(image):\n    image = tf.cast(image, dtype=tf.float32) / 255.0\n    return image","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:54.686089Z","iopub.execute_input":"2021-08-11T20:58:54.686566Z","iopub.status.idle":"2021-08-11T20:58:54.699624Z","shell.execute_reply.started":"2021-08-11T20:58:54.686524Z","shell.execute_reply":"2021-08-11T20:58:54.697851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_aug(aug_fun, with_mask):\n    rows = 2\n    cols = 5\n    n_imgs = rows * cols\n\n    _, _, images, masks, _ = next(iter(\n        raw_ds.take(1).repeat(n_imgs).batch(n_imgs)))\n    images = image_to_float_0_1(images)\n    \n    plt.figure(figsize=(12, 4))\n    aug_images, aug_masks = aug_fun(images, masks)\n    for i, aug_image in enumerate(aug_images):\n        plt.subplot(rows, cols, i+1)\n        plt.imshow(aug_image)\n        plt.axis(\"off\")\n    plt.tight_layout()\n    plt.show()        \n    \n    if with_mask:\n        plt.figure(figsize=(12, 4))\n        for i, aug_mask in enumerate(aug_masks):\n            plt.subplot(rows, cols, i+1)\n            plt.imshow(aug_mask, cmap='gray')\n            plt.axis(\"off\")\n        plt.tight_layout()\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:54.701454Z","iopub.execute_input":"2021-08-11T20:58:54.702222Z","iopub.status.idle":"2021-08-11T20:58:54.717015Z","shell.execute_reply.started":"2021-08-11T20:58:54.702168Z","shell.execute_reply":"2021-08-11T20:58:54.715617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def random_int(shape=[], minval=0, maxval=1):\n    return tf.random.uniform(\n        shape=shape, minval=minval, maxval=maxval, dtype=tf.int32)\n\ndef random_float(shape=[], minval=0.0, maxval=1.0):\n    rnd = tf.random.uniform(\n        shape=shape, minval=minval, maxval=maxval, dtype=tf.float32)\n    return rnd","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:54.719124Z","iopub.execute_input":"2021-08-11T20:58:54.719478Z","iopub.status.idle":"2021-08-11T20:58:54.731372Z","shell.execute_reply.started":"2021-08-11T20:58:54.719444Z","shell.execute_reply":"2021-08-11T20:58:54.730208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BaseAug():\n    def __init__(self, p):\n        self.p = p\n        \n    def __call__(self, images, masks):\n        def _aug_one_data(i):\n            image = images[i]\n            mask = masks[i]\n            rnd = random_float()\n            return tf.cond(\n                rnd <= self.p, \n                lambda: self.aug_data(image, mask),\n                lambda: self._no_aug(image, mask))\n        \n        batch = tf.shape(images)[0]\n        batch_rng = tf.range(batch, dtype=tf.int64)\n        aug_images, aug_masks = tf.map_fn(\n            _aug_one_data, batch_rng,\n            fn_output_signature=(tf.float32, tf.float32))\n        return aug_images, aug_masks\n\n    def aug_data(self, image, mask):\n        raise NotImplemented(\"aug_data() needs to implement\")\n        \n    def _no_aug(self, image, mask):\n        return image, mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:54.733127Z","iopub.execute_input":"2021-08-11T20:58:54.733499Z","iopub.status.idle":"2021-08-11T20:58:54.745632Z","shell.execute_reply.started":"2021-08-11T20:58:54.733463Z","shell.execute_reply":"2021-08-11T20:58:54.744349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mirror_boundary(v, max_v):\n    # v % (max_v*2.0-2.0) ==> v % (512*2-2) ==> [0..1022]\n    # [0..1022] - (max_v-1.0) ==> [0..1022] - 511 ==> [-511..511]\n    # -1.0 * abs([-511..511]) ==> [-511..0]\n    # [-511..0] + max_v - 1.0 ==> [-511..0] + 511 ==> [0..511]\n    mirror_v = -1.0 * tf.math.abs(\n        v % (max_v*2.0-2.0) - (max_v-1.0)) + max_v-1.0\n    return mirror_v\n\ndef clip_boundary(v, max_v):\n    clip_v = tf.clip_by_value(v, 0.0, max_v-1.0)\n    return clip_v\n\ndef interpolate_bilinear(image, map_x, map_y):\n    def _gather(image, map_x, map_y):\n        map_stack = tf.stack([map_x, map_y]) # [ 2, height, width ]\n        map_indices = tf.transpose(\n            map_stack, perm=[1, 2, 0])       # [ height, width, 2 ]\n        map_indices = tf.cast(map_indices, dtype=tf.int32)\n        gather_image = tf.gather_nd(image, map_indices)\n        return gather_image\n    \n    ll = _gather(image, tf.math.floor(map_x), tf.math.floor(map_y))\n    lr = _gather(image, tf.math.ceil(map_x), tf.math.floor(map_y))\n    ul = _gather(image, tf.math.floor(map_x), tf.math.ceil(map_y))\n    ur = _gather(image, tf.math.ceil(map_x), tf.math.ceil(map_y))\n    \n    fraction_x = tf.expand_dims(map_x % 1.0, axis=-1) # [h, w, 1]\n    int_l = (lr - ll) * fraction_x + ll\n    int_u = (ur - ul) * fraction_x + ul\n    \n    fraction_y = tf.expand_dims(map_y % 1.0, axis=-1) # [h, w, 1]\n    interpolate_image = (int_u - int_l) * fraction_y + int_l\n    return interpolate_image\n\ndef remap(image, height, width, map_x, map_y, mode):\n    assert \\\n        mode in ('mirror', 'constant'), \\\n        \"mode is neither 'mirror' nor 'constant'\"\n\n    height_f = tf.cast(height, dtype=tf.float32)\n    width_f = tf.cast(width, dtype=tf.float32)\n    map_x = tf.reshape(map_x, shape=[height, width])\n    map_y = tf.reshape(map_y, shape=[height, width])\n    if mode == 'mirror':\n        b_map_x = mirror_boundary(map_x, width_f)\n        b_map_y = mirror_boundary(map_y, height_f)\n    else:\n        b_map_x = clip_boundary(map_x, width_f)\n        b_map_y = clip_boundary(map_y, height_f)\n        \n    image_remap = interpolate_bilinear(image, b_map_x, b_map_y)\n    \n    if mode == 'constant':\n        map_stack = tf.stack([map_x, map_y])\n        map_indices = tf.transpose(map_stack, perm=[1, 2, 0])\n        x_ge_0 = (0.0 <= map_indices[ : , : , 0])    # [h, w]\n        x_lt_w = (map_indices[ : , : , 0] < width_f)\n        y_ge_0 = (0.0 <= map_indices[ : , : , 1])\n        y_lt_h = (map_indices[ : , : , 1] < height_f)\n        inside_boundary = tf.math.reduce_all(\n            tf.stack([x_ge_0, x_lt_w, y_ge_0, y_lt_h]), axis=0) # [h, w]\n        inside_boundary = inside_boundary[ : , : , tf.newaxis]  # [h, w, 1]\n        image_remap = tf.where(inside_boundary, image_remap, 0.0)\n\n    return image_remap","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:54.74763Z","iopub.execute_input":"2021-08-11T20:58:54.748076Z","iopub.status.idle":"2021-08-11T20:58:54.77369Z","shell.execute_reply.started":"2021-08-11T20:58:54.748035Z","shell.execute_reply":"2021-08-11T20:58:54.772092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transpose","metadata":{}},{"cell_type":"code","source":"class Transpose(BaseAug):\n    def __init__(self, p):\n        super(Transpose, self).__init__(p)\n        \n    def aug_data(self, image, mask):\n        aug_image = tf.transpose(image, perm=[1, 0, 2])\n        aug_mask = tf.transpose(mask, perm=[1, 0, 2])\n        return aug_image, aug_mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:54.775811Z","iopub.execute_input":"2021-08-11T20:58:54.776646Z","iopub.status.idle":"2021-08-11T20:58:54.794427Z","shell.execute_reply.started":"2021-08-11T20:58:54.776594Z","shell.execute_reply":"2021-08-11T20:58:54.793399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transpose = Transpose(p=0.5)\ncheck_aug(transpose, with_mask=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:54.797165Z","iopub.execute_input":"2021-08-11T20:58:54.798255Z","iopub.status.idle":"2021-08-11T20:58:57.43179Z","shell.execute_reply.started":"2021-08-11T20:58:54.798197Z","shell.execute_reply":"2021-08-11T20:58:57.430486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### VerticalFlip","metadata":{}},{"cell_type":"code","source":"class VerticalFlip(BaseAug):\n    def __init__(self, p):\n        super(VerticalFlip, self).__init__(p)\n        \n    def aug_data(self, image, mask):\n        aug_image = tf.image.flip_up_down(image)\n        aug_mask = tf.image.flip_up_down(mask)\n        return aug_image, aug_mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:57.43755Z","iopub.execute_input":"2021-08-11T20:58:57.437922Z","iopub.status.idle":"2021-08-11T20:58:57.444218Z","shell.execute_reply.started":"2021-08-11T20:58:57.43789Z","shell.execute_reply":"2021-08-11T20:58:57.443106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vertical_flip = VerticalFlip(p=0.5)\ncheck_aug(vertical_flip, with_mask=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:58:57.446744Z","iopub.execute_input":"2021-08-11T20:58:57.447109Z","iopub.status.idle":"2021-08-11T20:59:00.017057Z","shell.execute_reply.started":"2021-08-11T20:58:57.447077Z","shell.execute_reply":"2021-08-11T20:59:00.015954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### HorizontalFlip","metadata":{}},{"cell_type":"code","source":"class HorizontalFlip(BaseAug):\n    def __init__(self, p):\n        super(HorizontalFlip, self).__init__(p)\n        \n    def aug_data(self, image, mask):\n        aug_image = tf.image.flip_left_right(image)\n        aug_mask = tf.image.flip_left_right(mask)\n        return aug_image, aug_mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:00.018502Z","iopub.execute_input":"2021-08-11T20:59:00.018813Z","iopub.status.idle":"2021-08-11T20:59:00.024237Z","shell.execute_reply.started":"2021-08-11T20:59:00.018781Z","shell.execute_reply":"2021-08-11T20:59:00.023282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"horizontal_flip = HorizontalFlip(p=0.5)\ncheck_aug(horizontal_flip, with_mask=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:00.025749Z","iopub.execute_input":"2021-08-11T20:59:00.026113Z","iopub.status.idle":"2021-08-11T20:59:02.37924Z","shell.execute_reply.started":"2021-08-11T20:59:00.02608Z","shell.execute_reply":"2021-08-11T20:59:02.37792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### RandomBrightness","metadata":{}},{"cell_type":"code","source":"class RandomBrightness(BaseAug):\n    def __init__(self, max_delta, p):\n        super(RandomBrightness, self).__init__(p)\n        self.max_delta = max_delta\n        \n    def aug_data(self, image, mask):\n        aug_image = tf.image.random_brightness(image, self.max_delta)\n        return aug_image, mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:02.380642Z","iopub.execute_input":"2021-08-11T20:59:02.380973Z","iopub.status.idle":"2021-08-11T20:59:02.386805Z","shell.execute_reply.started":"2021-08-11T20:59:02.38094Z","shell.execute_reply":"2021-08-11T20:59:02.385664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_brightness = RandomBrightness(max_delta=0.2, p=0.75)\ncheck_aug(random_brightness, with_mask=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:02.388307Z","iopub.execute_input":"2021-08-11T20:59:02.388619Z","iopub.status.idle":"2021-08-11T20:59:03.966668Z","shell.execute_reply.started":"2021-08-11T20:59:02.388589Z","shell.execute_reply":"2021-08-11T20:59:03.965904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### RandomContrast","metadata":{}},{"cell_type":"code","source":"class RandomContrast(BaseAug):\n    def __init__(self, lower, upper, p):\n        super(RandomContrast, self).__init__(p)\n        self.lower = lower\n        self.upper = upper\n\n    def aug_data(self, image, mask):\n        aug_image = tf.image.random_contrast(\n            image, self.lower, self.upper)\n        return aug_image, mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:03.96771Z","iopub.execute_input":"2021-08-11T20:59:03.968099Z","iopub.status.idle":"2021-08-11T20:59:03.973731Z","shell.execute_reply.started":"2021-08-11T20:59:03.96807Z","shell.execute_reply":"2021-08-11T20:59:03.972949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_contrast = RandomContrast(lower=0.2, upper=0.8, p=0.75)\ncheck_aug(random_contrast, with_mask=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:03.974928Z","iopub.execute_input":"2021-08-11T20:59:03.975324Z","iopub.status.idle":"2021-08-11T20:59:05.676743Z","shell.execute_reply.started":"2021-08-11T20:59:03.975292Z","shell.execute_reply":"2021-08-11T20:59:05.67573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Blur","metadata":{}},{"cell_type":"code","source":"class Blur(BaseAug):\n    def __init__(self, blur_limit, p):\n        super(Blur, self).__init__(p)\n        self.blur_limit = blur_limit\n        \n    def aug_data(self, image, mask):\n        filter_size = random_int([], 3, self.blur_limit + 1)\n        filter_shape = (filter_size, filter_size)\n        aug_image = tfa.image.gaussian_filter2d(\n            image, filter_shape=filter_shape)\n        aug_image = tf.reshape(aug_image, [IMAGE_SIZE, IMAGE_SIZE, 3])\n        return aug_image, mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:05.677845Z","iopub.execute_input":"2021-08-11T20:59:05.678255Z","iopub.status.idle":"2021-08-11T20:59:05.684936Z","shell.execute_reply.started":"2021-08-11T20:59:05.678224Z","shell.execute_reply":"2021-08-11T20:59:05.684024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"blur = Blur(blur_limit=5, p=1.0)\ncheck_aug(blur, with_mask=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:05.686354Z","iopub.execute_input":"2021-08-11T20:59:05.686897Z","iopub.status.idle":"2021-08-11T20:59:08.463848Z","shell.execute_reply.started":"2021-08-11T20:59:05.686836Z","shell.execute_reply":"2021-08-11T20:59:08.462555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### MedianBlur","metadata":{}},{"cell_type":"code","source":"class MedianBlur(BaseAug):\n    def __init__(self, blur_limit, p):\n        super(MedianBlur, self).__init__(p)\n        self.blur_limit = blur_limit\n        \n    def aug_data(self, image, mask):\n#         filter_size = random_int([], 3, self.blur_limit + 1)\n#         filter_shape = (filter_size, filter_size)\n        filter_shape = (3, 3)\n        aug_image = tfa.image.median_filter2d(\n            image, filter_shape=filter_shape)\n        aug_image = tf.reshape(aug_image, [IMAGE_SIZE, IMAGE_SIZE, 3])\n        return aug_image, mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:08.465263Z","iopub.execute_input":"2021-08-11T20:59:08.465563Z","iopub.status.idle":"2021-08-11T20:59:08.473499Z","shell.execute_reply.started":"2021-08-11T20:59:08.465534Z","shell.execute_reply":"2021-08-11T20:59:08.471265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"median_blur = MedianBlur(blur_limit=3, p=1.0)\ncheck_aug(median_blur, with_mask=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:08.476851Z","iopub.execute_input":"2021-08-11T20:59:08.477237Z","iopub.status.idle":"2021-08-11T20:59:10.660759Z","shell.execute_reply.started":"2021-08-11T20:59:08.477205Z","shell.execute_reply":"2021-08-11T20:59:10.659775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### OneOf","metadata":{}},{"cell_type":"code","source":"class OneOf(BaseAug):\n    def __init__(self, trans1, trans2, p):\n        super(OneOf, self).__init__(p)\n        self.trans1 = trans1\n        self.trans2 = trans2\n        \n    def aug_data(self, image, mask):\n        rnd = random_float()\n        aug_image, aug_mask = tf.cond(\n            rnd <= 0.5,\n            lambda: self.trans1.aug_data(image, mask),\n            lambda: self.trans2.aug_data(image, mask))\n        return aug_image, aug_mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:10.662272Z","iopub.execute_input":"2021-08-11T20:59:10.662804Z","iopub.status.idle":"2021-08-11T20:59:10.669512Z","shell.execute_reply.started":"2021-08-11T20:59:10.662771Z","shell.execute_reply":"2021-08-11T20:59:10.668215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_of_blur_median_blur = OneOf(\n    blur, median_blur, p=0.7)\ncheck_aug(one_of_blur_median_blur, with_mask=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:10.671081Z","iopub.execute_input":"2021-08-11T20:59:10.671403Z","iopub.status.idle":"2021-08-11T20:59:12.245226Z","shell.execute_reply.started":"2021-08-11T20:59:10.671372Z","shell.execute_reply":"2021-08-11T20:59:12.244202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### JpegCompression","metadata":{}},{"cell_type":"code","source":"class JpegCompression(BaseAug):\n    def __init__(self, quality_lower, quality_upper, p):\n        super(JpegCompression, self).__init__(p)\n        self.quality_lower = quality_lower\n        self.quality_upper = quality_upper\n        \n    def aug_data(self, image, mask):\n        jpeg_quality = random_int(\n            [], self.quality_lower, self.quality_upper + 1)\n        aug_image = tf.image.adjust_jpeg_quality(image, jpeg_quality)\n        aug_image = tf.reshape(aug_image, [IMAGE_SIZE, IMAGE_SIZE, 3])\n        return aug_image, mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:12.246563Z","iopub.execute_input":"2021-08-11T20:59:12.246954Z","iopub.status.idle":"2021-08-11T20:59:12.253885Z","shell.execute_reply.started":"2021-08-11T20:59:12.246923Z","shell.execute_reply":"2021-08-11T20:59:12.252771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jpeg_compression = JpegCompression(\n    quality_lower=85, quality_upper=95, p=0.5)\ncheck_aug(jpeg_compression, with_mask=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:12.255315Z","iopub.execute_input":"2021-08-11T20:59:12.255602Z","iopub.status.idle":"2021-08-11T20:59:13.823335Z","shell.execute_reply.started":"2021-08-11T20:59:12.255574Z","shell.execute_reply":"2021-08-11T20:59:13.822168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### OpticalDistortion","metadata":{}},{"cell_type":"code","source":"def initUndistortRectifyMap(height, width, k, dx, dy):\n    height = tf.cast(height, dtype=tf.float32)\n    width = tf.cast(width, dtype=tf.float32)\n    \n    f_x = width\n    f_y = height\n    c_x = width * 0.5 + dx\n    c_y = height * 0.5 + dy\n    \n    f_dash_x = f_x\n    c_dash_x = (width - 1.0) * 0.5\n    f_dash_y = f_y\n    c_dash_y = (height - 1.0) * 0.5\n\n    h_rng = tf.range(height, dtype=tf.float32)\n    w_rng = tf.range(width, dtype=tf.float32)\n    v, u = tf.meshgrid(h_rng, w_rng)\n    \n    x = (u - c_dash_x) / f_dash_x\n    y = (v - c_dash_y) / f_dash_y\n    x_dash = x\n    y_dash = y\n    \n    r_2 = x_dash * x_dash + y_dash * y_dash\n    r_4 = r_2 * r_2\n    x_dash_dash = x_dash * (1 + k*r_2 + k*r_4)\n    y_dash_dash = y_dash * (1 + k*r_2 + k*r_4)\n\n    map_x = x_dash_dash * f_x + c_x\n    map_y = y_dash_dash * f_y + c_y\n    return map_x, map_y","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:13.824888Z","iopub.execute_input":"2021-08-11T20:59:13.825296Z","iopub.status.idle":"2021-08-11T20:59:13.836079Z","shell.execute_reply.started":"2021-08-11T20:59:13.825254Z","shell.execute_reply":"2021-08-11T20:59:13.835011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class OpticalDistortion(BaseAug):\n    def __init__(self, distort_limit, shift_limit, p=1.0):\n        super(OpticalDistortion, self).__init__(p)\n        self.distort_limit = distort_limit\n        self.shift_limit= shift_limit\n        \n    def aug_data(self, image, mask):\n        k = random_float([], -self.distort_limit, self.distort_limit)\n        dx = random_float([], -self.shift_limit, self.shift_limit)\n        dy = random_float([], -self.shift_limit, self.shift_limit)\n        image_shape = tf.shape(image)\n        height = image_shape[0]\n        width = image_shape[1]\n        map_x, map_y = initUndistortRectifyMap(\n            height, width, k, dx, dy)\n        aug_image = remap(\n            image, height, width, map_x, map_y, mode='mirror')\n        aug_mask = remap(\n            mask, height, width, map_x, map_y, mode='mirror')\n        return aug_image, aug_mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:13.837565Z","iopub.execute_input":"2021-08-11T20:59:13.838159Z","iopub.status.idle":"2021-08-11T20:59:13.851122Z","shell.execute_reply.started":"2021-08-11T20:59:13.838113Z","shell.execute_reply":"2021-08-11T20:59:13.850275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optical_distortion = OpticalDistortion(\n    distort_limit=1.0, shift_limit=0.05, p=0.75)\ncheck_aug(optical_distortion, with_mask=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:13.852106Z","iopub.execute_input":"2021-08-11T20:59:13.852483Z","iopub.status.idle":"2021-08-11T20:59:17.410802Z","shell.execute_reply.started":"2021-08-11T20:59:13.852455Z","shell.execute_reply":"2021-08-11T20:59:17.40999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### GridDistortion","metadata":{}},{"cell_type":"code","source":"def make_grid_distorted_maps(height, width, num_steps, xsteps, ysteps):\n    def _make_maps_before_last(size, step, steps): # size=512, step=102,\n                                                   # steps.shape=[num_steps]\n        step_rep = tf.repeat(step, num_steps)  # [102, 102, 102, 102, 102]\n        step_rep_f = tf.cast(step_rep, dtype=tf.float32)\n        step_inc = step_rep_f * steps          # [102*s_0, ..., 102*s_4]\n        cur = tf.math.cumsum(step_inc)         # [si_0, si_0 + si_1, ... ]\n        zero = tf.zeros([1], dtype=tf.float32)\n        prev = tf.concat([ zero, cur[ :-1] ], axis=0) # [0, c_0, ..., c_3]\n        prev_cur = tf.stack([prev, cur])       # [[p_0, p_1, ...], [c_0, c_1, ...]]\n        ranges = tf.transpose(prev_cur)        # [[p_0, c_0], [p_1, c_1], ... ]\n\n        def _linspace_range(rng):\n            return tf.linspace(rng[0], rng[1], step)\n \n        maps_stack = tf.map_fn(_linspace_range, ranges)\n        maps = tf.reshape(maps_stack, [-1])    # [-1] flatten into 1-D\n        return maps\n    \n    def _make_last_map(size, step, last_start):\n        last_step = size - step * num_steps  # 512 - 102*5 = 2 \n        size_f = tf.cast(size, dtype=tf.float32)\n        last_map = tf.linspace(last_start, size_f-1.0, last_step)\n        return last_map\n    \n    def _make_distorted_map(size, steps):\n        step = size // num_steps               # step=102 \n        maps_before_last = _make_maps_before_last(size, step, steps[ :-1 ])\n        last_map = _make_last_map(size, step, maps_before_last[-1])\n        distorted_map = tf.concat([maps_before_last, last_map], axis=0)\n        return distorted_map\n\n    xx = _make_distorted_map(width, xsteps)\n    yy = _make_distorted_map(height, ysteps)\n    map_y, map_x = tf.meshgrid(xx, yy)\n    return map_x, map_y\n\nclass GridDistortion(BaseAug):\n    def __init__(self, num_steps, distort_limit, p=1.0):\n        super(GridDistortion, self).__init__(p)\n        self.num_steps = num_steps\n        self.distort_limit = distort_limit\n        \n    def aug_data(self, image, mask):\n        xsteps = tf.random.uniform(\n            [self.num_steps + 1],\n            minval=1.0 - self.distort_limit,\n            maxval=1.0 + self.distort_limit)\n        ysteps = tf.random.uniform(\n            [self.num_steps + 1],\n            minval=1.0 - self.distort_limit,\n            maxval=1.0 + self.distort_limit)\n\n        image_shape = tf.shape(image)\n        height = image_shape[0]\n        width = image_shape[1]\n        map_x, map_y = make_grid_distorted_maps(\n            height, width, self.num_steps, xsteps, ysteps)\n        aug_image = remap(\n            image, height, width, map_x, map_y, mode='mirror')\n        aug_mask = remap(\n            mask, height, width, map_x, map_y, mode='mirror')\n        return aug_image, aug_mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:17.411845Z","iopub.execute_input":"2021-08-11T20:59:17.412261Z","iopub.status.idle":"2021-08-11T20:59:17.429599Z","shell.execute_reply.started":"2021-08-11T20:59:17.41223Z","shell.execute_reply":"2021-08-11T20:59:17.428555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_distortion = GridDistortion(\n    num_steps=5, distort_limit=1.0, p=0.75)\ncheck_aug(grid_distortion, with_mask=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:17.431412Z","iopub.execute_input":"2021-08-11T20:59:17.431742Z","iopub.status.idle":"2021-08-11T20:59:22.153695Z","shell.execute_reply.started":"2021-08-11T20:59:17.431712Z","shell.execute_reply":"2021-08-11T20:59:22.152358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### OneOf","metadata":{}},{"cell_type":"code","source":"one_of_opt_grid_distortion = OneOf(\n    optical_distortion, grid_distortion, p=0.75)\ncheck_aug(one_of_opt_grid_distortion, with_mask=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:22.155021Z","iopub.execute_input":"2021-08-11T20:59:22.155321Z","iopub.status.idle":"2021-08-11T20:59:25.171153Z","shell.execute_reply.started":"2021-08-11T20:59:22.155284Z","shell.execute_reply":"2021-08-11T20:59:25.169757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### HeuSaturationValue","metadata":{}},{"cell_type":"code","source":"class HueSaturationValue(BaseAug):\n    def __init__(\n            self, hue_shift_limit, sat_shift_limit,\n            val_shift_limit, p):\n        super(HueSaturationValue, self).__init__(p)\n        self.hue_shift_limit = hue_shift_limit\n        self.sat_shift_limit = sat_shift_limit\n        self.val_shift_limit = val_shift_limit\n        \n    def aug_data(self, image, mask):\n        hue_shift = random_float(\n            [], -self.hue_shift_limit, self.hue_shift_limit)\n        sat_shift = random_float(\n            [], -self.sat_shift_limit, self.sat_shift_limit)\n        val_shift = random_float(\n            [], -self.val_shift_limit, self.val_shift_limit)\n\n        hsv_image = tf.image.rgb_to_hsv(image)\n        hue_value = (hsv_image[ ... , :1 ] + hue_shift) % 1.0\n        sat_value = tf.clip_by_value(\n            hsv_image[ ... , 1:2 ] + sat_shift, 0.0, 1.0)\n        val_value = tf.clip_by_value(\n            hsv_image[ ... , 2: ] + val_shift, 0.0, 1.0)\n        hsv_image = tf.concat(\n            [hue_value, sat_value, val_value], axis=-1)\n        aug_image = tf.image.hsv_to_rgb(hsv_image)\n        return aug_image, mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:25.172566Z","iopub.execute_input":"2021-08-11T20:59:25.17297Z","iopub.status.idle":"2021-08-11T20:59:25.182926Z","shell.execute_reply.started":"2021-08-11T20:59:25.172935Z","shell.execute_reply":"2021-08-11T20:59:25.181735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hue_saturation_value = HueSaturationValue(\n    hue_shift_limit=0.2, sat_shift_limit=0.3,\n    val_shift_limit=0.2, p=0.75)\ncheck_aug(hue_saturation_value, with_mask=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:25.1844Z","iopub.execute_input":"2021-08-11T20:59:25.184801Z","iopub.status.idle":"2021-08-11T20:59:26.943117Z","shell.execute_reply.started":"2021-08-11T20:59:25.184768Z","shell.execute_reply":"2021-08-11T20:59:26.942067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ShiftScaleRotate","metadata":{}},{"cell_type":"code","source":"def affine_transform(height, width, tx, ty, z, theta):\n    cx = (width - 1.0) * 0.5\n    cy = (height - 1.0) * 0.5\n    \n    center_shift_mat = tf.convert_to_tensor([\n        [1.0, 0.0, -cx],\n        [0.0, 1.0, -cy],\n        [0.0, 0.0, 1.0]], dtype=tf.float32)\n    trans_mat = center_shift_mat\n    \n    rot_rad = -2.0 * math.pi * theta / 360.0\n    roration_mat = tf.convert_to_tensor([\n        [tf.math.cos(rot_rad), tf.math.sin(rot_rad), 0.0],\n        [-tf.math.sin(rot_rad), tf.math.cos(rot_rad), 0.0],\n        [0.0, 0.0, 1.0]], dtype=tf.float32)\n    trans_mat = tf.linalg.matmul(roration_mat, trans_mat)\n\n    zoom_mat = tf.convert_to_tensor([\n        [1.0 / z, 0.0, 0.0],\n        [0.0, 1.0 / z, 0.0],\n        [0.0, 0.0, 1.0]], dtype=tf.float32)\n    trans_mat = tf.linalg.matmul(zoom_mat, trans_mat)\n    \n    shift_mat = tf.convert_to_tensor([\n        [1.0, 0.0, cx - tx],\n        [0.0, 1.0, cy - ty],\n        [0.0, 0.0, 1.0]], dtype=tf.float32)\n    trans_mat = tf.linalg.matmul(shift_mat, trans_mat)\n    \n    h_rng = tf.range(height, dtype=tf.float32)\n    w_rng = tf.range(width, dtype=tf.float32)\n    y, x = tf.meshgrid(h_rng, w_rng)\n    x = tf.reshape(x, [-1])\n    y = tf.reshape(y, [-1])\n    ones = tf.ones_like(x)\n    coord_mat = tf.stack([x, y, ones])\n    \n    res_mat = tf.linalg.matmul(trans_mat, coord_mat)\n    map_x = res_mat[0]\n    map_y = res_mat[1]\n    return map_x, map_y","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:26.944589Z","iopub.execute_input":"2021-08-11T20:59:26.944945Z","iopub.status.idle":"2021-08-11T20:59:27.249709Z","shell.execute_reply.started":"2021-08-11T20:59:26.944896Z","shell.execute_reply":"2021-08-11T20:59:27.248427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ShiftScaleRotate(BaseAug):\n    def __init__(\n            self, shift_limit, scale_limit, rotate_limit, p):\n        super(ShiftScaleRotate, self).__init__(p)\n        self.shift_limit = shift_limit\n        self.scale_limit = scale_limit\n        self.rotate_limit = rotate_limit\n\n    def aug_data(self, image, mask):\n        image_shape = tf.shape(image)\n        height_i = image_shape[0]\n        width_i = image_shape[1]\n        height_f = tf.cast(height_i, dtype=tf.float32)\n        width_f = tf.cast(width_i, dtype=tf.float32)\n        rnd_shift = random_float(\n            [2], -self.shift_limit, self.shift_limit)\n        tx = width_f * rnd_shift[0]\n        ty = height_f * rnd_shift[1]\n        z = random_float(\n            [], 1.0 - self.scale_limit, 1.0 + self.scale_limit)\n        theta = random_float(\n            [], -self.rotate_limit, self.rotate_limit)\n\n        map_x, map_y = affine_transform(\n            height_f, width_f, tx, ty, z, theta)\n        aug_image = remap(\n            image, height_i, width_i, map_x, map_y, mode='constant')\n        aug_mask = remap(\n            mask, height_i, width_i, map_x, map_y, mode='constant')\n        return aug_image, aug_mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:27.251176Z","iopub.execute_input":"2021-08-11T20:59:27.251504Z","iopub.status.idle":"2021-08-11T20:59:27.270552Z","shell.execute_reply.started":"2021-08-11T20:59:27.251469Z","shell.execute_reply":"2021-08-11T20:59:27.269315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shift_scale_rotate = ShiftScaleRotate(\n    shift_limit=0.2, scale_limit=0.3, rotate_limit=30, p=0.75)\ncheck_aug(shift_scale_rotate, with_mask=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:27.271913Z","iopub.execute_input":"2021-08-11T20:59:27.272304Z","iopub.status.idle":"2021-08-11T20:59:30.44049Z","shell.execute_reply.started":"2021-08-11T20:59:27.272186Z","shell.execute_reply":"2021-08-11T20:59:30.439471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Cutout","metadata":{}},{"cell_type":"code","source":"class Cutout(BaseAug):\n    def __init__(self, num_cuts, mask_factor, p):\n        super(Cutout, self).__init__(p)\n        self.num_cuts = num_cuts\n        self.mask_factor = mask_factor\n\n    def aug_data(self, image, mask):\n        image_shape = tf.shape(image)\n        height_i = image_shape[0]\n        width_i = image_shape[1]\n        height_f = tf.cast(height_i, dtype=tf.float32)\n        width_f = tf.cast(width_i, dtype=tf.float32)\n        cut_h = tf.cast(height_f * self.mask_factor, dtype=tf.int32)\n        cut_w = tf.cast(width_f * self.mask_factor, dtype=tf.int32)\n\n        y_centers = random_int([self.num_cuts], 0, height_i)\n        x_centers = random_int([self.num_cuts], 0, width_i)\n        tops = tf.math.maximum(y_centers - cut_h//2, 0)\n        lefts = tf.math.maximum(x_centers - cut_w//2, 0)\n        bottoms = tf.math.minimum(tops + cut_h, height_i - 1)\n        rights = tf.math.minimum(lefts + cut_w, width_i - 1)\n\n        def _make_one_mask(i):\n            mask_height = bottoms[i] - tops[i] + 1\n            mask_width = rights[i] - lefts[i] + 1\n            mask_shape = [mask_height, mask_width]\n            mask = tf.ones(mask_shape, dtype=tf.bool)\n\n            paddings = [\n                [tops[i], height_i - bottoms[i] - 1],\n                [lefts[i], width_i - rights[i] - 1]]\n            mask = tf.pad(mask, paddings, mode='CONSTANT')\n            return mask\n\n        num_cuts_rng = tf.range(self.num_cuts, dtype=tf.int64)\n        cut_masks = tf.map_fn(\n            _make_one_mask, num_cuts_rng,\n            fn_output_signature=tf.bool)\n        cut_mask = tf.reduce_any(cut_masks, axis=0)\n        cut_mask = cut_mask[ ..., tf.newaxis ]\n\n        mask_value = tf.constant(0.0, dtype=tf.float32)\n        aug_image = tf.where(cut_mask, mask_value, image)\n        return aug_image, mask","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:30.441909Z","iopub.execute_input":"2021-08-11T20:59:30.442465Z","iopub.status.idle":"2021-08-11T20:59:30.457886Z","shell.execute_reply.started":"2021-08-11T20:59:30.442422Z","shell.execute_reply":"2021-08-11T20:59:30.456934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cut_out = Cutout(num_cuts=1, mask_factor=0.4, p=0.75)\ncheck_aug(cut_out, with_mask=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:30.459194Z","iopub.execute_input":"2021-08-11T20:59:30.459766Z","iopub.status.idle":"2021-08-11T20:59:32.570211Z","shell.execute_reply.started":"2021-08-11T20:59:30.459722Z","shell.execute_reply":"2021-08-11T20:59:32.569122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Do Augment","metadata":{}},{"cell_type":"code","source":"def do_augment(image, mask, labels):\n    image, mask = transpose(image, mask)\n    image, mask = vertical_flip(image, mask)\n    image, mask = horizontal_flip(image, mask)\n    image, mask = random_brightness(image, mask)\n    image, mask = random_contrast(image, mask)\n    image, mask = one_of_blur_median_blur(image, mask)\n    image, mask = jpeg_compression(image, mask)\n    image, mask = one_of_opt_grid_distortion(image, mask)\n    image, mask = hue_saturation_value(image, mask)\n    image, mask = shift_scale_rotate(image, mask)\n    image, mask = cut_out(image, mask)\n    return image, mask, labels","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:32.571683Z","iopub.execute_input":"2021-08-11T20:59:32.572118Z","iopub.status.idle":"2021-08-11T20:59:32.580424Z","shell.execute_reply.started":"2021-08-11T20:59:32.572071Z","shell.execute_reply":"2021-08-11T20:59:32.579286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset 2","metadata":{}},{"cell_type":"code","source":"def select_train(ds, fold_i):\n    ds = ds.filter(\n        lambda ids, fold, image, mask, study_label: fold != fold_i)\n    return ds\n    \ndef select_val(ds, fold_i):\n    ds = ds.filter(\n        lambda ids, fold, image, mask, study_label: fold == fold_i)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:32.582096Z","iopub.execute_input":"2021-08-11T20:59:32.582433Z","iopub.status.idle":"2021-08-11T20:59:32.602423Z","shell.execute_reply.started":"2021-08-11T20:59:32.582403Z","shell.execute_reply":"2021-08-11T20:59:32.601119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_data(ids, fold, image, mask, study_label):\n    return image, mask, study_label\n\ndef rescale_image(image, mask, study_label):\n    image = image_to_float_0_1(image)\n    return image, mask, study_label","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:32.604338Z","iopub.execute_input":"2021-08-11T20:59:32.604899Z","iopub.status.idle":"2021-08-11T20:59:32.616784Z","shell.execute_reply.started":"2021-08-11T20:59:32.604736Z","shell.execute_reply":"2021-08-11T20:59:32.615434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reform_for_model(image, mask, labels):\n    return image, {'sigmoid': mask, 'study_label': labels}","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:32.618338Z","iopub.execute_input":"2021-08-11T20:59:32.618847Z","iopub.status.idle":"2021-08-11T20:59:32.631132Z","shell.execute_reply.started":"2021-08-11T20:59:32.618811Z","shell.execute_reply":"2021-08-11T20:59:32.629901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\n\ndef build_dataset(\n        dset, augment=True, repeat=True, shuffle=1024):\n    dset = dset.map(get_data, num_parallel_calls=AUTOTUNE)\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(BATCH_SIZE)\n    dset = dset.map(\n        rescale_image, num_parallel_calls=AUTOTUNE)\n    dset = dset.map(\n        do_augment, num_parallel_calls=AUTOTUNE) if augment else dset\n    dset = dset.map(\n        reform_for_model, num_parallel_calls=AUTOTUNE)\n    dset = dset.prefetch(AUTOTUNE)\n    return dset","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:32.632591Z","iopub.execute_input":"2021-08-11T20:59:32.633051Z","iopub.status.idle":"2021-08-11T20:59:32.64422Z","shell.execute_reply.started":"2021-08-11T20:59:32.633003Z","shell.execute_reply":"2021-08-11T20:59:32.643096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_datasets(fold_i):\n    train_ds = select_train(raw_ds, fold_i)\n    train_ds = build_dataset(\n        train_ds, augment=True, repeat=True, shuffle=1024)\n\n    val_ds = select_val(raw_ds, fold_i)\n    val_ds = build_dataset(\n        val_ds, augment=False, repeat=False, shuffle=None)\n\n    train_steps = get_train_count(fold_i) // BATCH_SIZE\n    val_steps = get_val_count(fold_i) // BATCH_SIZE\n\n    return train_ds, val_ds, train_steps, val_steps","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:32.645425Z","iopub.execute_input":"2021-08-11T20:59:32.645699Z","iopub.status.idle":"2021-08-11T20:59:32.657554Z","shell.execute_reply.started":"2021-08-11T20:59:32.645671Z","shell.execute_reply":"2021-08-11T20:59:32.656451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization","metadata":{}},{"cell_type":"code","source":"train_ds, val_ds, train_steps, val_steps = make_datasets(0)\n\nprint(train_ds)\nprint(val_ds)\nprint(train_steps)\nprint(val_steps)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:32.65906Z","iopub.execute_input":"2021-08-11T20:59:32.659345Z","iopub.status.idle":"2021-08-11T20:59:35.065807Z","shell.execute_reply.started":"2021-08-11T20:59:32.659317Z","shell.execute_reply":"2021-08-11T20:59:35.064792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_images(ds):\n    rows = 4\n    cols = 5\n    n_imgs = (rows//2) * cols\n\n    images, label_dict = next(iter(\n        ds.unbatch().take(n_imgs).batch(n_imgs)))\n    masks = label_dict['sigmoid']\n    plt.figure(figsize=(12, 8))\n    for i, image in enumerate(images):\n        plt.subplot(rows, cols, i+1)\n        plt.imshow(image)\n        plt.axis(\"off\")\n    for i, mask in enumerate(masks):\n        plt.subplot(rows, cols, n_imgs+i+1)\n        plt.imshow(mask, cmap='gray')\n        plt.axis(\"off\")\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:35.067193Z","iopub.execute_input":"2021-08-11T20:59:35.067505Z","iopub.status.idle":"2021-08-11T20:59:35.076108Z","shell.execute_reply.started":"2021-08-11T20:59:35.067472Z","shell.execute_reply":"2021-08-11T20:59:35.074887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_images(train_ds)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:35.077545Z","iopub.execute_input":"2021-08-11T20:59:35.077901Z","iopub.status.idle":"2021-08-11T20:59:55.48528Z","shell.execute_reply.started":"2021-08-11T20:59:35.077835Z","shell.execute_reply":"2021-08-11T20:59:55.484107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_images(val_ds)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T20:59:55.48682Z","iopub.execute_input":"2021-08-11T20:59:55.487243Z","iopub.status.idle":"2021-08-11T21:00:01.068757Z","shell.execute_reply.started":"2021-08-11T20:59:55.4872Z","shell.execute_reply":"2021-08-11T21:00:01.06774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"def make_model():\n    base_model = SEG_MODEL(\n        BACKBONE, encoder_weights='imagenet', \n        input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3),\n        classes=1, activation='sigmoid')\n    \n    x = base_model.get_layer(name='top_activation').output \n    x = L.GlobalAveragePooling2D(name='avgpool')(x)\n    study_label = L.Dense(\n        N_STUDY_LABELS, activation='sigmoid', name='study_label')(x)\n\n    model = tf.keras.Model(\n        inputs=base_model.input, \n        outputs=[base_model.output, study_label]) \n    \n    pr_auc = tf.keras.metrics.AUC(\n        curve=\"PR\", multi_label=True, name=\"pr_auc\")\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(),\n        loss={\n            'sigmoid': sm.losses.bce_jaccard_loss,\n            'study_label': \"binary_crossentropy\" },\n        loss_weights = [1.0, 1.0],\n        metrics={\n            'sigmoid': sm.metrics.iou_score,\n            'study_label': pr_auc },\n        # overheads and allows the XLA compiler to unroll the loop on TPU\n        # and optimize hardware utilization.\n        # needs to be commented out for Tensorflow 2.3\n        steps_per_execution=8)\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:00:01.069996Z","iopub.execute_input":"2021-08-11T21:00:01.070279Z","iopub.status.idle":"2021-08-11T21:00:01.079952Z","shell.execute_reply.started":"2021-08-11T21:00:01.070252Z","shell.execute_reply":"2021-08-11T21:00:01.078651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope(): \n    model = make_model()\n    \ninitial_weights = model.get_weights()\n# model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:00:01.08115Z","iopub.execute_input":"2021-08-11T21:00:01.081583Z","iopub.status.idle":"2021-08-11T21:00:28.70813Z","shell.execute_reply.started":"2021-08-11T21:00:01.081545Z","shell.execute_reply":"2021-08-11T21:00:28.706967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"LR_START = INIT_LR\nLR_MAX = 1e-3\nLR_MIN = 1e-5\nLR_RAMPUP_EPOCHS = WARMUP_EPO\nLR_SUSTAIN_EPOCHS = 0\nEPOCHS = N_EPOCHS\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        decay_total_epochs = EPOCHS - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\n        decay_epoch_index = epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n        phase = math.pi * decay_epoch_index / decay_total_epochs\n        cosine_decay = 0.5 * (1 + math.cos(phase))\n        lr = (LR_MAX - LR_MIN) * cosine_decay + LR_MIN\n    return lr\n\nrng = [i for i in range(EPOCHS)]\nlr_y = [lrfn(x) for x in rng]\nplt.figure(figsize=(10, 4))\nplt.plot(rng, lr_y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\". \\\n      format(lr_y[0], max(lr_y), lr_y[-1]))","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:00:28.709798Z","iopub.execute_input":"2021-08-11T21:00:28.710236Z","iopub.status.idle":"2021-08-11T21:00:28.917907Z","shell.execute_reply.started":"2021-08-11T21:00:28.71019Z","shell.execute_reply":"2021-08-11T21:00:28.91685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cb_monitor = 'val_study_label_pr_auc'\n\nclass RestoreBestWeights(tf.keras.callbacks.Callback):\n    def __init__(self):\n        super(RestoreBestWeights, self).__init__()\n        self.best_monitor = -np.Inf\n        self.best_weights = None\n        self.best_epoch = None\n        \n    def on_epoch_end(self, epoch, logs=None):\n        current_monitor = logs.get(cb_monitor)\n        if current_monitor > self.best_monitor:\n            self.best_monitor = current_monitor\n            self.best_weights = self.model.get_weights()\n            self.best_epoch = epoch\n            \n    def on_train_end(self, logs=None):\n        print(\"Restoring best weights on epoch {0}, {1} was {2:.5f}\".format(\n            self.best_epoch + 1, cb_monitor, self.best_monitor))\n        self.model.set_weights(self.best_weights)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:00:28.919221Z","iopub.execute_input":"2021-08-11T21:00:28.919534Z","iopub.status.idle":"2021-08-11T21:00:28.94178Z","shell.execute_reply.started":"2021-08-11T21:00:28.919503Z","shell.execute_reply":"2021-08-11T21:00:28.939928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_callbacks(fold_i):\n    best_model_file_name = \\\n        \"study_aux_loss_model_{0}_{1}.hdf5\".format(VID, fold_i)\n    cb_mode = 'max'\n    cb_min_delta = 1e-4\n    cb_verbose = 1\n\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(\n        best_model_file_name, save_best_only=True,\n        save_weights_only=False, monitor=cb_monitor, mode=cb_mode,\n        verbose=cb_verbose)\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = False)\n    restore_best_weights = RestoreBestWeights()\n    \n    return checkpoint, lr_callback, restore_best_weights","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:00:28.943641Z","iopub.execute_input":"2021-08-11T21:00:28.944231Z","iopub.status.idle":"2021-08-11T21:00:28.957616Z","shell.execute_reply.started":"2021-08-11T21:00:28.944169Z","shell.execute_reply":"2021-08-11T21:00:28.956115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_one_fold(fold_i):\n    train_dataset, val_dataset, train_steps, val_steps = \\\n        make_datasets(fold_i)\n    checkpoint, lr_callback, restore_best_weights = \\\n        make_callbacks(fold_i)\n    history = model.fit(\n        train_dataset, \n        epochs=EPOCHS,\n        verbose=1,\n        callbacks=[checkpoint, lr_callback, restore_best_weights],\n        steps_per_epoch=train_steps,\n        validation_data=val_dataset,\n        validation_steps=val_steps)\n    return history, val_dataset, val_steps","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:00:28.95939Z","iopub.execute_input":"2021-08-11T21:00:28.959733Z","iopub.status.idle":"2021-08-11T21:00:28.976492Z","shell.execute_reply.started":"2021-08-11T21:00:28.9597Z","shell.execute_reply":"2021-08-11T21:00:28.975385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(history, title, labels, subplot):\n    plt.subplot(*subplot)\n    plt.title(title)\n    for label in labels:\n        plt.plot(history.history[label], label=label)\n    plt.legend()","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:00:28.983217Z","iopub.execute_input":"2021-08-11T21:00:28.983618Z","iopub.status.idle":"2021-08-11T21:00:28.990493Z","shell.execute_reply.started":"2021-08-11T21:00:28.98357Z","shell.execute_reply":"2021-08-11T21:00:28.988659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_fit_result(history):\n    plt.figure(figsize=(12, 8))\n    plot_history(\n        history, \"Loss\",\n        ['loss', 'val_loss'],\n        (2, 2, 1))\n    plot_history(\n        history, \"PR_AUC\", \n        ['study_label_pr_auc', 'val_study_label_pr_auc'],\n        (2, 2, 2))\n    plot_history(\n        history, \"IOU\", \n        ['sigmoid_iou_score', 'val_sigmoid_iou_score'],\n        (2, 2, 3))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:00:28.992834Z","iopub.execute_input":"2021-08-11T21:00:28.993306Z","iopub.status.idle":"2021-08-11T21:00:29.006911Z","shell.execute_reply.started":"2021-08-11T21:00:28.993258Z","shell.execute_reply":"2021-08-11T21:00:29.005025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_one_fold(model, val_dataset, val_steps):\n    val_true_list = [] \n    for _, label_dict in val_dataset:\n        study_labels = label_dict['study_label']\n        val_true_list.append(study_labels)\n    val_true = np.concatenate(val_true_list, axis=0)\n   \n    val_pred_list = []\n    for images, _ in val_dataset:\n        pred_list = model(images, training=False)\n        val_preds = pred_list[1]\n        val_pred_list.append(val_preds)\n    val_pred = np.concatenate(val_pred_list, axis=0)\n    \n    val_true = val_true[ : len(val_pred) ]\n    return val_true, val_pred","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:00:29.008471Z","iopub.execute_input":"2021-08-11T21:00:29.008872Z","iopub.status.idle":"2021-08-11T21:00:29.021393Z","shell.execute_reply.started":"2021-08-11T21:00:29.008813Z","shell.execute_reply":"2021-08-11T21:00:29.019959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_labels = [\n    'Negative for Pneumonia',\n    'Typical Appearance',\n    'Indeterminate Appearance',\n    'Atypical Appearance',\n]","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:00:29.02289Z","iopub.execute_input":"2021-08-11T21:00:29.023248Z","iopub.status.idle":"2021-08-11T21:00:29.040212Z","shell.execute_reply.started":"2021-08-11T21:00:29.023202Z","shell.execute_reply":"2021-08-11T21:00:29.038846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import average_precision_score\n\ndef show_average_precision_score(val_true, val_pred):\n    average_precision_list = []\n    for i in range(val_true.shape[-1]):\n        average_precision = average_precision_score(\n            val_true[ : , i], val_pred[ : , i])\n        print(\"{0:30s}: {1:.4f}\".format(\n            study_labels[i], average_precision))\n        average_precision_list.append(average_precision)\n\n    mean_average_precision = np.mean(average_precision_list)\n    print(\"{0:30s}: {1:.4f}\".format(\n        \"Mean\", mean_average_precision))\n    \n    plt.figure(figsize=(8, 4))\n    plt.plot(average_precision_list)\n    ticks = np.arange(len(study_labels))\n    plt.xticks(ticks=ticks, labels=study_labels, rotation=45)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:00:29.041952Z","iopub.execute_input":"2021-08-11T21:00:29.042316Z","iopub.status.idle":"2021-08-11T21:00:29.226961Z","shell.execute_reply.started":"2021-08-11T21:00:29.042283Z","shell.execute_reply":"2021-08-11T21:00:29.225838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_pred_str(pred):\n    labels = ['negative', 'typical', 'indeterminate', 'atypical']\n    pred_list = []\n    for i, label in enumerate(labels):\n        s = \"{0} {1:.6f} 0 0 1 1\".format(label, pred[i])\n        pred_list.append(s)\n    return ' '.join(pred_list)\n\ndef make_submission(fold_i, val_pred):\n    val_study_ids = get_val_study_ids(fold_i)\n    fold_sub_list = []\n    for study_id, pred in zip(val_study_ids, val_pred):\n        pred_str = make_pred_str(pred)\n        fold_sub_list.append([study_id + \"_study\", pred_str])\n    return fold_sub_list","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:00:29.22953Z","iopub.execute_input":"2021-08-11T21:00:29.229962Z","iopub.status.idle":"2021-08-11T21:00:29.237982Z","shell.execute_reply.started":"2021-08-11T21:00:29.229927Z","shell.execute_reply":"2021-08-11T21:00:29.23691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_sub_list = []\nfor fold_i in FOLD_I_LIST:\n    print(\"####################\")\n    print(\"# Fold {0}\".format(fold_i))\n    model.set_weights(initial_weights)\n    history, val_dataset, val_steps = fit_one_fold(fold_i)\n    plot_fit_result(history)\n    val_true, val_pred = predict_one_fold(model, val_dataset, val_steps)\n    show_average_precision_score(val_true, val_pred)\n    fold_sub_list = make_submission(fold_i, val_pred)\n    study_sub_list.extend(fold_sub_list)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:00:29.239585Z","iopub.execute_input":"2021-08-11T21:00:29.240034Z","iopub.status.idle":"2021-08-11T21:08:30.229633Z","shell.execute_reply.started":"2021-08-11T21:00:29.239999Z","shell.execute_reply":"2021-08-11T21:08:30.227551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_sub_df = pd.DataFrame(\n    study_sub_list, columns=['id', 'PredictionString'])\n\nstudy_sub_df","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:08:30.231277Z","iopub.execute_input":"2021-08-11T21:08:30.231595Z","iopub.status.idle":"2021-08-11T21:08:30.249391Z","shell.execute_reply.started":"2021-08-11T21:08:30.231564Z","shell.execute_reply":"2021-08-11T21:08:30.248008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_sub_file_name = \"study_sub_{0}.csv\".format(VID)\nstudy_sub_df.to_csv(study_sub_file_name, index=False)\n\n! head study_sub_*.csv","metadata":{"execution":{"iopub.status.busy":"2021-08-11T21:08:30.251316Z","iopub.execute_input":"2021-08-11T21:08:30.251694Z","iopub.status.idle":"2021-08-11T21:08:31.041394Z","shell.execute_reply.started":"2021-08-11T21:08:30.25166Z","shell.execute_reply":"2021-08-11T21:08:31.040143Z"},"trusted":true},"execution_count":null,"outputs":[]}]}