{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nimport math\nimport tensorflow.keras.backend as K\n#import tensorflow_addons as tda\n#import tensorflow_hub as hub\nimport tensorflow_datasets as tfds\n#import efficientnet.tfkeras as efn\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tarfile\nimport os\nimport cv2\nfrom functools import partial\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\n!pip install keras-unet-collection >>/dev/null\nfrom keras_unet_collection import models\nfrom keras_unet_collection import losses\nfrom keras_unet_collection import base\n\nAUTO = tf.data.experimental.AUTOTUNE\nDEVICE = \"TPU\"","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:09:40.700319Z","iopub.execute_input":"2021-07-05T09:09:40.700871Z","iopub.status.idle":"2021-07-05T09:09:56.469071Z","shell.execute_reply.started":"2021-07-05T09:09:40.700784Z","shell.execute_reply":"2021-07-05T09:09:56.467905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    print(\"Could not connect to TPU\")\n    tpu = None\n\nif tpu:\n    try:\n        print(\"initializing  TPU ...\")\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.TPUStrategy(tpu)\n        print(\"TPU initialized\")\n    except Exception:\n        print(\"failed to initialize TPU\")\nelse:\n    DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n\n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:09:56.470844Z","iopub.execute_input":"2021-07-05T09:09:56.471195Z","iopub.status.idle":"2021-07-05T09:10:02.126768Z","shell.execute_reply.started":"2021-07-05T09:09:56.471160Z","shell.execute_reply":"2021-07-05T09:10:02.126121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_img = plt.imread(\"../input/isic2017-and-ph2/ISIC_2017 + PH2/ISIC_2017/trainx/ISIC_0000000.jpg\")\nimage = cv2.imread(\"../input/isic2017-and-ph2/ISIC_2017 + PH2/ISIC_2017/trainx/ISIC_0000000.jpg\")\nimage = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n#image = tf.image.resize(image, [512, 512])\nimage = cv2.imencode('.jpg', image, (cv2.IMWRITE_JPEG_QUALITY, 94))[1]#.tobytes()\nprint(image)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:10:02.128205Z","iopub.execute_input":"2021-07-05T09:10:02.128482Z","iopub.status.idle":"2021-07-05T09:10:02.241325Z","shell.execute_reply.started":"2021-07-05T09:10:02.128455Z","shell.execute_reply":"2021-07-05T09:10:02.240165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_img = plt.imread(\"../input/isic2017-and-ph2/ISIC_2017 + PH2/ISIC_2017/trainx/ISIC_0000000.jpg\")\ntest_img = tf.convert_to_tensor(test_img)\ntest_img = tf.image.resize(test_img, [512, 512])/255\ntest_img = tf.image.convert_image_dtype(test_img, tf.uint8, saturate=True, name=None)\ntest_img = tf.io.encode_jpeg(test_img)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:10:02.242874Z","iopub.execute_input":"2021-07-05T09:10:02.243437Z","iopub.status.idle":"2021-07-05T09:10:02.293562Z","shell.execute_reply.started":"2021-07-05T09:10:02.243402Z","shell.execute_reply":"2021-07-05T09:10:02.292330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## READING TFRECORDS:","metadata":{}},{"cell_type":"code","source":"def get_seg_paths(data_type=\"train\", tfrec_roots=None, img_root_paths=None):\n    if data_type == \"tfrecords\":\n        test_paths = []\n        train_paths = []\n        for tfrec_root in tfrec_roots:  \n            test_paths += tf.io.gfile.glob(tfrec_root+'/test*.tfrec')\n            train_paths += tf.io.gfile.glob(tfrec_root+'/train*.tfrec')\n        test_paths = np.sort(np.array(test_paths))\n        train_paths = np.sort(np.array(train_paths))\n        return train_paths, test_paths\n    else:\n        complete_img_paths = [0]*len(img_root_paths)\n        for index, img_root_path in enumerate(img_root_paths):\n            if index == 0:\n                complete_img_paths[index] = np.sort(np.array(tf.io.gfile.glob(img_root_path + '/*.jpg')))\n                complete_img_paths[index] = np.sort(np.array(tf.io.gfile.glob(img_root_path + '/*.png')))\n            else:\n                complete_img_paths[index] = np.sort(np.array(tf.io.gfile.glob(img_root_path + '/*.jpg')))\n                complete_img_paths[index] = np.sort(np.array(tf.io.gfile.glob(img_root_path + '/*.png')))\n        return complete_img_paths\n    \n    \n#All the code below comes from TensorFlow's docs here\n\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef serialize_example(feature0, feature1, feature2):\n  feature = {\n      'image': _bytes_feature(feature0),\n      'image_name': _bytes_feature(feature1),\n      'mask': _bytes_feature(feature2),\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()\n\ndef create_seg_tfrecords(tfrecord_type=\"train\", SIZE=500, tfrec_roots=None, img_root_paths=None):\n    image_paths, mask_paths = get_seg_paths(data_type=tfrecord_type,\n                                            tfrec_roots=tfrec_roots,\n                                            img_root_paths=img_root_paths)\n    folder_name = f'{tfrecord_type}_tfrecords'\n    path = os.path.join(os.getcwd(), folder_name)\n    os.makedirs(path, exist_ok=True)\n    path_zip = zip(image_paths, mask_paths)\n    tfrecord_nums = image_paths.size // 500 + 1\n    for tfrecord_counter in range(tfrecord_nums):\n        tfrecord_size = min(SIZE, image_paths.size-tfrecord_counter*SIZE)\n        print('\\nCreating {tfrecord_type}_{tfrecord_counter}.tfrec......'.format(\n                            tfrecord_type=tfrecord_type,tfrecord_counter=tfrecord_counter))\n        with tf.io.TFRecordWriter(os.path.join(path, f'{tfrecord_type}{tfrecord_counter}.tfrec')) as writer:\n            for k in range(tfrecord_size):\n                # processing image\n                image = cv2.imread(image_paths[tfrecord_size * tfrecord_counter + k])\n                image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n                image = tf.convert_to_tensor(image)\n                image = tf.image.resize(image, [512, 512])/255\n                image = tf.image.convert_image_dtype(image, tf.uint8, saturate=True, name=None)\n                image = tf.io.encode_jpeg(image)\n\n                # processing mask\n                mask = cv2.imread(mask_paths[tfrecord_size * tfrecord_counter + k])\n                mask = cv2.cvtColor(mask, cv2.COLOR_RGB2BGR)\n                mask = tf.convert_to_tensor(mask)\n                mask = tf.image.resize(mask, [512, 512])/255\n                mask = tf.image.convert_image_dtype(mask, tf.uint8, saturate=True, name=None)\n                mask = tf.io.encode_jpeg(mask)\n\n                # extracting image name\n                image_name = os.path.split(image_paths[tfrecord_size*tfrecord_counter+k])[1]\n                image_name = image_name.split('.')[0]\n                \n                # writing the example\n                example = serialize_example(image, str.encode(image_name), mask)\n                writer.write(example)\n                if k%100==0: print(k,', ',end='')\n\ndef read_seg_tfrecord(example, labeled=True, return_image_names=False):\n    if labeled:\n        tfrec_format = {\n            'image'                        : tf.io.FixedLenFeature([], tf.string),\n            'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n            'mask'                       : tf.io.FixedLenFeature([], tf.string)\n        }      \n    else:\n        tfrec_format = {\n            'image'                        : tf.io.FixedLenFeature([], tf.string),\n            'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n        }\n        \n    example = tf.io.parse_single_example(example, tfrec_format)\n    if labeled:\n        return ({\"seg_input\": example['image']}, example['mask'])\n    else:\n        return ({\"seg_input\": example['image']},\n                example['image_name'] if return_image_names else 0)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:10:02.295029Z","iopub.execute_input":"2021-07-05T09:10:02.295350Z","iopub.status.idle":"2021-07-05T09:10:02.323206Z","shell.execute_reply.started":"2021-07-05T09:10:02.295322Z","shell.execute_reply":"2021-07-05T09:10:02.321501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_image(images, dim=384, mask=False, mask_channels=3): \n    if mask:\n        img = images\n        img = tf.image.decode_jpeg(img, channels=mask_channels)\n    else:\n        img = images['seg_input']\n        img = tf.image.decode_jpeg(img, channels=3)\n        \n    channels = img.shape.as_list()[-1]\n    img = tf.cast(img, tf.float32) / 255.0\n    \n    img = tf.reshape(img, [dim, dim, channels])\n    if mask:\n        return img\n    else:\n        images['seg_input'] = img\n        return images","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:10:02.325169Z","iopub.execute_input":"2021-07-05T09:10:02.325849Z","iopub.status.idle":"2021-07-05T09:10:02.338816Z","shell.execute_reply.started":"2021-07-05T09:10:02.325801Z","shell.execute_reply":"2021-07-05T09:10:02.337621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef transform_grid_mark(image, inv_mat, image_shape):\n    h, w, c = image_shape\n    \n    cx, cy = w//2, h//2\n\n    new_xs = tf.repeat( tf.range(-cx, cx, 1), h)\n    new_ys = tf.tile( tf.range(-cy, cy, 1), [w])\n    new_zs = tf.ones([h*w], dtype=tf.int32)\n\n    old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))\n    old_coords_x, old_coords_y = tf.round(old_coords[0, :] + tf.cast(w, tf.float32)//2.), tf.round(old_coords[1, :] + tf.cast(h, tf.float32)//2.)\n    old_coords_x = tf.cast(old_coords_x, tf.int32)\n    old_coords_y = tf.cast(old_coords_y, tf.int32)    \n\n    clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)\n    clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)\n    clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)\n\n    old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))\n    old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))\n    new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))\n    new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))\n\n    old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)\n    new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)\n    rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))\n    rotated_image_channel = list()\n    for i in range(c):\n        vals = rotated_image_values[:,i]\n        sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])\n        rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))\n\n    return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])\n\n\n@tf.function\ndef random_rotate(image, angle, image_shape):\n    def get_rotation_mat_inv(angle):\n          #transform to radian\n        angle = math.pi * angle / 180\n\n        cos_val = tf.math.cos(angle)\n        sin_val = tf.math.sin(angle)\n        one = tf.constant([1], tf.float32)\n        zero = tf.constant([0], tf.float32)\n\n        rot_mat_inv = tf.concat([cos_val, sin_val, zero,\n                                     -sin_val, cos_val, zero,\n                                     zero, zero, one], axis=0)\n        rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])\n\n        return rot_mat_inv\n    angle = float(angle) * tf.random.normal([1],dtype='float32')\n    rot_mat_inv = get_rotation_mat_inv(angle)\n    return transform_grid_mark(image, rot_mat_inv, image_shape)\n\n\n@tf.function\ndef get_grid_mask(DIM=384):\n    h = tf.constant(DIM, dtype=tf.float32)\n    w = tf.constant(DIM, dtype=tf.float32)\n    \n    image_height, image_width = (h, w)\n\n    # CHANGE THESE PARAMETER\n    d1 = int(DIM / 6)\n    d2 = int(DIM / 4)\n    rotate_angle = 45\n    ratio = 0.4 # this is delete ratio, so keep ratio = 1 - delete\n\n    hh = tf.math.ceil(tf.math.sqrt(h*h+w*w))\n    hh = tf.cast(hh, tf.int32)\n    hh = hh+1 if hh%2==1 else hh\n    d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)\n    l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)\n\n    st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n    st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n\n    y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)\n    x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)\n\n    for i in range(0, hh//d+1):\n        s1 = i * d + st_h\n        s2 = i * d + st_w\n        y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)\n        x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)\n\n    x_clip_mask = tf.logical_or(x_ranges <0 , x_ranges > hh-1)\n    y_clip_mask = tf.logical_or(y_ranges <0 , y_ranges > hh-1)\n    clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)\n\n    x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))\n    y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))\n\n    hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)), tf.int32)])\n    x_ranges = tf.repeat(x_ranges, hh)\n    y_ranges = tf.repeat(y_ranges, hh)\n\n    y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))\n    x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))\n\n    y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64),  tf.zeros_like(y_ranges), [hh, hh])\n    y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)\n\n    x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])\n    x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)\n\n    mask = tf.expand_dims( tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)\n\n    mask = random_rotate(mask, rotate_angle, [hh, hh, 1])\n    mask = tf.image.crop_to_bounding_box(mask, (hh-tf.cast(h, tf.int32))//2, (hh-tf.cast(w, tf.int32))//2, tf.cast(image_height, tf.int32), tf.cast(image_width, tf.int32))\n\n    return mask\n\n\n@tf.function\ndef apply_grid_mask(image, mask, DIM=384):\n    #mask = grid_mask(DIM=DIM)\n    mask = tf.concat([mask, mask, mask], axis=-1)\n    return image * tf.cast(mask, 'float32')\n\n\n\ndef augmenter(image, mask, dim=384, grid_mask=True, grid_mask_aug=True):\n    \"\"\"\n    position_change_func_list = [\n        tf.image.stateless_random_flip_left_right,\n        tf.image.stateless_random_flip_up_down,\n    ]\n\n    color_change_func_list =[\n        partial(tf.image.stateless_random_brightness, max_delta=0.95),\n        partial(tf.image.stateless_random_contrast, upper=0.5, lower=0.1),\n        partial(tf.image.stateless_random_hue, max_delta=0.3),\n        partial(tf.image.stateless_random_saturation, upper=0.6, lower=0.1), \n    ] \n    \n    #print(image[\"seg_input\"].shape)\n    img_cum_mask_array = tf.stack([image[\"seg_input\"], mask], 3)\n    print(img_cum_mask_array.shape)\n    #print(asdad)\n    for i, aug_func in enumerate(position_change_func_list):\n        rand_seed = tf.random.uniform(shape=[2], minval=-10**5, maxval=10**5, dtype=tf.int64)\n        aug_img = aug_func(image[\"seg_input\"], seed=rand_seed)\n        aug_mask = aug_func(mask, seed=rand_seed)\n        augmented_img_cum_mask = tf.stack([aug_img, aug_mask], 3)\n        if i == 0:\n            img_cum_mask_array = tf.stack([img_cum_mask_array,\n                                           augmented_img_cum_mask], 0)\n        else:\n            img_cum_mask_array = tf.concat([img_cum_mask_array,\n                                            tf.expand_dims(augmented_img_cum_mask, 0)], 0)\n        print(img_cum_mask_array.shape)\n    \n    for i, aug_func in enumerate(color_change_func_list):\n        rand_seed = tf.random.uniform(shape=[2], minval=-10**5, maxval=10**5, dtype=tf.int64)\n        aug_img = aug_func(image[\"seg_input\"], seed=rand_seed)\n        augmented_img_cum_mask = tf.stack([aug_img, mask], 3)\n        img_cum_mask_array = tf.concat([img_cum_mask_array,\n                                        tf.expand_dims(augmented_img_cum_mask, 0)], 0)\n        print(img_cum_mask_array.shape)\n            \n    augmented_ds = tf.data.Dataset.from_tensor_slices(img_cum_mask_array)\n    \"\"\"\n\n    position_change_func_list = [\n        tf.image.stateless_random_flip_left_right,\n        tf.image.stateless_random_flip_up_down,\n    ]\n    \n    color_change_func_list =[\n        partial(tf.image.stateless_random_brightness, max_delta=0.95),\n        partial(tf.image.stateless_random_contrast, upper=0.5, lower=0.1),\n        partial(tf.image.stateless_random_hue, max_delta=0.3),\n        partial(tf.image.stateless_random_saturation, upper=0.6, lower=0.1), \n    ] \n    \n    img_cum_mask_array = tf.stack([image[\"seg_input\"], mask], 3)\n\n    for i, aug_func in enumerate(position_change_func_list):\n        rand_seed = tf.random.uniform(shape=[2], minval=-10**5, maxval=10**5, dtype=tf.int64)\n        aug_img = aug_func(image[\"seg_input\"], seed=rand_seed)\n        aug_mask = aug_func(mask, seed=rand_seed)\n        augmented_img_cum_mask = tf.stack([aug_img, aug_mask], 3)\n        if i == 0:\n            img_cum_mask_array = tf.stack([img_cum_mask_array,\n                                           augmented_img_cum_mask], 0)\n        else:\n            img_cum_mask_array = tf.concat([img_cum_mask_array,\n                                            tf.expand_dims(augmented_img_cum_mask, 0)], 0)\n    \n    for i, aug_func in enumerate(color_change_func_list):\n        rand_seed = tf.random.uniform(shape=[2], minval=-10**5, maxval=10**5, dtype=tf.int64)\n        aug_img = aug_func(image[\"seg_input\"], seed=rand_seed)\n        augmented_img_cum_mask = tf.stack([aug_img, mask], 3)\n        img_cum_mask_array = tf.concat([img_cum_mask_array,\n                                        tf.expand_dims(augmented_img_cum_mask, 0)], 0)\n    \n    if grid_mask:\n        grid_mask = get_grid_mask(DIM=dim)\n        grid_masked_img = apply_grid_mask(image[\"seg_input\"], grid_mask)\n        grid_masked_mask = apply_grid_mask(mask, grid_mask)\n        augmented_img_cum_mask = tf.stack([grid_masked_img, grid_masked_mask], 3)\n        img_cum_mask_array = tf.concat([img_cum_mask_array,\n                                        tf.expand_dims(augmented_img_cum_mask, 0)], 0)\n        if grid_mask_aug:\n            \"\"\"\n            all_aug_func_list = position_change_func_list + color_change_func_list\n            for i, aug_func in enumerate(all_aug_func_list):\n                rand_seed = tf.random.uniform(shape=[2], minval=-10**5, maxval=10**5, dtype=tf.int64)\n                grid_mask_aug_img = aug_func(grid_masked_img, seed=rand_seed)\n                aug_img_array = tf.concat([aug_img_array, tf.expand_dims(aug_img, 0)], 0)\n            \"\"\"\n            for i, aug_func in enumerate(position_change_func_list):\n                rand_seed = tf.random.uniform(shape=[2], minval=-10**5, maxval=10**5, dtype=tf.int64)\n                aug_img = aug_func(grid_masked_img, seed=rand_seed)\n                aug_mask = aug_func(grid_masked_mask, seed=rand_seed)\n                augmented_img_cum_mask = tf.stack([aug_img, aug_mask], 3)\n                img_cum_mask_array = tf.concat([img_cum_mask_array,\n                                                tf.expand_dims(augmented_img_cum_mask, 0)], 0)\n\n            for i, aug_func in enumerate(color_change_func_list):\n                rand_seed = tf.random.uniform(shape=[2], minval=-10**5, maxval=10**5, dtype=tf.int64)\n                aug_img = aug_func(grid_masked_img, seed=rand_seed)\n                augmented_img_cum_mask = tf.stack([aug_img, grid_masked_mask], 3)\n                img_cum_mask_array = tf.concat([img_cum_mask_array,\n                                                tf.expand_dims(augmented_img_cum_mask, 0)], 0)\n        \n    augmented_ds = tf.data.Dataset.from_tensor_slices(img_cum_mask_array)\n    \n    def input_handler(img_cum_mask):\n        return {\"seg_input\": img_cum_mask[:, :, :, 0]}, img_cum_mask[:, :, :, 1]\n    \n    augmented_ds = augmented_ds.map(lambda img_cum_mask: input_handler(img_cum_mask),\n                                    num_parallel_calls=AUTO)\n    return augmented_ds","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:10:02.340645Z","iopub.execute_input":"2021-07-05T09:10:02.341096Z","iopub.status.idle":"2021-07-05T09:10:02.397236Z","shell.execute_reply.started":"2021-07-05T09:10:02.341036Z","shell.execute_reply":"2021-07-05T09:10:02.395920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_seg_dataset(files, shuffle = False, repeat = False, labeled=True, augment=True,\n                    return_image_names=False, batch_size=32, dim=384, mask_channels=3,\n                    grid_mask=True, grid_mask_aug=True):\n    \n    read_labeled_unlabeled = partial(read_seg_tfrecord, labeled=labeled)\n\n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n    \n    ds = ds.map(read_labeled_unlabeled, num_parallel_calls=AUTO)\n    ds = ds.map(lambda image, mask_or_image_name: (prepare_image(image, dim=dim),\n                                                   mask_or_image_name),\n                num_parallel_calls=AUTO)\n    # for image dataset\n    if labeled: \n        ds = ds.map(lambda image, mask: (image, prepare_image(mask, dim=dim,\n                                                              mask=True,\n                                                              mask_channels=mask_channels)),\n                    num_parallel_calls=AUTO)\n        \n        if augment: # I am not using Test Time Augmentation, this is for Train only.\n            aug_func = partial(augmenter, dim=dim, grid_mask=grid_mask,\n                               grid_mask_aug=grid_mask_aug)\n            ds = ds.flat_map(lambda image, mask: aug_func(image, mask))\n            ds = ds.shuffle(1024*8)\n            \n    ds = ds.batch(batch_size * REPLICAS, drop_remainder=True)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:10:02.398921Z","iopub.execute_input":"2021-07-05T09:10:02.399236Z","iopub.status.idle":"2021-07-05T09:10:02.410305Z","shell.execute_reply.started":"2021-07-05T09:10:02.399207Z","shell.execute_reply":"2021-07-05T09:10:02.409154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#IMG_ROOT_PATHS = [\"../input/isic2017-and-ph2/ISIC_2017 + PH2/ISIC_2017/trainx\",\n#                  \"../input/isic2017-and-ph2/ISIC_2017 + PH2/ISIC_2017/trainy\"]\n\n#image_paths, mask_paths = get_seg_paths(\"train\", img_root_paths=IMG_ROOT_PATHS)\n\n#image_paths","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:10:02.412618Z","iopub.execute_input":"2021-07-05T09:10:02.412913Z","iopub.status.idle":"2021-07-05T09:10:02.424096Z","shell.execute_reply.started":"2021-07-05T09:10:02.412884Z","shell.execute_reply":"2021-07-05T09:10:02.423133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create_seg_tfrecords(tfrecord_type=\"train\", SIZE=500, tfrec_roots=None, img_root_paths=IMG_ROOT_PATHS)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:10:02.426209Z","iopub.execute_input":"2021-07-05T09:10:02.426708Z","iopub.status.idle":"2021-07-05T09:10:02.434634Z","shell.execute_reply.started":"2021-07-05T09:10:02.426664Z","shell.execute_reply":"2021-07-05T09:10:02.433746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TFREC_ROOT_PATHS = [KaggleDatasets().get_gcs_path(\"isic2018andph2384x384tfrecords\"),\n                    KaggleDatasets().get_gcs_path(\"isic2017384x384tfrecords\")]\n\ntrain_paths_segs, test_paths_segs = get_seg_paths(\"tfrecords\", tfrec_roots=TFREC_ROOT_PATHS)\nvalid_paths_segs = train_paths_segs[-1]\ntrain_paths_segs = train_paths_segs[:-1]\n\nBATCH_SIZE = 16\nMASK_CHANNELS = 3\ntrain_dataset_seg = get_seg_dataset(train_paths_segs, batch_size=BATCH_SIZE,# augment=False,\n                                    labeled=True, mask_channels=MASK_CHANNELS,\n                                    grid_mask_aug=False)\nvalid_dataset_seg = get_seg_dataset(valid_paths_segs, batch_size=BATCH_SIZE,\n                                    labeled=True, mask_channels=MASK_CHANNELS,\n                                    grid_mask_aug=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:10:02.435685Z","iopub.execute_input":"2021-07-05T09:10:02.435946Z","iopub.status.idle":"2021-07-05T09:10:13.054295Z","shell.execute_reply.started":"2021-07-05T09:10:02.435921Z","shell.execute_reply":"2021-07-05T09:10:13.053299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset_seg = get_seg_dataset(test_paths_segs, batch_size=BATCH_SIZE,\n                                   labeled=True, mask_channels=MASK_CHANNELS,\n                                   grid_mask=False, grid_mask_aug=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T16:31:22.422153Z","iopub.execute_input":"2021-07-05T16:31:22.422504Z","iopub.status.idle":"2021-07-05T16:31:22.597256Z","shell.execute_reply.started":"2021-07-05T16:31:22.422474Z","shell.execute_reply":"2021-07-05T16:31:22.596105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for item in train_dataset_seg.take(1):\n#    print(item)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:10:13.092005Z","iopub.execute_input":"2021-07-05T09:10:13.092490Z","iopub.status.idle":"2021-07-05T09:10:13.095119Z","shell.execute_reply.started":"2021-07-05T09:10:13.092458Z","shell.execute_reply":"2021-07-05T09:10:13.094395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## METRICS:","metadata":{}},{"cell_type":"code","source":"def iou(y_true, y_pred, smooth = 100):\n    intersection = K.sum(K.abs(y_true * y_pred), axis=-1)\n    sum_ = K.sum(K.square(y_true), axis = -1) + K.sum(K.square(y_pred), axis=-1)\n    jac = (intersection + smooth) / (sum_ - intersection + smooth)\n    return jac\n\n\ndef dice_coe(y_true, y_pred, smooth = 1):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n\n\ndef precision(y_true, y_pred):\n    '''Calculates the precision, a metric for multi-label classification of\n    how many selected items are relevant.\n    '''\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\n\ndef recall(y_true, y_pred):\n    '''Calculates the recall, a metric for multi-label classification of\n    how many relevant items are selected.\n    '''\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\n\ndef accuracy(y_true, y_pred):\n    '''Calculates the mean accuracy rate across all predictions for binary\n    classification problems.\n    '''\n    return K.mean(K.equal(y_true, K.round(y_pred)))","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:10:13.096122Z","iopub.execute_input":"2021-07-05T09:10:13.096754Z","iopub.status.idle":"2021-07-05T09:10:13.110373Z","shell.execute_reply.started":"2021-07-05T09:10:13.096722Z","shell.execute_reply":"2021-07-05T09:10:13.109074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BUILDING AND TRAINING THE MODEL:","metadata":{}},{"cell_type":"code","source":"help(base.unet_3plus_2d_base)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:10:13.112235Z","iopub.execute_input":"2021-07-05T09:10:13.112701Z","iopub.status.idle":"2021-07-05T09:10:13.127702Z","shell.execute_reply.started":"2021-07-05T09:10:13.112651Z","shell.execute_reply":"2021-07-05T09:10:13.126628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(batch_size=8, REPLICAS=1):\n    lr_start   = 0.00005\n    lr_max     = 0.0000125 * REPLICAS * batch_size\n    lr_min     = 0.00001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\n\ndef binary_jaccard_index(y_true, y_pred, smooth=100):\n    y_true = K.round(y_true)\n    y_pred = K.round(y_pred)\n    intersection = K.sum(K.abs(y_true * y_pred), axis=[1, 2, 3])\n    #print(intersection)\n    union = K.sum(K.abs(y_true) + K.abs(y_pred), axis=[1, 2, 3])\n    #print(union)\n    #print(tf.clip_by_value(union - intersection, K.epsilon(), None))\n    iou = intersection / K.clip(union - intersection, K.epsilon(), None)\n    return iou\n\ndef jaccard_distance(y_true, y_pred, smooth=100):\n    intersection = K.sum(K.abs(y_true * y_pred), axis=-1)\n    sum_ = K.sum(K.square(y_true), axis = -1) + K.sum(K.square(y_pred), axis=-1)\n    jac = (intersection + smooth) / (sum_ - intersection + smooth)\n    return (1 - jac)\n      \nDIM = 384 \n\nif DEVICE=='TPU':\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n        \n\nactivation = 'ReLU'\nfilter_num = [32, 64, 128, 256, 512, 1024]\nstack_num_down = 2\nstack_num_up = 1\nrecur_num=2\n\nwith strategy.scope():\n    input_layer = keras.layers.Input(shape=(DIM, DIM, 3), name=\"seg_input\")\n    unet_base = base.r2_unet_2d_base(input_layer, filter_num=filter_num, stack_num_down=stack_num_down,\n                                     stack_num_up=stack_num_up, recur_num=recur_num, activation=\"ReLU\",\n                                     batch_norm=True, pool=\"max\", unpool=\"nearest\", name=\"res_unet_base\")\n    unet_output = keras.layers.Conv2D(MASK_CHANNELS, (1, 1), activation=\"sigmoid\")(unet_base)\n    unet_model = keras.Model(inputs=[input_layer], outputs=[unet_output])\n    unet_model.compile(optimizer=keras.optimizers.Nadam(0.0001),\n                       loss=[losses.focal_tversky],\n                       metrics=[dice_coe])\n    \n#unet_model.summary()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-05T09:10:13.129271Z","iopub.execute_input":"2021-07-05T09:10:13.129887Z","iopub.status.idle":"2021-07-05T09:10:28.665919Z","shell.execute_reply.started":"2021-07-05T09:10:13.129839Z","shell.execute_reply":"2021-07-05T09:10:28.664918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#keras.utils.plot_model(unet_model)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:10:28.667047Z","iopub.execute_input":"2021-07-05T09:10:28.667346Z","iopub.status.idle":"2021-07-05T09:10:28.671194Z","shell.execute_reply.started":"2021-07-05T09:10:28.667312Z","shell.execute_reply":"2021-07-05T09:10:28.670151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = unet_model.fit(\n    train_dataset_seg,\n    epochs=50,\n    validation_data=valid_dataset_seg,\n    callbacks=[#get_lr_callback(BATCH_SIZE, REPLICAS),\n               keras.callbacks.ModelCheckpoint(\"r2_unet.h5\", monitor='val_loss',\n                                               verbose=2, save_best_only=True,\n                                               save_weights_only=True, mode='min',\n                                               save_freq='epoch'),\n               keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10)],\n    #steps_per_epoch=count_data_items(train_paths_segs)/32//REPLICAS,\n    verbose=2,\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T09:10:28.672726Z","iopub.execute_input":"2021-07-05T09:10:28.673336Z","iopub.status.idle":"2021-07-05T16:28:10.461262Z","shell.execute_reply.started":"2021-07-05T09:10:28.673291Z","shell.execute_reply":"2021-07-05T16:28:10.458168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weight_dir = os.path.join(os.getcwd(), \"weights\")\nos.makedirs(weight_dir, exist_ok=True)\nunet_model.save_weights(os.path.join(weight_dir, \"r2_unet_weights.h5\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unet_model.load_weights(\"./r2_unet.h5\")","metadata":{"execution":{"iopub.status.busy":"2021-07-05T16:28:29.211472Z","iopub.execute_input":"2021-07-05T16:28:29.211883Z","iopub.status.idle":"2021-07-05T16:28:36.664645Z","shell.execute_reply.started":"2021-07-05T16:28:29.211849Z","shell.execute_reply":"2021-07-05T16:28:36.663704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\npickle.dump(history.history, open(\"r2_unet_history.pkl\", \"wb\"))","metadata":{"execution":{"iopub.status.busy":"2021-07-05T16:28:38.213028Z","iopub.execute_input":"2021-07-05T16:28:38.213452Z","iopub.status.idle":"2021-07-05T16:28:38.222684Z","shell.execute_reply.started":"2021-07-05T16:28:38.213412Z","shell.execute_reply":"2021-07-05T16:28:38.221336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights_list = [\"../input/r2-unet-weight/gm_aug_r2_unet_extreme.h5\",\n                \"../input/r2-unet-weight/r2_unet_extreme.h5\",\n                \"../input/r2-unet-weight/r2_unet_better.h5\",\n                \"../input/r2-unet-weight/r2_unet.h5\",]\nfor weights in weights_list:\n    unet_model.load_weights(weights)\n    unet_model.evaluate(test_dataset_seg)","metadata":{"execution":{"iopub.status.busy":"2021-07-05T16:41:41.991185Z","iopub.execute_input":"2021-07-05T16:41:41.991666Z","iopub.status.idle":"2021-07-05T16:45:59.519768Z","shell.execute_reply.started":"2021-07-05T16:41:41.991632Z","shell.execute_reply":"2021-07-05T16:45:59.518721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_preds = unet_model.predict(train_dataset_seg)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(train_preds[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(preds[10])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for item in test_dataset_seg.take(1):\n    images = item[0][\"seg_input\"]\n    image_names = item[1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(images[20])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(preds[20])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## If the predictions are blurry, then use this function for better thresholding\ndef enhance_preds(img_data, threshold=0.5, dim_x=384, dim_y=384, channels=3):\n    \n    if len(img_data.shape) == 3:\n        real_img_data_dims = 3\n        img_data = tf.expand_dims(img_data, axis=0)\n    else:\n        real_img_data_dims = 4\n        \n    preds = unet_model.predict(img_data)\n    plt.imshow(preds[0])\n    if real_img_data_dims == 3:\n        preds = preds.flatten()\n        for i in range(len(preds)):\n            if preds[i] > 0.5:\n                preds[i] = 1\n            else:\n                preds[i] = 0 \n        return tf.reshape(preds, [dim_x, dim_y, channels])\n    else:\n        for i in range(preds.shape[0]):\n            pred_img = preds[i].flatten() \n            for j in range(len(pred_img)):\n                if pred_img[j] > 0.5:\n                    pred_img[j] = 1\n                else:\n                    pred_img[j] = 0\n            preds[i] = tf.reshape(pred_img, [dim_x, dim_y, channels])\n        return preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}