{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Hello Fellow Kagglers,\n\nThis notebook demonstrated the training process on a TPU using EfficientNet B7 with the famous UNet model architecture widely used in this competition.\n\nThis notebook is heavily inspired by the amazing notebook [Training - FastAI Baseline](https://www.kaggle.com/code/thedevastator/training-fastai-baseline) from [The Devastator](https://www.kaggle.com/thedevastator).\n\n[Preprocessing Notebook](https://www.kaggle.com/code/markwijkhuizen/hubmap-patched-tfrecord-generation-visualization)\n\n[Inference Notebook](https://www.kaggle.com/code/masterray/hubmap-inference-tf-tpu-efficientnet-b8-640-640)","metadata":{}},{"cell_type":"markdown","source":"Copied from: https://www.kaggle.com/code/markwijkhuizen/hubmap-training-tf-tpu-efficientnet-b7-640x640","metadata":{}},{"cell_type":"code","source":"# Import EfficientNet models with intermediate endpoints\nimport sys\nsys.path.append('/kaggle/input/efficientnetv2-head-1x1-endpoint-v2/')\nsys.path.append('/kaggle/input/efficientnetv2-head-1x1-endpoint-v2/efficientnetv2/')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:39:34.53824Z","iopub.execute_input":"2022-08-23T12:39:34.539375Z","iopub.status.idle":"2022-08-23T12:39:34.573991Z","shell.execute_reply.started":"2022-08-23T12:39:34.539253Z","shell.execute_reply":"2022-08-23T12:39:34.572733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nimport tensorflow_addons as tfa\nimport matplotlib.pyplot as plt\n\nfrom kaggle_datasets import KaggleDatasets\nfrom tqdm.notebook import tqdm\nfrom multiprocessing import cpu_count\nfrom sklearn import metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom albumentations import *\n\nimport effnetv2_model\nimport re\nimport os\nimport io\nimport time\nimport pickle\nimport math\nimport random\nimport sys\nimport cv2\nimport gc\n\nimport matplotlib as mpl\nmpl.rcParams['figure.dpi'] = 150\n\nprint(f'tensorflow version: {tf.__version__}')\nprint(f'tensorflow keras version: {tf.keras.__version__}')\nprint(f'python version: P{sys.version}')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:39:36.647506Z","iopub.execute_input":"2022-08-23T12:39:36.647985Z","iopub.status.idle":"2022-08-23T12:39:46.983703Z","shell.execute_reply.started":"2022-08-23T12:39:36.647947Z","shell.execute_reply":"2022-08-23T12:39:46.981787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    TPU = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', TPU.master())\nexcept ValueError:\n    print('Running on GPU')\n    TPU = None\n\nif TPU:\n    IS_TPU = True\n    tf.config.experimental_connect_to_cluster(TPU)\n    tf.tpu.experimental.initialize_tpu_system(TPU)\n    strategy = tf.distribute.experimental.TPUStrategy(TPU)\nelse:\n    IS_TPU = False\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}, IS_TPU: {IS_TPU}')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:44:14.269451Z","iopub.execute_input":"2022-08-23T12:44:14.269856Z","iopub.status.idle":"2022-08-23T12:44:14.278909Z","shell.execute_reply.started":"2022-08-23T12:44:14.269825Z","shell.execute_reply":"2022-08-23T12:44:14.277788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For TPU's the dataset needs to be stored in Google Cloud\n# Retrieve the Google Cloud location of the dataset\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('hubmap-patched-tfrecords-300x300')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:46:30.280809Z","iopub.execute_input":"2022-08-23T12:46:30.281343Z","iopub.status.idle":"2022-08-23T12:46:38.157389Z","shell.execute_reply.started":"2022-08-23T12:46:30.281299Z","shell.execute_reply":"2022-08-23T12:46:38.156069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 43\nDEBUG = False\n\n# Image dimensions\nIMG_SIZE_FULL = 640\nIMG_SIZE = 640\nN_PATCHES_PER_IMAGE = (IMG_SIZE_FULL // IMG_SIZE) ** 2\nN_CHANNELS = 3\nINPUT_SHAPE = (IMG_SIZE, IMG_SIZE, N_CHANNELS)\n\n# EfficientNet version, b0/b1/b2/b3/b4/b5/b6/b7/b8\nEFN_SIZE = 'b8'\n# Peak Learning Rate\nLR_MAX_WHOLE = 8e-4\nN_FOLDS = 4\nTRAIN_FOLDS = [1]\n\n# Epochs are run 10 at a time due to low training samples\nCOMBINE_EPOCHS = 8\nEPOCHS_WHOLE = 600 // COMBINE_EPOCHS\n\n# Batch size\nBATCH_SIZE = 8 * REPLICAS\n\n# Dataset Mean and Standard Deviation\nMEAN = np.load('/kaggle/input/hubmap-patched-tfrecords-300x300/MEAN.npy')\nSTD = np.load('/kaggle/input/hubmap-patched-tfrecords-300x300/STD.npy')\n\n# Tensorflow AUTO flag\nAUTO = tf.data.experimental.AUTOTUNE\nif TPU:\n    NUM_PARALLEL_CALLS = AUTO\nelse:\n    NUM_PARALLEL_CALLS = cpu_count()\n\nprint(f'BATCH_SIZE: {BATCH_SIZE}, NUM_PARALLEL_CALLS: {NUM_PARALLEL_CALLS}')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:37.471409Z","iopub.execute_input":"2022-08-23T12:07:37.471679Z","iopub.status.idle":"2022-08-23T12:07:37.493209Z","shell.execute_reply.started":"2022-08-23T12:07:37.471646Z","shell.execute_reply":"2022-08-23T12:07:37.491876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'MEAN: {MEAN}, STD: {STD}')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:37.495066Z","iopub.execute_input":"2022-08-23T12:07:37.495492Z","iopub.status.idle":"2022-08-23T12:07:37.502242Z","shell.execute_reply.started":"2022-08-23T12:07:37.495466Z","shell.execute_reply":"2022-08-23T12:07:37.500659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Seed","metadata":{}},{"cell_type":"code","source":"# Seed all random number generators\ndef seed_everything(seed=SEED):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \n\nseed_everything()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:37.503431Z","iopub.execute_input":"2022-08-23T12:07:37.503818Z","iopub.status.idle":"2022-08-23T12:07:37.513089Z","shell.execute_reply.started":"2022-08-23T12:07:37.503793Z","shell.execute_reply":"2022-08-23T12:07:37.511901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"# Train DataFrame\ntrain = pd.read_csv('/kaggle/input/hubmap-organ-segmentation/train.csv')\nN_SAMPLES = len(train)\nprint(f'N_SAMPLES: {N_SAMPLES}')\n\n# Add Ordinal Encoded Organ\ntrain['organ_ordinal'] = train['organ'].astype('category').cat.codes\nN_ORGANS = train['organ'].nunique()\nORGANS = sorted(train['organ'].unique())\norg_ord2org = dict(enumerate(train['organ'].astype('category').cat.categories))\nprint(f'N_ORGANS: {N_ORGANS}, ORGANS: {ORGANS}')\n\ndisplay(train.head())\ndisplay(train.info())","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:37.51411Z","iopub.execute_input":"2022-08-23T12:07:37.5144Z","iopub.status.idle":"2022-08-23T12:07:37.848447Z","shell.execute_reply.started":"2022-08-23T12:07:37.514375Z","shell.execute_reply":"2022-08-23T12:07:37.847478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{}},{"cell_type":"code","source":"# One in K chance function\ndef tf_rand_int(minval, maxval, dtype=tf.int64):\n    minval = tf.cast(minval, dtype)\n    maxval = tf.cast(maxval, dtype)\n    return tf.random.uniform(shape=(), minval=minval, maxval=maxval, dtype=dtype)\n\ndef one_in(k):\n    return 0 == tf_rand_int(0, k)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:37.850019Z","iopub.execute_input":"2022-08-23T12:07:37.850302Z","iopub.status.idle":"2022-08-23T12:07:37.856194Z","shell.execute_reply.started":"2022-08-23T12:07:37.850254Z","shell.execute_reply":"2022-08-23T12:07:37.854961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss\n\nThe following code provides a wide variety of semantic segmentations losses and metrics. Surprisingly, dice loss did perform worse than good old binary crossnetropy. The only code used in this notebook is the IoU metric.","metadata":{}},{"cell_type":"code","source":"# Source: https://github.com/shruti-jadon/Semantic-Segmentation-Loss-Functions/blob/master/loss_functions.py\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.losses import binary_crossentropy\n\nbeta = 0.25\nalpha = 0.25\ngamma = 2\nepsilon = 1e-5\nsmooth = 1\nthreshold=0.50\n\nclass Semantic_loss_functions(object):\n    def __init__(self):\n        print ('semantic loss functions initialized')\n        \n    def iou(self, y_true, y_pred):\n        y_true = tf.cast(y_true, tf.float32)\n        y_pred = tf.where(y_pred > threshold, x=1.0, y=0.0)\n        y_true_f = K.flatten(y_true)\n        y_pred_f = K.flatten(y_pred)\n        intersection = K.sum(y_true_f * y_pred_f)\n        union = K.sum(y_true_f + y_pred_f) - intersection\n        return intersection / (union + epsilon)\n\n    def dice_coef(self, y_true, y_pred):\n        y_true = tf.cast(y_true, tf.float32)\n        y_true_f = K.flatten(y_true)\n        y_pred_f = K.flatten(y_pred)\n        intersection = K.sum(y_true_f * y_pred_f, axis=1)\n        return (2. * intersection + K.epsilon()) / (\n                    K.sum(y_true_f) + K.sum(y_pred_f) + K.epsilon())\n\n    def sensitivity(self, y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        return true_positives / (possible_positives + K.epsilon())\n\n    def specificity(self, y_true, y_pred):\n        true_negatives = K.sum(\n            K.round(K.clip((1 - y_true) * (1 - y_pred), 0, 1)))\n        possible_negatives = K.sum(K.round(K.clip(1 - y_true, 0, 1)))\n        return true_negatives / (possible_negatives + K.epsilon())\n\n    def convert_to_logits(self, y_pred):\n        y_pred = tf.clip_by_value(y_pred, tf.keras.backend.epsilon(),\n                                  1 - tf.keras.backend.epsilon())\n        return tf.math.log(y_pred / (1 - y_pred))\n\n    def weighted_cross_entropyloss(self, y_true, y_pred):\n        y_pred = self.convert_to_logits(y_pred)\n        pos_weight = beta / (1 - beta)\n        loss = tf.nn.weighted_cross_entropy_with_logits(logits=y_pred,\n                                                        targets=y_true,\n                                                        pos_weight=pos_weight)\n        return tf.reduce_mean(loss)\n\n    def focal_loss_with_logits(self, logits, targets, alpha, gamma, y_pred):\n        weight_a = alpha * (1 - y_pred) ** gamma * targets\n        weight_b = (1 - alpha) * y_pred ** gamma * (1 - targets)\n\n        return (tf.math.log1p(tf.exp(-tf.abs(logits))) + tf.nn.relu(\n            -logits)) * (weight_a + weight_b) + logits * weight_b\n\n    def focal_loss(self, y_true, y_pred):\n        y_pred = tf.clip_by_value(y_pred, tf.keras.backend.epsilon(),\n                                  1 - tf.keras.backend.epsilon())\n        logits = tf.math.log(y_pred / (1 - y_pred))\n\n        loss = self.focal_loss_with_logits(logits=logits, targets=y_true,\n                                      alpha=alpha, gamma=gamma, y_pred=y_pred)\n\n        return tf.reduce_mean(loss)\n\n    def depth_softmax(self, matrix):\n        sigmoid = lambda x: 1 / (1 + K.exp(-x))\n        sigmoided_matrix = sigmoid(matrix)\n        softmax_matrix = sigmoided_matrix / K.sum(sigmoided_matrix, axis=0)\n        return softmax_matrix\n\n    def generalized_dice_coefficient(self, y_true, y_pred):\n        smooth = 1e0\n        y_true = tf.cast(y_true, tf.float32)\n        y_true_f = K.flatten(y_true)\n        y_pred_f = K.flatten(y_pred)\n        intersection = K.sum(y_true_f * y_pred_f)\n        return (2. * intersection + smooth) / (\n                    K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n        \n    def dice_loss(self, y_true, y_pred):\n        loss = 1 - self.generalized_dice_coefficient(y_true, y_pred)\n        return loss\n    \n    def dice_loss_symmetric(self, y_true, y_pred):\n        loss = 1 - self.generalized_dice_coefficient(y_true, y_pred)\n        loss_neg = 1 - self.generalized_dice_coefficient(1 - y_true, 1 - y_pred)\n        return 0.50 * (loss + loss_neg)\n\n    def bce_dice_loss(self, y_true, y_pred):\n        loss = binary_crossentropy(y_true, y_pred) + \\\n               self.dice_loss(y_true, y_pred)\n        return loss / 2.0\n\n    def confusion(self, y_true, y_pred):\n        smooth = 1\n        y_pred_pos = K.clip(y_pred, 0, 1)\n        y_pred_neg = 1 - y_pred_pos\n        y_pos = K.clip(y_true, 0, 1)\n        y_neg = 1 - y_pos\n        tp = K.sum(y_pos * y_pred_pos)\n        fp = K.sum(y_neg * y_pred_pos)\n        fn = K.sum(y_pos * y_pred_neg)\n        prec = (tp + smooth) / (tp + fp + smooth)\n        recall = (tp + smooth) / (tp + fn + smooth)\n        return prec, recall\n\n    def true_positive(self, y_true, y_pred):\n        smooth = 1\n        y_pred_pos = K.round(K.clip(y_pred, 0, 1))\n        y_pos = K.round(K.clip(y_true, 0, 1))\n        tp = (K.sum(y_pos * y_pred_pos) + smooth) / (K.sum(y_pos) + smooth)\n        return tp\n\n    def true_negative(self, y_true, y_pred):\n        smooth = 1\n        y_pred_pos = K.round(K.clip(y_pred, 0, 1))\n        y_pred_neg = 1 - y_pred_pos\n        y_pos = K.round(K.clip(y_true, 0, 1))\n        y_neg = 1 - y_pos\n        tn = (K.sum(y_neg * y_pred_neg) + smooth) / (K.sum(y_neg) + smooth)\n        return tn\n\n    def tversky_index(self, y_true, y_pred):\n        y_true = tf.cast(y_true, tf.float32)\n        y_true_pos = K.flatten(y_true)\n        y_pred_pos = K.flatten(y_pred)\n        true_pos = K.sum(y_true_pos * y_pred_pos)\n        false_neg = K.sum(y_true_pos * (1 - y_pred_pos))\n        false_pos = K.sum((1 - y_true_pos) * y_pred_pos)\n        alpha = 0.75\n        return (true_pos + smooth) / (true_pos + alpha * false_neg + (\n                    1 - alpha) * false_pos + smooth)\n\n    def tversky_loss(self, y_true, y_pred):\n        return 1 - self.tversky_index(y_true, y_pred)\n\n    def focal_tversky(self, y_true, y_pred, gamme=2.0):\n        pt_1 = self.tversky_index(y_true, y_pred)\n        return K.pow((1 - pt_1), gamma)\n\n    def log_cosh_dice_loss(self, y_true, y_pred):\n        y_true = tf.cast(y_true, tf.float32)\n        x = self.dice_loss(y_true, y_pred)\n        return tf.math.log((tf.exp(x) + tf.exp(-x)) / 2.0)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:37.861152Z","iopub.execute_input":"2022-08-23T12:07:37.862245Z","iopub.status.idle":"2022-08-23T12:07:37.896823Z","shell.execute_reply.started":"2022-08-23T12:07:37.862214Z","shell.execute_reply":"2022-08-23T12:07:37.89576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEMANTIC_LOSS_FUNCTIONS = Semantic_loss_functions()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:37.898258Z","iopub.execute_input":"2022-08-23T12:07:37.899633Z","iopub.status.idle":"2022-08-23T12:07:37.912818Z","shell.execute_reply.started":"2022-08-23T12:07:37.899576Z","shell.execute_reply":"2022-08-23T12:07:37.911479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Pyramid Network (FPN)\n\n(Pytorch Source)[https://www.kaggle.com/code/thedevastator/training-fastai-baseline#Model}","metadata":{}},{"cell_type":"code","source":"def FPN(xs, output_channels, last_layer, debug=False):\n    def _conv(x):\n        x = tf.keras.layers.ZeroPadding2D(padding=1)(x)\n        x = tf.keras.layers.Conv2D(output_channels * 2, 3, padding='SAME', kernel_initializer='he_normal', activation='relu')(x)\n        x = tf.keras.layers.BatchNormalization()(x)\n        x = tf.keras.layers.ZeroPadding2D(padding=1)(x)\n        x = tf.keras.layers.Conv2D(output_channels, 3, padding='SAME', kernel_initializer='he_normal')(x)\n        x = tf.image.resize(x, size=target_size, method=tf.image.ResizeMethod.BILINEAR)\n        x = tf.nn.relu(x)\n        return x\n\n    target_size = last_layer.shape[1:3]\n    xs = tf.keras.layers.Concatenate()([_conv(x) for x in xs])\n    x = tf.keras.layers.Concatenate()([xs, last_layer])\n\n    if debug:\n        return x, xs\n    else:\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:37.914677Z","iopub.execute_input":"2022-08-23T12:07:37.91603Z","iopub.status.idle":"2022-08-23T12:07:37.924012Z","shell.execute_reply.started":"2022-08-23T12:07:37.915984Z","shell.execute_reply":"2022-08-23T12:07:37.923328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Atrous Spatial Pyramid Pooling (ASPP)\n\n(Pytorch Source)[https://www.kaggle.com/code/thedevastator/training-fastai-baseline#Model]","metadata":{}},{"cell_type":"code","source":"def ASPP(x, mid_c=320, dilations=[1, 2, 3, 4], out_c=640, debug=False):\n    def _aspp_module(x, filters, kernel_size, padding, dilation, groups=1):\n        x = tf.keras.layers.ZeroPadding2D(padding=padding)(x)\n        x = tf.keras.layers.Conv2D(\n                filters=filters,\n                kernel_size=kernel_size,\n                dilation_rate=dilation,\n                groups=1 if IS_TPU else groups,\n                kernel_initializer='he_uniform',\n            )(x)\n        x = tf.keras.layers.BatchNormalization()(x)\n        x = tf.nn.relu(x)\n        \n        return x\n    \n    x0 = tf.math.reduce_max(x, axis=(1,2), keepdims=True)\n    x0 = tf.keras.layers.Conv2D(filters=mid_c, kernel_size=1, strides=1, kernel_initializer='he_uniform', use_bias=False)(x0)\n    x0 = tf.keras.layers.BatchNormalization(gamma_initializer=tf.constant_initializer(value=0.25))(x0)\n    x0 = tf.nn.relu(x0)\n                                  \n                                  \n    xs = (\n        [_aspp_module(x, mid_c, 1, padding=0, dilation=1)] +\n        [_aspp_module(x, mid_c, 3, padding=d, dilation=d, groups=4) for d in dilations]\n    )\n    \n    x0= tf.image.resize(x0, size=xs[0].shape[1:3])\n    x = tf.keras.layers.Concatenate()([x0] + xs)\n    x = tf.keras.layers.Conv2D(filters=out_c, kernel_size=1, kernel_initializer='he_uniform', use_bias=False)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.nn.relu(x)\n                       \n    if debug:\n        return x, x0, xs\n    else:\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:37.924983Z","iopub.execute_input":"2022-08-23T12:07:37.9259Z","iopub.status.idle":"2022-08-23T12:07:37.940034Z","shell.execute_reply.started":"2022-08-23T12:07:37.925838Z","shell.execute_reply":"2022-08-23T12:07:37.938428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Upsample","metadata":{}},{"cell_type":"code","source":"# PixelShuffle implemented in Tensorflow, not used\ndef PixelShuffle(x, upscale_factor=2):\n    _, w, h, c = x.shape\n    n = -1\n\n    c_out = c // upscale_factor ** 2\n    w_out = w * upscale_factor\n    h_out = h * upscale_factor\n\n    x = tf.reshape(x, [-1, upscale_factor, upscale_factor, w, h, c_out])\n    x = tf.transpose(x, [0, 3, 1, 4, 2, 5])\n    x = tf.reshape(x, [-1, w_out, h_out, c_out])\n\n    return x","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:37.941523Z","iopub.execute_input":"2022-08-23T12:07:37.941913Z","iopub.status.idle":"2022-08-23T12:07:37.958087Z","shell.execute_reply.started":"2022-08-23T12:07:37.94188Z","shell.execute_reply":"2022-08-23T12:07:37.956618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Inspiration: https://www.tensorflow.org/tutorials/generative/pix2pix#build_an_input_pipeline_with_tfdata\ndef upsample(x, concat, target_filters, name, conv2dt_kernel_init_max, relu=True, dropout=0, debug=False):\n    filters = concat.shape[-1]\n    x_up = tf.keras.layers.Conv2DTranspose(\n            filters, # Number of Convolutional Filters\n            kernel_size=4, # Kernel Size\n            strides=2, # Kernel Steps\n            padding='SAME', # linear scaling\n            name=f'Conv2DTranspose_{name}', # Name of Layer\n            kernel_initializer='he_uniform',\n            use_bias=False,\n        )(x)\n    \n    concat = tf.keras.layers.BatchNormalization(\n        gamma_initializer=tf.constant_initializer(value=0.25),\n        name=f'BatchNormalization_{name}'\n    )(concat)\n    x = tf.keras.layers.Concatenate(name=f'Concatenate_{name}')([x_up, concat])\n    x = tf.nn.relu(x)\n    \n        \n    x = tf.keras.layers.Conv2D(target_filters, 3, padding='SAME', kernel_initializer='he_uniform', activation='relu', name=f'Conv2D_1_{name}')(x)\n    x = tf.keras.layers.Conv2D(target_filters, 3, padding='SAME', kernel_initializer='he_uniform', name=f'Conv2D_2_{name}')(x)\n    \n    if relu:\n        x = tf.nn.relu(x)\n    \n    x = tf.keras.layers.Dropout(dropout, name=f'Dropout_{name}')(x)\n\n    if debug:\n        return x, x_up, concat\n    else:\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:37.959365Z","iopub.execute_input":"2022-08-23T12:07:37.959666Z","iopub.status.idle":"2022-08-23T12:07:37.972311Z","shell.execute_reply.started":"2022-08-23T12:07:37.959641Z","shell.execute_reply":"2022-08-23T12:07:37.971459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\n\nA lot of debug code is present in the model construction, which will be later used to show the values during a forward pass. ","metadata":{}},{"cell_type":"code","source":"GCS_WEIGHTS_PATH = KaggleDatasets().get_gcs_path('efficientnetv2-head-1x1-endpoint-v2')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:37.973462Z","iopub.execute_input":"2022-08-23T12:07:37.974265Z","iopub.status.idle":"2022-08-23T12:07:38.493958Z","shell.execute_reply.started":"2022-08-23T12:07:37.974228Z","shell.execute_reply":"2022-08-23T12:07:38.49284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(dropout_decoder=0, dropout_cnn=0, file_path=None, lr=1e-3, eps=1e-7, clipnorm=5.0, wd_coef=1e-2, cnn_trainable=True, debug=DEBUG):\n    # enable XLA optmizations\n    tf.config.optimizer.set_jit(True)\n    # Set seed for deterministic weights initialization\n    seed_everything()\n    \n    with strategy.scope():\n        # EfficientNetV2 Backbone # \n        cnn = effnetv2_model.get_model(f'efficientnet-{EFN_SIZE}', include_top=False, weights=None if IS_TPU else 'jft', model_config={ 'conv_dropout': dropout_cnn })\n        if IS_TPU:\n            WEIGHT_PATH = f'{GCS_WEIGHTS_PATH}/noisy_student_efficientnet-{EFN_SIZE}'\n            ckpt = tf.train.latest_checkpoint(WEIGHT_PATH)\n            cnn.load_weights(ckpt)\n\n        # Inputs, note the names are equal to the dictionary keys in the dataset\n        image = tf.keras.layers.Input(INPUT_SHAPE, name='image', dtype=tf.float32)\n        image_norm = tf.cast(image, tf.float32) / 255\n        image_norm = tf.keras.layers.experimental.preprocessing.Normalization(mean=MEAN, variance=STD, dtype=tf.float32)(image_norm)\n\n        embedding, up6, up5, up4, up3, up2, up1 = cnn(image_norm, with_endpoints=True)\n        \n        if debug:\n            print(f'embedding shape: {embedding.shape} up1 shape: {up1.shape}, up2 shape: {up2.shape}')\n            print(f'up3 shape: {up3.shape}, up4 shape: {up4.shape}, up5 shape: {up5.shape}, up6 shape: {up6.shape}')\n                \n        if debug:\n            dec0, x0, (xs0, xs1, xs2, xs3, xs4) = ASPP(up2, debug=True)\n        else:\n            dec0 = ASPP(up2)\n        \n        dec0 = tf.keras.layers.Dropout(0.30)(dec0)\n\n        if debug:\n            dec1, dec1_up, dec1_concat = upsample(dec0, up3, up4.shape[-1] * 4, 'upsample1', 0.02, dropout=dropout_decoder, debug=True)\n            dec2, dec2_up, dec2_concat = upsample(dec1, up4, up5.shape[-1] * 2, 'upsample2', 0.02, dropout=dropout_decoder, debug=True)\n            dec3, dec3_up, dec3_concat = upsample(dec2, up5, up6.shape[-1] * 2, 'upsample3', 0.02, debug=True)\n            dec4, dec4_up, dec4_concat = upsample(dec3, up6, 32, 'upsample4', 0.02, debug=True)\n        else:\n            dec1 = upsample(dec0, up3, up4.shape[-1] * 4, 'upsample1', 0.02, dropout=dropout_decoder)\n            dec2 = upsample(dec1, up4, up5.shape[-1] * 2, 'upsample2', 0.02, dropout=dropout_decoder)\n            dec3 = upsample(dec2, up5, up6.shape[-1] * 2, 'upsample3', 0.02)\n            dec4 = upsample(dec3, up6, 64, 'upsample4', 0.02)\n        \n        if debug:\n            dec_fpn, dec_fpn_xs = FPN([dec0, dec1, dec2, dec3], 32, dec4, debug=True)\n        else:\n            dec_fpn = FPN([dec0, dec1, dec2, dec3], 32, dec4)\n        \n        if debug:\n            print(f'dec0 shape: {dec0.shape}, dec1 shape: {dec1.shape}, dec2 shape: {dec2.shape}, dec3 shape: {dec3.shape}, dec4 shape: {dec4.shape}')\n            print(f'dec_fpn shape: {dec_fpn.shape}')\n\n        # Head\n        x = tf.keras.layers.Dropout(0.10)(dec_fpn)\n        x = tf.keras.layers.Conv2D(\n            filters=1,\n            kernel_size=1,\n            padding='SAME',\n            kernel_initializer=tf.random_normal_initializer(0.00, 0.05),\n            activation=None if debug else 'sigmoid',\n            name='Conv2D_3_head'\n        )(x)\n        output = tf.image.resize(x, size=[IMG_SIZE, IMG_SIZE], method=tf.image.ResizeMethod.BILINEAR)\n\n        # We will use the famous Adam optimizer for fast learning\n        optimizer = tf.optimizers.Adam(learning_rate=lr, epsilon=eps, clipnorm=clipnorm)\n\n        # Loss\n        loss = tf.keras.losses.BinaryCrossentropy()\n        \n        # Metrics\n        metrics = [\n            SEMANTIC_LOSS_FUNCTIONS.iou,\n            tf.keras.metrics.Precision(),\n            tf.keras.metrics.Recall(),\n            tf.keras.metrics.AUC(),\n            tf.keras.metrics.BinaryAccuracy(),\n        ]\n\n        if debug:\n            model = tf.keras.models.Model(inputs=image, outputs=[\n                image_norm,\n                embedding, up6, up5, up4, up3, up2, up1,\n                dec0, x0, xs0, xs1, xs2, xs3, xs4,\n                dec1, dec1_up, dec1_concat,\n                dec2, dec2_up, dec2_concat,\n                dec3, dec3_up, dec3_concat,\n                dec4, dec4_up, dec4_concat,\n                dec_fpn, dec_fpn_xs, output\n                \n            ])\n        else:\n            model = tf.keras.models.Model(inputs=image, outputs=[output])\n        \n        model.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n\n        if file_path:\n            print('Loading pretrained weights...')\n            model.load_weights(file_path)\n\n        return model","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:38.495266Z","iopub.execute_input":"2022-08-23T12:07:38.496329Z","iopub.status.idle":"2022-08-23T12:07:38.51975Z","shell.execute_reply.started":"2022-08-23T12:07:38.496263Z","shell.execute_reply":"2022-08-23T12:07:38.518341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pretrained File Path: '/kaggle/input/sartorius-training-dataset/model.h5'\ntf.keras.backend.clear_session()\ngc.collect()\n    \nmodel = get_model(file_path=None, debug=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:07:38.521743Z","iopub.execute_input":"2022-08-23T12:07:38.522408Z","iopub.status.idle":"2022-08-23T12:08:11.501239Z","shell.execute_reply.started":"2022-08-23T12:07:38.522368Z","shell.execute_reply":"2022-08-23T12:08:11.500533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot model summary\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:08:11.503572Z","iopub.execute_input":"2022-08-23T12:08:11.504193Z","iopub.status.idle":"2022-08-23T12:08:11.50786Z","shell.execute_reply.started":"2022-08-23T12:08:11.504158Z","shell.execute_reply":"2022-08-23T12:08:11.507149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:08:11.509103Z","iopub.execute_input":"2022-08-23T12:08:11.509695Z","iopub.status.idle":"2022-08-23T12:08:16.04393Z","shell.execute_reply.started":"2022-08-23T12:08:11.509638Z","shell.execute_reply":"2022-08-23T12:08:16.042845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentations\n\nSince the train set is different from the train set heavy augmentations are required to make the model generalize well.","metadata":{}},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"# Function to benchmark the dataset\ndef benchmark_dataset(dataset, num_epochs=3, n_steps_per_epoch=10, bs=BATCH_SIZE):\n    start_time = time.perf_counter()\n    for epoch_num in range(num_epochs):\n        for idx, (images, labels) in enumerate(dataset.take(n_steps_per_epoch + 1)):\n            if idx == 0:\n                epoch_start = time.perf_counter()\n            elif idx == 1 and epoch_num == 0:\n                print(f'image shape: {images.shape}, image dtype: {images.dtype}')\n            else:\n                pass\n        epoch_t = time.perf_counter() - epoch_start\n        mean_step_t = round(epoch_t / n_steps_per_epoch * 1000, 1)\n        n_imgs_per_s = int(1 / (mean_step_t / 1000) * bs)\n        print(f'epoch {epoch_num} took: {round(epoch_t, 2)} sec, mean step duration: {mean_step_t}ms, images/s: {n_imgs_per_s}')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:08:16.045017Z","iopub.execute_input":"2022-08-23T12:08:16.045327Z","iopub.status.idle":"2022-08-23T12:08:16.052965Z","shell.execute_reply.started":"2022-08-23T12:08:16.045264Z","shell.execute_reply":"2022-08-23T12:08:16.051902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plots a batch of images\ndef show_batch(dataset, rows=4, cols=4):\n    imgs, lbls = next(iter(dataset))\n    imgs = imgs.numpy()\n    # Plot\n    fig, axes = plt.subplots(nrows=rows, ncols=cols, figsize=(rows*4, cols*4))\n    for r in range(rows):\n        for c in range(cols // 2):\n            img = imgs[r*cols+c]\n            axes[r, c*2].imshow(img)\n            axes[r, c*2].set_title(f'std: {img.std():.1f}')\n            lbl = lbls[r*cols+c]\n            axes[r, c*2+1].imshow(lbl)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:08:16.054162Z","iopub.execute_input":"2022-08-23T12:08:16.054614Z","iopub.status.idle":"2022-08-23T12:08:16.064993Z","shell.execute_reply.started":"2022-08-23T12:08:16.054582Z","shell.execute_reply":"2022-08-23T12:08:16.063939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Decodes the TFRecords\ndef decode_image(record_bytes, val):\n    features = tf.io.parse_single_example(record_bytes, {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'mask': tf.io.FixedLenFeature([], tf.string),\n        'organ': tf.io.FixedLenFeature([], tf.string),\n    })\n\n    image = tf.io.parse_tensor(features['image'], out_type=tf.uint8)\n    image = tf.reshape(image, [IMG_SIZE, IMG_SIZE, N_CHANNELS])\n       \n    mask = tf.io.parse_tensor(features['mask'], out_type=tf.uint8)\n    mask = tf.reshape(mask, [IMG_SIZE, IMG_SIZE, 1])\n    \n    # Ogran\n    organ = features['organ']\n    \n    return image, mask, organ","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:08:16.066226Z","iopub.execute_input":"2022-08-23T12:08:16.067084Z","iopub.status.idle":"2022-08-23T12:08:16.078054Z","shell.execute_reply.started":"2022-08-23T12:08:16.067027Z","shell.execute_reply":"2022-08-23T12:08:16.077251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augment_image(image, mask, organ, val, return_organ):\n    \n    if not val:\n        rotations = tf_rand_int(0, 4, dtype=tf.int32)\n\n        image = tf.image.rot90(image, rotations)\n        mask = tf.image.rot90(mask, rotations)\n        \n        if one_in(2):\n            image = tf.image.transpose(image)\n            mask = tf.image.transpose(mask)\n        \n        # Pixel Level Augmentations\n        if one_in(2):\n            image = tf.image.random_hue(image, 0.2)\n        if one_in(2):\n            image = tf.image.random_saturation(image, 0.80, 1.20)\n        if one_in(2):\n            image = tf.image.random_contrast(image, 0.80, 1.20)\n        if one_in(2):\n            image = tf.image.random_brightness(image, 0.10)\n        if one_in(2):\n            image = tf.image.random_jpeg_quality(image, 75, 100)\n        \n        # Random Crop\n        offset_x = tf.random.uniform([], 0, tf.cast(IMG_SIZE * 0.50, tf.int32), dtype=tf.int32)\n        img_size_crop = IMG_SIZE - offset_x\n        if offset_x > 0:\n            offset_y = tf.random.uniform([], 0, offset_x, dtype=tf.int32)\n        else:\n            offset_y = tf.constant(0, dtype=tf.int32)\n\n        # Crop\n        if one_in(2):\n            image = tf.slice(image, [offset_x, offset_y, 0], [img_size_crop, img_size_crop, N_CHANNELS])\n            mask = tf.slice(mask, [offset_x, offset_y, 0], [img_size_crop, img_size_crop, 1])\n            \n            image = tf.image.resize(image, [IMG_SIZE, IMG_SIZE], method=tf.image.ResizeMethod.BICUBIC)\n            image = tf.cast(image / tf.reduce_max(image) * 255, tf.uint8)\n            \n            mask = tf.image.resize(mask, [IMG_SIZE, IMG_SIZE], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)\n        \n        # Rotate\n        if one_in(2):\n            angle = tf.random.uniform([], -45 * math.pi / 180, 45 * math.pi / 180, dtype=tf.float32)\n            image = tfa.image.rotate(image, angle, interpolation='bilinear', fill_mode='reflect')\n            mask = tfa.image.rotate(mask, angle, interpolation='nearest', fill_mode='reflect')\n        \n        # Resize\n        image = tf.cast(image / tf.reduce_max(image) * 255, tf.uint8)\n        \n        # Explicit Reshape for TPU\n        image = tf.reshape(image, [IMG_SIZE, IMG_SIZE, N_CHANNELS])\n        mask = tf.reshape(mask, [IMG_SIZE, IMG_SIZE, 1])\n\n    if return_organ:\n        return image, mask, organ\n    else:\n        return image, mask","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:08:16.079203Z","iopub.execute_input":"2022-08-23T12:08:16.079571Z","iopub.status.idle":"2022-08-23T12:08:16.096125Z","shell.execute_reply.started":"2022-08-23T12:08:16.079547Z","shell.execute_reply":"2022-08-23T12:08:16.094967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path('hubmap-patched-tfrecords-300x300')\nprint(f'GCS_DS_PATH: {GCS_DS_PATH}')\n\nTFRECORDS_FILE_PATHS = np.array(tf.io.gfile.glob(f'{GCS_DS_PATH}/*.tfrecords'))\nTFRECORDS_FILE_PATHS = np.array(\n    sorted(TFRECORDS_FILE_PATHS, key=lambda fp: int(fp.split('.')[-2].split('_')[-1]))\n)\nprint(f'Found {len(TFRECORDS_FILE_PATHS)} TFRecords')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:08:16.097222Z","iopub.execute_input":"2022-08-23T12:08:16.097463Z","iopub.status.idle":"2022-08-23T12:08:16.184877Z","shell.execute_reply.started":"2022-08-23T12:08:16.097441Z","shell.execute_reply":"2022-08-23T12:08:16.183792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check whether file paths are correctly ordered\npd.options.display.max_colwidth = 999\ndisplay(pd.DataFrame(TFRECORDS_FILE_PATHS, columns=['File Path']).sample(10, random_state=SEED))","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:08:16.185966Z","iopub.execute_input":"2022-08-23T12:08:16.186337Z","iopub.status.idle":"2022-08-23T12:08:16.192801Z","shell.execute_reply.started":"2022-08-23T12:08:16.186265Z","shell.execute_reply":"2022-08-23T12:08:16.19155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_SAMPLES_PER_TFRECORD = np.load('/kaggle/input/hubmap-patched-tfrecords-300x300/N_SAMPLES_PER_TFRECORD.npy')\nprint(f'N_SAMPLES_PER_TFRECORD shape: {N_SAMPLES_PER_TFRECORD.shape}, dtype: {N_SAMPLES_PER_TFRECORD.dtype}')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:08:16.193927Z","iopub.execute_input":"2022-08-23T12:08:16.194704Z","iopub.status.idle":"2022-08-23T12:08:16.207985Z","shell.execute_reply.started":"2022-08-23T12:08:16.194674Z","shell.execute_reply":"2022-08-23T12:08:16.207324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(tfrecord_idxs=None, bs=BATCH_SIZE, idxs=None, return_steps=False, val=False, return_organ=False, debug=False):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False\n    if IS_TPU:\n        npr = 1 if val else AUTO\n    else:\n        npr = 1 if val else cpu_count()\n    \n    if tfrecord_idxs is None:\n        dataset = tf.data.TFRecordDataset(TFRECORDS_FILE_PATHS, num_parallel_reads=npr)\n    else:\n        dataset = tf.data.TFRecordDataset(TFRECORDS_FILE_PATHS[tfrecord_idxs], num_parallel_reads=npr)\n    \n    dataset = dataset.map(\n        lambda record_bytes: decode_image(record_bytes, val), num_parallel_calls=AUTO if IS_TPU else 1\n    )\n    \n    # Cache dataset to speedup dataloader\n    if not debug:\n        dataset = dataset.cache()\n    \n    if not val and not debug:\n        dataset = dataset.with_options(ignore_order)\n        dataset = dataset.shuffle(128)\n        dataset = dataset.repeat()\n    \n    # Augment\n    dataset = dataset.map(\n        lambda image, mask, organ: augment_image(image, mask, organ, val, return_organ), num_parallel_calls=npr\n    )\n\n    dataset = dataset.batch(bs, drop_remainder=True)\n    dataset = dataset.prefetch(AUTO)\n    \n    if return_steps:\n        return dataset, math.ceil(N_SAMPLES_PER_TFRECORD[tfrecord_idxs].sum() / bs)\n    else:\n        return dataset","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:08:16.214034Z","iopub.execute_input":"2022-08-23T12:08:16.214903Z","iopub.status.idle":"2022-08-23T12:08:16.223582Z","shell.execute_reply.started":"2022-08-23T12:08:16.214876Z","shell.execute_reply":"2022-08-23T12:08:16.222676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Benchmark Dataset\nbenchmark_dataset(get_dataset(debug=True))","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:08:16.224709Z","iopub.execute_input":"2022-08-23T12:08:16.225323Z","iopub.status.idle":"2022-08-23T12:08:24.236946Z","shell.execute_reply.started":"2022-08-23T12:08:16.225264Z","shell.execute_reply":"2022-08-23T12:08:24.236236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sanity check\nimages, labels = next(iter(get_dataset(debug=True)))\nprint(f'images shape: {images.shape}, labels shape: {labels.shape}')\nprint(f'images dtype: {images.dtype}, labels dtype: {labels.dtype}')\n\npercentiles = [0.01, 0.05, 0.10, 0.25, 0.40 ,0.50, 0.60, 0.75, 0.90, 0.95, 0.99]\ndisplay(pd.Series(images.numpy().flatten()).describe(percentiles=percentiles).to_frame(name='images').T)\ndisplay(pd.Series(labels.numpy().flatten()).value_counts().to_frame(name='labels').T)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:08:24.238853Z","iopub.execute_input":"2022-08-23T12:08:24.239145Z","iopub.status.idle":"2022-08-23T12:08:25.260263Z","shell.execute_reply.started":"2022-08-23T12:08:24.239122Z","shell.execute_reply":"2022-08-23T12:08:25.258944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show what we will be training on\nshow_batch(get_dataset(bs=16, debug=True))","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:08:25.261714Z","iopub.execute_input":"2022-08-23T12:08:25.261978Z","iopub.status.idle":"2022-08-23T12:08:29.1397Z","shell.execute_reply.started":"2022-08-23T12:08:25.261955Z","shell.execute_reply":"2022-08-23T12:08:29.138353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weights Initialization\n\nThe following function shows the intermediate values during a forward pass. It is important for values to not explode (become huge) or vanish (become tiny), as this will affect the gradient update by having too large or too small updates. Moreover, many values are concatenated in a UNet architecture, those values should have equal distributions, otherwise consecutive layers will only pay attention to a single part of the input. Lastly, the output sigmoid function should be initialized to be centered around 0.50 to start with large gradients. If all values are near 0 or near 1 at the start gradients will be tiny as the slope of the sigmoid function is near straight at 0 or 1.\n\nIt is a lot of work to get statistics on all intermediate values and plot the output, however it gives you a good understanding of how a neural network behaves, and, more importantly, why it could not behave the way you expect it to do.","metadata":{}},{"cell_type":"code","source":"def weight_init_analysis():\n    # Get large batch\n    images, _ = next(iter(get_dataset(bs=8, debug=True)))\n    print(f'images shape: {images.shape}')\n    \n    tf.keras.backend.clear_session()\n    gc.collect()\n    model = get_model(file_path=None, debug=True)\n    \n    (\n        image_norm,\n        embedding, up6, up5, up4, up3, up2, up1,\n        dec0, x0, xs0, xs1, xs2, xs3, xs4, \n        dec1, dec1_up, dec1_concat,\n        dec2, dec2_up, dec2_concat,\n        dec3, dec3_up, dec3_concat,\n        dec4, dec4_up, dec4_concat,\n        dec_fpn, dec_fpn_xs, output\n    ) = model(images, training=False) \n\n    percentiles = [0.01, 0.05, 0.10, 0.25, 0.40, 0.50, 0.60, 0.75, 0.90, 0.95, 0.99]\n\n    for v, v_name in zip(\n        [image_norm, embedding, up6, up5, up4, up3, up2, up1, dec0, x0, xs0, xs1, xs2, xs3, xs4, dec1, dec1_up, dec1_concat, dec2, dec2_up, dec2_concat, dec3, dec3_up, dec3_concat, dec4, dec4_up, dec4_concat, dec_fpn, dec_fpn_xs, output],\n        ['image_norm', 'embedding', 'up6', 'up5', 'up4', 'up3', 'up2', 'up1', 'dec0', 'x0', 'xs0', 'xs1', 'xs2', 'xs3', 'xs4', 'dec1', 'dec1_up', 'dec1_concat', 'dec2', 'dec2_up', 'dec2_concat', 'dec3', 'dec3_up', 'dec3_concat', 'dec4', 'dec4_up', 'dec4_concat', 'dec_fpn', 'dec_fpn_xs', 'output'],\n    ):\n        display(pd.Series(v.numpy().flatten()).describe(percentiles=percentiles).astype(float).round(2).to_frame(name=v_name).T)\n        print('=' * 50)\n\n    # Histogram\n    plt.figure(figsize=(15,5))\n    plt.title('Logits Output', size=24)\n    pd.Series(output.numpy().flatten()).plot(kind='hist', bins=100)\n    plt.grid()\n    plt.show()\n\n    # Histogram\n    plt.figure(figsize=(15,5))\n    plt.title('Sigmoid Output', size=24)\n    pd.Series(tf.math.sigmoid(output).numpy().flatten()).plot(kind='hist', bins=100)\n    plt.xticks(np.arange(0.0, 1.1, 0.1))\n    plt.grid()\n    plt.show()\n    \nweight_init_analysis()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:08:29.14205Z","iopub.execute_input":"2022-08-23T12:08:29.142724Z","iopub.status.idle":"2022-08-23T12:09:29.415542Z","shell.execute_reply.started":"2022-08-23T12:08:29.142689Z","shell.execute_reply":"2022-08-23T12:09:29.414307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:29.41713Z","iopub.execute_input":"2022-08-23T12:09:29.417799Z","iopub.status.idle":"2022-08-23T12:09:29.846303Z","shell.execute_reply.started":"2022-08-23T12:09:29.417769Z","shell.execute_reply":"2022-08-23T12:09:29.844839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Rate Scheduler\n\nAn learning rate scheduler is used with a cosine decay and exponential warmup.","metadata":{}},{"cell_type":"code","source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=EPOCHS_WHOLE):\n    \n    if current_step < num_warmup_steps:\n        return lr_max * 0.50 ** (num_warmup_steps - current_step)\n    else:\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:29.849441Z","iopub.execute_input":"2022-08-23T12:09:29.849767Z","iopub.status.idle":"2022-08-23T12:09:29.856671Z","shell.execute_reply.started":"2022-08-23T12:09:29.849739Z","shell.execute_reply":"2022-08-23T12:09:29.855247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_lr_schedule(lr_schedule, epochs):\n    fig = plt.figure(figsize=(20, 10))\n    plt.plot([None] + lr_schedule + [None])\n    # X Labels\n    x = np.arange(1, epochs + 1)\n    x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n    plt.xlim([1, epochs])\n    plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n    \n    # Increase y-limit for better readability\n    plt.ylim([0, max(lr_schedule) * 1.1])\n    \n    # Title\n    schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n    plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n    \n    # Plot Learning Rates\n    for x, val in enumerate(lr_schedule):\n        if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n            if x < len(lr_schedule) - 1:\n                if lr_schedule[x - 1] < val:\n                    ha = 'right'\n                else:\n                    ha = 'left'\n            elif x == 0:\n                ha = 'right'\n            else:\n                ha = 'left'\n            plt.plot(x + 1, val, 'o', color='black');\n            offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n            plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n    \n    plt.xlabel('Epoch', size=16, labelpad=5)\n    plt.ylabel('Learning Rate', size=16, labelpad=5)\n    plt.grid()\n    plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=3, lr_max=LR_MAX_WHOLE, num_cycles=0.50) for step in range(EPOCHS_WHOLE)]\nplot_lr_schedule(LR_SCHEDULE, epochs=EPOCHS_WHOLE)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:29.860225Z","iopub.execute_input":"2022-08-23T12:09:29.860564Z","iopub.status.idle":"2022-08-23T12:09:30.419722Z","shell.execute_reply.started":"2022-08-23T12:09:29.860539Z","shell.execute_reply":"2022-08-23T12:09:30.418726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Whole Model","metadata":{}},{"cell_type":"code","source":"ORGAN_PER_TFRECORD = np.load('../input/hubmap-patched-tfrecords-300x300/ORGAN_PER_TFRECORD.npy')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:30.420963Z","iopub.execute_input":"2022-08-23T12:09:30.42133Z","iopub.status.idle":"2022-08-23T12:09:30.432069Z","shell.execute_reply.started":"2022-08-23T12:09:30.421269Z","shell.execute_reply":"2022-08-23T12:09:30.430549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:30.433824Z","iopub.execute_input":"2022-08-23T12:09:30.434627Z","iopub.status.idle":"2022-08-23T12:09:30.438783Z","shell.execute_reply.started":"2022-08-23T12:09:30.434599Z","shell.execute_reply":"2022-08-23T12:09:30.437813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create KFOLDS\nFOLDS = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\nHISTORIES = dict()\nIDXS = { 'train_idxs': [], 'val_idxs': [] }\nfor fold, (train_idxs, val_idxs) in enumerate(FOLDS.split(X=TFRECORDS_FILE_PATHS, y=ORGAN_PER_TFRECORD)):\n    # Only train selected folds\n    if fold not in TRAIN_FOLDS:\n        continue\n    \n    IDXS['train_idxs'].append(train_idxs)\n    IDXS['val_idxs'].append(val_idxs)\n    train_dataset, train_steps_per_epoch = get_dataset(tfrecord_idxs=train_idxs, bs=BATCH_SIZE, return_steps=True)\n    val_dataset, val_steps_per_epoch = get_dataset(tfrecord_idxs=val_idxs, bs=N_PATCHES_PER_IMAGE * REPLICAS, return_steps=True, val=True)\n    print('=' * 80)\n    print(f'FOLD {fold}, train_steps_per_epoch: {train_steps_per_epoch}, val_steps_per_epoch: {val_steps_per_epoch}')\n    print('=' * 80)\n    \n    tf.keras.backend.clear_session()\n    gc.collect()\n    \n    model = get_model(file_path=None, cnn_trainable=True, lr=LR_MAX_WHOLE, eps=1e-5)\n    #model = get_model(file_path='../input/hubmap-training-tf-tpu-efficientnet-b8-640640-p/model_0.h5', cnn_trainable=True, lr=LR_MAX_WHOLE, eps=1e-5)\n    if fold == TRAIN_FOLDS[0]:\n        print(model.summary())\n    \n    HISTORIES[fold] = model.fit(\n        train_dataset,\n        steps_per_epoch = train_steps_per_epoch * COMBINE_EPOCHS,\n        validation_data = val_dataset,\n        epochs = EPOCHS_WHOLE,\n        verbose = 2,\n        callbacks = [\n            lr_callback,\n        ],\n    )\n    \n    model.save_weights(f'model_{fold}.h5')\n    \n    print('\\n' * 3)\n    \n    break","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:30.440248Z","iopub.execute_input":"2022-08-23T12:09:30.440668Z","iopub.status.idle":"2022-08-23T12:09:55.97467Z","shell.execute_reply.started":"2022-08-23T12:09:30.440645Z","shell.execute_reply":"2022-08-23T12:09:55.972972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Visualisation\n\nTraining and validation predictions are plotted as a sanity check.","metadata":{}},{"cell_type":"code","source":"def plot_results(dataset, nrows, ncols=4):\n    images, labels = next(iter(dataset))\n    \n    # Predict Masks\n    labels_pred = model(images, training=False)\n    \n    fig, axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=(ncols*8, nrows*6))\n    \n    for r, (img, lbl, lbl_pred) in enumerate(zip(images, labels, labels_pred)):\n        if r > nrows - 1: # zero indexed\n            break\n        # Plot Image\n        axes[r, 0].imshow(img)\n        axes[r, 0].set_title('Image', size=18)\n        axes[r, 0].axis(False)\n        \n        # Mask\n        axes[r, 1].imshow(lbl)\n        axes[r, 1].set_title('Mask', size=18)\n        axes[r, 1].axis(False)\n        \n        # Predicted Mask with Threshold\n        axes[r, 2].imshow(lbl_pred)\n        axes[r, 2].set_title('Mask Predicted', size=18)\n        axes[r, 2].axis(False)\n        \n        # Predicted Mask with Threshold\n        lbl_pred_th50 =tf.cast(lbl_pred > 0.50, tf.uint8)\n        axes[r, 3].imshow(lbl_pred_th50)\n        axes[r, 3].set_title('Mask Predicted Threshold 0.50', size=18)\n        axes[r, 3].axis(False)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.976608Z","iopub.status.idle":"2022-08-23T12:09:55.976953Z","shell.execute_reply.started":"2022-08-23T12:09:55.976799Z","shell.execute_reply":"2022-08-23T12:09:55.976817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Visualisation\nplot_results(get_dataset(bs=8, idxs=train_idxs), 8)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.978073Z","iopub.status.idle":"2022-08-23T12:09:55.978398Z","shell.execute_reply.started":"2022-08-23T12:09:55.978223Z","shell.execute_reply":"2022-08-23T12:09:55.978237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation Visualisation\nplot_results(get_dataset(bs=8, idxs=val_idxs, val=True), 8)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.979593Z","iopub.status.idle":"2022-08-23T12:09:55.979923Z","shell.execute_reply.started":"2022-08-23T12:09:55.979752Z","shell.execute_reply":"2022-08-23T12:09:55.979766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training History\n\nPlot training history metrics to get an understanding on how the model is learning.","metadata":{}},{"cell_type":"code","source":"def plot_history_metric(metric, f_best=np.argmax, ylim=None, yscale=None, yticks=None):\n    plt.figure(figsize=(20, 10))\n    \n    for fold, history in HISTORIES.items():\n        values = history.history[metric]\n        N_EPOCHS = len(values)\n        val = 'val' in ''.join(history.history.keys())\n        # Epoch Ticks\n        if N_EPOCHS <= 20:\n            x = np.arange(1, N_EPOCHS + 1)\n        else:\n            x = [1, 5] + [10 + 5 * idx for idx in range((N_EPOCHS - 10) // 5 + 1)]\n        x_ticks = np.arange(1, N_EPOCHS+1)\n\n        # Validation\n        if val:\n            val_values = history.history[f'val_{metric}']\n            val_argmin = f_best(val_values)\n            plt.plot(x_ticks, val_values, label=f'val_fold_{fold}')\n\n        # summarize history for accuracy\n        plt.plot(x_ticks, values, label=f'train_fold_{fold}')\n        argmin = f_best(values)\n        plt.scatter(argmin + 1, values[argmin], color='red', s=75, marker='o', label=f'train_best_fold_{fold}')\n        if val:\n            plt.scatter(val_argmin + 1, val_values[val_argmin], color='purple', s=75, marker='o', label=f'val_best_fold_{fold}')\n\n    plt.title(f'Model {metric}', fontsize=24, pad=10)\n    plt.ylabel(metric, fontsize=20, labelpad=10)\n\n    if ylim:\n        plt.ylim(ylim)\n\n    if yscale is not None:\n        plt.yscale(yscale)\n        \n    if yticks is not None:\n        plt.yticks(yticks, fontsize=16)\n\n    plt.xlabel('epoch', fontsize=20, labelpad=10)        \n    plt.tick_params(axis='x', labelsize=8)\n    plt.xticks(x, fontsize=16) # set tick step to 1 and let x axis start at 1\n    plt.yticks(fontsize=16)\n    \n    plt.legend(prop={'size': 10})\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.981223Z","iopub.status.idle":"2022-08-23T12:09:55.981535Z","shell.execute_reply.started":"2022-08-23T12:09:55.981396Z","shell.execute_reply":"2022-08-23T12:09:55.981409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('loss', f_best=np.argmin, yscale='log')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.982499Z","iopub.status.idle":"2022-08-23T12:09:55.982783Z","shell.execute_reply.started":"2022-08-23T12:09:55.982639Z","shell.execute_reply":"2022-08-23T12:09:55.982652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('precision', ylim=(0,1), yticks=np.arange(0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.983579Z","iopub.status.idle":"2022-08-23T12:09:55.983889Z","shell.execute_reply.started":"2022-08-23T12:09:55.983715Z","shell.execute_reply":"2022-08-23T12:09:55.983736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('recall', ylim=(0,1), yticks=np.arange(0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.984902Z","iopub.status.idle":"2022-08-23T12:09:55.985172Z","shell.execute_reply.started":"2022-08-23T12:09:55.985039Z","shell.execute_reply":"2022-08-23T12:09:55.985052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('iou', ylim=(0,1), yticks=np.arange(0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.985969Z","iopub.status.idle":"2022-08-23T12:09:55.986233Z","shell.execute_reply.started":"2022-08-23T12:09:55.986101Z","shell.execute_reply":"2022-08-23T12:09:55.986114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('auc', ylim=(0,1), yticks=np.arange(0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.987321Z","iopub.status.idle":"2022-08-23T12:09:55.987595Z","shell.execute_reply.started":"2022-08-23T12:09:55.987465Z","shell.execute_reply":"2022-08-23T12:09:55.987478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('binary_accuracy', ylim=(0,1), yticks=np.arange(0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.988398Z","iopub.status.idle":"2022-08-23T12:09:55.98907Z","shell.execute_reply.started":"2022-08-23T12:09:55.988898Z","shell.execute_reply":"2022-08-23T12:09:55.988914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation Predictions","metadata":{}},{"cell_type":"code","source":"# Validation Dataset\ndataset = get_dataset(\n        tfrecord_idxs=val_idxs,\n        bs=len(val_idxs),\n        val=True,\n        return_organ=True,\n        debug=True,\n    )\n\n# Validation images, masks and organs\nval_images, val_masks, val_organs = next(iter(dataset))\n# Cast from Tensorflow to Numpy\nval_images = val_images.numpy()\nval_masks = val_masks.numpy()\nval_organs = val_organs.numpy()\nprint(f'val_images shape: {val_images.shape}, val_masks shape: {val_masks.shape}, val_organs shape: {val_organs.shape}')\n\n# Validation Predictions\nVAL_Y_PREDS = model.predict(val_images, verbose=1, batch_size=len(val_idxs))\nprint(f'VAL_Y_PREDS shape: {VAL_Y_PREDS.shape}, VAL_Y_PREDS dtype: {VAL_Y_PREDS.dtype}')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.990335Z","iopub.status.idle":"2022-08-23T12:09:55.990618Z","shell.execute_reply.started":"2022-08-23T12:09:55.990479Z","shell.execute_reply":"2022-08-23T12:09:55.990492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mean Intersection Over Union By Threshold\n\nImportant analysis on the actual leaderboard metric, intersection over union, in general and per organ at different thresholds. The mean IoU per organ will vary widely, with the lung being by far the hardest to predict with a ~0.10 IoU.","metadata":{}},{"cell_type":"code","source":"def iou(y_true, y_pred):\n    intersection = np.count_nonzero(y_true * y_pred)\n    union = np.count_nonzero(y_true + y_pred)\n    return intersection / union","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.991945Z","iopub.status.idle":"2022-08-23T12:09:55.992228Z","shell.execute_reply.started":"2022-08-23T12:09:55.992088Z","shell.execute_reply":"2022-08-23T12:09:55.992101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predictions and true labels of validation dataset\ndef get_y_true_y_pred():\n    thresholds = np.arange(0, 1.01, 0.01)\n    IoUs = {}\n    for t in thresholds:\n        IoUs[t] = []\n        \n    IoUsOrgans = {}\n    for o in ORGANS:\n        IoUsOrgans[o] = {}\n        for t in thresholds:\n            IoUsOrgans[o][t] = []\n    \n    for idx, (image, y_true, organ, y_pred) in enumerate(zip(tqdm(val_images), val_masks, val_organs, VAL_Y_PREDS)):\n        if idx == 0:\n            print(f'image shape: {image.shape}, y_true shape: {y_true.shape}')\n            print(f'organs: {organ.decode()}, y_pred shape: {y_pred.shape}')\n        \n        # Compute IoU for each threshold\n        o = organ.decode()\n        for t in thresholds:\n            IoU = iou(y_true, (y_pred > t).astype(np.int8))\n            IoUs[t].append(IoU)\n            IoUsOrgans[o][t].append(IoU)\n    \n    return IoUs, IoUsOrgans\n\nIoUs, IoUsOrgans = get_y_true_y_pred()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.993555Z","iopub.status.idle":"2022-08-23T12:09:55.993844Z","shell.execute_reply.started":"2022-08-23T12:09:55.993695Z","shell.execute_reply":"2022-08-23T12:09:55.993708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mean IoU at Threshold","metadata":{}},{"cell_type":"code","source":"def plot_iou_by_threshold(ious, name):\n    thresholds = list(ious.keys())\n    MeanIoUs = [np.mean(v)for v in ious.values()]\n\n    plt.figure(figsize=(12,8))\n    plt.title(f'Mean IoU by Threshold {name}', size=24)\n    plt.plot(thresholds, MeanIoUs)\n    plt.grid()\n    plt.xlabel('Threshold', size=16)\n    plt.ylabel('Mean IoU', size=16)\n    plt.xticks(size=12)\n    plt.yticks(size=12)\n    plt.ylim(0,1)\n\n    # Best Threshold\n    arg_best = np.argmax(MeanIoUs)\n    threshold_best = thresholds[arg_best]\n    mean_iou_best = MeanIoUs[arg_best]\n    plt.scatter(threshold_best, mean_iou_best, color='red', s=100, marker='o', label=f'Best Mean IoU ({mean_iou_best:.3f}) at Threshold {threshold_best:.3f}')\n    plt.legend(prop={'size': 16})\n\n    plt.show()\n    \n    return threshold_best","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.995381Z","iopub.status.idle":"2022-08-23T12:09:55.995664Z","shell.execute_reply.started":"2022-08-23T12:09:55.995525Z","shell.execute_reply":"2022-08-23T12:09:55.995538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Global Mean Intersection over Union at Threshold\nthreshold_best = plot_iou_by_threshold(IoUs, 'All')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.996623Z","iopub.status.idle":"2022-08-23T12:09:55.996905Z","shell.execute_reply.started":"2022-08-23T12:09:55.99677Z","shell.execute_reply":"2022-08-23T12:09:55.996783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Per Organ Mean Intersection over Union at Threshold\nfor organ, ious in IoUsOrgans.items():\n    plot_iou_by_threshold(ious, organ)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.997889Z","iopub.status.idle":"2022-08-23T12:09:55.998163Z","shell.execute_reply.started":"2022-08-23T12:09:55.998027Z","shell.execute_reply":"2022-08-23T12:09:55.99804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# IoU Distribution Best Threshold","metadata":{}},{"cell_type":"code","source":"display(pd.Series(IoUs[threshold_best]).describe().apply(lambda v: f'{v:.2f}').to_frame(name='Value'))","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:55.999362Z","iopub.status.idle":"2022-08-23T12:09:55.999644Z","shell.execute_reply.started":"2022-08-23T12:09:55.99951Z","shell.execute_reply":"2022-08-23T12:09:55.999523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\npd.Series(IoUs[threshold_best]).plot(kind='hist')\nplt.title('IoU Distribution at Best Threshold', size=24)\nplt.grid()\nplt.xlabel('Threshold', size=16)\nplt.ylabel('Count', size=16)\nplt.xticks(size=12)\nplt.yticks(size=12)\nplt.xlim(0,1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:56.000679Z","iopub.status.idle":"2022-08-23T12:09:56.000989Z","shell.execute_reply.started":"2022-08-23T12:09:56.000826Z","shell.execute_reply":"2022-08-23T12:09:56.000839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation Prediction Visualization\n\nVisualize all validation predictions with the following color codes:\n\n* **RED** False Positive: model predicted pixel to belong to mask but the pixel is actually a background pixel\n\n* **BLUE** False Negative: model predicted pixel to be background but it actually is a pixel belonging to the mask\n\n* **GREEN** True Positive: model predicted the pixel to belong to the mask, and it actually is a mask belonging to the mask!","metadata":{}},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:56.004176Z","iopub.status.idle":"2022-08-23T12:09:56.004498Z","shell.execute_reply.started":"2022-08-23T12:09:56.004354Z","shell.execute_reply":"2022-08-23T12:09:56.004369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def merge_patches(patches):\n    if len(patches.shape) == 3:\n        return patches\n    \n    image = np.zeros(shape=[IMG_SIZE_FULL, IMG_SIZE_FULL, patches.shape[-1]], dtype=patches.dtype)\n    s = int(N_PATCHES_PER_IMAGE ** 0.50)\n    for r in range(s):\n        for c in range(s):\n            start_x = r * IMG_SIZE\n            end_x = (r + 1) * IMG_SIZE\n            start_y = c * IMG_SIZE\n            end_y = (c + 1) * IMG_SIZE\n            image[start_x:end_x, start_y:end_y] = patches[r * s + c]\n            \n    return image","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:56.005447Z","iopub.status.idle":"2022-08-23T12:09:56.00572Z","shell.execute_reply.started":"2022-08-23T12:09:56.005585Z","shell.execute_reply":"2022-08-23T12:09:56.005597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_validation_predictions():\n    for idx, (image, y_true, organ, y_pred) in enumerate(zip(tqdm(val_images), val_masks, val_organs, VAL_Y_PREDS)):\n        organ = organ.decode()\n        # Predicted Mask\n        y_pred = merge_patches(y_pred)\n        y_pred_binary = (y_pred > threshold_best).astype(np.uint8)\n        # Merge Image and Label patches\n        image = merge_patches(image)\n        y_true = merge_patches(y_true)\n        # Red = False Positive\n        r = ((y_pred_binary == 1) * (y_true == 0)).astype(np.uint8) \n        # Green = True Positive\n        g = ((y_pred_binary == 1) * (y_true == 1)).astype(np.uint8)\n        # Blue = False Negative\n        b = ((y_pred_binary == 0) * (y_true == 1)).astype(np.uint8)\n        # Error Visualization Using RGB\n        mask_error = np.stack((r, g, b), axis=2).squeeze() * 255\n\n        fig, axes = plt.subplots(1, 5, figsize=(20, 4))\n        axes[0].imshow(image)\n        axes[0].set_title(f'Image {organ} IoU: {IoUs[threshold_best][idx]:.2f}')\n        axes[1].imshow(y_true)\n        axes[1].set_title('Mask True')\n        axes[2].imshow(y_pred)\n        axes[2].set_title('Mask Pred')\n        axes[3].imshow(y_pred_binary)\n        axes[3].set_title('Mask Pred Binary')\n        axes[4].imshow(mask_error)\n        axes[4].set_title('Mask Error')\n        \n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:56.007025Z","iopub.status.idle":"2022-08-23T12:09:56.007334Z","shell.execute_reply.started":"2022-08-23T12:09:56.007161Z","shell.execute_reply":"2022-08-23T12:09:56.007174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\nplot_validation_predictions()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T12:09:56.008251Z","iopub.status.idle":"2022-08-23T12:09:56.008552Z","shell.execute_reply.started":"2022-08-23T12:09:56.008421Z","shell.execute_reply":"2022-08-23T12:09:56.008435Z"},"trusted":true},"execution_count":null,"outputs":[]}]}