{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Hello Fellow Kagglers,\n\nThis notebook demonstrates the inference process using an EfficientNet B8 based UNet model trained on a TPU.\n\n[Preprocessing Notebook](https://www.kaggle.com/code/markwijkhuizen/hubmap-patched-tfrecord-generation-visualization)\n\n[Training Notebook](https://www.kaggle.com/code/masterray/hubmap-training-tf-tpu-efficientnet-b8-640-640)","metadata":{}},{"cell_type":"markdown","source":"Copied from: https://www.kaggle.com/code/markwijkhuizen/hubmap-inference-tf-tpu-efficientnet-b7-640x640","metadata":{}},{"cell_type":"code","source":"!pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/torch-1.10.0+cu111-cp37-cp37m-linux_x86_64.whl\n!pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/torchvision-0.11.0+cu111-cp37-cp37m-linux_x86_64.whl\n!pip install ../input/mmdetection/addict-2.4.0-py3-none-any.whl\n!pip install ../input/mmdetection/yapf-0.31.0-py2.py3-none-any.whl\n!pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/mmcv_full-1.5.3-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/openmmlab-essential-repositories/openmmlab-repos/src/mmcls-0.23.1-py2.py3-none-any.whl\n!pip install ../input/mmdetection/terminaltables-3.1.0-py3-none-any.whl\n!pip install ../input/mmdetection/einops-0.4.1-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:26:43.489363Z","iopub.execute_input":"2022-09-22T14:26:43.490178Z","iopub.status.idle":"2022-09-22T14:31:47.444049Z","shell.execute_reply.started":"2022-09-22T14:26:43.490087Z","shell.execute_reply":"2022-09-22T14:31:47.443150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r ../input/mmseg-with-logits/mmsegmentation-master /kaggle/working/ && cd /kaggle/working/mmsegmentation-master && pip install -e . && cd ..","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:31:47.446232Z","iopub.execute_input":"2022-09-22T14:31:47.446491Z","iopub.status.idle":"2022-09-22T14:32:23.473351Z","shell.execute_reply.started":"2022-09-22T14:31:47.446454Z","shell.execute_reply":"2022-09-22T14:32:23.472429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import EfficientNet models with intermediate endpoints\nimport sys\nsys.path.append('../input/efficientnetv2-head-1x1-endpoint-v2/')\nsys.path.append('../input/efficientnetv2-head-1x1-endpoint-v2/efficientnetv2/')\nsys.path.append('./mmsegmentation-master')","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:23.475163Z","iopub.execute_input":"2022-09-22T14:32:23.475460Z","iopub.status.idle":"2022-09-22T14:32:23.481337Z","shell.execute_reply.started":"2022-09-22T14:32:23.475421Z","shell.execute_reply":"2022-09-22T14:32:23.480352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nimport tensorflow_addons as tfa\nimport matplotlib.pyplot as plt\n\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nfrom kaggle_datasets import KaggleDatasets\nfrom tqdm.notebook import tqdm\nfrom multiprocessing import cpu_count\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold\nimport albumentations as A\nfrom scipy.stats import mode\nfrom glob import glob\n\nimport effnetv2_model\nimport tifffile\nimport re\nimport os\nimport io\nimport time\nimport pickle\nimport math\nimport random\nimport sys\nimport cv2\nimport gc\nimport torch\n\nfrom mmseg.apis import init_segmentor, inference_segmentor\nfrom mmcv.utils import config\nimport mmcv\n\nprint(f'tensorflow version: {tf.__version__}')\nprint(f'tensorflow keras version: {tf.keras.__version__}')\nprint(f'python version: P{sys.version}')\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:23.483197Z","iopub.execute_input":"2022-09-22T14:32:23.483475Z","iopub.status.idle":"2022-09-22T14:32:34.145829Z","shell.execute_reply.started":"2022-09-22T14:32:23.483445Z","shell.execute_reply":"2022-09-22T14:32:34.145058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Seed all random number generators\ndef seed_everything(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nSEED = 42\nseed_everything(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.147303Z","iopub.execute_input":"2022-09-22T14:32:34.147622Z","iopub.status.idle":"2022-09-22T14:32:34.156661Z","shell.execute_reply.started":"2022-09-22T14:32:34.147578Z","shell.execute_reply":"2022-09-22T14:32:34.153843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Threshold to classify a pixel as mask\n# THRESHOLD = 0.1\n\nDEBUG = False\nIS_TPU = True\n\n# Image dimensions\nN_CHANNELS = 3\nN_PATCHES_PER_IMAGE = 1\n\n# INPUT_SHAPE = (PATCH_SIZE, PATCH_SIZE, N_CHANNELS)\n\n# EfficientNet version, b0/b1/b2/b3/s/m/l/xl/xxl\n# EFN_SIZE = 'b8'\nLR_MAX = 0.02\nEPOCHS = 30\nMOMENTUM = 0.00\n\n# Batch size\nBATCH_SIZE = 64\n\n# Dataset Mean and Standard Deviation\n# MEAN = np.load('/kaggle/input/hubmap-patched-tfrecords-300x300/MEAN.npy') # ?\n# STD = np.load('/kaggle/input/hubmap-patched-tfrecords-300x300/STD.npy') # ?\n\n# print(f'MEAN: {MEAN}, STD: {STD}')","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.158130Z","iopub.execute_input":"2022-09-22T14:32:34.158653Z","iopub.status.idle":"2022-09-22T14:32:34.186510Z","shell.execute_reply.started":"2022-09-22T14:32:34.158615Z","shell.execute_reply":"2022-09-22T14:32:34.185749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hardware Configuration","metadata":{}},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    TPU = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', TPU.master())\nexcept ValueError:\n    print('Running on GPU')\n    TPU = None\n\nif TPU:\n    tf.config.experimental_connect_to_cluster(TPU)\n    tf.tpu.experimental.initialize_tpu_system(TPU)\n    strategy = tf.distribute.experimental.TPUStrategy(TPU)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    gpus = tf.config.experimental.list_physical_devices('GPU')\n    for gpu in gpus:\n        tf.config.experimental.set_memory_growth(gpu, True)\n\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.188109Z","iopub.execute_input":"2022-09-22T14:32:34.188383Z","iopub.status.idle":"2022-09-22T14:32:34.313981Z","shell.execute_reply.started":"2022-09-22T14:32:34.188350Z","shell.execute_reply":"2022-09-22T14:32:34.312968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# FPN","metadata":{}},{"cell_type":"code","source":"def FPN(xs, output_channels, last_layer, debug=False):\n    def _conv(x):\n        x = tf.keras.layers.ZeroPadding2D(padding=1)(x)\n        x = tf.keras.layers.Conv2D(output_channels * 2, 3, padding='SAME', kernel_initializer='he_normal', activation='relu')(x)\n        x = tf.keras.layers.BatchNormalization()(x)\n        x = tf.keras.layers.ZeroPadding2D(padding=1)(x)\n        x = tf.keras.layers.Conv2D(output_channels, 3, padding='SAME', kernel_initializer='he_normal')(x)\n        x = tf.image.resize(x, size=target_size, method=tf.image.ResizeMethod.BILINEAR)\n        x = tf.nn.relu(x)\n        return x\n\n    target_size = last_layer.shape[1:3]\n    xs = tf.keras.layers.Concatenate()([_conv(x) for x in xs])\n    x = tf.keras.layers.Concatenate()([xs, last_layer])\n\n    if debug:\n        return x, xs\n    else:\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.316765Z","iopub.execute_input":"2022-09-22T14:32:34.317338Z","iopub.status.idle":"2022-09-22T14:32:34.326241Z","shell.execute_reply.started":"2022-09-22T14:32:34.317298Z","shell.execute_reply":"2022-09-22T14:32:34.325161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ASPP","metadata":{}},{"cell_type":"code","source":"def ASPP(x, mid_c=320, dilations=[1, 2, 3, 4], out_c=640, debug=False):\n    def _aspp_module(x, filters, kernel_size, padding, dilation, groups=1):\n        x = tf.keras.layers.ZeroPadding2D(padding=padding)(x)\n        x = tf.keras.layers.Conv2D(\n                filters=filters,\n                kernel_size=kernel_size,\n                dilation_rate=dilation,\n                groups=1 if IS_TPU else groups,\n                kernel_initializer='he_uniform',\n            )(x)\n        x = tf.keras.layers.BatchNormalization()(x)\n        x = tf.nn.relu(x)\n        \n        return x\n    \n    x0 = tf.math.reduce_max(x, axis=(1,2), keepdims=True)\n    x0 = tf.keras.layers.Conv2D(filters=mid_c, kernel_size=1, strides=1, kernel_initializer='he_uniform', use_bias=False)(x0)\n    x0 = tf.keras.layers.BatchNormalization(gamma_initializer=tf.constant_initializer(value=0.25))(x0)\n    x0 = tf.nn.relu(x0)\n                                  \n                                  \n    xs = (\n        [_aspp_module(x, mid_c, 1, padding=0, dilation=1)] +\n        [_aspp_module(x, mid_c, 3, padding=d, dilation=d, groups=4) for d in dilations]\n    )\n    \n    x0= tf.image.resize(x0, size=xs[0].shape[1:3])\n    x = tf.keras.layers.Concatenate()([x0] + xs)\n    x = tf.keras.layers.Conv2D(filters=out_c, kernel_size=1, kernel_initializer='he_uniform', use_bias=False)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.nn.relu(x)\n                       \n    if debug:\n        return x, x0, xs\n    else:\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.327820Z","iopub.execute_input":"2022-09-22T14:32:34.328209Z","iopub.status.idle":"2022-09-22T14:32:34.341341Z","shell.execute_reply.started":"2022-09-22T14:32:34.328173Z","shell.execute_reply":"2022-09-22T14:32:34.340185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Upsample","metadata":{}},{"cell_type":"code","source":"def PixelShuffle(x, upscale_factor=2):\n    _, w, h, c = x.shape\n    n = -1\n\n    c_out = c // upscale_factor ** 2\n    w_out = w * upscale_factor\n    h_out = h * upscale_factor\n\n    x = tf.reshape(x, [-1, upscale_factor, upscale_factor, w, h, c_out])\n    x = tf.transpose(x, [0, 3, 1, 4, 2, 5])\n    x = tf.reshape(x, [-1, w_out, h_out, c_out])\n\n    return x","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.345482Z","iopub.execute_input":"2022-09-22T14:32:34.345814Z","iopub.status.idle":"2022-09-22T14:32:34.354628Z","shell.execute_reply.started":"2022-09-22T14:32:34.345764Z","shell.execute_reply":"2022-09-22T14:32:34.353764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Inspiration: https://www.tensorflow.org/tutorials/generative/pix2pix#build_an_input_pipeline_with_tfdata\ndef upsample(x, concat, target_filters, name, conv2dt_kernel_init_max, relu=True, dropout=0, debug=False):\n#     x = PixelShuffle(x)\n\n    filters = concat.shape[-1]\n    x_up = tf.keras.layers.Conv2DTranspose(\n            filters, # Number of Convolutional Filters\n            kernel_size=4, # Kernel Size\n            strides=2, # Kernel Steps\n            padding='SAME', # linear scaling\n            name=f'Conv2DTranspose_{name}', # Name of Layer\n            kernel_initializer='he_uniform',\n            use_bias=False,\n        )(x)\n    \n    concat = tf.keras.layers.BatchNormalization(\n        gamma_initializer=tf.constant_initializer(value=0.25),\n        name=f'BatchNormalization_{name}'\n    )(concat)\n    x = tf.keras.layers.Concatenate(name=f'Concatenate_{name}')([x_up, concat])\n    x = tf.nn.relu(x)\n    \n        \n    x = tf.keras.layers.Conv2D(target_filters, 3, padding='SAME', kernel_initializer='he_uniform', activation='relu', name=f'Conv2D_1_{name}')(x)\n    x = tf.keras.layers.Conv2D(target_filters, 3, padding='SAME', kernel_initializer='he_uniform', name=f'Conv2D_2_{name}')(x)\n    \n    if relu:\n        x = tf.nn.relu(x)\n    \n    x = tf.keras.layers.Dropout(dropout, name=f'Dropout_{name}')(x)\n\n    if debug:\n        return x, x_up, concat\n    else:\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.355899Z","iopub.execute_input":"2022-09-22T14:32:34.356306Z","iopub.status.idle":"2022-09-22T14:32:34.367523Z","shell.execute_reply.started":"2022-09-22T14:32:34.356268Z","shell.execute_reply":"2022-09-22T14:32:34.366736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def get_model(dropout_decoder=0, dropout_cnn=0, file_path=None, efn_size=None, lr=1e-3, eps=1e-7, clipnorm=5.0, wd_coef=1e-2, cnn_trainable=True, mean_path=None, var_path=None, i_size=None):\n    with strategy.scope():\n        # EfficientNetV2 Backbone # \n        cnn = effnetv2_model.get_model(f'efficientnet-{efn_size}', include_top=False, weights=None, model_config={ 'conv_dropout': dropout_cnn })\n        cnn.trainable = cnn_trainable\n\n        # Inputs, note the names are equal to the dictionary keys in the dataset\n        image = tf.keras.layers.Input((i_size, i_size, N_CHANNELS), name='image', dtype=tf.float32)\n        image_norm = tf.cast(image, tf.float32) / 255\n        image_norm = tf.keras.layers.experimental.preprocessing.Normalization(\n            mean=np.load(mean_path),\n            variance=np.load(var_path),\n            dtype=tf.float32)(image_norm) # add mean and var\n\n        embedding, up6, up5, up4, up3, up2, up1 = cnn(image_norm, with_endpoints=True)\n        print(f'embedding shape: {embedding.shape} up1 shape: {up1.shape}, up2 shape: {up2.shape}')\n        print(f'up3 shape: {up3.shape}, up4 shape: {up4.shape}, up5 shape: {up5.shape}, up6 shape: {up6.shape}')\n        \n        dec0 = ASPP(up2)\n        dec0 = tf.keras.layers.Dropout(0.50)(dec0)\n\n        dec1 = upsample(dec0, up3, up4.shape[-1] * 4, 'upsample1', 0.02, dropout=dropout_decoder)\n        dec2 = upsample(dec1, up4, up5.shape[-1] * 2, 'upsample2', 0.02, dropout=dropout_decoder)\n        dec3 = upsample(dec2, up5, up6.shape[-1] * 2, 'upsample3', 0.02)\n        dec4 = upsample(dec3, up6, 64, 'upsample4', 0.02)\n        \n        print(f'dec0 shape: {dec0.shape}, dec1 shape: {dec1.shape}, dec2 shape: {dec2.shape}, dec3 shape: {dec3.shape}, dec4 shape: {dec4.shape}')\n        \n        dec_fpn = FPN([dec0, dec1, dec2, dec3], 32, dec4)\n        \n        print(f'dec_fpn shape: {dec_fpn.shape}')\n        \n        # Head\n        x = tf.keras.layers.Dropout(0.10)(dec_fpn)\n        x = tf.keras.layers.Conv2D(\n            filters=1,\n            kernel_size=1,\n            padding='SAME',\n            kernel_initializer=tf.random_normal_initializer(0.00, 0.05),\n            activation='sigmoid',\n            name='Conv2D_3_head'\n        )(x)\n        output = tf.image.resize(x, size=[i_size, i_size], method=tf.image.ResizeMethod.BILINEAR) # тоже заменить\n        \n        model = tf.keras.models.Model(inputs=image, outputs=output)\n\n        if file_path:\n            print('Loading pretrained weights...')\n            model.load_weights(file_path)\n            \n        model.trainable = False\n\n        return model","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.368873Z","iopub.execute_input":"2022-09-22T14:32:34.369284Z","iopub.status.idle":"2022-09-22T14:32:34.385809Z","shell.execute_reply.started":"2022-09-22T14:32:34.369246Z","shell.execute_reply":"2022-09-22T14:32:34.384653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Funtions","metadata":{}},{"cell_type":"code","source":"# Resized a tensor to the specified size\ndef resize_tensor(tensor, size=None, dtype=np.uint8):\n    return cv2.resize(tensor, [size, size], interpolation=cv2.INTER_CUBIC).astype(dtype)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.387282Z","iopub.execute_input":"2022-09-22T14:32:34.387567Z","iopub.status.idle":"2022-09-22T14:32:34.397768Z","shell.execute_reply.started":"2022-09-22T14:32:34.387531Z","shell.execute_reply":"2022-09-22T14:32:34.397008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\ndef get_mask(image_id, size):\n    row = train.loc[train['id'] == image_id].squeeze()\n    h, w = row[['img_height', 'img_width']]\n    mask = np.zeros(shape=[h * w], dtype=np.uint8)\n    s = row['rle'].split()\n    starts, lengths = [ np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2]) ]\n    starts -= 1\n    ends = starts + lengths\n    for lo, hi in zip(starts, ends):\n        mask[lo : hi] = 1\n        \n    mask = mask.reshape([h, w]).T\n        \n    mask = resize_tensor(mask, size)\n    \n    mask = np.expand_dims(mask, axis=2)\n        \n    return mask","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.399100Z","iopub.execute_input":"2022-09-22T14:32:34.399449Z","iopub.status.idle":"2022-09-22T14:32:34.412201Z","shell.execute_reply.started":"2022-09-22T14:32:34.399415Z","shell.execute_reply":"2022-09-22T14:32:34.411355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reads an image and returns the image and original image size\ndef get_image(image_id, folder, size, negative=True):\n    image = tifffile.imread(f'/kaggle/input/hubmap-organ-segmentation/{folder}_images/{image_id}.tiff')\n    if len(image.shape) == 5:\n        image = image.squeeze().transpose(1, 2, 0)\n    \n    # Image Size\n    image_size, _, _ = image.shape\n    \n    # Reverse pixels to make tissue colored and background black\n    if negative:\n        image = image - image.min()\n        image = image / (image.max() - image.min())\n        image = image * 255\n        image = 255 - image.astype(np.uint8)\n        \n    # Resize\n    image = resize_tensor(image, size)\n    \n    return image, image_size","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.413573Z","iopub.execute_input":"2022-09-22T14:32:34.413834Z","iopub.status.idle":"2022-09-22T14:32:34.422961Z","shell.execute_reply.started":"2022-09-22T14:32:34.413802Z","shell.execute_reply":"2022-09-22T14:32:34.422135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract patches from an image\ndef extract_patches(image, i_size):\n    _, _, c = image.shape\n    image = tf.expand_dims(image, 0)\n    image_patches = tf.image.extract_patches(image, [1,i_size,i_size,1], [1, i_size, i_size, 1], [1, 1, 1, 1], padding='SAME')\n    image_patches = tf.reshape(image_patches, [N_PATCHES_PER_IMAGE, i_size, i_size, c])\n    image_patches = image_patches.numpy()\n\n    return image_patches","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.424372Z","iopub.execute_input":"2022-09-22T14:32:34.424701Z","iopub.status.idle":"2022-09-22T14:32:34.433512Z","shell.execute_reply.started":"2022-09-22T14:32:34.424667Z","shell.execute_reply":"2022-09-22T14:32:34.432619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/bguberfain/memory-aware-rle-encoding\n#with transposed mask\ndef rle_encode_less_memory(img):\n    #the image should be transposed\n    pixels = img.T.flatten()\n    \n    # This simplified method requires first and last pixel to be zero\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] -= runs[::2]\n    \n    return ' '.join(str(x) for x in runs)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.434659Z","iopub.execute_input":"2022-09-22T14:32:34.434909Z","iopub.status.idle":"2022-09-22T14:32:34.444425Z","shell.execute_reply.started":"2022-09-22T14:32:34.434881Z","shell.execute_reply":"2022-09-22T14:32:34.443574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"# Test DataFrame\ntest = pd.read_csv('/kaggle/input/hubmap-organ-segmentation/test.csv')\n\ndisplay(test.head())","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.445776Z","iopub.execute_input":"2022-09-22T14:32:34.446149Z","iopub.status.idle":"2022-09-22T14:32:34.489634Z","shell.execute_reply.started":"2022-09-22T14:32:34.446111Z","shell.execute_reply":"2022-09-22T14:32:34.488881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reconstruct the original image from patches\ndef merge_patches(patches, size):\n    image = np.zeros(shape=[size, size, patches.shape[-1]], dtype=patches.dtype)\n    s = int(N_PATCHES_PER_IMAGE ** 0.50)\n    for r in range(s):\n        for c in range(s):\n            start_x = r * size\n            end_x = (r + 1) * size\n            start_y = c * size\n            end_y = (c + 1) * size\n            image[start_x:end_x, start_y:end_y] = patches[r * s + c]\n            \n    return image","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.490868Z","iopub.execute_input":"2022-09-22T14:32:34.491141Z","iopub.status.idle":"2022-09-22T14:32:34.498848Z","shell.execute_reply.started":"2022-09-22T14:32:34.491108Z","shell.execute_reply":"2022-09-22T14:32:34.498002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sanity Check","metadata":{}},{"cell_type":"code","source":"threshold_mapper = {\n    \"b8\": {\n        \"fold0\": {\n            \"mean\": 0.51,\n            \"kidney\": 0.63,\n            \"largeintestine\": 0.52,\n            \"lung\": 0.03,\n            \"prostate\": 0.56,\n            \"spleen\": 0.41\n        },\n        \"fold1\": {\n            \"mean\": 0.47,\n            \"kidney\": 0.63,\n            \"largeintestine\": 0.54,\n            \"lung\": 0.25,\n            \"prostate\": 0.42,\n            \"spleen\": 0.52\n        },\n        \"fold2\": {\n            \"mean\": 0.5,\n            \"kidney\": 0.58,\n            \"largeintestine\": 0.54,\n            \"lung\": 0.07,\n            \"prostate\": 0.42,\n            \"spleen\": 0.37\n        }\n    },\n    \"b7\": {\n        \"fold0\": { # val_iou: 0.6755\n            \"mean\": 0.44,\n            \"kidney\": 0.49,\n            \"largeintestine\": 0.48,\n            \"lung\": 0.02,\n            \"prostate\": 0.48,\n            \"spleen\": 0.37\n        },\n        \"fold1\": { # val_iou: 0.6813\n            \"mean\": 0.51,\n            \"kidney\": 0.56,\n            \"largeintestine\": 0.53,\n            \"lung\": 0.83,\n            \"prostate\": 0.42,\n            \"spleen\": 0.49\n        },\n        \"fold2\": { # val_iou: 0.6826\n            \"mean\": 0.44,\n            \"kidney\": 0.56,\n            \"largeintestine\": 0.5,\n            \"lung\": 0.01,\n            \"prostate\": 0.4,\n            \"spleen\": 0.41\n        }\n    },\n    \"b8_pseudo\": {\n        \"fold0\": { \n            \"mean\": 0.38,\n            \"kidney\": 0.54,\n            \"largeintestine\": 0.5,\n            \"lung\": 0.07,\n            \"prostate\": 0.48,\n            \"spleen\": 0.33\n        },\n        \"fold1\": { \n            \"mean\": 0.47,\n            \"kidney\": 0.54,\n            \"largeintestine\": 0.49,\n            \"lung\": 0.29,\n            \"prostate\": 0.3,\n            \"spleen\": 0.68\n        },\n        \"fold2\": { \n            \"mean\": 0.48,\n            \"kidney\": 0.52,\n            \"largeintestine\": 0.53,\n            \"lung\": 0.02,\n            \"prostate\": 0.39,\n            \"spleen\": 0.4\n        }\n    },\n    \"b7_pseudo\": {\n        \"fold0\": { \n            \"mean\": 0.45,\n            \"kidney\": 0.49,\n            \"largeintestine\": 0.46,\n            \"lung\": 0.01,\n            \"prostate\": 0.48,\n            \"spleen\": 0.25\n        },\n        \"fold1\": { \n            \"mean\": 0.48,\n            \"kidney\": 0.55,\n            \"largeintestine\": 0.47,\n            \"lung\": 0.58,\n            \"prostate\": 0.39,\n            \"spleen\": 0.35\n        },\n        \"fold2\": { \n            \"mean\": 0.45,\n            \"kidney\": 0.48,\n            \"largeintestine\": 0.5,\n            \"lung\": 0.01,\n            \"prostate\": 0.39,\n            \"spleen\": 0.27\n        }\n    },\n    \"b8_pseudo_768\": {\n        \"fold0\": { \n            \"mean\": 0.42,\n            \"kidney\": 0.52,\n            \"largeintestine\": 0.5,\n            \"lung\": 0.01,\n            \"prostate\": 0.4,\n            \"spleen\": 0.27\n        },\n        \"fold1\": { \n            \"mean\": 0.41,\n            \"kidney\": 0.55,\n            \"largeintestine\": 0.48,\n            \"lung\": 0.06,\n            \"prostate\": 0.32,\n            \"spleen\": 0.39\n        },\n        \"fold2\": { \n            \"mean\": 0.4,\n            \"kidney\": 0.5,\n            \"largeintestine\": 0.4,\n            \"lung\": 0.01,\n            \"prostate\": 0.28,\n            \"spleen\": 0.3\n        }\n    }\n}","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.500251Z","iopub.execute_input":"2022-09-22T14:32:34.500827Z","iopub.status.idle":"2022-09-22T14:32:34.514963Z","shell.execute_reply.started":"2022-09-22T14:32:34.500789Z","shell.execute_reply":"2022-09-22T14:32:34.514245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference Loop","metadata":{}},{"cell_type":"code","source":"torch.cuda.empty_cache()\n\nconfigs = [\n    '../input/hubmap-swinformer-large-stainaug-folds/swinformer_cfg.py',\n]\nmodels_with_folds = [\n    [\n        '../input/hubmap-swinformer-large-stainaug-folds/swinformer_fold_0.pth',\n        '../input/hubmap-swinformer-large-stainaug-folds/swinformer_fold_1.pth',\n    ],\n]\n\n\nmodels_mmseg = []\nfor idx,(cfg, folds) in enumerate(zip(configs, models_with_folds)):\n    print(f\"Model {idx+1} with config path: {cfg}\\n\")\n    cfg = config.Config.fromfile(cfg)\n    mdls = []\n    for fold in folds:\n        mdls.append(init_segmentor(cfg, fold, device='cuda'))\n    models_mmseg.append(mdls)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:32:34.517405Z","iopub.execute_input":"2022-09-22T14:32:34.518196Z","iopub.status.idle":"2022-09-22T14:33:41.894519Z","shell.execute_reply.started":"2022-09-22T14:32:34.518157Z","shell.execute_reply":"2022-09-22T14:33:41.892692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [\n    (\"../input/hubmap-640-b8-4folds\", \"b8\", \"b8\", \"../input/hubmap-patched-tfrecords-300x300/MEAN.npy\", \"../input/hubmap-patched-tfrecords-300x300/STD.npy\", 640),\n    (\"../input/hubmap-640-b7-4folds\", \"b7\", \"b7\", \"../input/hubmap-patched-tfrecords-300x300/MEAN.npy\", \"../input/hubmap-patched-tfrecords-300x300/STD.npy\", 640),\n    (\"../input/b8-pseudo-640\", \"b8\", \"b8_pseudo\", \"../input/data-pseudo-077/MEAN.npy\", \"../input/data-pseudo-077/STD.npy\", 640),\n    (\"../input/b7-pseudo-640\", \"b7\", \"b7_pseudo\", \"../input/data-pseudo-077/MEAN.npy\", \"../input/data-pseudo-077/STD.npy\", 640),\n    (\"../input/b8-pseudo-768\", \"b8\", \"b8_pseudo_768\", \"../input/data-768-pseudo/MEAN.npy\", \"../input/data-768-pseudo/STD.npy\", 768)]\nmodels_effnet = []\nfor model, size, mapper, mean_path, var_path, i_size in models:\n    mdls = []\n    for fold in range(2):\n        mdls.append(get_model(file_path=os.path.join(model, f\"model_{fold}.h5\"), efn_size=size, mean_path=mean_path, var_path=var_path, i_size=i_size))\n    models_effnet.append((mdls, size, mapper, i_size))","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:33:41.896014Z","iopub.execute_input":"2022-09-22T14:33:41.896293Z","iopub.status.idle":"2022-09-22T14:36:37.553015Z","shell.execute_reply.started":"2022-09-22T14:33:41.896257Z","shell.execute_reply":"2022-09-22T14:36:37.552223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\ntorch.cuda.empty_cache()\ngc.collect()\n\ntest_rows = []\ntf.keras.backend.clear_session()\n\n\n\nDATA = '../input/hubmap-organ-segmentation/test_images/'\nclasses_to_num = {'background':0, 'kidney':1, 'prostate':2, 'largeintestine':3, 'spleen':4, 'lung':5}\nfor row_idx, row in tqdm(test.iterrows(), total=len(test)):\n    \n#     image, image_size = get_image(row['id'], 'test')\n#     image_hflip = tf.image.flip_left_right(image)\n#     image_vflip = tf.image.flip_up_down(image)\n#     image_rot90 = tf.image.rot90(image, 1)\n\n#     image_patches = extract_patches(image)\n#     image_hflip_patches = extract_patches(image_hflip)\n#     image_vflip_patches = extract_patches(image_vflip)\n#     image_rot90_patches = extract_patches(image_rot90)\n\n    bins_from_models = []\n    for model, size, mapper, i_size in models_effnet:\n        bins_from_folds = []\n        \n        image, image_size = get_image(row['id'], 'test', i_size)\n        image_hflip = tf.image.flip_left_right(image)\n        image_vflip = tf.image.flip_up_down(image)\n        image_rot90 = tf.image.rot90(image, 1)\n\n        image_patches = extract_patches(image, i_size)\n        image_hflip_patches = extract_patches(image_hflip, i_size)\n        image_vflip_patches = extract_patches(image_vflip, i_size)\n        image_rot90_patches = extract_patches(image_rot90, i_size)\n        \n        for num_fold, fold in enumerate(model):\n            mask_patches_pred = fold.predict(image_patches)\n            mask_hflip_patches_pred = fold.predict(image_hflip_patches)\n            mask_vflip_patches_pred = fold.predict(image_vflip_patches)\n            mask_rot90_patches_pred = fold.predict(image_rot90_patches)\n\n            mask_pred = merge_patches(mask_patches_pred, i_size)\n            mask_hflip_pred = merge_patches(mask_hflip_patches_pred, i_size)\n            mask_vflip_pred = merge_patches(mask_vflip_patches_pred, i_size)\n            mask_rot90_pred = merge_patches(mask_rot90_patches_pred, i_size)\n\n            total_mask_pred = np.mean(\n                tf.squeeze(np.array(\n                    [mask_pred,\n                     tf.image.flip_left_right(mask_hflip_pred),\n                     tf.image.flip_up_down(mask_vflip_pred),\n                     tf.image.rot90(mask_rot90_pred, k=-1)\n                    ]\n                )), axis=0)\n\n            mask_binary = (resize_tensor(np.expand_dims(total_mask_pred, axis=-1), size=image_size, dtype=np.float32) > threshold_mapper[mapper][f\"fold{num_fold}\"][row[\"organ\"]]).astype(np.int8)\n            bins_from_folds.append(mask_binary)\n        bins_from_models.append( ((np.sum(np.array(bins_from_folds), axis=0) / len(model)) >= 0.5).astype(np.int8) )\n        \n        \n    # mmseg\n#     organ = row.organ\n#     data_source = row.data_source\n    class_num = classes_to_num[row.organ]\n    img = mmcv.imread(os.path.join(DATA,str(row.id)+'.tiff'))\n        \n    for model in models_mmseg:\n        torch.cuda.empty_cache()\n        bins_from_folds = []\n        for fold in model:\n            pred = inference_segmentor(fold, img)[0].argmax(axis=0) == class_num\n            mask_binary = (pred > 0).astype(np.uint8)\n            bins_from_folds.append(mask_binary)\n        bins_from_models.append( ((np.sum(np.array(bins_from_folds), axis=0) / len(model)) >= 0.5).astype(np.int8) )\n    \n    blend_bin_mask = ((np.sum(np.array(bins_from_models), axis=0) / ( len(models_effnet) + len(models_mmseg))) >= 0.5).astype(np.int8)\n    \n    test_rows.append({\n        'id': row['id'],\n        'rle': rle_encode_less_memory(blend_bin_mask)})","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:36:37.554281Z","iopub.execute_input":"2022-09-22T14:36:37.554965Z","iopub.status.idle":"2022-09-22T14:38:11.403915Z","shell.execute_reply.started":"2022-09-22T14:36:37.554919Z","shell.execute_reply":"2022-09-22T14:38:11.402935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:38:17.075651Z","iopub.execute_input":"2022-09-22T14:38:17.075938Z","iopub.status.idle":"2022-09-22T14:38:18.324220Z","shell.execute_reply.started":"2022-09-22T14:38:17.075907Z","shell.execute_reply":"2022-09-22T14:38:18.323334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Submission CSV","metadata":{}},{"cell_type":"code","source":"# Make submission DataFrame\ntest_df = pd.DataFrame(test_rows)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:38:18.326763Z","iopub.execute_input":"2022-09-22T14:38:18.327060Z","iopub.status.idle":"2022-09-22T14:38:18.332906Z","shell.execute_reply.started":"2022-09-22T14:38:18.327025Z","shell.execute_reply":"2022-09-22T14:38:18.332006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/*","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:38:18.819336Z","iopub.execute_input":"2022-09-22T14:38:18.819630Z","iopub.status.idle":"2022-09-22T14:38:20.001899Z","shell.execute_reply.started":"2022-09-22T14:38:18.819598Z","shell.execute_reply":"2022-09-22T14:38:20.000819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Write Submission CSV\ntest_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:38:20.556318Z","iopub.execute_input":"2022-09-22T14:38:20.556655Z","iopub.status.idle":"2022-09-22T14:38:20.571117Z","shell.execute_reply.started":"2022-09-22T14:38:20.556619Z","shell.execute_reply":"2022-09-22T14:38:20.570321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2022-09-22T14:38:20.923835Z","iopub.execute_input":"2022-09-22T14:38:20.924623Z","iopub.status.idle":"2022-09-22T14:38:20.934492Z","shell.execute_reply.started":"2022-09-22T14:38:20.924586Z","shell.execute_reply":"2022-09-22T14:38:20.933508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 416 92 2439 92 4461 93 6483 94 8504 96 10526 9...","metadata":{},"execution_count":null,"outputs":[]}]}