{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook I implement a U-Net model with less than 90,000 parameters. The model makes use of inverted residual blocks as the main processing blocks of the models and employs pyramid scene parsing in the 'horizontal' connections of U-Net. The model is pretrained on the LIVCell dataset for 3 epochs before being trained on the competition data. The model performs well on selecting out cells from the image, but performs poorly on the discriminatory masking required for the Sartorius competition. To put it simply, it does well at picking out all cells in an image but struggles to pick out a specific type of cell. \n\nMore detailed labeled images could help solve this problem. Multiple notebooks in this competition use Detectron2 to label all cell types present in the samples, but at that point one might as well use the Detectron2 model to mask your data.\n\nWhile the model is far from the top scoring, the fact that it scores at all is a win for me. It does well as a simple cell masker. It was fun to create such a small model! ","metadata":{}},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom scipy.io import loadmat\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow import keras\nimport tensorflow_io as tfio","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-30T16:01:31.534141Z","iopub.execute_input":"2021-12-30T16:01:31.534504Z","iopub.status.idle":"2021-12-30T16:01:31.539884Z","shell.execute_reply.started":"2021-12-30T16:01:31.534467Z","shell.execute_reply":"2021-12-30T16:01:31.539279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Generation","metadata":{}},{"cell_type":"code","source":"###\n### Defining the blocks which make up the model\n###\n\ndef inverted_residual_block(x, expand=64, squeeze=16):\n    m = keras.layers.Conv2D(expand, 1, padding='same')(x)\n    m = keras.layers.BatchNormalization()(m)\n    m = keras.layers.Activation('relu')(m)\n    \n    m = keras.layers.DepthwiseConv2D(3, padding='same')(m)\n    m = keras.layers.BatchNormalization()(m)\n    m = keras.layers.Activation('relu')(m)\n    \n    m = keras.layers.Conv2D(squeeze, 1, padding='same')(m)\n    m = keras.layers.BatchNormalization()(m)\n    m = keras.layers.Activation('relu')(m)\n    \n    return keras.layers.add([m, x])\n\ndef down_sampling_block(x,filters):\n    m = keras.layers.MaxPooling2D(3, strides=2, padding='same')(x)\n    # project residual\n    r = keras.layers.Conv2D(filters, 1, strides=2, padding='same')(x)\n    r = keras.layers.BatchNormalization()(r)\n    r = keras.layers.Activation('relu')(r)\n    return keras.layers.add([m, r])\n\ndef up_sampling_block(x, filters):\n    m = keras.layers.UpSampling2D(2)(x)\n    # project residual\n    r = keras.layers.UpSampling2D(2)(x)\n    # project residual\n    r = keras.layers.Conv2D(filters, 1, padding='same')(r)\n    r = keras.layers.BatchNormalization()(r)\n    r = keras.layers.Activation('relu')(r)\n    return keras.layers.add([m, r])\n\n\ndef pyramid_parsing(x, depth=0, img_size=(520, 704),\n                    filters=8, f_ratio=8, pyramid=[2,3,6,9]):\n    HEIGHT, WIDTH = img_size\n    HEIGHT, WIDTH = HEIGHT // 2**depth, WIDTH // 2**depth # the model down samples by 2 for each layer of given 'depth'\n        \n    y = x\n    for bin_val in pyramid:\n        \n        m = keras.layers.MaxPool2D(pool_size=(HEIGHT//bin_val, WIDTH//bin_val),\n                                   padding='same')(x)\n        # check if the bin size is larger due to the padding\n        m_HEIGHT, m_WIDTH = m.get_shape()[1:3]\n\n            \n        m = keras.layers.UpSampling2D(size=(HEIGHT//m_HEIGHT, WIDTH//m_WIDTH))(m)\n        # as it stands, with proper use of flooring values and calling the bins, \n        # the layers (if they do not match perfectly) are slightly undersized by several pixels. \n        # resize allows to use nearest neighbor interpolation to extend the edge bins inplace of zero padding. \n        m = keras.layers.Resizing(HEIGHT, WIDTH, interpolation='nearest')(m)\n        \n        \n        # scale down the amount of filters\n        m = keras.layers.Conv2D(filters//f_ratio, 1, padding='same')(m)\n        m = keras.layers.BatchNormalization()(m)\n        m = keras.layers.Activation('relu')(m)\n        y = keras.layers.Concatenate()([y, m])\n        \n    return y","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:01:31.541038Z","iopub.execute_input":"2021-12-30T16:01:31.541496Z","iopub.status.idle":"2021-12-30T16:01:31.556328Z","shell.execute_reply.started":"2021-12-30T16:01:31.541470Z","shell.execute_reply":"2021-12-30T16:01:31.555873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(img_size, channels, num_classes, filter_list = [8, 16, 32, 64], PSP=False):\n    \n    \n    \"\"\"\n    Creates the U-net model.\n    \n    Parameters\n    ----------\n    img_size: tuple\n        The dimensions of the input samples (x,y)\n    channels: int\n        The channels of the input samples\n    filter_list: list\n        A list of the desired filters to construct the model with. The length of the list defines the depth of the model,\n        with the final value being the 'bottom' layer.\n    num_classes: int\n        The amount of desired output channels.\n    PSP: bool\n        Defines if the model should use Pyramid Scene Parsing in its layers.\n        \n    Returns\n    -------\n    tf.keras Functional API model, samples have shape of (img_size[0], img_size[1], num_classes.)\n    \n    \"\"\"\n    max_filter = filter_list[-1]\n    filter_list = filter_list[:-1]\n    \n    \n    inputs = keras.Input(shape=img_size + (channels,))\n    x = inputs # redefine for the loop\n\n    ### Downsampling layers ###\n    \n    horizontal_connections = [] # collect the outputs of layers to use in the horizontal connection of U-net\n    \n    depth = 0\n    # Used in pyramid parsing\n    # for repeating halfing of size\n    # 2**depth\n    \n    for filters in filter_list:\n        # point wise convolution to expand filters. \n        x = keras.layers.Conv2D(filters, 1, padding='same')(x)\n        x = keras.layers.BatchNormalization()(x)\n        x = keras.layers.Activation('relu')(x)\n        \n        x = inverted_residual_block(x, expand=filters*2, squeeze=filters)\n        x = inverted_residual_block(x, expand=filters*2, squeeze=filters)\n\n        \n        if PSP:\n            context_stack = pyramid_parsing(x, depth=depth, img_size=img_size,\n                                        filters=filters, f_ratio=filters)\n            horizontal_connections.append(context_stack)\n            depth += 1\n        \n        else:\n            horizontal_connections.append(x)\n        \n        x = down_sampling_block(x,filters=filters)\n        \n        \n    ### Bottom layer ### \n    # no horizontal connection\n    bottom_filter = max_filter\n    x = keras.layers.Conv2D(bottom_filter, 1, padding='same')(x)\n    x = keras.layers.BatchNormalization()(x)\n    x = keras.layers.Activation('relu')(x)\n    \n    x = inverted_residual_block(x, expand=bottom_filter*2, squeeze=bottom_filter)\n    x = inverted_residual_block(x, expand=bottom_filter*2, squeeze=bottom_filter)\n        \n    ### Upsampling layers ###\n\n        \n    for filters, h_con in zip(filter_list[::-1], horizontal_connections[::-1]):\n        # upsample, add the horizontal components\n        x = up_sampling_block(x, filters=filters*2)\n        x = keras.layers.Concatenate()([x,h_con])\n        \n\n        # shrink filters to desired size\n        x = keras.layers.Conv2D(filters, 1, padding='same')(x)\n        x = keras.layers.BatchNormalization()(x)\n        x = keras.layers.Activation('relu')(x)\n\n        \n        x = inverted_residual_block(x, expand=filters*2, squeeze=filters)\n        x = inverted_residual_block(x, expand=filters*2, squeeze=filters)\n        \n        \n        \n     # Add a per-pixel classification layer\n\n    x = keras.layers.Conv2D(num_classes, 3, padding='same')(x)\n    x = keras.layers.BatchNormalization()(x)\n    outputs = keras.layers.Activation('sigmoid')(x)\n    \n\n\n    model = keras.Model(inputs, outputs)\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:01:31.557323Z","iopub.execute_input":"2021-12-30T16:01:31.557816Z","iopub.status.idle":"2021-12-30T16:01:31.572059Z","shell.execute_reply.started":"2021-12-30T16:01:31.557788Z","shell.execute_reply":"2021-12-30T16:01:31.571386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HEIGHT, WIDTH = 520, 704\nimage_size = (HEIGHT, WIDTH)\nmodel = get_model(image_size, channels=1, num_classes=1, PSP=True)\nmodel.load_weights('/kaggle/input/cell-model-weights/spyramid_8/spyramid_8_epoch') # load the weights from training\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:01:31.573259Z","iopub.execute_input":"2021-12-30T16:01:31.573534Z","iopub.status.idle":"2021-12-30T16:01:34.390969Z","shell.execute_reply.started":"2021-12-30T16:01:31.573501Z","shell.execute_reply":"2021-12-30T16:01:34.390237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# A Look at the mask performance","metadata":{}},{"cell_type":"code","source":"sample_files = ['../input/sartorius-cell-instance-segmentation/test/7ae19de7bc2a.png',\n               '../input/sartorius-cell-instance-segmentation/test/d48ec7815252.png',\n               '../input/sartorius-cell-instance-segmentation/test/d8bfd1dafdc4.png']\n\n\nfor file_path in sample_files:\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16,16), sharey=True)\n\n    input_test = plt.imread(str(file_path)).reshape(1, 520, 704, 1)\n    input_test = (input_test) - np.mean(input_test) # \n    \n    pred = model.predict(input_test) \n    \n    ax1.imshow(input_test[0])\n    ax2.imshow(pred[0])","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:01:34.392660Z","iopub.execute_input":"2021-12-30T16:01:34.392863Z","iopub.status.idle":"2021-12-30T16:01:38.546558Z","shell.execute_reply.started":"2021-12-30T16:01:34.392836Z","shell.execute_reply":"2021-12-30T16:01:38.545710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As mentioned, the model can actually select out the cells present in the sample. If one wanted to simply select all cells in an image, this is not a bad masker.","metadata":{}},{"cell_type":"markdown","source":"# Creating Mask output","metadata":{}},{"cell_type":"code","source":"import skimage.morphology\n\n############ Submission\ndef rle_encode(img):\n    '''\n    img: numpy array. \n        1 - mask, 0 - background\n        \n    Returns run length as string formated\n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef get_regions(x, THRESHOLD, min_size=25):\n    \"\"\"\n    Converts predictions to rle encoded masks.\n    \n    Parameters\n    --------------\n    x: numpy array\n        Predictions from our model.\n    THRESHOLD: float\n        The threshold value were we consider a pixel a cell.\n    min_size: int\n        The minimum size (length) of a mask to be included in our final output.\n        \n    Returns\n    --------------\n    res: list\n        list of masks encoded into rle encodings. \n    \"\"\"\n    res = []\n    regions = skimage.morphology.label(x>THRESHOLD) # select out seperate regions.\n    for i in range(1, regions.max() +1):\n        rle = rle_encode(regions==i)\n        if len(rle) >= min_size: # skip predictions with small area\n#             print(len(rle))\n            res.append(rle)\n        else:\n            continue\n    return res","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:01:38.547667Z","iopub.execute_input":"2021-12-30T16:01:38.547931Z","iopub.status.idle":"2021-12-30T16:01:38.556738Z","shell.execute_reply.started":"2021-12-30T16:01:38.547894Z","shell.execute_reply":"2021-12-30T16:01:38.556109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files = glob('../input/sartorius-cell-instance-segmentation/test/*') # grab all test files\n\nsample_submission = pd.read_csv('../input/sartorius-cell-instance-segmentation/sample_submission.csv')\noutput_df = pd.DataFrame(data = None, columns = sample_submission.columns)\n\n\ncount = 0 # running count to idx the output df\n\n### Loop over all test files ###\nfor file_path in test_files:\n    tag = file_path.split('/')[-1][:-4] # splits the sample name off from the filepath\n    print(tag)\n    input_test = plt.imread(str(file_path)).reshape(1, 520, 704, 1) # extra channel in the front so easy input into the model.\n    input_test = (input_test) - np.mean(input_test) # zero center the data\n\n    pred = model.predict(input_test)\n    regions = get_regions(pred[0], 0.5, min_size=25)\n    \n    \n    for mask in regions:\n        output_df.loc[count] = tag, mask\n        count+= 1","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:01:38.557802Z","iopub.execute_input":"2021-12-30T16:01:38.559162Z","iopub.status.idle":"2021-12-30T16:01:39.565422Z","shell.execute_reply.started":"2021-12-30T16:01:38.559132Z","shell.execute_reply":"2021-12-30T16:01:39.564798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_df","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:01:39.567465Z","iopub.execute_input":"2021-12-30T16:01:39.567765Z","iopub.status.idle":"2021-12-30T16:01:39.578473Z","shell.execute_reply.started":"2021-12-30T16:01:39.567727Z","shell.execute_reply":"2021-12-30T16:01:39.577877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_df.to_csv('/kaggle/working/submission.csv', index = False)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-30T16:01:39.579378Z","iopub.execute_input":"2021-12-30T16:01:39.579563Z","iopub.status.idle":"2021-12-30T16:01:39.592244Z","shell.execute_reply.started":"2021-12-30T16:01:39.579541Z","shell.execute_reply":"2021-12-30T16:01:39.591555Z"},"trusted":true},"execution_count":null,"outputs":[]}]}