{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true},"cell_type":"markdown","source":"This is my idea for using a U-Net like architecture for instance segmentation. I use ResNet50 for encoder and add my own decoder.<br>\nThe instance classification task is implicitly done by placing instances from each category into one of 7 categorical masks. I've seen others (in literature) using a separate fully connected classification layer on top of the final feature map for instance classification. However, I want to experiment with this implicit classification approach (and to remove dedicated classification layer all together). Instance segmentation is done by predicting boundaries of each masks, then separate the masks in the post.<br>\nThis is still work in progress, I only got the model to train properly and still need more training. I also need to work on the post processing. Please let me know if you have any suggestion to improve."},{"metadata":{"trusted":false,"_uuid":"9f4a142b310f920305e33ea345724cc2d544b677"},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Model","execution_count":2,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"2a90cceb314378323255937098a632de231f5d14"},"cell_type":"code","source":"from keras.layers import Input, UpSampling2D, BatchNormalization\nfrom keras.layers.core import Lambda\nfrom keras.layers.convolutional import Conv2D, SeparableConv2D\nfrom keras.layers.pooling import MaxPooling2D\nfrom keras.layers.merge import concatenate\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, Callback\nfrom keras import backend as K\nfrom keras.applications.resnet50 import ResNet50,preprocess_input","execution_count":3,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"3410ad2321f7d176e805438efdcb69d1247e56bf"},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore', category=UserWarning, module='skimage')\n\nimport numpy as np\nimport pandas as pd\nimport os\nfrom skimage.io import imread, imshow\nfrom skimage.transform import resize\nfrom skimage.morphology import label\nfrom skimage.util import crop\nfrom skimage.transform import resize\nfrom skimage.segmentation import find_boundaries, mark_boundaries\nfrom scipy.ndimage.morphology import binary_erosion as erosion\nfrom scipy.ndimage.morphology import binary_dilation as dilation\nfrom scipy.ndimage import distance_transform_edt\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":4,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"32a54741922dcbff3ca415fb7cfa59832999fb0a"},"cell_type":"code","source":"train_path = '../input/train_color'\nlabel_path = '../input/train_label'\ndef get_data():\n    label = os.listdir(label_path)\n    label.sort()\n    train = os.listdir(train_path)\n    train.sort()\n    df_id = pd.DataFrame()\n    df_id['label'] = label\n    df_id['train'] = train\n    df_id['label_path'] = df_id['label'].apply(lambda x: os.path.join(label_path, x))\n    df_id['train_path'] = df_id['train'].apply(lambda x: os.path.join(train_path, x))\n    return df_id\n\ndf_id = get_data()\nix = np.random.randint(0, len(df_id.index))\ntrain_ex = imread(df_id.loc[ix, 'train_path'])\nlabel_ex = imread(df_id.loc[ix, 'label_path'])\nf, ax = plt.subplots(1,2, figsize = (10,10))\nax[0].imshow(train_ex)\nax[1].imshow(np.squeeze(label_ex))\nax[0].axis('off')\nax[1].axis('off')\nax[0].set_title('Photo')\nax[1].set_title('Mask')\nplt.show()\nprint(df_id.info())\ndf_id.head()","execution_count":5,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e536b9c68af4432401b947486f1092664f03765f"},"cell_type":"code","source":"test_path = '../input/test'\ndef get_test_data():\n    test = os.listdir(test_path)\n    test.sort()\n    df_id = pd.DataFrame()\n    df_id['test'] = test\n    df_id['test_path'] = df_id['test'].apply(lambda x: os.path.join(test_path, x))\n    return df_id\n\ndf_test_id = get_test_data()\nplt.imshow(imread(df_test_id.sample(n = 1)['test_path'].values[0]))\nplt.axis('off')\nprint(df_test_id.info())\ndf_test_id.head()","execution_count":6,"outputs":[]},{"metadata":{"_uuid":"85059f93322cbeb39b84a95877c5a02524da689d"},"cell_type":"markdown","source":"Only 7 classes are considered: car, motorcycle, bicycle, pedestrian (or people, 36), truck, bus, and tricycle. The following is the complete class dictionary and the targeted class dictionary.<br>\nlabelmap_all = {0:'others', \n            1:'rover', \n            17:'sky', \n            33:'car', \n            34:'motorbicycle', \n            35:'bicycle', \n            36:'person', \n            37:'rider', \n            38:'truck', \n            39:'bus', \n            40:'tricycle', \n            49:'road', \n            50:'siderwalk', \n            65:'traffic_cone', \n            66:'road_pile', \n            67:'fence', \n            81:'traffic_light', \n            82:'pole', \n            83:'traffic_sign', \n            84:'wall', \n            85:'dustbin', \n            86:'billboard', \n            97:'building', \n            98:'bridge', \n            99:'tunnel', \n            100:'overpass', \n            113:'vegatation', \n            161:'car_groups', \n            162:'motorbicycle_group', \n            163:'bicycle_group', \n            164:'person_group', \n            165:'rider_group', \n            166:'truck_group', \n            167:'bus_group', \n            168:'tricycle_group'}"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"ef6aa3663b460440aecbb1fc1be2dea9b9c2879f"},"cell_type":"code","source":"labelmap_target = {33:'car', 34:'motorbicycle', 35:'bicycle', 36:'person', 38:'truck', 39:'bus', 40:'tricycle'}","execution_count":7,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"2e14d52f2a0d50cd7029cea023fd00bd2cb6c355"},"cell_type":"code","source":"datagen_arg = dict(horizontal_flip = True)\n\nmaskgen_arg = dict(horizontal_flip = True)\n\nx_gen = ImageDataGenerator(**datagen_arg)\nmask_gen = ImageDataGenerator(**maskgen_arg)\n\n#a custom generator that outputs x and y\n#x is the raw input unit 8 image\n# y is the 7 dims array of masks\ndef label_to_mask(y, classes):\n    mask_core = np.zeros((y.shape[0], y.shape[1], len(classes)))\n    mask_edge = np.zeros((y.shape[0], y.shape[1], 1))\n    foreground = (y*((y >= classes[0]*1000) & (y < (classes[-1]+1)*1000))).astype(np.uint16)\n    unique_objects = np.delete(np.unique(foreground), 0)\n    # mask core\n    for i, class_ in enumerate(classes):\n        mask_core[:,:,i] = np.squeeze(((foreground/1000).astype(np.int32) == class_).astype(np.bool))\n    # This operation makes the edge ~2 pixels thick,\n    # remember to fill the pixels back in post processing\n    mask_edge = find_boundaries(foreground, mode = 'outer').astype(np.bool)\n    return mask_core, mask_edge\n\ndef resize_random_crop(images, random_state, resize_w = 1280, resize_h = 720, crop_w = 224, crop_h = 224):\n    np.random.seed(random_state)\n    # function to shrink and random crop input to desired size for training\n    # resize is done so random crop won't output images with no targets too often\n    images = np.array([resize(image, (resize_h, resize_w), mode = 'constant', preserve_range = True) for image in images])\n    height = images.shape[1]\n    width = images.shape[2]\n    # The randn function is to prevent data augmentation from producing too many crop with sky only\n    rand_h_start = (np.clip(0.9*np.random.randn()+0.7, 0, 1)*(height - crop_h)).astype(np.uint16)\n    rand_w_start = np.random.randint(0, (width - crop_w))\n    return crop(images, ((0, 0), (rand_h_start, height - (rand_h_start + crop_h)),\n                        (rand_w_start, width - (rand_w_start + crop_w)), (0, 0)))\n\ndef augmentation_checker(mask, classes, percentage = 0.005):\n    # function to ensure that the cropped input has at least some mask in it\n    mask_core, mask_edge = label_to_mask(mask, classes)\n    while np.mean(mask_core) < percentage:\n        mask_core, mask_edge = label_to_mask(mask, classes)\n    return mask_core, mask_edge\n\ndef data_generator(x_gen_, mask_gen_, df_data, mini_bat_size, classes = sorted(labelmap_target.keys())):\n    while True:\n        seed = np.random.randint(0,1000)\n        sampled_set = df_data.sample(n = mini_bat_size)\n        sampled_train = sampled_set['train_path']\n        sampled_label = sampled_set['label_path']\n        X = preprocess_input(np.array([imread(train_path) for train_path in sampled_train]).astype(np.float32))\n        y = np.expand_dims(np.array([imread(label_path) for label_path in sampled_label]), axis = -1)\n\n        x_generator = x_gen_.flow(X, batch_size = mini_bat_size, seed = seed)\n        mask_generator = mask_gen_.flow(y, batch_size = mini_bat_size, seed = seed)\n        X = resize_random_crop(x_generator.next(), seed)\n        y = resize_random_crop(mask_generator.next(), seed)\n        y_mask = []\n        y_edge = []\n        for mask in y:\n            mask_core, mask_edge = label_to_mask(mask, classes)\n            y_mask.append(mask_core)\n            y_edge.append(mask_edge)\n        yield (X, {'masks': np.array(y_mask), 'edges': np.array(y_edge)})","execution_count":8,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"5d7bc0f20477e0a1f61b3d6cc40d0c059aa9d332"},"cell_type":"code","source":"mini_bat_size = 2\nX, y= next(data_generator(x_gen, mask_gen, df_id, mini_bat_size))","execution_count":9,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d4cd6c2492d132786af9b846ba8f0546a6381a7a"},"cell_type":"code","source":"ix = 1\nX_cropped = X[ix]\ny_edge_cropped = np.squeeze(y['edges'][ix])\ny_mask_cropped = np.squeeze(y['masks'][ix,:,:,0])\ny_mask_seg = y_mask_cropped - y_edge_cropped\nf, ax = plt.subplots(1,4, figsize = (20,20))\nax[0].imshow(X_cropped/255 + 150) # This is my lazy work to roughly undo preprocessing for this visualization\nax[0].axis('off')\nax[0].set_title('X cropped')\nax[1].imshow(y_edge_cropped)\nax[1].axis('off')\nax[1].set_title('edge: all')\nax[2].imshow(y_mask_cropped)\nax[2].axis('off')\nax[2].set_title('mask: car')\nax[3].imshow(y_mask_seg > 0)\nax[3].axis('off')\nax[3].set_title('post: segmented by instance')","execution_count":10,"outputs":[]},{"metadata":{"_uuid":"2819cc2ed4a8860a6070088fde1e198d18f904a0"},"cell_type":"markdown","source":"Next, we will construct our U-net like architecture for instance segmentation.<br>\nThe encoder is a [ResNet 50](https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/He_Deep_Residual_Learning_CVPR_2016_paper.pdf). The decoder has skipped connection from each of residual blocks from ResNet 50.<br>\nThe full architecture looks like this:<br>\n![](https://github.com/IshootLaser/modified-ResNet50/blob/master/resnet50.JPG?raw=true)\n![](https://github.com/IshootLaser/modified-ResNet50/blob/master/net%20architecture.JPG?raw=true)\nIt seems like Kaggle kernel cannot download ResNet50 weight from github, so we will use weight = None for now.<br>\nBefore proceeding with the following code, please note that the original keras implementation (resnet50.py) has first 5 layers like this: <br>\n```python\n    x = ZeroPadding2D(padding=(3, 3), name='conv1_pad')(img_input)\n    x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(img_input)\n    x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)\n    x = Activation('relu')(x)\n    x = MaxPooling2D((3, 3), strides=(2, 2))(x)```\n\nI believe the maxpooling is incorrect because it will produce feature map of shape 55 x 55 (unlike 56 x 56 as stated in paper). Need to change it before proceeding: <br>\n\n```python\n    x = Conv2D(64, (7, 7), strides=(2, 2), padding='same', name='conv1')(img_input)\n    x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)\n    x = Activation('relu')(x)\n    x = MaxPooling2D((3, 3), strides=(2, 2), padding = 'same')(x)\n    ```\nAfter we got our network to produce the mask and the edge, we can use edge to separate instances in the post. I am still working on this.<br>"},{"metadata":{"trusted":false,"_uuid":"d03b3e25af943ee61eee9682ad62703e53d67f65"},"cell_type":"code","source":"try:\n    R50 = ResNet50(include_top = False, input_shape = (224, 224, 3))\nexcept:\n    # I can't get the pretrained weight on Kaggle kernel, so we will set weights = None for now.\n    R50 = ResNet50(include_top = False, weights = None, input_shape = (224, 224, 3))\n# pop off the last pooling layer\nR50.layers.pop()\nfor layer in R50.layers:\n    layer.trainable = False\n    \n# Layers from ResNet50 to make skip connections\nskip_ix = [172, 140, 78, 36, 3]\n# Layers in decoder to connect to encoder\nskip_end = []\nfor i in skip_ix:\n    skip_end.append(R50.layers[i])\nR50.summary()","execution_count":13,"outputs":[]},{"metadata":{"_uuid":"51bf40efee3c5985b33b57f7fe368cd484b35a55"},"cell_type":"markdown","source":"Here is information about [BAU](https://arxiv.org/abs/1707.05847) and [separable convolution](https://towardsdatascience.com/types-of-convolutions-in-deep-learning-717013397f4d). I use them to reduce the total amount of hyperparameters. If Conv2D is used, I will end up with 10E9 hyperparameters. With separable convolution it is only 10E8 which fits in 12GB GPU memory."},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"3c61beea919984f6d6e94bdd5615a8ee35f97e53"},"cell_type":"code","source":"# Use billinear additive upsampling (BAU) and separable convolution to reduce the total amount of hyperparameters\ndef BAU_layer(last_layer, channel_num):\n    additive = []\n    depth = int(last_layer.get_shape()[-1])\n    step = int(depth / channel_num)\n    last_layer[:, :, :, 1*step:(1*step+step)]\n    for i in range(channel_num):\n        layersum = K.mean(last_layer[:, :, :, i*step:(i*step+step)], axis = -1)\n        additive.append(layersum)\n    additive = K.stack(additive, axis = -1)\n    return additive\n\ndef upsampling_step(skipped_conv, num_output_filters, prev_conv = None):\n    num_filters = skipped_conv.output_shape[-1]\n    if prev_conv != None:\n        concat_layer = concatenate([skipped_conv.output, prev_conv])\n    else:\n        concat_layer = skipped_conv.output\n    conv1 = SeparableConv2D(num_filters, 3, padding = 'same', activation = 'relu',\n                           depthwise_initializer = 'he_normal', pointwise_initializer = 'he_normal')(concat_layer)\n    conv2 = SeparableConv2D(num_filters, 3, padding = 'same', activation = 'relu',\n                           depthwise_initializer = 'he_normal', pointwise_initializer = 'he_normal')(conv1)\n    up = UpSampling2D()(conv2)\n    BAU = Lambda(BAU_layer, arguments = {'channel_num': num_output_filters})(up)\n    conv3 = SeparableConv2D(num_output_filters, 2, padding = 'same', activation = 'relu',\n                           depthwise_initializer = 'he_normal', pointwise_initializer = 'he_normal')(BAU)\n    return conv3\n\ndef output(feature_map, mask = True):\n    if mask:\n        conv3 = Conv2D(7, 1, padding = 'same', activation = 'sigmoid', kernel_initializer = 'he_normal', name = 'masks')(feature_map)\n    else:\n        conv3 = Conv2D(1, 1, padding = 'same', activation = 'sigmoid', kernel_initializer = 'he_normal', name = 'edges')(feature_map)\n    return conv3","execution_count":12,"outputs":[]},{"metadata":{"_uuid":"9b41ff7b12f2cbe6803c6ea0df1a121253fd3e85"},"cell_type":"markdown","source":"The rest of code is for demonstration only because I don't know how to access resnet50.py on Kaggle kernel to make this modification, but the above mod works on my local machine."},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"365fcd414d8d4af1f7de963efcc9cac3156b939d"},"cell_type":"code","source":"up_num_filters = [1024, 512, 256, 64, 64]\nfor n, i in enumerate(up_num_filters):\n    if n == 0:\n        conv_layer = upsampling_step(skip_end[n], i)\n    else:\n        conv_layer = upsampling_step(skip_end[n], i, conv_layer)\nconv1 = Conv2D(64, 3, padding = 'same', activation = 'relu', kernel_initializer = 'he_normal')(conv_layer)\nconv2 = Conv2D(64, 3, padding = 'same', activation = 'relu', kernel_initializer = 'he_normal')(conv1)\n\nmasks = output(conv2)\nedges = output(conv2, False)\nmodel = Model(inputs = R50.inputs, outputs = [masks, edges])\nmodel.compile(optimizer = 'adam', loss = {'masks': binary_crossentropy, 'edges': binary_crossentropy})\n\n# Optional callbacks\n# checkpointer = ModelCheckpoint('./models/ResNet50_U-Net_chkpt.h5', verbose = 1, save_best_only = True)\n# tbCallback = TensorBoard(log_dir = './Graph', histogram_freq = 0, write_graph = True, write_images = True)\n# earlystopper = EarlyStopping(patience = 5, verbose = 1)\nbat_size = 2\nresult = model.fit_generator(data_generator(x_gen, mask_gen, df_id, bat_size), steps_per_epoch = bat_size, epochs = 50,\n                            verbose = 1)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}