{"cells":[{"metadata":{"_uuid":"97544de77c49e1a5fa3e9fe86a722288541bab5a"},"cell_type":"markdown","source":"# Introduction\n* This is part 2 for this competitions submit"},{"metadata":{"_uuid":"aa8401d73c7a19e1a43fdd6a992ea9dcb60039a2"},"cell_type":"markdown","source":"##  U-net base on ResNet34 transfer learning \n* This kernel use Unet Base on ResNet34 model\n* For pre data process and data augumatation reference by Kevin's [kernel](https://www.kaggle.com/kmader/baseline-u-net-model-part-1)\n* Use bce_log+dice_loss for ResNet34+Unet loss function\n* ResNet34+ Unet model reference by this [GITHUB]( https://github.com/qubvel/segmentation_models/blob/master/segmentation_models/unet)"},{"metadata":{"_uuid":"b9eba6bafd215783594b3432c8c6d8d7056c4690"},"cell_type":"markdown","source":"##  Part1 - Simple tansfer learning to detect ship exist\n* https://www.kaggle.com/super13579/simple-transfer-learning-detect-ship-exist-keras"},{"metadata":{"_uuid":"e18f351a92929144c380c09d86dfad5f365d3991"},"cell_type":"markdown","source":"## Part3 - Submittion result with predict ship + Unet34 (0.628)\n* https://www.kaggle.com/super13579/unet34-predict-result"},{"metadata":{"_uuid":"a6cd9d5ad61ffe3b8858769f20a5f9493f024a56"},"cell_type":"markdown","source":"## Model Parameters\nWe might want to adjust these later (or do some hyperparameter optimizations)"},{"metadata":{"trusted":true,"_uuid":"301a5d939c566d1487a049bb2554d09b592b18b1"},"cell_type":"code","source":"BATCH_SIZE = 4\nEDGE_CROP = 16\nNB_EPOCHS = 5\nGAUSSIAN_NOISE = 0.1\nUPSAMPLE_MODE = 'SIMPLE'\n# downsampling inside the network\nNET_SCALING = None\n# downsampling in preprocessing\nIMG_SCALING = (1, 1)\n# number of validation images to use\nVALID_IMG_COUNT = 400\n# maximum number of steps_per_epoch in training\nMAX_TRAIN_STEPS = 200\nAUGMENT_BRIGHTNESS = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c826bd059a97f34ef3549a393d140dd1451d657"},"cell_type":"code","source":"!ls ../input","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom skimage.io import imread\nimport matplotlib.pyplot as plt\nfrom skimage.segmentation import mark_boundaries\nfrom skimage.util.montage import montage2d as montage\nmontage_rgb = lambda x: np.stack([montage(x[:, :, :, i]) for i in range(x.shape[3])], -1)\nship_dir = '../input'\ntrain_image_dir = os.path.join(ship_dir, 'train_v2')\ntest_image_dir = os.path.join(ship_dir, 'test_v2')\nimport gc; gc.enable() # memory is tight\n\nfrom skimage.morphology import label\ndef multi_rle_encode(img):\n    labels = label(img[:, :, 0])\n    return [rle_encode(labels==k) for k in np.unique(labels[labels>0])]\n\n# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef rle_decode(mask_rle, shape=(768, 768)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T  # Needed to align to RLE direction\n\ndef masks_as_image(in_mask_list):\n    # Take the individual ship masks and create a single mask array for all ships\n    all_masks = np.zeros((768, 768), dtype = np.int16)\n    #if isinstance(in_mask_list, list):\n    for mask in in_mask_list:\n        if isinstance(mask, str):\n            all_masks += rle_decode(mask)\n    return np.expand_dims(all_masks, -1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ca7119188fbb4c6540d9df55f5833b55435287e"},"cell_type":"code","source":"masks = pd.read_csv(os.path.join('../input/',\n                                 'train_ship_segmentations_v2.csv'))\nprint(masks.shape[0], 'masks found')\nprint(masks['ImageId'].value_counts().shape[0])\nmasks.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40cb72e241c0c3d8bc245b4e3c663b4a835b0011"},"cell_type":"markdown","source":"# Split into training and validation groups\nWe stratify by the number of boats appearing so we have nice balances in each set"},{"metadata":{"trusted":true,"_uuid":"c4f008bf6898518fd371de013418f936edaa09f8"},"cell_type":"code","source":"masks['ships'] = masks['EncodedPixels'].map(lambda c_row: 1 if isinstance(c_row, str) else 0)\nunique_img_ids = masks.groupby('ImageId').agg({'ships': 'sum'}).reset_index()\nunique_img_ids['has_ship'] = unique_img_ids['ships'].map(lambda x: 1.0 if x>0 else 0.0)\nunique_img_ids['has_ship_vec'] = unique_img_ids['has_ship'].map(lambda x: [x])\n# some files are too small/corrupt\nunique_img_ids['file_size_kb'] = unique_img_ids['ImageId'].map(lambda c_img_id: \n                                                               os.stat(os.path.join(train_image_dir, \n                                                                                    c_img_id)).st_size/1024)\nunique_img_ids = unique_img_ids[unique_img_ids['file_size_kb']>50] # keep only 50kb files\nunique_img_ids['file_size_kb'].hist()\nmasks.drop(['ships'], axis=1, inplace=True)\nunique_img_ids.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"871720221ac25f7f9408bfe01aeb4ccb95edbd1f"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_ids, valid_ids = train_test_split(unique_img_ids, \n                 test_size = 0.3, \n                 stratify = unique_img_ids['ships'])\ntrain_df = pd.merge(masks, train_ids)\nvalid_df = pd.merge(masks, valid_ids)\nprint(train_df.shape[0], 'training masks')\nprint(valid_df.shape[0], 'validation masks')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef8115a80749ac47f295e9a70217a5553970c2b3"},"cell_type":"markdown","source":"# Undersample Empty Images\nHere we undersample the empty images to get a better balanced group with more ships to try and segment"},{"metadata":{"trusted":true,"_uuid":"0cf0bb261eda957cb0a12a330260e1390c57c8c9"},"cell_type":"code","source":"train_df['grouped_ship_count'] = train_df['ships'].map(lambda x: (x+1)//2).clip(0, 7)\ndef sample_ships(in_df, base_rep_val=1500):\n    if in_df['ships'].values[0]==0:\n        return in_df.sample(base_rep_val//3) # even more strongly undersample no ships\n    else:\n        return in_df.sample(base_rep_val, replace=(in_df.shape[0]<base_rep_val))\n    \nbalanced_train_df = train_df.groupby('grouped_ship_count').apply(sample_ships)\nbalanced_train_df['ships'].hist(bins=np.arange(10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1282446dc1a67c4c50ad220703a18973135aac4"},"cell_type":"code","source":"balanced_train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a3fb9fe33d81374c7bd836f5bc86a1df89190805"},"cell_type":"markdown","source":"# Decode all the RLEs into Images\nWe make a generator to produce batches of images"},{"metadata":{"trusted":true,"_uuid":"6181ac51577e5636995e38a9e29311cf47f513ca"},"cell_type":"code","source":"def make_image_gen(in_df, batch_size = BATCH_SIZE):\n    all_batches = list(in_df.groupby('ImageId'))\n    out_rgb = []\n    out_mask = []\n    while True:\n        np.random.shuffle(all_batches)\n        for c_img_id, c_masks in all_batches:\n            rgb_path = os.path.join(train_image_dir, c_img_id)\n            c_img = imread(rgb_path)\n            c_mask = masks_as_image(c_masks['EncodedPixels'].values)\n            if IMG_SCALING is not None:\n                c_img = c_img[::IMG_SCALING[0], ::IMG_SCALING[1]]\n                c_mask = c_mask[::IMG_SCALING[0], ::IMG_SCALING[1]]\n            out_rgb += [c_img]\n            out_mask += [c_mask]\n            if len(out_rgb)>=batch_size:\n                yield np.stack(out_rgb, 0)/255.0, np.stack(out_mask, 0)\n                out_rgb, out_mask=[], []","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f47639c987a10ebcb53e51f55aa8a11c98fa860"},"cell_type":"markdown","source":"# Make the Validation Set"},{"metadata":{"trusted":true,"_uuid":"30cb02a2a7103a9d66e90f701991199de1e5b73e"},"cell_type":"code","source":"valid_x, valid_y = next(make_image_gen(valid_df, VALID_IMG_COUNT))\nprint(valid_x.shape, valid_y.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8f65e7942816fb75b687a549dc1d5cc48d00e21"},"cell_type":"markdown","source":"# Augment Data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\ndg_args = dict(featurewise_center = False, \n                  samplewise_center = False,\n                  rotation_range = 15, \n                  width_shift_range = 0.1, \n                  height_shift_range = 0.1, \n                  shear_range = 0.01,\n                  zoom_range = [0.9, 1.25],  \n                  horizontal_flip = True, \n                  vertical_flip = True,\n                  fill_mode = 'reflect',\n                   data_format = 'channels_last')\n# brightness can be problematic since it seems to change the labels differently from the images \nif AUGMENT_BRIGHTNESS:\n    dg_args[' brightness_range'] = [0.5, 1.5]\nimage_gen = ImageDataGenerator(**dg_args)\n\nif AUGMENT_BRIGHTNESS:\n    dg_args.pop('brightness_range')\nlabel_gen = ImageDataGenerator(**dg_args)\n\ndef create_aug_gen(in_gen, seed = None):\n    np.random.seed(seed if seed is not None else np.random.choice(range(9999)))\n    for in_x, in_y in in_gen:\n        seed = np.random.choice(range(9999))\n        # keep the seeds syncronized otherwise the augmentation to the images is different from the masks\n        g_x = image_gen.flow(255*in_x, \n                             batch_size = in_x.shape[0], \n                             seed = seed, \n                             shuffle=True)\n        g_y = label_gen.flow(in_y, \n                             batch_size = in_x.shape[0], \n                             seed = seed, \n                             shuffle=True)\n\n        yield next(g_x)/255.0, next(g_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33300c4f03b6600da7b418f775d11d7ebf76a35a"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ba08494eb9736ec3556b7c879143cdcdea89febf"},"cell_type":"markdown","source":"# Build ResNet34+ Unet Model\nHere we use a slight deviation on the U-Net standard"},{"metadata":{"trusted":true,"_uuid":"bc694385625f3a7a49372e211f7298436d027ee4"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport six\n\nfrom random import randint\n\nimport matplotlib.pyplot as plt\nplt.style.use('seaborn-white')\nimport seaborn as sns\nsns.set_style(\"white\")\n\nfrom sklearn.model_selection import train_test_split\n\nfrom skimage.transform import resize\n\n\nfrom keras import Model\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom keras.models import load_model\nfrom keras.optimizers import Adam\nfrom keras.utils.vis_utils import plot_model\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers import Input, Conv2D, Conv2DTranspose, MaxPooling2D, concatenate, Dropout,BatchNormalization\nfrom keras.layers import Conv2D, Concatenate, MaxPooling2D\nfrom keras.layers import UpSampling2D, Dropout, BatchNormalization\nfrom tqdm import tqdm_notebook\nfrom keras import initializers\nfrom keras import regularizers\nfrom keras import constraints\nfrom keras.utils import conv_utils\nfrom keras.utils.data_utils import get_file\nfrom keras.engine.topology import get_source_inputs\nfrom keras.engine import InputSpec\nfrom keras import backend as K\nfrom keras.regularizers import l2\n\nfrom keras.engine.topology import Input\nfrom keras.engine.training import Model\nfrom keras.layers.convolutional import Conv2D, UpSampling2D, Conv2DTranspose\nfrom keras.layers.core import Activation, SpatialDropout2D\nfrom keras.layers.merge import concatenate,add\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers.pooling import MaxPooling2D","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e54a6b4396f146c1663f3ecf4196e6d6e4a0a2a3"},"cell_type":"markdown","source":"## Create Upsample layer\n"},{"metadata":{"trusted":true,"_uuid":"ed9721b45b3ea28752cb0c685ceef80451dec4c0"},"cell_type":"code","source":"from keras.layers import Conv2DTranspose\nfrom keras.layers import UpSampling2D\nfrom keras.layers import Conv2D\nfrom keras.layers import BatchNormalization\nfrom keras.layers import Activation\nfrom keras.layers import Concatenate\n\ndef handle_block_names_decode(stage):\n    conv_name = 'decoder_stage{}_conv'.format(stage)\n    bn_name = 'decoder_stage{}_bn'.format(stage)\n    relu_name = 'decoder_stage{}_relu'.format(stage)\n    up_name = 'decoder_stage{}_upsample'.format(stage)\n    return conv_name, bn_name, relu_name, up_name\n\n\ndef Upsample2D_block(filters, stage, kernel_size=(3,3), upsample_rate=(2,2),\n                     batchnorm=False, skip=None):\n\n    def layer(input_tensor):\n\n        conv_name, bn_name, relu_name, up_name = handle_block_names_decode(stage)\n\n        x = UpSampling2D(size=upsample_rate, name=up_name)(input_tensor)\n\n        if skip is not None:\n            x = Concatenate()([x, skip])\n\n        x = Conv2D(filters, kernel_size, padding='same', name=conv_name+'1')(x)\n        if batchnorm:\n            x = BatchNormalization(name=bn_name+'1')(x)\n        x = Activation('relu', name=relu_name+'1')(x)\n\n        x = Conv2D(filters, kernel_size, padding='same', name=conv_name+'2')(x)\n        if batchnorm:\n            x = BatchNormalization(name=bn_name+'2')(x)\n        x = Activation('relu', name=relu_name+'2')(x)\n\n        return x\n    return layer\n\n\ndef Transpose2D_block(filters, stage, kernel_size=(3,3), upsample_rate=(2,2),\n                      transpose_kernel_size=(4,4), batchnorm=False, skip=None):\n\n    def layer(input_tensor):\n\n        conv_name, bn_name, relu_name, up_name = handle_block_names_decode(stage)\n\n        x = Conv2DTranspose(filters, transpose_kernel_size, strides=upsample_rate,\n                            padding='same', name=up_name)(input_tensor)\n        if batchnorm:\n            x = BatchNormalization(name=bn_name+'1')(x)\n        x = Activation('relu', name=relu_name+'1')(x)\n\n        if skip is not None:\n            x = Concatenate()([x, skip])\n\n        x = Conv2D(filters, kernel_size, padding='same', name=conv_name+'2')(x)\n        if batchnorm:\n            x = BatchNormalization(name=bn_name+'2')(x)\n        x = Activation('relu', name=relu_name+'2')(x)\n\n        return x\n    return layer","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a3e6f145db7c5eb92fe521c7e32f5d4fe8aadb9"},"cell_type":"markdown","source":"## Define Unet model (Decoder)"},{"metadata":{"trusted":true,"_uuid":"e22ae327bed4fbb6fcf604b8755f61f2efbf066e"},"cell_type":"code","source":"def build_unet(backbone, classes, last_block_filters, skip_layers,\n               n_upsample_blocks=5, upsample_rates=(2,2,2,2,2),\n               block_type='upsampling', activation='sigmoid',\n               **kwargs):\n\n    input = backbone.input\n    x = backbone.output\n    print(x)\n    if block_type == 'transpose':\n        up_block = Transpose2D_block\n    else:\n        up_block = Upsample2D_block\n\n    # convert layer names to indices\n    skip_layers = ([get_layer_number(backbone, l) if isinstance(l, str) else l\n                    for l in skip_layers])\n    for i in range(n_upsample_blocks):\n\n        # check if there is a skip connection\n        if i < len(skip_layers):\n            print(backbone.layers[skip_layers[i]])\n            print(backbone.layers[skip_layers[i]].output)\n            skip = backbone.layers[skip_layers[i]].output\n        else:\n            skip = None\n\n        up_size = (upsample_rates[i], upsample_rates[i])\n        filters = last_block_filters * 2**(n_upsample_blocks-(i+1))\n\n        x = up_block(filters, i, upsample_rate=up_size, skip=skip, **kwargs)(x)\n\n    if classes < 2:\n        activation = 'sigmoid'\n\n    x = Conv2D(classes, (3,3), padding='same', name='final_conv')(x)\n    x = Activation(activation, name=activation)(x)\n\n    model = Model(input, x)\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"53ffd058a6d40cfd9036146f82467fe3bd0fc2b3"},"cell_type":"markdown","source":"## Built ResNet34 model\n* Reference this [github](https://github.com/qubvel/classification_models/blob/master/classification_models/resnet/)"},{"metadata":{"trusted":true,"_uuid":"5334dd03afa3f361b1c3226916fbe6dbaabe85fb"},"cell_type":"code","source":"from keras.layers import Conv2D\nfrom keras.layers import BatchNormalization\nfrom keras.layers import Activation\nfrom keras.layers import Add\nfrom keras.layers import ZeroPadding2D\n\ndef handle_block_names(stage, block):\n    name_base = 'stage{}_unit{}_'.format(stage + 1, block + 1)\n    conv_name = name_base + 'conv'\n    bn_name = name_base + 'bn'\n    relu_name = name_base + 'relu'\n    sc_name = name_base + 'sc'\n    return conv_name, bn_name, relu_name, sc_name\n\n\ndef basic_identity_block(filters, stage, block):\n\n    def layer(input_tensor):\n        conv_params = get_conv_params()\n        bn_params = get_bn_params()\n        conv_name, bn_name, relu_name, sc_name = handle_block_names(stage, block)\n\n        x = BatchNormalization(name=bn_name + '1', **bn_params)(input_tensor)\n        x = Activation('relu', name=relu_name + '1')(x)\n        x = ZeroPadding2D(padding=(1, 1))(x)\n        x = Conv2D(filters, (3, 3), name=conv_name + '1', **conv_params)(x)\n\n        x = BatchNormalization(name=bn_name + '2', **bn_params)(x)\n        x = Activation('relu', name=relu_name + '2')(x)\n        x = ZeroPadding2D(padding=(1, 1))(x)\n        x = Conv2D(filters, (3, 3), name=conv_name + '2', **conv_params)(x)\n\n        x = Add()([x, input_tensor])\n        return x\n\n    return layer\n\n\ndef basic_conv_block(filters, stage, block, strides=(2, 2)):\n\n    def layer(input_tensor):\n        conv_params = get_conv_params()\n        bn_params = get_bn_params()\n        conv_name, bn_name, relu_name, sc_name = handle_block_names(stage, block)\n\n        x = BatchNormalization(name=bn_name + '1', **bn_params)(input_tensor)\n        x = Activation('relu', name=relu_name + '1')(x)\n        shortcut = x\n        x = ZeroPadding2D(padding=(1, 1))(x)\n        x = Conv2D(filters, (3, 3), strides=strides, name=conv_name + '1', **conv_params)(x)\n\n        x = BatchNormalization(name=bn_name + '2', **bn_params)(x)\n        x = Activation('relu', name=relu_name + '2')(x)\n        x = ZeroPadding2D(padding=(1, 1))(x)\n        x = Conv2D(filters, (3, 3), name=conv_name + '2', **conv_params)(x)\n\n        shortcut = Conv2D(filters, (1, 1), name=sc_name, strides=strides, **conv_params)(shortcut)\n        x = Add()([x, shortcut])\n        return x\n\n    return layer\n\n\ndef conv_block(filters, stage, block, strides=(2, 2)):\n    def layer(input_tensor):\n        conv_params = get_conv_params()\n        bn_params = get_bn_params()\n        conv_name, bn_name, relu_name, sc_name = handle_block_names(stage, block)\n\n        x = BatchNormalization(name=bn_name + '1', **bn_params)(input_tensor)\n        x = Activation('relu', name=relu_name + '1')(x)\n        shortcut = x\n        x = Conv2D(filters, (1, 1), name=conv_name + '1', **conv_params)(x)\n\n        x = BatchNormalization(name=bn_name + '2', **bn_params)(x)\n        x = Activation('relu', name=relu_name + '2')(x)\n        x = ZeroPadding2D(padding=(1, 1))(x)\n        x = Conv2D(filters, (3, 3), strides=strides, name=conv_name + '2', **conv_params)(x)\n\n        x = BatchNormalization(name=bn_name + '3', **bn_params)(x)\n        x = Activation('relu', name=relu_name + '3')(x)\n        x = Conv2D(filters*4, (1, 1), name=conv_name + '3', **conv_params)(x)\n\n        shortcut = Conv2D(filters*4, (1, 1), name=sc_name, strides=strides, **conv_params)(shortcut)\n        x = Add()([x, shortcut])\n        return x\n\n    return layer\n\n\ndef identity_block(filters, stage, block):\n\n    def layer(input_tensor):\n        conv_params = get_conv_params()\n        bn_params = get_bn_params()\n        conv_name, bn_name, relu_name, sc_name = handle_block_names(stage, block)\n\n        x = BatchNormalization(name=bn_name + '1', **bn_params)(input_tensor)\n        x = Activation('relu', name=relu_name + '1')(x)\n        x = Conv2D(filters, (1, 1), name=conv_name + '1', **conv_params)(x)\n\n        x = BatchNormalization(name=bn_name + '2', **bn_params)(x)\n        x = Activation('relu', name=relu_name + '2')(x)\n        x = ZeroPadding2D(padding=(1, 1))(x)\n        x = Conv2D(filters, (3, 3), name=conv_name + '2', **conv_params)(x)\n\n        x = BatchNormalization(name=bn_name + '3', **bn_params)(x)\n        x = Activation('relu', name=relu_name + '3')(x)\n        x = Conv2D(filters*4, (1, 1), name=conv_name + '3', **conv_params)(x)\n\n        x = Add()([x, input_tensor])\n        return x\n\n    return layer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"619b56e2a6480fcd094c8fa3711e9e7fb2ce1857"},"cell_type":"code","source":"def get_conv_params(**params):\n    default_conv_params = {\n        'kernel_initializer': 'glorot_uniform',\n        'use_bias': False,\n        'padding': 'valid',\n    }\n    default_conv_params.update(params)\n    return default_conv_params\n\n\ndef get_bn_params(**params):\n    default_bn_params = {\n        'axis': 3,\n        'momentum': 0.99,\n        'epsilon': 2e-5,\n        'center': True,\n        'scale': True,\n    }\n    default_bn_params.update(params)\n    return default_bn_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f43c24643a591b5a15280e64a2d343f543eaad0"},"cell_type":"code","source":"import keras.backend as K\nfrom keras.layers import Input\nfrom keras.layers import Conv2D\nfrom keras.layers import MaxPooling2D\nfrom keras.layers import BatchNormalization\nfrom keras.layers import Activation\nfrom keras.layers import GlobalAveragePooling2D\nfrom keras.layers import ZeroPadding2D\nfrom keras.layers import Dense\nfrom keras.models import Model\nfrom keras.engine import get_source_inputs\n\nimport keras\nfrom distutils.version import StrictVersion\n\nif StrictVersion(keras.__version__) < StrictVersion('2.2.0'):\n    from keras.applications.imagenet_utils import _obtain_input_shape\nelse:\n    from keras_applications.imagenet_utils import _obtain_input_shape\n    \ndef build_resnet(\n     repetitions=(2, 2, 2, 2),\n     include_top=True,\n     input_tensor=None,\n     input_shape=None,\n     classes=1000,\n     block_type='usual'):\n\n    # Determine proper input shape\n    input_shape = _obtain_input_shape(input_shape,\n                                      default_size=224,\n                                      min_size=197,\n                                      data_format='channels_last',\n                                      require_flatten=include_top)\n\n    if input_tensor is None:\n        img_input = Input(shape=input_shape, name='data')\n    else:\n        if not K.is_keras_tensor(input_tensor):\n            img_input = Input(tensor=input_tensor, shape=input_shape)\n        else:\n            img_input = input_tensor\n    \n    # get parameters for model layers\n    no_scale_bn_params = get_bn_params(scale=False)\n    bn_params = get_bn_params()\n    conv_params = get_conv_params()\n    init_filters = 64\n\n    if block_type == 'basic':\n        conv_block = basic_conv_block\n        identity_block = basic_identity_block\n    else:\n        conv_block = usual_conv_block\n        identity_block = usual_identity_block\n    \n    # resnet bottom\n    x = BatchNormalization(name='bn_data', **no_scale_bn_params)(img_input)\n    x = ZeroPadding2D(padding=(3, 3))(x)\n    x = Conv2D(init_filters, (7, 7), strides=(2, 2), name='conv0', **conv_params)(x)\n    x = BatchNormalization(name='bn0', **bn_params)(x)\n    x = Activation('relu', name='relu0')(x)\n    x = ZeroPadding2D(padding=(1, 1))(x)\n    x = MaxPooling2D((3, 3), strides=(2, 2), padding='valid', name='pooling0')(x)\n    \n    # resnet body\n    for stage, rep in enumerate(repetitions):\n        for block in range(rep):\n            \n            filters = init_filters * (2**stage)\n            \n            # first block of first stage without strides because we have maxpooling before\n            if block == 0 and stage == 0:\n                x = conv_block(filters, stage, block, strides=(1, 1))(x)\n                \n            elif block == 0:\n                x = conv_block(filters, stage, block, strides=(2, 2))(x)\n                \n            else:\n                x = identity_block(filters, stage, block)(x)\n                \n    x = BatchNormalization(name='bn1', **bn_params)(x)\n    x = Activation('relu', name='relu1')(x)\n\n    # resnet top\n    if include_top:\n        x = GlobalAveragePooling2D(name='pool1')(x)\n        x = Dense(classes, name='fc1')(x)\n        x = Activation('softmax', name='softmax')(x)\n\n    # Ensure that the model takes into account any potential predecessors of `input_tensor`.\n    if input_tensor is not None:\n        inputs = get_source_inputs(input_tensor)\n    else:\n        inputs = img_input\n        \n    # Create model.\n    model = Model(inputs, x)\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5a37cb24b97c790f0884d864711409b2e7b0f41f"},"cell_type":"markdown","source":"## Load pretrain Weight by using ImageNet trained\n* Reference this [GITHUB](https://github.com/qubvel/classification_models/blob/9e438e2e133897b115148c737abdda3e1db31787/classification_models/weights.py)"},{"metadata":{"trusted":true,"_uuid":"5c1b6f8ccfe706af1f7b64576910ea97168cc8fd"},"cell_type":"code","source":"from keras.utils import get_file\n\n\ndef find_weights(weights_collection, model_name, dataset, include_top):\n    w = list(filter(lambda x: x['model'] == model_name, weights_collection))\n    w = list(filter(lambda x: x['dataset'] == dataset, w))\n    w = list(filter(lambda x: x['include_top'] == include_top, w))\n    return w\n\n\ndef load_model_weights(weights_collection, model, dataset, classes, include_top):\n    weights = find_weights(weights_collection, model.name, dataset, include_top)\n\n    if weights:\n        weights = weights[0]\n\n        if include_top and weights['classes'] != classes:\n            raise ValueError('If using `weights` and `include_top`'\n                             ' as true, `classes` should be {}'.format(weights['classes']))\n\n        weights_path = get_file(weights['name'],\n                                weights['url'],\n                                cache_subdir='models',\n                                md5_hash=weights['md5'])\n\n        model.load_weights(weights_path)\n\n    else:\n        raise ValueError('There is no weights for such configuration: ' +\n                         'model = {}, dataset = {}, '.format(model.name, dataset) +\n                         'classes = {}, include_top = {}.'.format(classes, include_top))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8000f2ee6e8482c1ba1bab2fa157830441f5ba6b"},"cell_type":"code","source":"weights_collection = [\n\n    # ResNet34\n    {\n        'model': 'resnet34',\n        'dataset': 'imagenet',\n        'classes': 1000,\n        'include_top': True,\n        'url': 'https://github.com/qubvel/classification_models/releases/download/0.0.1/resnet34_imagenet_1000.h5',\n        'name': 'resnet34_imagenet_1000.h5',\n        'md5': '2ac8277412f65e5d047f255bcbd10383',\n    },\n\n    {\n        'model': 'resnet34',\n        'dataset': 'imagenet',\n        'classes': 1000,\n        'include_top': False,\n        'url': 'https://github.com/qubvel/classification_models/releases/download/0.0.1/resnet34_imagenet_1000_no_top.h5',\n        'name': 'resnet34_imagenet_1000_no_top.h5',\n        'md5': '8caaa0ad39d927cb8ba5385bf945d582',\n    }]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6e3cf49751c8ea27a1cc623263d1781ff261cfb3"},"cell_type":"markdown","source":"## Buil Unet model base on ResNet34"},{"metadata":{"trusted":true,"_uuid":"b9f9fe78bb02e8ab116e94aaf540ae4a578b5f8b"},"cell_type":"code","source":"#Freeze Encoder weight if needed\n\ndef freeze_model(model):\n    for layer in model.layers:\n        layer.trainable = False\n    return\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f95ddf698c2afd39efa89a79fe5a436fada456a"},"cell_type":"code","source":"def UResNet34(input_shape=(None, None, 3), classes=1, decoder_filters=16, decoder_block_type='upsampling',\n                       encoder_weights=None, input_tensor=None, activation='sigmoid', **kwargs):\n\n    backbone = build_resnet(input_tensor=None,\n                         input_shape=input_shape,\n                         repetitions=(3, 4, 6, 3),\n                         classes=classes,\n                         include_top=False,\n                         block_type='basic')\n    backbone.name = 'resnet34'\n    \n    if encoder_weights == True:\n        load_model_weights(weights_collection, backbone , dataset= 'imagenet', classes = 1, include_top=False)\n    \n    skip_connections = list([129, 74, 37, 5]) # for resnet 34\n    model = build_unet(backbone, classes, decoder_filters,\n                       skip_connections, block_type=decoder_block_type,\n                       activation=activation, **kwargs)\n    model.name = 'u-resnet34'\n    \n    #freeze_model(backbone)\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7d7d7203c83a918386335584b8a0f378a48bfff"},"cell_type":"code","source":"seg_model = UResNet34(input_shape=(768,768,3),encoder_weights=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56d86c1e93cc35a8fbd65ada79c8eb9e92c8951a","_kg_hide-output":true},"cell_type":"code","source":"seg_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4710a5f4ebbb37d979acb0e9ddb18a0284d8e1e7"},"cell_type":"markdown","source":"## Define Loss function"},{"metadata":{"trusted":true,"_uuid":"4591a1e8577fb7a95a0c771d0ead24380dbd423a"},"cell_type":"code","source":"from keras.losses import binary_crossentropy\nfrom keras import backend as K\n\ndef dice_coef(y_true, y_pred):\n    y_true_f = K.flatten(y_true)\n    y_pred = K.cast(y_pred, 'float32')\n    y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')\n    intersection = y_true_f * y_pred_f\n    score = 2. * K.sum(intersection) / (K.sum(y_true_f) + K.sum(y_pred_f))\n    return score\n\ndef dice_loss(y_true, y_pred):\n    smooth = 1.\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = y_true_f * y_pred_f\n    score = (2. * K.sum(intersection) + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n    return 1. - score\n\ndef bce_dice_loss(y_true, y_pred):\n    return binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)\n\ndef bce_logdice_loss(y_true, y_pred):\n    return binary_crossentropy(y_true, y_pred) - K.log(1. - dice_loss(y_true, y_pred))\n\ndef weighted_bce_loss(y_true, y_pred, weight):\n    epsilon = 1e-7\n    y_pred = K.clip(y_pred, epsilon, 1. - epsilon)\n    logit_y_pred = K.log(y_pred / (1. - y_pred))\n    loss = weight * (logit_y_pred * (1. - y_true) + \n                     K.log(1. + K.exp(-K.abs(logit_y_pred))) + K.maximum(-logit_y_pred, 0.))\n    return K.sum(loss) / K.sum(weight)\n\ndef weighted_dice_loss(y_true, y_pred, weight):\n    smooth = 1.\n    w, m1, m2 = weight, y_true, y_pred\n    intersection = (m1 * m2)\n    score = (2. * K.sum(w * intersection) + smooth) / (K.sum(w * m1) + K.sum(w * m2) + smooth)\n    loss = 1. - K.sum(score)\n    return loss\n\ndef weighted_bce_dice_loss(y_true, y_pred):\n    y_true = K.cast(y_true, 'float32')\n    y_pred = K.cast(y_pred, 'float32')\n    # if we want to get same size of output, kernel size must be odd\n    averaged_mask = K.pool2d(\n            y_true, pool_size=(50, 50), strides=(1, 1), padding='same', pool_mode='avg')\n    weight = K.ones_like(averaged_mask)\n    w0 = K.sum(weight)\n    weight = 5. * K.exp(-5. * K.abs(averaged_mask - 0.5))\n    w1 = K.sum(weight)\n    weight *= (w0 / w1)\n    loss = weighted_bce_loss(y_true, y_pred, weight) + dice_loss(y_true, y_pred)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c9be1f5292b0b5e967ab47603a79651d2037f3a"},"cell_type":"code","source":"import keras.backend as K\nfrom keras.optimizers import Adam\nfrom keras.losses import binary_crossentropy\ndef dice_coef(y_true, y_pred, smooth=1):\n    intersection = K.sum(y_true * y_pred, axis=[1,2,3])\n    union = K.sum(y_true, axis=[1,2,3]) + K.sum(y_pred, axis=[1,2,3])\n    return K.mean( (2. * intersection + smooth) / (union + smooth), axis=0)\ndef dice_p_bce(in_gt, in_pred):\n    return 1e-3*binary_crossentropy(in_gt, in_pred) - dice_coef(in_gt, in_pred)\ndef true_positive_rate(y_true, y_pred):\n    return K.sum(K.flatten(y_true)*K.flatten(K.round(y_pred)))/K.sum(y_true)\n#seg_model.compile(optimizer=Adam(1e-4, decay=1e-6), loss=dice_p_bce, metrics=[dice_coef, 'binary_accuracy', true_positive_rate])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5149d0e0cf06775123a0c661bb5ea3c2fd818dc1"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a9f308bebfc7a8f067ca61bde886d33bd0e4649"},"cell_type":"markdown","source":"## Set Training Check Point"},{"metadata":{"trusted":true,"_uuid":"3128f14c1f1b86e8b383caea926b5f16d4f5094b"},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nweight_path=\"{}_weights.best.hdf5\".format('seg_model')\n\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_dice_coef', verbose=1, \n                             save_best_only=True, mode='max', save_weights_only = True)\n\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_dice_coef', factor=0.5, \n                                   patience=3, \n                                   verbose=1, mode='max', epsilon=0.0001, cooldown=2, min_lr=1e-6)\nearly = EarlyStopping(monitor=\"val_dice_coef\", \n                      mode=\"max\", \n                      patience=15) # probably needs to be more patient, but kaggle time is limited\ncallbacks_list = [checkpoint, early, reduceLROnPlat]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"50df57cf454854540c1594f0ed7118d3d0427470"},"cell_type":"markdown","source":"## Comiple model"},{"metadata":{"trusted":true,"_uuid":"698ff59ea2f49aebe8cd0c4af70692181cb3a9d5"},"cell_type":"code","source":"seg_model.compile(optimizer=Adam(1e-4, decay=1e-6), loss=bce_logdice_loss, metrics=[dice_coef, 'binary_accuracy', true_positive_rate])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07df3a3a549adec2c387b83d407be75a67308f68"},"cell_type":"code","source":"\"\"\"\nweight_path=\"{}_weights.best.hdf5\".format('seg_model')\n\nearly_stopping = EarlyStopping(patience=10, verbose=1)\nmodel_checkpoint = ModelCheckpoint(weight_path, save_best_only=True, verbose=1)\nreduce_lr = ReduceLROnPlateau(factor=0.1, patience=4, min_lr=0.00001, verbose=1)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d09b8deb89b59f2dfb10e664c34dd9514ec0a06d"},"cell_type":"markdown","source":"## Start Training"},{"metadata":{"trusted":true,"_uuid":"837b6e7589a725875f1d0ddfd410e8bd88989e2b"},"cell_type":"code","source":"\nepochs = 100\nbatch_size = 32\n\nstep_count = min(MAX_TRAIN_STEPS, balanced_train_df.shape[0]//BATCH_SIZE)\naug_gen = create_aug_gen(make_image_gen(balanced_train_df))\nloss_history = [seg_model.fit_generator(aug_gen,\n                            steps_per_epoch=step_count,\n                            validation_data=(valid_x, valid_y), \n                            epochs=5,\n                            callbacks=callbacks_list,shuffle=True)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4438fd660d8ef48310e890415da0b549971b9884"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce1167e9f09200f537e61f93f486168a13be1711"},"cell_type":"code","source":"seg_model.load_weights(weight_path)\nseg_model.save('seg_model.h5')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0688eacb61c50be9b73f854592dd4066e66300a8"},"cell_type":"markdown","source":"## Plot loss history"},{"metadata":{"trusted":true,"_uuid":"1fc067d4e714b7aa13ab2e5e31ded120fd45ff56"},"cell_type":"code","source":"    epich = np.cumsum(np.concatenate(\n        [np.linspace(0.5, 1, len(mh.epoch)) for mh in loss_history]))\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(22, 10))\n    _ = ax1.plot(epich,\n                 np.concatenate([mh.history['loss'] for mh in loss_history]),\n                 'b-',\n                 epich, np.concatenate(\n            [mh.history['val_loss'] for mh in loss_history]), 'r-')\n    ax1.legend(['Training', 'Validation'])\n    ax1.set_title('Loss')\n    \n    _ = ax2.plot(epich, np.concatenate(\n        [mh.history['dice_coef'] for mh in loss_history]), 'b-',\n                     epich, np.concatenate(\n            [mh.history['val_dice_coef'] for mh in loss_history]),\n                     'r-')\n    ax2.legend(['Training', 'Validation'])\n    ax2.set_title('DICE')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0018ab172d18936f8cc2c5df33d2f840dc16bf4f"},"cell_type":"markdown","source":"# Prepare Full Resolution Model\nHere we account for the scaling so everything can happen in the model itself"},{"metadata":{"trusted":true,"_uuid":"17408f0ee8dc16149b8eff0447a1427ab3ed82ba"},"cell_type":"code","source":"from keras import models, layers\nif IMG_SCALING is not None:\n    fullres_model = models.Sequential()\n    fullres_model.add(layers.AvgPool2D(IMG_SCALING, input_shape = (None, None, 3)))\n    fullres_model.add(seg_model)\n    fullres_model.add(layers.UpSampling2D(IMG_SCALING))\nelse:\n    fullres_model = seg_model\nfullres_model.save('fullres_model_net34.h5')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17edb177402ae51651692511827a7e9d60646533"},"cell_type":"markdown","source":"# Run the test data"},{"metadata":{"trusted":true,"_uuid":"4911811f267f9f3397a58902da9e75c6f261ad40"},"cell_type":"code","source":"test_paths = os.listdir(test_image_dir)\nprint(len(test_paths), 'test images found')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73ef7b3b2a74bf64968c79b4005075d4f0e23143"},"cell_type":"code","source":"fig, m_axs = plt.subplots(20, 2, figsize = (10, 40))\n[c_ax.axis('off') for c_ax in m_axs.flatten()]\nfor (ax1, ax2), c_img_name in zip(m_axs, test_paths):\n    c_path = os.path.join(test_image_dir, c_img_name)\n    c_img = imread(c_path)\n    first_img = np.expand_dims(c_img, 0)/255.0\n    first_seg = fullres_model.predict(first_img)\n    ax1.imshow(first_img[0])\n    ax1.set_title('Image')\n    ax2.imshow(first_seg[0,:, :, 0])\n    ax2.set_title('Prediction')\nfig.savefig('test_predictions.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"11a6c6615131ff8c317f95a5097b46565ef21121"},"cell_type":"markdown","source":"# Submission\n* We merge [part1 ](https://www.kaggle.com/super13579/simple-transfer-learning-detect-ship-exist-keras)trainsfer learning for detect ship exist result to generate submit result \n* Please refer the submission result kernel : https://www.kaggle.com/super13579/unet34-predict-result"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}