{"cells":[{"metadata":{"_uuid":"0867fe7c0ceddf83bef7ff2609158acbbb5c4fa0"},"cell_type":"markdown","source":"# Overview\n\n## Using a pretrained model to segment\nHere is an example kernel where we use a pretrained VGG16 model as the encoder portion of a U-Net and thus can benefit from the features already created in the model and only focus on learning the specific decoding features. The strategy was used with LinkNet by one of the top placers in the competition. I wanted to see how well it worked in particular comparing it to standard or non-pretrained approaches, the code is setup now for VGG16 but can be easily adapted to other problems"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"492a630ff3962eee98089482f3b6a2e730ac828c"},"cell_type":"code","source":"# copy the weights and configurations for the pre-trained models (before any imports)\n!rm -rf ~/.keras\n!mkdir ~/.keras\n!mkdir ~/.keras/models\n!cp ../input/keras-pretrained-models/*notop* ~/.keras/models/\n!cp ../input/keras-pretrained-models/imagenet_class_index.json ~/.keras/models/","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport numpy as np\nimport skimage.transform\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport os\nimport matplotlib.pyplot as plt # plotting\nfrom skimage.io import imread # read in images\nfrom skimage.segmentation import mark_boundaries # mark labels\nfrom sklearn.metrics import roc_curve, auc # roc curve tools\nfrom skimage.color import label2rgb\nimport numpy as np # linear algebra / matrices\nimport pandas as pd\n\nfrom IPython.display import display\nfrom skimage.util.montage import montage2d\nfrom glob import glob\nfrom os.path import split, splitext, join\n\nbase_dir = '../input/carvana-image-masking-challenge/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dfb4c826f1025b5e35177424ab60ebb3f07f3331"},"cell_type":"code","source":"all_img_df = pd.DataFrame(dict(path = glob(os.path.join(base_dir, 'train', '*.*'))))\nall_img_df['key_id'] = all_img_df['path'].map(lambda x: splitext(os.path.basename(x))[0])\nall_img_df['car_id'] = all_img_df['key_id'].map(lambda x: x.split('_')[0])\nall_img_df['mask_path'] = all_img_df['path'].map(lambda x: x.replace('train', 'train_masks').replace('.jpg', '_mask.gif'))\nall_img_df['exists'] = all_img_df['mask_path'].map(os.path.exists)\nprint(all_img_df['exists'].value_counts())\nall_img_df.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def read_diff_img(c_row):\n    t0_img = imread(c_row['path'])[:,:,0:3]\n    cg_img = imread(c_row['mask_path'], as_grey=True)\n    return (t0_img, cg_img)\ndef make_change_figure(c_row):\n    a,c = read_diff_img(c_row)\n    fig, (ax1, ax3) = plt.subplots(1, 2, figsize = (21,7))\n    ax1.imshow(a)\n    ax1.set_title('Before')\n    ax1.axis('off')\n    d = skimage.measure.label(c)\n    ax3.imshow(d, cmap = 'nipy_spectral_r')\n    ax3.set_title('Changes')\n    ax3.axis('off')\n    return fig\n_, t_row = next(all_img_df.sample(1).iterrows())\nmake_change_figure(t_row).savefig('overview.png', dpi = 300)\na,c = read_diff_img(t_row)\nprint(a.shape, c.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e8a695000380ccca55cf259138c7243caf65992c"},"cell_type":"markdown","source":"# Training and Validation Split\nHere we split based on scene so the model doesn't overfit the individual images"},{"metadata":{"trusted":true,"_uuid":"9bc812ebf9d79e4c717baea165659f63d6f90344"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ndef train_test_split_on_group(in_df, col_id, **kwargs):\n    group_val = np.unique(in_df[col_id])\n    train_ids, test_ids = train_test_split(group_val, **kwargs)\n    return in_df[in_df[col_id].isin(train_ids)], in_df[in_df[col_id].isin(test_ids)],\ntrain_df, valid_df = train_test_split_on_group(all_img_df, col_id = 'car_id', random_state = 2018, test_size = 0.2)\nvalid_df, test_df = train_test_split_on_group(valid_df, col_id = 'car_id', random_state = 2018, test_size = 0.5)\nprint(train_df.shape[0], 'training images')\nprint(valid_df.shape[0], 'validation images')\nprint(test_df.shape[0], 'test images')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4e05eaf1553628e4ca327c8af2c906f5f4f04e5e"},"cell_type":"markdown","source":"# Augmenting Data"},{"metadata":{"trusted":true,"_uuid":"31128ad302ea0789b4a3dba782c4902fc1dbb113"},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.applications.vgg16 import preprocess_input\ndg_args = dict(featurewise_center = False, \n                  samplewise_center = False,\n                  rotation_range = 5, \n                  width_shift_range = 0.01, \n                  height_shift_range = 0.01, \n                  shear_range = 0.01,\n                  zoom_range = [0.9, 1.1],  \n                  horizontal_flip = True, \n                  vertical_flip = False, # no upside down cars\n                  fill_mode = 'nearest',\n                   data_format = 'channels_last',\n               preprocessing_function = preprocess_input)\nIMG_SIZE = (512, 512) # slightly smaller than vgg16 normally expects\ndefault_batch_size = 8\ncore_idg = ImageDataGenerator(**dg_args)\nmask_args = dg_args.copy()\nmask_args['preprocessing_function'] = lambda x: x/255.0\nmask_idg = ImageDataGenerator(**mask_args)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e7c70251379d3bf2757a64e60368417fe67644e9"},"cell_type":"code","source":"def flow_from_dataframe(img_data_gen, in_df, path_col, y_col, **dflow_args):\n    base_dir = os.path.dirname(in_df[path_col].values[0])\n    print('## Ignore next message from keras, values are replaced anyways')\n    df_gen = img_data_gen.flow_from_directory(base_dir, \n                                     class_mode = 'sparse',\n                                    **dflow_args)\n    df_gen.filenames = in_df[path_col].values\n    df_gen.classes = np.stack(in_df[y_col].values)\n    df_gen.samples = in_df.shape[0]\n    df_gen.n = in_df.shape[0]\n    df_gen._set_index_array()\n    df_gen.directory = '' # since we have the full path\n    print('Reinserting dataframe: {} images'.format(in_df.shape[0]))\n    return df_gen","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3699616c98a7a90c14c9ad5e0f3b37c7e5dd768a"},"cell_type":"code","source":"def make_gen(img_gen, mask_gen, in_df, batch_size = default_batch_size, seed = None, shuffle = True):\n    if seed is None:\n        seed = np.random.choice(range(9999))\n    flow_args = dict(target_size = IMG_SIZE, \n                     batch_size = batch_size, \n                     seed = seed,\n                     shuffle = shuffle,\n                    y_col = 'key_id')\n    t0_gen = flow_from_dataframe(img_gen, in_df, \n                                 path_col = 'path',\n                                 color_mode = 'rgb',\n                                **flow_args)\n    dm_gen = flow_from_dataframe(mask_gen, in_df, \n                                 path_col = 'mask_path',\n                                 color_mode = 'grayscale',\n                                **flow_args)\n    for (t0_img, _), (dm_img, _) in zip(t0_gen, dm_gen):\n        yield [t0_img], dm_img\n\ntrain_gen = make_gen(core_idg, mask_idg, train_df)\nvalid_gen = make_gen(core_idg, mask_idg, valid_df, seed = 0, shuffle = False)\ntest_gen = make_gen(core_idg, mask_idg, test_df, seed = 0, shuffle = False, batch_size = 2*default_batch_size)\n[t0_img], dm_img = next(train_gen)\nprint(t0_img.shape, t0_img.max())\nprint(dm_img.shape, dm_img.max(), dm_img.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4ae92af877a1ea31edd89093b23d83bbc9229cb"},"cell_type":"code","source":"n_rgb = lambda x: np.stack([(x[:, :, i]-x[:, :, i].min())/(x[:, :, i].max()-x[:, :, i].min()) for i in range(x.shape[2])], 2)[:, :, ::-1]\nnn_rgb = lambda x: n_rgb(np.stack([montage2d(x[:, :, :, i]) for i in range(x.shape[3])], -1))\nfig, (ax1, ax3) = plt.subplots(1, 2, figsize = (25, 10))\nax1.imshow(nn_rgb(t0_img[:, :, :, :]), cmap = 'bone')\nax1.set_title('$T_0$ Image')\nax3.imshow(montage2d(dm_img[:, :, :, 0]), cmap = 'bone')\nax3.set_title('$\\Delta$T Mask');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cb59a9e624546484705b6f0d48ed95d2b0545c49"},"cell_type":"markdown","source":"# Building the Model"},{"metadata":{"trusted":true,"_uuid":"2e9f8a5821fcd736ff41ae4b66ac7f9640e76c7a"},"cell_type":"code","source":"from keras.applications.vgg16 import VGG16 as PTModel\nbase_pretrained_model = PTModel(input_shape =  t0_img.shape[1:], include_top = False, weights = 'imagenet')\nbase_pretrained_model.trainable = False\nbase_pretrained_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e3a57a98ddc49e441107457dd9eab0def4fbbf34"},"cell_type":"markdown","source":"## Collect Interesting Layers for Model\nHere we collect layers by size so we can make an encoder from them"},{"metadata":{"trusted":true,"_uuid":"46dc350649db92be02f90099c6c643769879cb2f"},"cell_type":"code","source":"from collections import defaultdict, OrderedDict\nfrom keras.models import Model\nlayer_size_dict = defaultdict(list)\ninputs = []\nfor lay_idx, c_layer in enumerate(base_pretrained_model.layers):\n    if not c_layer.__class__.__name__ == 'InputLayer':\n        layer_size_dict[c_layer.get_output_shape_at(0)[1:3]] += [c_layer]\n    else:\n        inputs += [c_layer]\n# freeze dict\nlayer_size_dict = OrderedDict(layer_size_dict.items())\nfor k,v in layer_size_dict.items():\n    print(k, [w.__class__.__name__ for w in v])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f990a9a7c59da91b5aa047c1bb313ea9a5452fbe"},"cell_type":"code","source":"%%time\n# take the last layer of each shape and make it into an output\npretrained_encoder = Model(inputs = base_pretrained_model.get_input_at(0), \n                           outputs = [v[-1].get_output_at(0) for k, v in layer_size_dict.items()])\npretrained_encoder.trainable = False\nn_outputs = pretrained_encoder.predict([t0_img])\nfor c_out, (k, v) in zip(n_outputs, layer_size_dict.items()):\n    print(c_out.shape, 'expected', k)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80616b511ba30f98a380db3538fef0f76ce69dd0","collapsed":true},"cell_type":"markdown","source":"# Build the U-Net"},{"metadata":{"trusted":true,"_uuid":"ae5e04f6cd54772327a2f1eda1dff4861cad3d14"},"cell_type":"code","source":"from keras.layers import Input, Conv2D, concatenate, UpSampling2D, BatchNormalization, Activation, Cropping2D, ZeroPadding2D\nx_wid, y_wid = t0_img.shape[1:3]\nin_t0 = Input(t0_img.shape[1:], name = 'T0_Image')\nwrap_encoder = lambda i_layer: {k: v for k, v in zip(layer_size_dict.keys(), pretrained_encoder(i_layer))}\n\nt0_outputs = wrap_encoder(in_t0)\nlay_dims = sorted(t0_outputs.keys(), key = lambda x: x[0])\nskip_layers = 2\nlast_layer = None\nfor k in lay_dims[skip_layers:]:\n    cur_layer = t0_outputs[k]\n    channel_count = cur_layer._keras_shape[-1]\n    cur_layer = Conv2D(channel_count//2, kernel_size=(3,3), padding = 'same', activation = 'linear')(cur_layer)\n    cur_layer = BatchNormalization()(cur_layer) # gotta keep an eye on that internal covariant shift\n    cur_layer = Activation('relu')(cur_layer)\n    \n    if last_layer is None:\n        x = cur_layer\n    else:\n        last_channel_count = last_layer._keras_shape[-1]\n        x = Conv2D(last_channel_count//2, kernel_size=(3,3), padding = 'same')(last_layer)\n        x = UpSampling2D((2, 2))(x)\n        x = concatenate([cur_layer, x])\n    last_layer = x\nfinal_output = Conv2D(dm_img.shape[-1], kernel_size=(1,1), padding = 'same', activation = 'sigmoid')(last_layer)\ncrop_size = 20\nfinal_output = Cropping2D((crop_size, crop_size))(final_output)\nfinal_output = ZeroPadding2D((crop_size, crop_size))(final_output)\nunet_model = Model(inputs = [in_t0],\n                  outputs = [final_output])\nunet_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"2ba1083e1777032a2598230604b0feb5258740a8"},"cell_type":"code","source":"import keras.backend as K\nfrom keras.optimizers import Adam\nfrom keras.losses import binary_crossentropy\ndef dice_coef(y_true, y_pred, smooth=1):\n    intersection = K.sum(y_true * y_pred, axis=[1,2,3])\n    union = K.sum(y_true, axis=[1,2,3]) + K.sum(y_pred, axis=[1,2,3])\n    return K.mean( (2. * intersection + smooth) / (union + smooth), axis=0)\ndef dice_p_bce(in_gt, in_pred):\n    return 0.0*binary_crossentropy(in_gt, in_pred) - dice_coef(in_gt, in_pred)\ndef true_positive_rate(y_true, y_pred):\n    return K.sum(K.flatten(y_true)*K.flatten(K.round(y_pred)))/K.sum(y_true)\n\nunet_model.compile(optimizer=Adam(1e-3, decay = 1e-6), \n                   loss=dice_p_bce, \n                   metrics=[dice_coef, 'binary_accuracy', true_positive_rate])\nloss_history = []","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"05b0e650cbed55f59e2a304ed52ae6560b64128c"},"cell_type":"markdown","source":"## Callbacks for training\nMake sure the training works well and doesnt run too long or overfit too much"},{"metadata":{"trusted":true,"_uuid":"b6bef5fae1fba66ca77f93a9a6d2d3a09eae2252"},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nweight_path=\"{}_weights.best.hdf5\".format('vgg_unet')\n\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\n\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=10, verbose=1, mode='auto', epsilon=0.0001, cooldown=5, min_lr=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=5) # probably needs to be more patient, but kaggle time is limited\ncallbacks_list = [checkpoint, early, reduceLROnPlat]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6467854b46e25e1f4b31f88d9646cf315b273d71"},"cell_type":"code","source":"loss_history += [unet_model.fit_generator(make_gen(core_idg, mask_idg, train_df), \n                                         steps_per_epoch=min(100, train_df.shape[0]//t0_img.shape[0]), \n                                         epochs = 5, \n                                         validation_data = valid_gen,\n                                          validation_steps = valid_df.shape[0]//t0_img.shape[0],\n                                         callbacks = callbacks_list,\n                                        workers = 2\n                                       )]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db6f26a26bddd4b676b0b3cf799d11e0910f1413","collapsed":true},"cell_type":"code","source":"def show_loss(loss_history):\n    epich = np.cumsum(np.concatenate(\n        [np.linspace(0.5, 1, len(mh.epoch)) for mh in loss_history]))\n    fig, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4, figsize=(22, 10))\n    _ = ax1.plot(epich,\n                 np.concatenate([mh.history['loss'] for mh in loss_history]),\n                 'b-',\n                 epich, np.concatenate(\n            [mh.history['val_loss'] for mh in loss_history]), 'r-')\n    ax1.legend(['Training', 'Validation'])\n    ax1.set_title('Loss')\n\n    _ = ax2.plot(epich, np.concatenate(\n        [mh.history['true_positive_rate'] for mh in loss_history]), 'b-',\n                     epich, np.concatenate(\n            [mh.history['val_true_positive_rate'] for mh in loss_history]),\n                     'r-')\n    ax2.legend(['Training', 'Validation'])\n    ax2.set_title('True Positive Rate\\n(Positive Accuracy)')\n    \n    _ = ax3.plot(epich, np.concatenate(\n        [mh.history['binary_accuracy'] for mh in loss_history]), 'b-',\n                     epich, np.concatenate(\n            [mh.history['val_binary_accuracy'] for mh in loss_history]),\n                     'r-')\n    ax3.legend(['Training', 'Validation'])\n    ax3.set_title('Binary Accuracy (%)')\n    \n    _ = ax4.plot(epich, np.concatenate(\n        [mh.history['dice_coef'] for mh in loss_history]), 'b-',\n                     epich, np.concatenate(\n            [mh.history['val_dice_coef'] for mh in loss_history]),\n                     'r-')\n    ax4.legend(['Training', 'Validation'])\n    ax4.set_title('DICE')\n\nshow_loss(loss_history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c33e706efc7e7b577322bc7c7546f9eb70f883e"},"cell_type":"code","source":"unet_model.load_weights(weight_path)\nunet_model.save('full_seg_model.h5')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e1c9f6ca2eff605fd1af44cc4fe735a71833d6c3"},"cell_type":"markdown","source":"# Show the results"},{"metadata":{"trusted":true,"_uuid":"01d238e8ab95b70212b111040b88972a7310d076"},"cell_type":"code","source":"[t0_img], dm_img = next(train_gen)\ndm_pred = unet_model.predict([t0_img])\nfig, (ax1, ax3, ax4) = plt.subplots(1, 3, figsize = (40, 10))\nax1.imshow(montage2d(t0_img[:, :, :, 0]), cmap = 'bone')\nax1.set_title('$T_0$ Image')\nax3.imshow(montage2d(dm_pred[:, :, :, 0]), cmap = 'nipy_spectral')\nax3.set_title('$\\Delta$T Prediction');\nax4.imshow(montage2d(dm_img[:, :, :, 0]), cmap = 'bone')\nax4.set_title('$\\Delta$T Mask');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6022b1f3580e928f5537d167711905b24ff0b316"},"cell_type":"code","source":"fig, m_axs = plt.subplots(5, 4, figsize = (40, 40))\nfor (ax1, ax3, ax4, ax5), x_t0, x_pred, x_dm in zip(m_axs, t0_img, dm_pred, dm_img):\n    n_rgb = lambda x: np.stack([(x[:, :, i]-x[:, :, i].min())/(x[:, :, i].max()-x[:, :, i].min()) for i in range(x.shape[2])], 2)[:, :, ::-1]\n    ax1.imshow(n_rgb(x_t0), cmap = 'bone')\n    ax1.set_title('$T_0$ Image')\n    ax3.imshow(x_pred[:, :, 0], cmap = 'nipy_spectral')\n    ax3.set_title('Mask Prediction');\n    ax4.imshow(x_dm[:, :, 0], cmap = 'bone')\n    ax4.set_title('Mask Reality');\n    ax5.imshow(x_pred[:, :, 0]-x_dm[:, :, 0], cmap = 'RdBu', vmin = -1, vmax = 1)\n    ax5.set_title('$\\Delta$ Mask');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3340be9585f79c593efe9743e4e673d94c91436"},"cell_type":"markdown","source":"# Show results on hold-out data"},{"metadata":{"trusted":true,"_uuid":"25ba41dca13c2144ae877b96714be7b38e715c17"},"cell_type":"code","source":"out_parms = unet_model.evaluate_generator(test_gen, steps=test_df.shape[0] // (2*default_batch_size))\nprint('\\n')\nfor k,v in zip(unet_model.metrics_names, out_parms):\n    print(k,'%2.2f' % v)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"56ec958b214cc042f4c6046f99937b4d4b3232b5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}