{"cells":[{"metadata":{},"cell_type":"markdown","source":"\n\n## Information\n* In this code are compared the outcome of using Augmentation. \n* The networks are trained in 60 epochs.\n* The results can be checked individualy for each network and compared in the last cells.\n\n## log\n* **V8**. Checking suffling data while training\n* **V2**. Checking different batch sizes\n\n\n## References\n\n* EDA of albumentations: https://www.kaggle.com/artgor/segmentation-in-pytorch-using-convenient-tools\n* Data generator: https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly\n* RLE encoding and decoding: https://www.kaggle.com/paulorzp/rle-functions-run-lenght-encode-decode\n* Architecture: https://www.kaggle.com/jesperdramsch/intro-chest-xray-dicom-viz-u-nets-full-data\n* Mask encoding: https://www.kaggle.com/c/siim-acr-pneumothorax-segmentation/data\n* My original Kernel U-Net: https://www.kaggle.com/xhlulu/severstal-simple-keras-u-net-boilerplate"},{"metadata":{},"cell_type":"markdown","source":"# Imports "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"import os\nimport json\n\nimport gc\n\nimport albumentations as albu\nimport cv2\nimport keras\nfrom keras import backend as K\nfrom keras.models import Model\nfrom keras.layers import Input\nfrom keras.layers.convolutional import Conv2D, Conv2DTranspose\nfrom keras.layers.pooling import MaxPooling2D\nfrom keras.layers.merge import concatenate\nfrom keras.losses import binary_crossentropy\nfrom keras.optimizers import Adam, Nadam\nfrom keras.callbacks import Callback, ModelCheckpoint\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\n#import segmentation_models as sm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/understanding_cloud_organization/train.csv')\ntrain_df['ImageId'] = train_df['Image_Label'].apply(lambda x: x.split('_')[0])\ntrain_df['ClassId'] = train_df['Image_Label'].apply(lambda x: x.split('_')[1])\ntrain_df['hasMask'] = ~ train_df['EncodedPixels'].isna()\n\nprint(train_df.shape)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mask_count_df = train_df.groupby('ImageId').agg(np.sum).reset_index()\nmask_count_df.sort_values('hasMask', ascending=False, inplace=True)\nprint(mask_count_df.shape)\nmask_count_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"sub_df = pd.read_csv('/kaggle/input/understanding_cloud_organization/sample_submission.csv')\nsub_df['ImageId'] = sub_df['Image_Label'].apply(lambda x: x.split('_')[0])\ntest_imgs = pd.DataFrame(sub_df['ImageId'].unique(), columns=['ImageId'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utility Functions\n\nSource: https://www.kaggle.com/paulorzp/rle-functions-run-lenght-encode-decode\n\nUnhide below for the definition of `np_resize`, `build_masks`, `build_rles`."},{"metadata":{"_kg_hide-input":true,"trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"def np_resize(img, input_shape):\n    \"\"\"\n    Reshape a numpy array, which is input_shape=(height, width), \n    as opposed to input_shape=(width, height) for cv2\n    \"\"\"\n    height, width = input_shape\n    return cv2.resize(img, (width, height))\n    \ndef mask2rle(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels= img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef rle2mask(rle, input_shape):\n    width, height = input_shape[:2]\n    \n    mask= np.zeros( width*height ).astype(np.uint8)\n    \n    array = np.asarray([int(x) for x in rle.split()])\n    starts = array[0::2]\n    lengths = array[1::2]\n\n    current_position = 0\n    for index, start in enumerate(starts):\n        mask[int(start):int(start+lengths[index])] = 1\n        current_position += lengths[index]\n        \n    return mask.reshape(height, width).T\n\ndef build_masks(rles, input_shape, reshape=None):\n    depth = len(rles)\n    if reshape is None:\n        masks = np.zeros((*input_shape, depth))\n    else:\n        masks = np.zeros((*reshape, depth))\n    \n    for i, rle in enumerate(rles):\n        if type(rle) is str:\n            if reshape is None:\n                masks[:, :, i] = rle2mask(rle, input_shape)\n            else:\n                mask = rle2mask(rle, input_shape)\n                reshaped_mask = np_resize(mask, reshape)\n                masks[:, :, i] = reshaped_mask\n    \n    return masks\n\ndef build_rles(masks, reshape=None):\n    width, height, depth = masks.shape\n    \n    rles = []\n    \n    for i in range(depth):\n        mask = masks[:, :, i]\n        \n        if reshape:\n            mask = mask.astype(np.float32)\n            mask = np_resize(mask, reshape).astype(np.int64)\n        \n        rle = mask2rle(mask)\n        rles.append(rle)\n        \n    return rles","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualization functions\nUnhide below to see definition of `visualize`:"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def visualize(image, mask, mask_prediction):\n    fontsize = 14\n    class_dict = {0: 'Fish', 1: 'Flower', 2: 'Gravel', 3: 'Sugar'}\n    f, ax = plt.subplots(2, 5, figsize=(24,8))\n\n    ax[0, 0].imshow(image.reshape(image.shape[0],image.shape[1]))\n    ax[0, 0].set_title('Original image', fontsize=fontsize)\n\n    for i in range(4):\n        ax[0, i + 1].imshow(mask[:, :, i],vmin = 0, vmax = 1)\n        ax[0, i + 1].set_title(f'Original mask {class_dict[i]}', fontsize=fontsize)\n    \n    ax[1, 0].imshow(image.reshape(image.shape[0],image.shape[1]))\n    ax[1, 0].set_title('Original image', fontsize=fontsize)\n\n    for i in range(4):\n        ax[1, i + 1].imshow(mask_prediction[:, :, i],vmin = 0, vmax = 1)\n        ax[1, i + 1].set_title(f'Prediction {class_dict[i]}', fontsize=fontsize)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## RAdam\n\nUnhide below to see definition of `RAdam`:"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class RAdam(keras.optimizers.Optimizer):\n    \"\"\"RAdam optimizer.\n    # Arguments\n        lr: float >= 0. Learning rate.\n        beta_1: float, 0 < beta < 1. Generally close to 1.\n        beta_2: float, 0 < beta < 1. Generally close to 1.\n        epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`.\n        decay: float >= 0. Learning rate decay over each update.\n        weight_decay: float >= 0. Weight decay for each param.\n        amsgrad: boolean. Whether to apply the AMSGrad variant of this\n            algorithm from the paper \"On the Convergence of Adam and\n            Beyond\".\n        total_steps: int >= 0. Total number of training steps. Enable warmup by setting a positive value.\n        warmup_proportion: 0 < warmup_proportion < 1. The proportion of increasing steps.\n        min_lr: float >= 0. Minimum learning rate after warmup.\n    # References\n        - [Adam - A Method for Stochastic Optimization](https://arxiv.org/abs/1412.6980v8)\n        - [On the Convergence of Adam and Beyond](https://openreview.net/forum?id=ryQu7f-RZ)\n        - [On The Variance Of The Adaptive Learning Rate And Beyond](https://arxiv.org/pdf/1908.03265v1.pdf)\n    \"\"\"\n\n    def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,\n                 epsilon=None, decay=0., weight_decay=0., amsgrad=False,\n                 total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):\n        super(RAdam, self).__init__(**kwargs)\n        with K.name_scope(self.__class__.__name__):\n            self.iterations = K.variable(0, dtype='int64', name='iterations')\n            self.lr = K.variable(lr, name='lr')\n            self.beta_1 = K.variable(beta_1, name='beta_1')\n            self.beta_2 = K.variable(beta_2, name='beta_2')\n            self.decay = K.variable(decay, name='decay')\n            self.weight_decay = K.variable(weight_decay, name='weight_decay')\n            self.total_steps = K.variable(total_steps, name='total_steps')\n            self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')\n            self.min_lr = K.variable(min_lr, name='min_lr')\n        if epsilon is None:\n            epsilon = K.epsilon()\n        self.epsilon = epsilon\n        self.initial_decay = decay\n        self.initial_weight_decay = weight_decay\n        self.initial_total_steps = total_steps\n        self.amsgrad = amsgrad\n\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = [K.update_add(self.iterations, 1)]\n\n        lr = self.lr\n\n        if self.initial_decay > 0:\n            lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay))))\n\n        t = K.cast(self.iterations, K.floatx()) + 1\n\n        if self.initial_total_steps > 0:\n            warmup_steps = self.total_steps * self.warmup_proportion\n            lr = K.switch(\n                t <= warmup_steps,\n                lr * (t / warmup_steps),\n                self.min_lr + (lr - self.min_lr) * (1.0 - K.minimum(t, self.total_steps) / self.total_steps),\n            )\n\n        ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for (i, p) in enumerate(params)]\n        vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for (i, p) in enumerate(params)]\n\n        if self.amsgrad:\n            vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for (i, p) in enumerate(params)]\n        else:\n            vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params))]\n\n        self.weights = [self.iterations] + ms + vs + vhats\n\n        beta_1_t = K.pow(self.beta_1, t)\n        beta_2_t = K.pow(self.beta_2, t)\n\n        sma_inf = 2.0 / (1.0 - self.beta_2) - 1.0\n        sma_t = sma_inf - 2.0 * t * beta_2_t / (1.0 - beta_2_t)\n\n        for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):\n            m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n            v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n\n            m_corr_t = m_t / (1.0 - beta_1_t)\n            if self.amsgrad:\n                vhat_t = K.maximum(vhat, v_t)\n                v_corr_t = K.sqrt(vhat_t / (1.0 - beta_2_t) + self.epsilon)\n                self.updates.append(K.update(vhat, vhat_t))\n            else:\n                v_corr_t = K.sqrt(v_t / (1.0 - beta_2_t) + self.epsilon)\n\n            r_t = K.sqrt((sma_t - 4.0) / (sma_inf - 4.0) *\n                         (sma_t - 2.0) / (sma_inf - 2.0) *\n                         sma_inf / sma_t)\n\n            p_t = K.switch(sma_t >= 5, r_t * m_corr_t / v_corr_t, m_corr_t)\n\n            if self.initial_weight_decay > 0:\n                p_t += self.weight_decay * p\n\n            p_t = p - lr * p_t\n\n            self.updates.append(K.update(m, m_t))\n            self.updates.append(K.update(v, v_t))\n            new_p = p_t\n\n            # Apply constraints.\n            if getattr(p, 'constraint', None) is not None:\n                new_p = p.constraint(new_p)\n\n            self.updates.append(K.update(p, new_p))\n        return self.updates\n\n    def get_config(self):\n        config = {\n            'lr': float(K.get_value(self.lr)),\n            'beta_1': float(K.get_value(self.beta_1)),\n            'beta_2': float(K.get_value(self.beta_2)),\n            'decay': float(K.get_value(self.decay)),\n            'weight_decay': float(K.get_value(self.weight_decay)),\n            'epsilon': self.epsilon,\n            'amsgrad': self.amsgrad,\n            'total_steps': float(K.get_value(self.total_steps)),\n            'warmup_proportion': float(K.get_value(self.warmup_proportion)),\n            'min_lr': float(K.get_value(self.min_lr)),\n        }\n        base_config = super(RAdam, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss function\n\nSource for `bce_dice_loss`: https://lars76.github.io/neural-networks/object-detection/losses-for-segmentation/"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def dice_coef(y_true, y_pred, smooth=1):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n\ndef dice_loss(y_true, y_pred):\n    smooth = 1.\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = y_true_f * y_pred_f\n    score = (2. * K.sum(intersection) + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n    return 1. - score\n\ndef bce_dice_loss(y_true, y_pred):\n    return binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Generator /edited to charge images before start training"},{"metadata":{},"cell_type":"markdown","source":"Unhide below for the definition of `DataGenerator`:"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class DataGenerator(keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, list_IDs, df, target_df=None, mode='fit',\n                 base_path='/kaggle/input/understanding_cloud_organization/train_images',\n                 batch_size=32, dim=(1400, 2100), n_channels=1, reshape=None,\n                 augment=False, n_classes=4, random_state=2019, shuffle=True):\n        self.dim = dim\n        self.batch_size = batch_size\n        self.df = df\n        self.mode = mode\n        self.base_path = base_path\n        self.target_df = target_df\n        self.list_IDs = list_IDs\n        self.reshape = reshape\n        self.n_channels = n_channels\n        self.augment = augment\n        self.n_classes = n_classes\n        self.shuffle = shuffle\n        self.random_state = random_state\n        \n        self.on_epoch_end()\n        np.random.seed(self.random_state)\n        \n        \n        ###\n        self.imgs = {}\n        keys = list_IDs\n        \n        for k in keys:\n            im_name = self.df['ImageId'].iloc[k]\n            img_path = f\"{self.base_path}/{im_name}\"\n            if self.reshape is None:\n                self.imgs[k] = self.__load_grayscale(img_path)\n            else:\n                self.imgs[k] = np_resize(self.__load_grayscale(img_path), self.reshape)\n            \n            self.imgs[k] = self.imgs[k].reshape((self.imgs[k].shape[0],self.imgs[k].shape[1],1))\n\n        #\n        \n        self.masks = {}\n        \n        for k in keys:\n            im_name = self.df['ImageId'].iloc[k]\n            img_path = f\"{self.base_path}/{im_name}\"\n            if self.reshape is None:\n                self.imgs[k] = self.__load_grayscale(img_path)\n            else:\n                self.imgs[k] = np_resize(self.__load_grayscale(img_path), self.reshape)\n                \n            self.imgs[k] = self.imgs[k].reshape((self.imgs[k].shape[0],self.imgs[k].shape[1],1))\n            \n        for k in keys:\n            im_name = self.df['ImageId'].iloc[k]\n            image_df = self.target_df[self.target_df['ImageId'] == im_name]\n            \n            rles = image_df['EncodedPixels'].values\n            \n            if self.reshape is not None:\n                masks = build_masks(rles, input_shape=self.dim, reshape=self.reshape)\n            else:\n                masks = build_masks(rles, input_shape=self.dim)\n            \n            self.masks[k] = masks\n\n        #\n        \n\n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        return int(np.floor(len(self.list_IDs) / self.batch_size))\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        # Generate indexes of the batch\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n\n        # Find list of IDs\n        list_IDs_batch = [self.list_IDs[k] for k in indexes]\n        \n        X = self.__generate_X(list_IDs_batch)\n        \n        if self.mode == 'fit':\n            y = self.__generate_y(list_IDs_batch)\n            \n            if self.augment:\n                X, y = self.__augment_batch(X, y)\n            \n            return X, y\n        \n        elif self.mode == 'predict':\n            return X\n\n        else:\n            raise AttributeError('The mode parameter should be set to \"fit\" or \"predict\".')\n        \n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange(len(self.list_IDs))\n        if self.shuffle == True:\n            np.random.seed(self.random_state)\n            np.random.shuffle(self.indexes)\n    \n    def __generate_X(self, list_IDs_batch):\n        'Generates data containing batch_size samples'\n        # Initialization\n        if self.reshape is None:\n            X = np.empty((self.batch_size, *self.dim, self.n_channels))\n        else:\n            X = np.empty((self.batch_size, *self.reshape, self.n_channels))\n        \n        # Generate data\n        for i, ID in enumerate(list_IDs_batch):\n            im_name = self.df['ImageId'].iloc[ID]\n            img_path = f\"{self.base_path}/{im_name}\"\n            img = self.imgs[ID]\n\n            X[i,] = img\n\n        return X\n    \n    def __generate_y(self, list_IDs_batch):\n        if self.reshape is None:\n            y = np.empty((self.batch_size, *self.dim, self.n_classes), dtype=int)\n        else:\n            y = np.empty((self.batch_size, *self.reshape, self.n_classes), dtype=int)\n        \n        for i, ID in enumerate(list_IDs_batch):\n            \n            y[i, ] = self.masks[ID]\n\n        return y\n    \n    def __load_grayscale(self, img_path):\n        img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)\n        img = img.astype(np.float32) / 255.\n        img = np.expand_dims(img, axis=-1)\n\n        return img\n    \n    def __load_rgb(self, img_path):\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = img.astype(np.float32) / 255.\n\n        return img\n    \n    def __random_transform(self, img, masks):\n        composition = albu.Compose([\n            albu.HorizontalFlip(),\n            albu.VerticalFlip(),\n            albu.ShiftScaleRotate(rotate_limit=45, shift_limit=0.15, scale_limit=0.15)\n        ])\n        \n        composed = composition(image=img, mask=masks)\n        aug_img = composed['image']\n        aug_masks = composed['mask']\n        \n        return aug_img, aug_masks\n    \n    def __augment_batch(self, img_batch, masks_batch):\n        for i in range(img_batch.shape[0]):\n            img_batch[i, ], masks_batch[i, ] = self.__random_transform(\n                img_batch[i, ], masks_batch[i, ])\n        \n        return img_batch, masks_batch\n    \n    def getitem(self, index):\n        'Generate one batch of data'\n        # Generate indexes of the batch\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n\n        # Find list of IDs\n        list_IDs_batch = [self.list_IDs[k] for k in indexes]\n        \n        X = self.__generate_X(list_IDs_batch)\n        \n        if self.mode == 'fit':\n            y = self.__generate_y(list_IDs_batch)\n            \n            if self.augment:\n                X, y = self.__augment_batch(X, y)\n            \n            return X, y\n        \n        elif self.mode == 'predict':\n            return X\n\n        else:\n            raise AttributeError('The mode parameter should be set to \"fit\" or \"predict\".')\n\n    def setBatchSize(self, value):\n        self.batch_size = value\n    def getBatchSize(self):\n        return self.batch_size\n    \n    def setSuffle(self, value):\n        self.shuffle = value\n    def getSuffle(self):\n        return self.shuffle\n    \n    def setAugment(self, value):\n        self.shuffle = value\n    def getAugment(self):\n        return self.shuffle\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Architectures"},{"metadata":{},"cell_type":"markdown","source":"** Vanilla Unet**"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def vanilla_unet(input_shape):\n\n    inputs = Input(input_shape)\n    c1 = Conv2D(8, (3, 3), activation='elu', padding='same') (inputs)\n    c1 = Conv2D(8, (3, 3), activation='elu', padding='same') (c1)\n    p1 = MaxPooling2D((2, 2), padding='same') (c1)\n\n    c2 = Conv2D(16, (3, 3), activation='elu', padding='same') (p1)\n    c2 = Conv2D(16, (3, 3), activation='elu', padding='same') (c2)\n    p2 = MaxPooling2D((2, 2), padding='same') (c2)\n\n    c3 = Conv2D(32, (3, 3), activation='elu', padding='same') (p2)\n    c3 = Conv2D(32, (3, 3), activation='elu', padding='same') (c3)\n    p3 = MaxPooling2D((2, 2), padding='same') (c3)\n\n    c4 = Conv2D(64, (3, 3), activation='elu', padding='same') (p3)\n    c4 = Conv2D(64, (3, 3), activation='elu', padding='same') (c4)\n    p4 = MaxPooling2D((2, 2), padding='same') (c4)\n\n    c5 = Conv2D(64, (3, 3), activation='elu', padding='same') (p4)\n    c5 = Conv2D(64, (3, 3), activation='elu', padding='same') (c5)\n    p5 = MaxPooling2D((2, 2), padding='same') (c5)\n\n    c55 = Conv2D(128, (3, 3), activation='elu', padding='same') (p5)\n    c55 = Conv2D(128, (3, 3), activation='elu', padding='same') (c55)\n\n    u6 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same') (c55)\n    u6 = concatenate([u6, c5])\n    c6 = Conv2D(64, (3, 3), activation='elu', padding='same') (u6)\n    c6 = Conv2D(64, (3, 3), activation='elu', padding='same') (c6)\n\n    u71 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same') (c6)\n    u71 = concatenate([u71, c4])\n    c71 = Conv2D(32, (3, 3), activation='elu', padding='same') (u71)\n    c61 = Conv2D(32, (3, 3), activation='elu', padding='same') (c71)\n\n    u7 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same') (c61)\n    u7 = concatenate([u7, c3])\n    c7 = Conv2D(32, (3, 3), activation='elu', padding='same') (u7)\n    c7 = Conv2D(32, (3, 3), activation='elu', padding='same') (c7)\n\n    u8 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same') (c7)\n    u8 = concatenate([u8, c2])\n    c8 = Conv2D(16, (3, 3), activation='elu', padding='same') (u8)\n    c8 = Conv2D(16, (3, 3), activation='elu', padding='same') (c8)\n\n    u9 = Conv2DTranspose(8, (2, 2), strides=(2, 2), padding='same') (c8)\n    u9 = concatenate([u9, c1], axis=3)\n    c9 = Conv2D(8, (3, 3), activation='elu', padding='same') (u9)\n    c9 = Conv2D(8, (3, 3), activation='elu', padding='same') (c9)\n\n    outputs = Conv2D(4, (1, 1), activation='sigmoid') (c9)\n\n    model = Model(inputs=[inputs], outputs=[outputs])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load dataset"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"%%time\nBATCH_SIZE = 16\nSUFFLE = False\nAUGMENT = False\n\ntrain_idx, val_idx = train_test_split(\n    mask_count_df.index, random_state=2019, test_size=0.2\n)\ntrain_generator = DataGenerator(\n    train_idx, \n    df=mask_count_df,\n    target_df=train_df,\n    batch_size=BATCH_SIZE,\n    reshape=(128, 128),\n    augment=AUGMENT,\n    shuffle=SUFFLE,\n    n_channels=1,\n    n_classes=4\n)\nprint(\"Train generator loaded\")\n\n\nval_generator = DataGenerator(\n    val_idx, \n    df=mask_count_df,\n    target_df=train_df,\n    batch_size=BATCH_SIZE, \n    reshape=(128, 128),\n    augment=AUGMENT,\n    shuffle=SUFFLE,\n    n_channels=1,\n    n_classes=4,\n    #batch_size=1,\n\n)\nprint(\"Validation generator loaded\")\n\ncheck_generator = DataGenerator(\n    #val_idx[0:10],\n    val_idx,\n    df=mask_count_df, \n    target_df=train_df,\n    #mode='predict',\n    shuffle=False,\n    reshape=(128, 128),\n    augment=False,\n    n_channels=1,\n    n_classes=4,\n    batch_size=1,\n)\nprint(\"Check generator loaded\")\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train batch 16 shuffling with no augmentation"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"model = vanilla_unet((128, 128,1))\n\nmodel.compile(optimizer=Nadam(lr=0.0002), loss=bce_dice_loss, metrics=[dice_coef])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"%%time\ncheckpoint = ModelCheckpoint('model_0.h5', save_best_only=True)\n\nhistory0 = model.fit_generator(\n    train_generator,\n    validation_data=val_generator,\n    callbacks=[checkpoint],\n    epochs=60\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('history_0.json', 'w') as f:\n    json.dump(str(history0.history), f)\n\nhistory_df = pd.DataFrame(history0.history)\nhistory_df[['loss', 'val_loss']].plot()\nhistory_df[['dice_coef', 'val_dice_coef']].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('model_0.h5')\n\nbatch_pred_masks = model.predict_generator(\n    check_generator, \n    workers=1,\n    verbose=1\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(4):\n    visualize(check_generator.getitem(i)[0][0,:,:,:],check_generator.getitem(i)[1][0,:,:,:],batch_pred_masks[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#evaluation\nevaluation = model.evaluate_generator(\n    check_generator, \n    workers=1,\n    verbose=1\n)\nevaluation0 = evaluation\nprint(f\"best val loss: {evaluation[0]} dice : {evaluation[1]}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train batch 16 shuffling with augmentation"},{"metadata":{"trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"%%time\nBATCH_SIZE = 16\nSUFFLE = True\nAUGMENT = True\n\ntrain_generator.setSuffle(SUFFLE)\ntrain_generator.setBatchSize(BATCH_SIZE)\ntrain_generator.setAugment(AUGMENT)\n\nprint(\"Train generator load\")\n\n\nval_generator.setSuffle(SUFFLE)\nval_generator.setBatchSize(BATCH_SIZE)\n\nprint(\"Validation generator load\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"%%time\nmodel = vanilla_unet((128, 128,1))\n\nmodel.compile(optimizer=Nadam(lr=0.0002), loss=bce_dice_loss, metrics=[dice_coef])\nmodel.summary()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"%%time\ncheckpoint = ModelCheckpoint('model_1.h5', save_best_only=True)\n\nhistory1 = model.fit_generator(\n    train_generator,\n    validation_data=val_generator,\n    callbacks=[checkpoint],\n    epochs=60\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('history_1.json', 'w') as f:\n    json.dump(str(history1.history), f)\n\nhistory_df = pd.DataFrame(history1.history)\nhistory_df[['loss', 'val_loss']].plot()\nhistory_df[['dice_coef', 'val_dice_coef']].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('model_1.h5')\n\nbatch_pred_masks = model.predict_generator(\n    check_generator, \n    workers=1,\n    verbose=1\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(4):\n    visualize(check_generator.getitem(i)[0][0,:,:,:],check_generator.getitem(i)[1][0,:,:,:],batch_pred_masks[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#evaluation\nevaluation = model.evaluate_generator(\n    check_generator, \n    workers=1,\n    verbose=1\n)\nevaluation1 = evaluation\nprint(f\"Best val loss: {evaluation[0]:.3f} dice : {evaluation[1]:.3f}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Comparison"},{"metadata":{"trusted":true},"cell_type":"code","source":"data = (history0.history,history1.history)\ntitles = (\"Batch 16 Not Augmented\",\"Batch 16 Augmented\")\ncolors = ('r','b')\n\n#f, ax = plt.subplots(2, len(data), figsize=(24,8))\nf, ax = plt.subplots(2, 1, figsize=(24,24))\nax[0].set_title(\"train loss\")\n\nax[1].set_title(\"dice coeficient\")\nfor i in range(len(data)):\n    pd.DataFrame(data[i])[['loss', 'val_loss']].plot(ax = ax[0], c = colors[i])\nfor i in range(len(data)):\n    pd.DataFrame(data[i])[['dice_coef', 'val_dice_coef']].plot(ax = ax[1],c = colors[i])\nlegend = [\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\"]\nfor l in range(len(data)):\n    legend[l*2] = titles[l] +\"  loss\" \n    legend[l*2+1] = titles[l] + \"  val_loss\"  \nax[0].legend(legend,prop={'size': 20})\n\nfor l in range(len(data)):\n    legend[l*2] = titles[l] +\"  dice_coef\" \n    legend[l*2+1] = titles[l] + \"  val_dice_coef\"  \nax[1].legend(legend, prop={'size': 20})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"objects = (\"Batch 16 Not Augmented\",\"Batch 16 Augmented\")\ny_pos = np.arange(len(objects))\nperformance = [evaluation0[1],evaluation1[1]]\n\nplt.bar(y_pos, performance, align='center', alpha=0.5)\nplt.xticks(y_pos, objects)\nplt.ylabel('Dice Loss')\nplt.title('Performance of different trainings')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}