{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview\nThe notebook shows how to extract the segmentation map for the ships, augment the images and train a Unet model with pretrained encoder to detect them. A few additional tweaks like balancing the ship-count out a little better have been done.","metadata":{"_uuid":"aa8401d73c7a19e1a43fdd6a992ea9dcb60039a2"}},{"cell_type":"markdown","source":"## Importing Packages","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nfrom skimage.io import imread\nimport matplotlib.pyplot as plt\nfrom skimage.segmentation import mark_boundaries\nfrom skimage.util.montage import montage2d as montage\nimport gc; gc.enable() \nfrom skimage.morphology import label\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-09-12T02:27:05.986690Z","iopub.execute_input":"2023-09-12T02:27:05.987028Z","iopub.status.idle":"2023-09-12T02:27:07.022149Z","shell.execute_reply.started":"2023-09-12T02:27:05.986969Z","shell.execute_reply":"2023-09-12T02:27:07.021296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ship_dir = '../input'\ntrain_image_dir = os.path.join(ship_dir, 'train_v2')\ntest_image_dir = os.path.join(ship_dir, 'test_v2')\nmodel_dir = '/kaggle/working/Unet_model'","metadata":{"execution":{"iopub.status.busy":"2023-09-12T02:27:36.219294Z","iopub.execute_input":"2023-09-12T02:27:36.219682Z","iopub.status.idle":"2023-09-12T02:27:36.225687Z","shell.execute_reply.started":"2023-09-12T02:27:36.219619Z","shell.execute_reply":"2023-09-12T02:27:36.224421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Parameters\nWe might want to adjust these later (or do some hyperparameter optimizations)","metadata":{"_uuid":"a6cd9d5ad61ffe3b8858769f20a5f9493f024a56"}},{"cell_type":"code","source":"BATCH_SIZE = 48\nNB_EPOCHS = 200\nVALID_IMG_COUNT = 900\nMAX_TRAIN_STEPS = 9\nIMG_SCALING = (3, 3)\nBATCH_NORMALIZATION = None\nAUGMENT_BRIGHTNESS = False\nSEED = None","metadata":{"_uuid":"301a5d939c566d1487a049bb2554d09b592b18b1","execution":{"iopub.status.busy":"2023-09-12T02:27:16.157075Z","iopub.execute_input":"2023-09-12T02:27:16.157473Z","iopub.status.idle":"2023-09-12T02:27:16.162953Z","shell.execute_reply.started":"2023-09-12T02:27:16.157403Z","shell.execute_reply":"2023-09-12T02:27:16.161882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating Our Masks\nWe decode and encode our pictures using the following functions. <br>\nmasks_as_images is used to gather all masks of ships belonging to one image in one image. <br>\nResource: https://www.kaggle.com/code/inversion/run-length-decoding-quick-start","metadata":{}},{"cell_type":"code","source":"def multi_rle_encode(img):\n    labels = label(img[:, :, 0])\n    return [rle_encode(labels==k) for k in np.unique(labels[labels>0])]\n\ndef rle_encode(img):\n    pixels = img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef rle_decode(mask_rle, shape=(768, 768)):\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T  # Needed to align to RLE direction\n\ndef masks_as_image(in_mask_list):\n    all_masks = np.zeros((768, 768), dtype = np.int16)\n    for mask in in_mask_list:\n        if isinstance(mask, str):\n            all_masks += rle_decode(mask)\n    return np.expand_dims(all_masks, -1)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-12T02:27:38.482032Z","iopub.execute_input":"2023-09-12T02:27:38.482397Z","iopub.status.idle":"2023-09-12T02:27:38.497537Z","shell.execute_reply.started":"2023-09-12T02:27:38.482321Z","shell.execute_reply":"2023-09-12T02:27:38.496582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"masks = pd.read_csv(os.path.join('../input/',\n                                 'train_ship_segmentations_v2.csv'))\nprint(masks.shape[0], 'masks found')\nprint(masks['ImageId'].value_counts().shape[0])\nmasks.head()","metadata":{"_uuid":"3ca7119188fbb4c6540d9df55f5833b55435287e","execution":{"iopub.status.busy":"2023-09-12T02:27:40.360812Z","iopub.execute_input":"2023-09-12T02:27:40.361138Z","iopub.status.idle":"2023-09-12T02:27:41.153307Z","shell.execute_reply.started":"2023-09-12T02:27:40.361079Z","shell.execute_reply":"2023-09-12T02:27:41.152479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make sure encode/decode works\nGiven the process\n$$  RLE_0 \\stackrel{Decode}{\\longrightarrow} \\textrm{Image}_0 \\stackrel{Encode}{\\longrightarrow} RLE_1 \\stackrel{Decode}{\\longrightarrow} \\textrm{Image}_1 $$\nWe want to check if/that\n$ \\textrm{Image}_0 \\stackrel{?}{=} \\textrm{Image}_1 $\nWe could check the RLEs as well but that is more tedious. Also depending on how the objects have been labeled we might have different counts.\n\n","metadata":{"_uuid":"fdedd5965f47f84aa8f3aab1cad978512781a1cc"}},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize = (10, 5))\nrle_0 = masks.query('ImageId==\"00021ddc3.jpg\"')['EncodedPixels']\nimg_0 = masks_as_image(rle_0)\nax1.imshow(img_0[:, :, 0])\nax1.set_title('Image$_0$')\nrle_1 = multi_rle_encode(img_0)\nimg_1 = masks_as_image(rle_1)\nax2.imshow(img_1[:, :, 0])\nax2.set_title('Image$_1$')\nprint('Check Decoding->Encoding',\n      'RLE_0:', len(rle_0), '->',\n      'RLE_1:', len(rle_1))","metadata":{"_uuid":"0081fd6f387abd7c05eb35f29575a2ee6ddc2236","execution":{"iopub.status.busy":"2023-09-12T02:27:41.784739Z","iopub.execute_input":"2023-09-12T02:27:41.785071Z","iopub.status.idle":"2023-09-12T02:27:42.435524Z","shell.execute_reply.started":"2023-09-12T02:27:41.785012Z","shell.execute_reply":"2023-09-12T02:27:42.434285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split into training and validation groups\nWe stratify by the number of boats appearing so we have nice balances in each set","metadata":{"_uuid":"40cb72e241c0c3d8bc245b4e3c663b4a835b0011"}},{"cell_type":"code","source":"masks['ships'] = masks['EncodedPixels'].map(lambda c_row: 1 if isinstance(c_row, str) else 0)\nunique_img_ids = masks.groupby('ImageId').agg({'ships': 'sum'}).reset_index()\nunique_img_ids['has_ship'] = unique_img_ids['ships'].map(lambda x: 1.0 if x>0 else 0.0)\n# some files are too small/corrupt\nunique_img_ids['file_size_kb'] = unique_img_ids['ImageId'].map(lambda c_img_id: \n                                                               os.stat(os.path.join(train_image_dir, \n                                                                                    c_img_id)).st_size/1024)\nunique_img_ids = unique_img_ids[unique_img_ids['file_size_kb']>50] # keep only 50kb files\nunique_img_ids['file_size_kb'].hist()\nmasks.drop(['ships'], axis=1, inplace=True)\nunique_img_ids.sample(5)","metadata":{"_uuid":"c4f008bf6898518fd371de013418f936edaa09f8","execution":{"iopub.status.busy":"2023-09-12T02:27:43.132892Z","iopub.execute_input":"2023-09-12T02:27:43.133228Z","iopub.status.idle":"2023-09-12T02:38:46.773410Z","shell.execute_reply.started":"2023-09-12T02:27:43.133167Z","shell.execute_reply":"2023-09-12T02:38:46.772361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Undersample Empty Images\nHere we undersample the empty images to get a better balanced group with more ships to try and segment","metadata":{"_uuid":"ef8115a80749ac47f295e9a70217a5553970c2b3"}},{"cell_type":"code","source":"SAMPLES_PER_GROUP = 2000\nbalanced_df = unique_img_ids.groupby('ships').apply(lambda x: x.sample(SAMPLES_PER_GROUP) if len(x) > SAMPLES_PER_GROUP else x)\nbalanced_df['ships'].hist(bins=balanced_df['ships'].max()+1)\nprint(balanced_df.shape[0], 'masks')","metadata":{"execution":{"iopub.status.busy":"2023-09-12T02:42:30.916895Z","iopub.execute_input":"2023-09-12T02:42:30.917236Z","iopub.status.idle":"2023-09-12T02:42:31.373049Z","shell.execute_reply.started":"2023-09-12T02:42:30.917177Z","shell.execute_reply":"2023-09-12T02:42:31.372012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ids, valid_ids = train_test_split(balanced_df, \n                 test_size = 0.2, \n                 stratify = balanced_df['ships'],\n                 random_state=SEED)\ntrain_df = pd.merge(masks, train_ids)\nvalid_df = pd.merge(masks, valid_ids)\nprint(train_df.shape[0], 'training masks')\nprint(valid_df.shape[0], 'validation masks')","metadata":{"_uuid":"871720221ac25f7f9408bfe01aeb4ccb95edbd1f","execution":{"iopub.status.busy":"2023-09-12T03:47:26.272506Z","iopub.execute_input":"2023-09-12T03:47:26.272854Z","iopub.status.idle":"2023-09-12T03:47:26.429485Z","shell.execute_reply.started":"2023-09-12T03:47:26.272796Z","shell.execute_reply":"2023-09-12T03:47:26.428528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Examine Number of Ship Images\nHere we examine how often ships appear and replace the ones without any ships with 0","metadata":{"_uuid":"c21d5bff04bf9180463969ac120379345745ed03"}},{"cell_type":"code","source":"train_df['ships'].hist()","metadata":{"_uuid":"2612fa47c7e9fdcaa7aa720c4e15fc86fd65d69a","execution":{"iopub.status.busy":"2023-09-12T02:42:35.636110Z","iopub.execute_input":"2023-09-12T02:42:35.636668Z","iopub.status.idle":"2023-09-12T02:42:35.942900Z","shell.execute_reply.started":"2023-09-12T02:42:35.636597Z","shell.execute_reply":"2023-09-12T02:42:35.941526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Decode all the RLEs into Images\nWe make a generator to produce batches of images","metadata":{"_uuid":"a3fb9fe33d81374c7bd836f5bc86a1df89190805"}},{"cell_type":"code","source":"def make_image_gen(in_df, batch_size = BATCH_SIZE):\n    all_batches = list(in_df.groupby('ImageId'))\n    out_rgb = []\n    out_mask = []\n    while True:\n        np.random.shuffle(all_batches)\n        for c_img_id, c_masks in all_batches:\n            rgb_path = os.path.join(train_image_dir, c_img_id)\n            c_img = imread(rgb_path)\n            c_mask = masks_as_image(c_masks['EncodedPixels'].values)\n            if IMG_SCALING is not None:\n                c_img = c_img[::IMG_SCALING[0], ::IMG_SCALING[1]]\n                c_mask = c_mask[::IMG_SCALING[0], ::IMG_SCALING[1]]\n            out_rgb += [c_img]\n            out_mask += [c_mask]\n            if len(out_rgb)>=batch_size:\n                yield np.stack(out_rgb, 0)/255.0, np.stack(out_mask, 0)\n                out_rgb, out_mask=[], []","metadata":{"_uuid":"6181ac51577e5636995e38a9e29311cf47f513ca","execution":{"iopub.status.busy":"2023-09-12T02:42:51.034515Z","iopub.execute_input":"2023-09-12T02:42:51.034850Z","iopub.status.idle":"2023-09-12T02:42:51.044690Z","shell.execute_reply.started":"2023-09-12T02:42:51.034791Z","shell.execute_reply":"2023-09-12T02:42:51.043715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_gen = make_image_gen(train_df)\nprint(train_gen)\ntrain_x, train_y = next(train_gen)\nprint('x', train_x.shape, train_x.min(), train_x.max())\nprint('y', train_y.shape, train_y.min(), train_y.max())","metadata":{"_uuid":"1983738da75b031f2bec8ba36db01c095e7c5d59","execution":{"iopub.status.busy":"2023-09-12T02:49:32.147715Z","iopub.execute_input":"2023-09-12T02:49:32.148044Z","iopub.status.idle":"2023-09-12T02:49:35.445492Z","shell.execute_reply.started":"2023-09-12T02:49:32.147985Z","shell.execute_reply":"2023-09-12T02:49:35.444524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create a Montage of Random Sample to Check their Masks","metadata":{}},{"cell_type":"code","source":"montage_rgb = lambda x: np.stack([montage(x[:, :, :, i]) for i in range(x.shape[3])], -1)\n\nfig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize = (30, 10))\nbatch_rgb = montage_rgb(train_x)\nbatch_seg = montage(train_y[:, :, :, 0])\nax1.imshow(batch_rgb)\nax1.set_title('Images')\nax2.imshow(batch_seg)\nax2.set_title('Segmentations')\nax3.imshow(mark_boundaries(batch_rgb, \n                           batch_seg.astype(int)))\nax3.set_title('Outlined Ships')\nfig.savefig('overview.png')","metadata":{"_uuid":"b4396cd28ddd2e4c8076fcb165e9b61e3baeeeb7","execution":{"iopub.status.busy":"2023-09-05T23:49:43.844736Z","iopub.execute_input":"2023-09-05T23:49:43.845079Z","iopub.status.idle":"2023-09-05T23:49:47.825106Z","shell.execute_reply.started":"2023-09-05T23:49:43.845013Z","shell.execute_reply":"2023-09-05T23:49:47.824151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make the Validation Set","metadata":{"_uuid":"8f47639c987a10ebcb53e51f55aa8a11c98fa860"}},{"cell_type":"code","source":"valid_x, valid_y = next(make_image_gen(valid_df, VALID_IMG_COUNT))\nprint(valid_x.shape, valid_y.shape)","metadata":{"_uuid":"30cb02a2a7103a9d66e90f701991199de1e5b73e","execution":{"iopub.status.busy":"2023-09-12T02:43:33.162084Z","iopub.execute_input":"2023-09-12T02:43:33.162440Z","iopub.status.idle":"2023-09-12T02:43:55.789898Z","shell.execute_reply.started":"2023-09-12T02:43:33.162378Z","shell.execute_reply":"2023-09-12T02:43:55.789082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"_uuid":"33300c4f03b6600da7b418f775d11d7ebf76a35a","execution":{"iopub.status.busy":"2023-09-12T02:49:38.113132Z","iopub.execute_input":"2023-09-12T02:49:38.113481Z","iopub.status.idle":"2023-09-12T02:49:38.261553Z","shell.execute_reply.started":"2023-09-12T02:49:38.113417Z","shell.execute_reply":"2023-09-12T02:49:38.260664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image Augmentation <br>\nResource: https://www.kaggle.com/code/kmader/baseline-u-net-model-part-1?scriptVersionId=6617956","metadata":{}},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\ndg_args = dict(featurewise_center = False, \n                  samplewise_center = False,\n                  rotation_range = 45, \n                  width_shift_range = 0.1, \n                  height_shift_range = 0.1, \n                  shear_range = 0.01,\n                  zoom_range = [0.9, 1.25],  \n                  horizontal_flip = True, \n                  vertical_flip = True,\n                  fill_mode = 'reflect',\n                   data_format = 'channels_last')\n# brightness can be problematic since it seems to change the labels differently from the images \nif AUGMENT_BRIGHTNESS:\n    dg_args[' brightness_range'] = [0.5, 1.5]\nimage_gen = ImageDataGenerator(**dg_args)\n\nif AUGMENT_BRIGHTNESS:\n    dg_args.pop('brightness_range')\nlabel_gen = ImageDataGenerator(**dg_args)\n\ndef create_aug_gen(in_gen, seed=SEED):\n    np.random.seed(seed if seed is not None else np.random.choice(range(9999)))\n    for in_x, in_y in in_gen:\n        seed = np.random.choice(range(9999))\n        # keep the seeds syncronized otherwise the augmentation to the images is different from the masks\n        g_x = image_gen.flow(255*in_x, \n                             batch_size = in_x.shape[0], \n                             seed = seed, \n                             shuffle=True)\n        g_y = label_gen.flow(in_y, \n                             batch_size = in_x.shape[0], \n                             seed = seed, \n                             shuffle=True)\n\n        yield next(g_x)/255.0, next(g_y)","metadata":{"execution":{"iopub.status.busy":"2023-09-12T02:48:01.750815Z","iopub.execute_input":"2023-09-12T02:48:01.751171Z","iopub.status.idle":"2023-09-12T02:48:02.087515Z","shell.execute_reply.started":"2023-09-12T02:48:01.751114Z","shell.execute_reply":"2023-09-12T02:48:02.086421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Unet Model using ResNet50 as Encoder","metadata":{"_uuid":"ba08494eb9736ec3556b7c879143cdcdea89febf"}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers import Input, Conv2D, BatchNormalization, MaxPool2D, Conv2DTranspose, Concatenate, Activation\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.applications import ResNet50, VGG16, VGG19, DenseNet121, DenseNet169","metadata":{"execution":{"iopub.status.busy":"2023-09-12T02:48:04.984524Z","iopub.execute_input":"2023-09-12T02:48:04.984853Z","iopub.status.idle":"2023-09-12T02:48:04.990356Z","shell.execute_reply.started":"2023-09-12T02:48:04.984797Z","shell.execute_reply":"2023-09-12T02:48:04.989541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def conv_block(inputs, no_filters):\n    x = Conv2D(no_filters, 3, padding=\"same\")(inputs)\n    if BATCH_NORMALIZATION is not None:\n        x = BatchNormalization()(x)\n    x = Activation(\"relu\")(x)\n    \n    x = Conv2D(no_filters, 3, padding=\"same\")(x)\n    if BATCH_NORMALIZATION is not None:\n        x = BatchNormalization()(x)\n    x = Activation(\"relu\")(x)\n    \n    return x\n\ndef decoder_block(inputs, skip_features, no_filters):\n    x = Conv2DTranspose(no_filters, (2, 2), strides=2, padding=\"same\")(inputs)\n\n    x = Concatenate(axis=3)([skip_features, x])\n    x = conv_block(x, no_filters)\n    return x","metadata":{"execution":{"iopub.status.busy":"2023-09-12T02:48:06.621567Z","iopub.execute_input":"2023-09-12T02:48:06.621911Z","iopub.status.idle":"2023-09-12T02:48:06.630883Z","shell.execute_reply.started":"2023-09-12T02:48:06.621839Z","shell.execute_reply":"2023-09-12T02:48:06.629959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_resnet50_unet(input_shape):\n    \"\"\" Input \"\"\"\n    input_imgs = Input(input_shape, name = 'RGB_Input')\n\n    \"\"\" Pretrained ResNet50 Model \"\"\"\n    resnet50 = ResNet50(include_top=False, weights=\"imagenet\", input_tensor=input_imgs)\n\n#     resnet50.trainable = False\n\n    \"\"\" Skip Features \"\"\"\n    e1 = input_imgs  # (265, 265, 3)\n    e2 = resnet50.layers[4].output # (128, 128, 64)\n    e3 = resnet50.layers[38].output # (64, 64, 265)\n    e4 = resnet50.layers[80].output # (32, 32, 512)\n\n    \"\"\" Bridge \"\"\"\n    b1 = resnet50.layers[142].output # (16, 16, 1024)\n\n    \"\"\" Decoder \"\"\"\n    d1 = decoder_block(b1, e4, 512)\n    d2 = decoder_block(d1, e3, 256)\n    d3 = decoder_block(d2, e2, 128)\n    d4 = decoder_block(d3, e1, 64)\n\n    \"\"\" Outputs \"\"\"\n    outputs = Conv2D(1, 1, padding=\"same\", activation=\"sigmoid\")(d4)\n\n    model = Model(input_imgs, outputs)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-09-12T02:48:08.766310Z","iopub.execute_input":"2023-09-12T02:48:08.766681Z","iopub.status.idle":"2023-09-12T02:48:08.776732Z","shell.execute_reply.started":"2023-09-12T02:48:08.766620Z","shell.execute_reply":"2023-09-12T02:48:08.775492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Unet Model using VGG19 as Encoder","metadata":{}},{"cell_type":"code","source":"def build_vgg19_unet(input_shape):\n    \"\"\" Input \"\"\"\n    input_imgs = Input(input_shape, name = 'RGB_Input')\n\n    \"\"\" Pretrained ResNet50 Model \"\"\"\n    vgg19 = VGG19(include_top=False, weights=\"imagenet\", input_tensor=input_imgs)\n    \n#     vgg19.trainable = False\n\n    \"\"\" Skip Features \"\"\"\n    e1 = vgg19.layers[2].output # (265, 265, 64)\n    e2 = vgg19.layers[5].output # (128, 128, 128)\n    e3 = vgg19.layers[10].output # (64, 64, 265)\n    e4 = vgg19.layers[15].output # (32, 32, 512)\n\n    \"\"\" Bridge \"\"\"\n    b1 = vgg19.layers[20].output # (16, 16, 512)\n\n    \"\"\" Decoder \"\"\"\n    d1 = decoder_block(b1, e4, 64)\n    d2 = decoder_block(d1, e3, 32)\n    d3 = decoder_block(d2, e2, 16)\n    d4 = decoder_block(d3, e1, 8)\n\n    \"\"\" Outputs \"\"\"\n    outputs = Conv2D(1, 1, padding=\"same\", activation=\"sigmoid\")(d4)\n\n    model = Model(input_imgs, outputs)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-09-12T03:47:08.447613Z","iopub.execute_input":"2023-09-12T03:47:08.447970Z","iopub.status.idle":"2023-09-12T03:47:08.548301Z","shell.execute_reply.started":"2023-09-12T03:47:08.447910Z","shell.execute_reply":"2023-09-12T03:47:08.547445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Unet Model using DenseNet121 as Encoder","metadata":{}},{"cell_type":"code","source":"def build_densenet169_unet(input_shape):\n    \"\"\" Input \"\"\"\n    input_imgs = Input(input_shape, name = 'RGB_Input')\n\n    \"\"\" Pretrained ResNet50 Model \"\"\"\n    densenet = DenseNet169(include_top=False, weights=\"imagenet\", input_tensor=input_imgs)\n    \n#     densenet.trainable = False\n\n    \"\"\" Skip Features \"\"\"\n    e1 = input_imgs # (265, 265, 3)\n    e2 = densenet.get_layer(\"conv1/relu\").output # (128, 128, 64)\n    e3 = densenet.get_layer(\"pool2_relu\").output # (64, 64, 265)\n    e4 = densenet.get_layer(\"pool3_relu\").output # (32, 32, 512)\n\n    \"\"\" Bridge \"\"\"\n    b1 = densenet.get_layer(\"pool4_relu\").output # (16, 16, 1280)\n\n    \"\"\" Decoder \"\"\"\n    d1 = decoder_block(b1, e4, 64)\n    d2 = decoder_block(d1, e3, 32)\n    d3 = decoder_block(d2, e2, 16)\n    d4 = decoder_block(d3, e1, 8)\n\n    \"\"\" Outputs \"\"\"\n    outputs = Conv2D(1, 1, padding=\"same\", activation=\"sigmoid\")(d4)\n\n    model = Model(input_imgs, outputs)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-09-12T02:49:45.736073Z","iopub.execute_input":"2023-09-12T02:49:45.736482Z","iopub.status.idle":"2023-09-12T02:49:45.751225Z","shell.execute_reply.started":"2023-09-12T02:49:45.736426Z","shell.execute_reply":"2023-09-12T02:49:45.748590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_vgg19_unet(train_x.shape[1:])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-09-12T03:19:44.365697Z","iopub.execute_input":"2023-09-12T03:19:44.366021Z","iopub.status.idle":"2023-09-12T03:19:46.389383Z","shell.execute_reply.started":"2023-09-12T03:19:44.365964Z","shell.execute_reply":"2023-09-12T03:19:46.388206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras.backend as K\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import binary_crossentropy\n\n## intersection over union\ndef IoU(y_true, y_pred, eps=1e-6):\n    if np.max(y_true) == 0.0:\n        return IoU(1-y_true, 1-y_pred) ## empty image; calc IoU of zeros\n    intersection = K.sum(y_true * y_pred, axis=[1,2,3])\n    union = K.sum(y_true, axis=[1,2,3]) + K.sum(y_pred, axis=[1,2,3]) - intersection\n    return -K.mean( (intersection + eps) / (union + eps), axis=0)\n\ndef dice_coef(y_true, y_pred, smooth=1):\n    intersection = K.sum(y_true * y_pred, axis=[1,2,3])\n    union = K.sum(y_true, axis=[1,2,3]) + K.sum(y_pred, axis=[1,2,3])\n    return K.mean( (2. * intersection + smooth) / (union + smooth), axis=0)\n\ndef combo_loss(y_true, y_pred):\n    return 1e-3 * binary_crossentropy(y_true, y_pred) - dice_coef(y_true, y_pred)\n\ndef jaccard_index(y_true, y_pred, smooth=1):\n    intersection = K.sum(y_true * y_pred, axis=[1,2,3])\n    union = K.sum(y_true, axis=[1,2,3]) + K.sum(y_pred, axis=[1,2,3]) - intersection\n    return -K.mean((intersection + smooth) / (union + smooth), axis=0)\n\nmodel.compile(optimizer=Adam(1e-3), loss=IoU, metrics=[dice_coef, 'binary_accuracy'])","metadata":{"_uuid":"1678069aa8013510264ba898291c6ae2dce88a76","execution":{"iopub.status.busy":"2023-09-12T03:48:38.762874Z","iopub.execute_input":"2023-09-12T03:48:38.763214Z","iopub.status.idle":"2023-09-12T03:48:38.883467Z","shell.execute_reply.started":"2023-09-12T03:48:38.763153Z","shell.execute_reply":"2023-09-12T03:48:38.882454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\n\nif not os.path.isdir(model_dir):\n    os.mkdir(model_dir)\n    \nweight_path=\"{}_weights.best.hdf5\".format(model_dir)\n\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\n\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.33, \n                                   patience=1, \n                                   verbose=1, mode='min', min_delta=0.0001, cooldown=0, min_lr=1e-8)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\",\n                      verbose = 2,\n                      patience=10) # probably needs to be more patient, but kaggle time is limited\ncallbacks_list = [checkpoint, early, reduceLROnPlat]","metadata":{"_uuid":"7282d18de3aff1cee12ff89b7d511a391702814f","execution":{"iopub.status.busy":"2023-09-12T03:48:40.671091Z","iopub.execute_input":"2023-09-12T03:48:40.671797Z","iopub.status.idle":"2023-09-12T03:48:40.690020Z","shell.execute_reply.started":"2023-09-12T03:48:40.671734Z","shell.execute_reply":"2023-09-12T03:48:40.689179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import shutil\n# shutil.rmtree('/kaggle/working/')","metadata":{"execution":{"iopub.status.busy":"2023-09-12T03:47:36.081307Z","iopub.execute_input":"2023-09-12T03:47:36.081796Z","iopub.status.idle":"2023-09-12T03:47:36.124423Z","shell.execute_reply.started":"2023-09-12T03:47:36.081725Z","shell.execute_reply":"2023-09-12T03:47:36.123396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"step_count = min(MAX_TRAIN_STEPS, train_df.shape[0]//BATCH_SIZE)\naug_gen = create_aug_gen(make_image_gen(train_df))\nloss_history = [model.fit_generator(aug_gen, \n                             steps_per_epoch=step_count, \n                             epochs=NB_EPOCHS, \n                             validation_data=(valid_x, valid_y),\n                             callbacks=callbacks_list,\n                             workers=1 # the generator is not very thread safe\n                                       )]","metadata":{"_uuid":"5b67d808c0b8c7e28bff41e6d3858ff6f09dd626","execution":{"iopub.status.busy":"2023-09-12T03:48:48.789588Z","iopub.execute_input":"2023-09-12T03:48:48.789919Z","iopub.status.idle":"2023-09-12T04:53:22.764641Z","shell.execute_reply.started":"2023-09-12T03:48:48.789859Z","shell.execute_reply":"2023-09-12T04:53:22.763803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_loss(loss_history):\n    epich = np.cumsum(np.concatenate(\n        [np.linspace(0.5, 1, len(mh.epoch)) for mh in loss_history]))\n    fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(22, 10))\n    _ = ax1.plot(epich,\n                 np.concatenate([mh.history['loss'] for mh in loss_history]),\n                 'b-',\n                 epich, np.concatenate(\n            [mh.history['val_loss'] for mh in loss_history]), 'r-')\n    ax1.legend(['Training', 'Validation'])\n    ax1.set_title('Loss')\n\n    _ = ax2.plot(epich, np.concatenate(\n        [mh.history['dice_coef'] for mh in loss_history]), 'b-',\n                     epich, np.concatenate(\n            [mh.history['val_dice_coef'] for mh in loss_history]),\n                     'r-')\n    ax2.legend(['Training', 'Validation'])\n    ax2.set_title('DICE')\n    \n    _ = ax3.plot(epich, np.concatenate(\n        [mh.history['binary_accuracy'] for mh in loss_history]), 'b-',\n                     epich, np.concatenate(\n            [mh.history['val_binary_accuracy'] for mh in loss_history]),\n                     'r-')\n    ax3.legend(['Training', 'Validation'])\n    ax3.set_title('Binary Accuracy (%)')\n    plt.savefig('losses.png')\n\nshow_loss(loss_history)","metadata":{"_uuid":"a168c8b1af446b800f6129104906003ededd61c4","execution":{"iopub.status.busy":"2023-09-12T05:09:53.613245Z","iopub.execute_input":"2023-09-12T05:09:53.613629Z","iopub.status.idle":"2023-09-12T05:09:54.945969Z","shell.execute_reply.started":"2023-09-12T05:09:53.613566Z","shell.execute_reply":"2023-09-12T05:09:54.945134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Taking Insight about Training and Validation Results","metadata":{}},{"cell_type":"code","source":"fig, m_axs = plt.subplots(20, 3, figsize = (10, 40))\n[c_ax.axis('off') for c_ax in m_axs.flatten()]\nvalid_imgs = list(valid_df.groupby('ImageId'))\nfor (ax1, ax2, ax3), (c_img_name, c_masks) in zip(m_axs, valid_imgs):\n    c_path = os.path.join(train_image_dir, c_img_name)\n    c_img = imread(c_path)\n    img = np.expand_dims(c_img, 0)/255.0\n    if IMG_SCALING is not None:\n        img = img[:, ::IMG_SCALING[0], ::IMG_SCALING[1]]\n    seg = model.predict(img)\n    mask = masks_as_image(c_masks['EncodedPixels'].values)\n    if IMG_SCALING is not None:\n        mask = mask[::IMG_SCALING[0], ::IMG_SCALING[1]]\n    ax1.imshow(img[0])\n    ax1.set_title('Image')\n    ax2.imshow(seg[0, :, :, 0], vmin = 0, vmax = 1)\n    ax2.set_title('Prediction')\n    ax3.imshow(mask[:, :, 0])\n    ax3.set_title('Ground Truth')\nplt.savefig('image_pred_ground_truth.png')","metadata":{"execution":{"iopub.status.busy":"2023-09-12T05:10:05.434327Z","iopub.execute_input":"2023-09-12T05:10:05.434690Z","iopub.status.idle":"2023-09-12T05:10:10.467852Z","shell.execute_reply.started":"2023-09-12T05:10:05.434629Z","shell.execute_reply":"2023-09-12T05:10:10.465505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_weights(weight_path)\nmodel.save('Unet_model.h5')","metadata":{"_uuid":"ce1167e9f09200f537e61f93f486168a13be1711","execution":{"iopub.status.busy":"2023-09-12T05:10:31.479300Z","iopub.execute_input":"2023-09-12T05:10:31.479684Z","iopub.status.idle":"2023-09-12T05:10:35.134435Z","shell.execute_reply.started":"2023-09-12T05:10:31.479622Z","shell.execute_reply":"2023-09-12T05:10:35.133434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_y = model.predict(valid_x)\nprint(pred_y.shape, pred_y.min(), pred_y.max(), pred_y.mean())","metadata":{"_uuid":"275b411dc97a350aacaba46c8562efcf2658b1a7","execution":{"iopub.status.busy":"2023-09-12T05:10:49.394929Z","iopub.execute_input":"2023-09-12T05:10:49.395297Z","iopub.status.idle":"2023-09-12T05:10:54.642525Z","shell.execute_reply.started":"2023-09-12T05:10:49.395225Z","shell.execute_reply":"2023-09-12T05:10:54.641524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize = (10, 10))\nax.hist(pred_y.ravel(), np.linspace(0, 1, 10))\nax.set_xlim(0, 1)\nax.set_yscale('log', nonposy='clip')","metadata":{"_uuid":"6a4fd2ca0cf47ba069a314356bf74c7b531c56ac","execution":{"iopub.status.busy":"2023-09-12T05:10:59.608796Z","iopub.execute_input":"2023-09-12T05:10:59.609130Z","iopub.status.idle":"2023-09-12T05:11:04.176572Z","shell.execute_reply.started":"2023-09-12T05:10:59.609070Z","shell.execute_reply":"2023-09-12T05:11:04.175635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run the test data","metadata":{"_uuid":"17edb177402ae51651692511827a7e9d60646533"}},{"cell_type":"code","source":"test_paths = os.listdir(test_image_dir)\nprint(len(test_paths), 'test images found')","metadata":{"_uuid":"4911811f267f9f3397a58902da9e75c6f261ad40","execution":{"iopub.status.busy":"2023-09-12T05:11:07.047551Z","iopub.execute_input":"2023-09-12T05:11:07.047881Z","iopub.status.idle":"2023-09-12T05:11:07.759522Z","shell.execute_reply.started":"2023-09-12T05:11:07.047825Z","shell.execute_reply":"2023-09-12T05:11:07.758519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize test predictions","metadata":{}},{"cell_type":"code","source":"def predict(img_name):\n    c_path = os.path.join(test_image_dir, img_name)\n    c_img = imread(c_path)\n    img = np.expand_dims(c_img, 0)/255.0\n    if IMG_SCALING is not None:\n        img = img[:, ::IMG_SCALING[0], ::IMG_SCALING[1]]\n    return img, model.predict(img) ","metadata":{"execution":{"iopub.status.busy":"2023-09-12T05:11:09.220361Z","iopub.execute_input":"2023-09-12T05:11:09.220694Z","iopub.status.idle":"2023-09-12T05:11:09.227157Z","shell.execute_reply.started":"2023-09-12T05:11:09.220632Z","shell.execute_reply":"2023-09-12T05:11:09.226306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, m_axs = plt.subplots(20, 2, figsize = (10, 40))\n[c_ax.axis('off') for c_ax in m_axs.flatten()]\nfor (ax1, ax2), c_img_name in zip(m_axs, test_paths):\n    img, pred = predict(c_img_name)\n    ax1.imshow(img[0])\n    ax1.set_title('Image')\n    ax2.imshow(pred[0, :, :, 0], vmin = 0, vmax = 1)\n    ax2.set_title('Prediction')\nfig.savefig('test_predictions.png')","metadata":{"_uuid":"73ef7b3b2a74bf64968c79b4005075d4f0e23143","execution":{"iopub.status.busy":"2023-09-12T05:11:11.289305Z","iopub.execute_input":"2023-09-12T05:11:11.289663Z","iopub.status.idle":"2023-09-12T05:11:15.135784Z","shell.execute_reply.started":"2023-09-12T05:11:11.289602Z","shell.execute_reply":"2023-09-12T05:11:15.134470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparing model predictions for submission","metadata":{}},{"cell_type":"code","source":"from scipy import ndimage\n\ndef split_mask(mask):\n    threshold = 0.6\n    threshold_obj = 8 #ignor predictions composed of \"threshold_obj\" pixels or less\n    labeled,n_objs = ndimage.label(mask > threshold)\n    result = []\n    for i in range(n_objs):\n        obj = (labeled == i + 1).astype(int)\n        if(obj.sum() > threshold_obj): result.append(obj)\n    return result","metadata":{"execution":{"iopub.status.busy":"2023-09-12T05:11:24.537802Z","iopub.execute_input":"2023-09-12T05:11:24.538157Z","iopub.status.idle":"2023-09-12T05:11:24.545414Z","shell.execute_reply.started":"2023-09-12T05:11:24.538095Z","shell.execute_reply":"2023-09-12T05:11:24.544367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_run_length_encoded_predictions(y_pred, img_name):\n    list_dict = []\n    masks = split_mask(y_pred)\n#     masks = multi_rle_encode(y_pred)\n    if len(masks) == 0:\n        list_dict.append({\"ImageId\": img_name, \"EncodedPixels\": np.nan})\n    for mask in masks:\n        list_dict.append({\"ImageId\": img_name, \"EncodedPixels\": rle_encode(mask)})\n    return list_dict","metadata":{"execution":{"iopub.status.busy":"2023-09-12T05:11:26.159372Z","iopub.execute_input":"2023-09-12T05:11:26.159722Z","iopub.status.idle":"2023-09-12T05:11:26.166377Z","shell.execute_reply.started":"2023-09-12T05:11:26.159657Z","shell.execute_reply":"2023-09-12T05:11:26.165317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_and_decode(test_img_names):\n    list_dict = []\n    for img_name in test_img_names:\n        _ , pred = predict(img_name)\n        rle_pred = get_run_length_encoded_predictions(pred[0], img_name)\n        list_dict += rle_pred\n    return pd.DataFrame(list_dict, columns=[\"ImageId\", \"EncodedPixels\"])    ","metadata":{"execution":{"iopub.status.busy":"2023-09-12T05:11:28.023433Z","iopub.execute_input":"2023-09-12T05:11:28.023761Z","iopub.status.idle":"2023-09-12T05:11:28.029786Z","shell.execute_reply.started":"2023-09-12T05:11:28.023704Z","shell.execute_reply":"2023-09-12T05:11:28.028896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = imread(os.path.join(test_image_dir, '0010551d9.jpg'))\nplt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2023-09-12T05:11:30.243488Z","iopub.execute_input":"2023-09-12T05:11:30.243818Z","iopub.status.idle":"2023-09-12T05:11:30.609653Z","shell.execute_reply.started":"2023-09-12T05:11:30.243762Z","shell.execute_reply":"2023-09-12T05:11:30.608453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_decoded_pred_df = predict_and_decode(test_paths)\ntest_decoded_pred_df.to_csv('submission.csv', index=False)\ntest_decoded_pred_df","metadata":{"execution":{"iopub.status.busy":"2023-09-12T05:11:33.941417Z","iopub.execute_input":"2023-09-12T05:11:33.941760Z","iopub.status.idle":"2023-09-12T05:21:12.370255Z","shell.execute_reply.started":"2023-09-12T05:11:33.941698Z","shell.execute_reply":"2023-09-12T05:21:12.369414Z"},"trusted":true},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}