{"cells":[{"metadata":{"_uuid":"aa8401d73c7a19e1a43fdd6a992ea9dcb60039a2"},"cell_type":"markdown","source":"# Overview\nInitial data-load code and submit code take from this awesome kernel https://www.kaggle.com/kmader/baseline-u-net-model-part-1 \n\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"BATCH_SIZE = 16\nimport os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom skimage.io import imread\nimport matplotlib.pyplot as plt\nfrom skimage.segmentation import mark_boundaries\nfrom skimage.util.montage import montage2d as montage\nimport cv2\nimport random\nfrom datetime import datetime\nimport json\nimport gc\n\nimport torch\nfrom torch import nn\nfrom torch.optim import Adam\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.backends.cudnn as cudnn\nimport torch.backends.cudnn\nfrom torch.autograd import Variable\nfrom torch.nn import functional as F\nfrom torchvision.transforms import ToTensor, Normalize, Compose\n\n\n\nfrom tqdm import tqdm\nfrom pathlib import Path\n\nfrom skimage.morphology import label\n\nship_dir = '../input/'\ntrain_image_dir = os.path.join(ship_dir, 'train')\ntest_image_dir = os.path.join(ship_dir, 'test')\n# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef multi_rle_encode(img):\n    labels = label(img)\n    return [rle_encode(labels==k) for k in np.unique(labels[labels>0])]\n\ndef rle_decode(mask_rle, shape=(768, 768)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T  # Needed to align to RLE direction\n\ndef masks_as_image(in_mask_list):\n    # Take the individual ship masks and create a single mask array for all ships\n    all_masks = np.zeros((768, 768), dtype = np.int16)\n    #if isinstance(in_mask_list, list):\n    for mask in in_mask_list:\n        if isinstance(mask, str):\n            all_masks += rle_decode(mask)\n    return np.expand_dims(all_masks, -1)\n\ndef mask_overlay(image, mask, color=(0, 1, 0)):\n    \"\"\"\n    Helper function to visualize mask on the top of the image\n    \"\"\"\n    mask = np.dstack((mask, mask, mask)) * np.array(color)\n    weighted_sum = cv2.addWeighted(mask, 0.5, image, 0.5, 0.)\n    img = image.copy()\n    ind = mask[:, :, 1] > 0\n    img[ind] = weighted_sum[ind]    \n    return img\n\ndef imshow(img, mask, title=None):\n    \"\"\"Imshow for Tensor.\"\"\"\n    img = img.numpy().transpose((1, 2, 0))\n    mean = np.array([0.485, 0.456, 0.406])\n    std = np.array([0.229, 0.224, 0.225])\n    img = std * img + mean\n    img = np.clip(img, 0, 1)\n    mask = mask.numpy().transpose((1, 2, 0))\n    mask = np.clip(mask, 0, 1)\n    fig = plt.figure(figsize = (6,6))\n    plt.imshow(mask_overlay(img, mask))\n    if title is not None:\n        plt.title(title)\n    plt.pause(0.001) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ca7119188fbb4c6540d9df55f5833b55435287e","collapsed":true},"cell_type":"code","source":"masks = pd.read_csv(os.path.join(ship_dir,\n                                 'train_ship_segmentations.csv'))\n#masks = masks.iloc[10000:12000]\nprint(masks.shape[0], 'masks found')\nprint(masks['ImageId'].value_counts().shape[0], 'unique images found')\n#masks.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c2ae52ded0129d129b32747517ce35f33d268904"},"cell_type":"markdown","source":"# Undersample Empty Images\nHere we undersample the empty images to get a better balanced group with more ships to try and segment"},{"metadata":{"trusted":true,"_uuid":"3088cfb8456143b025b8ed8dfd17f1e5b23fb833","collapsed":true},"cell_type":"code","source":"masks = masks.drop(masks[masks.EncodedPixels.isnull()].sample(70000,random_state=42).index)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40cb72e241c0c3d8bc245b4e3c663b4a835b0011"},"cell_type":"markdown","source":"# Split into training and validation groups\nWe stratify by the number of boats appearing so we have nice balances in each set"},{"metadata":{"trusted":true,"_uuid":"871720221ac25f7f9408bfe01aeb4ccb95edbd1f","collapsed":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nunique_img_ids = masks.groupby('ImageId').size().reset_index(name='counts')\ntrain_ids, valid_ids = train_test_split(unique_img_ids, \n                 test_size = 0.05, \n                 stratify = unique_img_ids['counts'],\n                 random_state=42\n                )\ntrain_df = pd.merge(masks, train_ids)\nvalid_df = pd.merge(masks, valid_ids)\nprint(train_df.shape[0], 'training masks')\nprint(valid_df.shape[0], 'validation masks')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0780b126671a906c97ea75d02143acaf9d8727f7","collapsed":true},"cell_type":"code","source":"train_df['counts'] = train_df.apply(lambda c_row: c_row['counts'] if \n                                    isinstance(c_row['EncodedPixels'], str) else\n                                    0, 1)\nvalid_df['counts'] = valid_df.apply(lambda c_row: c_row['counts'] if \n                                    isinstance(c_row['EncodedPixels'], str) else\n                                    0, 1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"817a2eafca90841991c1813acf6f95a13fe80d73","collapsed":true},"cell_type":"code","source":"train_df['counts'].hist(bins = 15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77fbfa514ceb831f747cf7fde644f05413eba21d","collapsed":true},"cell_type":"code","source":"valid_df['counts'].hist(bins = 15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d2025b2927c1a0cdb8b247480e84c3abf4bc9bb","collapsed":true},"cell_type":"code","source":"print(train_df.ImageId.nunique(), 'unique images in train set')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"763834f59772ba3494513984324be40923973789","collapsed":true},"cell_type":"code","source":"print(valid_df.ImageId.nunique(),'unique images in test set')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cff6b561467f272fbafada32bd36c248ae14e811"},"cell_type":"markdown","source":"## Dataset class for PyThorch dataloader"},{"metadata":{"trusted":true,"_uuid":"fe37496281ed3c4d31fe5a0e154d0c1ad8f2c1bd","collapsed":true},"cell_type":"code","source":"\nclass ShipDataset(Dataset):\n    def __init__(self, in_df, transform=None, mode='train'):\n        grp = list(in_df.groupby('ImageId'))\n        self.image_ids =  [_id for _id, _ in grp] \n        self.image_masks = [m['EncodedPixels'].values for _,m in grp]\n        self.transform = transform\n        self.mode = mode\n        self.img_transform = Compose([\n        ToTensor(),\n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n        ])\n\n    def __len__(self):\n        return len(self.image_ids)\n               \n    def __getitem__(self, idx):\n        img_file_name = self.image_ids[idx]\n        if self.mode == 'train':\n            rgb_path = os.path.join(train_image_dir, img_file_name)\n        else:\n            rgb_path = os.path.join(test_image_dir, img_file_name)\n        img = imread(rgb_path)\n        mask = masks_as_image(self.image_masks[idx])\n       \n        if self.transform is not None:\n            img, mask = self.transform(img, mask)\n\n        if self.mode == 'train':\n            #return self.to_float_tensor(img), self.to_float_tensor(mask)\n            #eturn img, mask\n            return self.img_transform(img), torch.from_numpy(np.moveaxis(mask, -1, 0)).float()\n        else:\n            return self.img_transform(img), str(img_file_name)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf2163cd39839dbdec41ecaea2939227db01a63f"},"cell_type":"markdown","source":"## Check some image from validation dataset"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e66c8200ee64255af88b8d00e062819928bc84ea"},"cell_type":"code","source":"#Create dataset\ndataset_valid = ShipDataset(valid_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f99ee770d577ce3662adf73826fcff1afe3b0e1d","collapsed":true},"cell_type":"code","source":"#To show image with its mask\nimshow(*dataset_valid[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8f65e7942816fb75b687a549dc1d5cc48d00e21"},"cell_type":"markdown","source":"# Augment Data\n\nWe should create custom classes for sumultaneous transformation images and  masks"},{"metadata":{"trusted":true,"_uuid":"f93ae564131b11562330d173a6f0b9e9118b4a23","collapsed":true},"cell_type":"code","source":"\"\"\"\n    Implementation from  https://github.com/ternaus/robot-surgery-segmentation\n\"\"\"\n\ndef clip(img, dtype, maxval):\n    return np.clip(img, 0, maxval).astype(dtype)\n\nclass DualCompose:\n    def __init__(self, transforms):\n        self.transforms = transforms\n\n    def __call__(self, x, mask=None):\n        for t in self.transforms:\n            x, mask = t(x, mask)\n        return x, mask\n    \nclass OneOf:\n    def __init__(self, transforms, prob=0.5):\n        self.transforms = transforms\n        self.prob = prob\n\n    def __call__(self, x, mask=None):\n        if random.random() < self.prob:\n            t = random.choice(self.transforms)\n            t.prob = 1.\n            x, mask = t(x, mask)\n        return x, mask\n\nclass OneOrOther:\n    def __init__(self, first, second, prob=0.5):\n        self.first = first\n        first.prob = 1.\n        self.second = second\n        second.prob = 1.\n        self.prob = prob\n\n    def __call__(self, x, mask=None):\n        if random.random() < self.prob:\n            x, mask = self.first(x, mask)\n        else:\n            x, mask = self.second(x, mask)\n        return x, mask\n\n\nclass ImageOnly:\n    def __init__(self, trans):\n        self.trans = trans\n\n    def __call__(self, x, mask=None):\n        return self.trans(x), mask\n\n\nclass VerticalFlip:\n    def __init__(self, prob=0.5):\n        self.prob = prob\n\n    def __call__(self, img, mask=None):\n        if random.random() < self.prob:\n            img = cv2.flip(img, 0)\n            if mask is not None:\n                mask = cv2.flip(mask, 0)\n        return img, mask\n\n\nclass HorizontalFlip:\n    def __init__(self, prob=0.5):\n        self.prob = prob\n\n    def __call__(self, img, mask=None):\n        if random.random() < self.prob:\n            img = cv2.flip(img, 1)\n            if mask is not None:\n                mask = cv2.flip(mask, 1)\n        return img, mask\n\n\nclass RandomFlip:\n    def __init__(self, prob=0.5):\n        self.prob = prob\n\n    def __call__(self, img, mask=None):\n        if random.random() < self.prob:\n            d = random.randint(-1, 1)\n            img = cv2.flip(img, d)\n            if mask is not None:\n                mask = cv2.flip(mask, d)\n        return img, mask\n\n\nclass Transpose:\n    def __init__(self, prob=0.5):\n        self.prob = prob\n\n    def __call__(self, img, mask=None):\n        if random.random() < self.prob:\n            img = img.transpose(1, 0, 2)\n            if mask is not None:\n                mask = mask.transpose(1, 0, 2)\n        return img, mask\n\n\nclass RandomRotate90:\n    def __init__(self, prob=0.5):\n        self.prob = prob\n\n    def __call__(self, img, mask=None):\n        if random.random() < self.prob:\n            factor = random.randint(0, 4)\n            img = np.rot90(img, factor)\n            if mask is not None:\n                mask = np.rot90(mask, factor)\n        return img.copy(), mask.copy()\n\n\nclass Rotate:\n    def __init__(self, limit=90, prob=0.5):\n        self.prob = prob\n        self.limit = limit\n\n    def __call__(self, img, mask=None):\n        if random.random() < self.prob:\n            angle = random.uniform(-self.limit, self.limit)\n\n            height, width = img.shape[0:2]\n            mat = cv2.getRotationMatrix2D((width / 2, height / 2), angle, 1.0)\n            img = cv2.warpAffine(img, mat, (height, width),\n                                 flags=cv2.INTER_LINEAR,\n                                 borderMode=cv2.BORDER_REFLECT_101)\n            if mask is not None:\n                mask = cv2.warpAffine(mask, mat, (height, width),\n                                      flags=cv2.INTER_LINEAR,\n                                      borderMode=cv2.BORDER_REFLECT_101)\n\n        return img, mask\n\n\nclass RandomCrop:\n    def __init__(self, size):\n        self.h = size[0]\n        self.w = size[1]\n\n    def __call__(self, img, mask=None):\n        height, width, _ = img.shape\n\n        h_start = np.random.randint(0, height - self.h)\n        w_start = np.random.randint(0, width - self.w)\n\n        img = img[h_start: h_start + self.h, w_start: w_start + self.w,:]\n\n        assert img.shape[0] == self.h\n        assert img.shape[1] == self.w\n\n        if mask is not None:\n            if mask.ndim == 2:\n                mask = np.expand_dims(mask, axis=2)\n            mask = mask[h_start: h_start + self.h, w_start: w_start + self.w,:]\n\n        return img, mask\n\n\nclass Shift:\n    def __init__(self, limit=4, prob=.5):\n        self.limit = limit\n        self.prob = prob\n\n    def __call__(self, img, mask=None):\n        if random.random() < self.prob:\n            limit = self.limit\n            dx = round(random.uniform(-limit, limit))\n            dy = round(random.uniform(-limit, limit))\n\n            height, width, channel = img.shape\n            y1 = limit + 1 + dy\n            y2 = y1 + height\n            x1 = limit + 1 + dx\n            x2 = x1 + width\n\n            img1 = cv2.copyMakeBorder(img, limit + 1, limit + 1, limit + 1, limit + 1,\n                                      borderType=cv2.BORDER_REFLECT_101)\n            img = img1[y1:y2, x1:x2, :]\n            if mask is not None:\n                msk1 = cv2.copyMakeBorder(mask, limit + 1, limit + 1, limit + 1, limit + 1,\n                                          borderType=cv2.BORDER_REFLECT_101)\n                mask = msk1[y1:y2, x1:x2, :]\n\n        return img, mask\n\n\nclass ShiftScale:\n    def __init__(self, limit=4, prob=.25):\n        self.limit = limit\n        self.prob = prob\n\n    def __call__(self, img, mask=None):\n        limit = self.limit\n        if random.random() < self.prob:\n            height, width, channel = img.shape\n            assert (width == height)\n            size0 = width\n            size1 = width + 2 * limit\n            size = round(random.uniform(size0, size1))\n\n            dx = round(random.uniform(0, size1 - size))\n            dy = round(random.uniform(0, size1 - size))\n\n            y1 = dy\n            y2 = y1 + size\n            x1 = dx\n            x2 = x1 + size\n\n            img1 = cv2.copyMakeBorder(img, limit, limit, limit, limit, borderType=cv2.BORDER_REFLECT_101)\n            img = (img1[y1:y2, x1:x2, :] if size == size0\n            else cv2.resize(img1[y1:y2, x1:x2, :], (size0, size0), interpolation=cv2.INTER_LINEAR))\n\n            if mask is not None:\n                msk1 = cv2.copyMakeBorder(mask, limit, limit, limit, limit, borderType=cv2.BORDER_REFLECT_101)\n                mask = (msk1[y1:y2, x1:x2, :] if size == size0\n                else cv2.resize(msk1[y1:y2, x1:x2, :], (size0, size0), interpolation=cv2.INTER_LINEAR))\n\n        return img, mask\n\n\nclass ShiftScaleRotate:\n    def __init__(self, shift_limit=0.0625, scale_limit=0.1, rotate_limit=45, prob=0.5):\n        self.shift_limit = shift_limit\n        self.scale_limit = scale_limit\n        self.rotate_limit = rotate_limit\n        self.prob = prob\n\n    def __call__(self, img, mask=None):\n        if random.random() < self.prob:\n            height, width, channel = img.shape\n\n            angle = random.uniform(-self.rotate_limit, self.rotate_limit)\n            scale = random.uniform(1 - self.scale_limit, 1 + self.scale_limit)\n            dx = round(random.uniform(-self.shift_limit, self.shift_limit)) * width\n            dy = round(random.uniform(-self.shift_limit, self.shift_limit)) * height\n\n            cc = math.cos(angle / 180 * math.pi) * scale\n            ss = math.sin(angle / 180 * math.pi) * scale\n            rotate_matrix = np.array([[cc, -ss], [ss, cc]])\n\n            box0 = np.array([[0, 0], [width, 0], [width, height], [0, height], ])\n            box1 = box0 - np.array([width / 2, height / 2])\n            box1 = np.dot(box1, rotate_matrix.T) + np.array([width / 2 + dx, height / 2 + dy])\n\n            box0 = box0.astype(np.float32)\n            box1 = box1.astype(np.float32)\n            mat = cv2.getPerspectiveTransform(box0, box1)\n            img = cv2.warpPerspective(img, mat, (width, height),\n                                      flags=cv2.INTER_LINEAR,\n                                      borderMode=cv2.BORDER_REFLECT_101)\n            if mask is not None:\n                mask = cv2.warpPerspective(mask, mat, (width, height),\n                                           flags=cv2.INTER_NEAREST,\n                                           borderMode=cv2.BORDER_REFLECT_101)\n\n        return img, mask\n\n\nclass CenterCrop:\n    def __init__(self, size):\n        self.height = size[0]\n        self.width = size[1]\n\n    def __call__(self, img, mask=None):\n        h, w, c = img.shape\n        dy = (h - self.height) // 2\n        dx = (w - self.width) // 2\n        y1 = dy\n        y2 = y1 + self.height\n        x1 = dx\n        x2 = x1 + self.width\n        img = img[y1:y2, x1:x2,:]\n        if mask is not None:\n            if mask.ndim == 2:\n                mask = np.expand_dims(mask, axis=2)\n            mask = mask[y1:y2, x1:x2,:]\n\n        return img, mask\n    \nclass RandomBrightness:\n    def __init__(self, limit=0.1, prob=0.5):\n        self.limit = limit\n        self.prob = prob\n\n    def __call__(self, img):\n        if random.random() < self.prob:\n            alpha = 1.0 + self.limit * random.uniform(-1, 1)\n\n            maxval = np.max(img[..., :3])\n            dtype = img.dtype\n            img[..., :3] = clip(alpha * img[..., :3], dtype, maxval)\n        return img\n\n\nclass RandomContrast:\n    def __init__(self, limit=.1, prob=.5):\n        self.limit = limit\n        self.prob = prob\n\n    def __call__(self, img):\n        if random.random() < self.prob:\n            alpha = 1.0 + self.limit * random.uniform(-1, 1)\n\n            gray = cv2.cvtColor(img[:, :, :3], cv2.COLOR_BGR2GRAY)\n            gray = (3.0 * (1.0 - alpha) / gray.size) * np.sum(gray)\n            maxval = np.max(img[..., :3])\n            dtype = img.dtype\n            img[:, :, :3] = clip(alpha * img[:, :, :3] + gray, dtype, maxval)\n        return img\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea4235de9bf5660000967af1c776e19c7f0e2005","collapsed":true},"cell_type":"code","source":"train_transform = DualCompose([\n        HorizontalFlip(),\n        VerticalFlip(),\n        RandomCrop((256,256,3)),\n        #ImageOnly(RandomBrightness()),\n        #ImageOnly(RandomContrast()),\n])\n\nval_transform = DualCompose([\n        CenterCrop((512,512,3)),\n      ])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0316b17da15e5e5495c67cbdfc7f35e72555544e"},"cell_type":"markdown","source":"## UNET model"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a2a06cab7c08985cf491630f081eb92c5b1f2707"},"cell_type":"code","source":"# Implementation from https://github.com/timctho/unet-pytorch/\nclass UNet_down_block(torch.nn.Module):\n    def __init__(self, input_channel, output_channel, down_size):\n        super(UNet_down_block, self).__init__()\n        self.conv1 = torch.nn.Conv2d(input_channel, output_channel, 3, padding=1)\n        self.bn1 = torch.nn.BatchNorm2d(output_channel)\n        self.conv2 = torch.nn.Conv2d(output_channel, output_channel, 3, padding=1)\n        self.bn2 = torch.nn.BatchNorm2d(output_channel)\n        self.conv3 = torch.nn.Conv2d(output_channel, output_channel, 3, padding=1)\n        self.bn3 = torch.nn.BatchNorm2d(output_channel)\n        self.max_pool = torch.nn.MaxPool2d(2, 2)\n        self.relu = torch.nn.ReLU()\n        self.down_size = down_size\n\n    def forward(self, x):\n        if self.down_size:\n            x = self.max_pool(x)\n        x = self.relu(self.bn1(self.conv1(x)))\n        x = self.relu(self.bn2(self.conv2(x)))\n        x = self.relu(self.bn3(self.conv3(x)))\n        return x\n\nclass UNet_up_block(torch.nn.Module):\n    def __init__(self, prev_channel, input_channel, output_channel):\n        super(UNet_up_block, self).__init__()\n        self.up_sampling = torch.nn.Upsample(scale_factor=2, mode='bilinear')\n        self.conv1 = torch.nn.Conv2d(prev_channel + input_channel, output_channel, 3, padding=1)\n        self.bn1 = torch.nn.BatchNorm2d(output_channel)\n        self.conv2 = torch.nn.Conv2d(output_channel, output_channel, 3, padding=1)\n        self.bn2 = torch.nn.BatchNorm2d(output_channel)\n        self.conv3 = torch.nn.Conv2d(output_channel, output_channel, 3, padding=1)\n        self.bn3 = torch.nn.BatchNorm2d(output_channel)\n        self.relu = torch.nn.ReLU()\n\n    def forward(self, prev_feature_map, x):\n        x = self.up_sampling(x)\n        x = torch.cat((x, prev_feature_map), dim=1)\n        x = self.relu(self.bn1(self.conv1(x)))\n        x = self.relu(self.bn2(self.conv2(x)))\n        x = self.relu(self.bn3(self.conv3(x)))\n        return x\n\n\nclass UNet(torch.nn.Module):\n    def __init__(self):\n        super(UNet, self).__init__()\n\n        self.down_block1 = UNet_down_block(3, 16, False)\n        self.down_block2 = UNet_down_block(16, 32, True)\n        self.down_block3 = UNet_down_block(32, 64, True)\n        self.down_block4 = UNet_down_block(64, 128, True)\n        self.down_block5 = UNet_down_block(128, 256, True)\n        self.down_block6 = UNet_down_block(256, 512, True)\n        self.down_block7 = UNet_down_block(512, 1024, True)\n\n        self.mid_conv1 = torch.nn.Conv2d(1024, 1024, 3, padding=1)\n        self.bn1 = torch.nn.BatchNorm2d(1024)\n        self.mid_conv2 = torch.nn.Conv2d(1024, 1024, 3, padding=1)\n        self.bn2 = torch.nn.BatchNorm2d(1024)\n        self.mid_conv3 = torch.nn.Conv2d(1024, 1024, 3, padding=1)\n        self.bn3 = torch.nn.BatchNorm2d(1024)\n\n        self.up_block1 = UNet_up_block(512, 1024, 512)\n        self.up_block2 = UNet_up_block(256, 512, 256)\n        self.up_block3 = UNet_up_block(128, 256, 128)\n        self.up_block4 = UNet_up_block(64, 128, 64)\n        self.up_block5 = UNet_up_block(32, 64, 32)\n        self.up_block6 = UNet_up_block(16, 32, 16)\n\n        self.last_conv1 = torch.nn.Conv2d(16, 16, 3, padding=1)\n        self.last_bn = torch.nn.BatchNorm2d(16)\n        self.last_conv2 = torch.nn.Conv2d(16, 1, 1, padding=0)\n        self.relu = torch.nn.ReLU()\n\n    def forward(self, x):\n        self.x1 = self.down_block1(x)\n        self.x2 = self.down_block2(self.x1)\n        self.x3 = self.down_block3(self.x2)\n        self.x4 = self.down_block4(self.x3)\n        self.x5 = self.down_block5(self.x4)\n        self.x6 = self.down_block6(self.x5)\n        self.x7 = self.down_block7(self.x6)\n        self.x7 = self.relu(self.bn1(self.mid_conv1(self.x7)))\n        self.x7 = self.relu(self.bn2(self.mid_conv2(self.x7)))\n        self.x7 = self.relu(self.bn3(self.mid_conv3(self.x7)))\n        x = self.up_block1(self.x6, self.x7)\n        x = self.up_block2(self.x5, x)\n        x = self.up_block3(self.x4, x)\n        x = self.up_block4(self.x3, x)\n        x = self.up_block5(self.x2, x)\n        x = self.up_block6(self.x1, x)\n        x = self.relu(self.last_bn(self.last_conv1(x)))\n        x = self.last_conv2(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0fa3ff2ec41f63d60b36f0d7fefb953f8a07be75"},"cell_type":"markdown","source":"## Loss function\nDue to non-differentiability of the competition evaluation metric (F2 Score at different intersection over union (IoU) thresholds) we use combination of  the BinnaryCrossEntropy loss and minus Jaccard index (https://en.wikipedia.org/wiki/Jaccard_index)\n"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4a2a0b420fe9113d24348ccc2d40d85ece70c343"},"cell_type":"code","source":" class LossBinary:\n    \"\"\"\n     Implementation from  https://github.com/ternaus/robot-surgery-segmentation\n    \"\"\"\n\n    def __init__(self, jaccard_weight=0):\n        self.nll_loss = nn.BCEWithLogitsLoss()\n        self.jaccard_weight = jaccard_weight\n\n    def __call__(self, outputs, targets):\n        loss = self.nll_loss(outputs, targets)\n\n        if self.jaccard_weight:\n            eps = 1e-15\n            jaccard_target = (targets == 1.0).float()\n            jaccard_output = F.sigmoid(outputs)\n\n            intersection = (jaccard_output * jaccard_target).sum()\n            union = jaccard_output.sum() + jaccard_target.sum()\n\n            loss -= self.jaccard_weight * torch.log((intersection + eps) / (union - intersection + eps))\n        return loss\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9215afb7f69a97aedcec4b0af06aae8a29c4c7b1"},"cell_type":"markdown","source":"## Validation routine"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"fea9ca85fa0df8718bfed351c68638884e392a7d"},"cell_type":"code","source":"def validation(model: nn.Module, criterion, valid_loader):\n    print(\"Validation on hold-out....\")\n    model.eval()\n    losses = []\n    jaccard = []\n    for inputs, targets in valid_loader:\n        inputs = variable(inputs, volatile=True)\n        targets = variable(targets)\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        losses.append(loss.data[0])\n        jaccard += [get_jaccard(targets, (outputs > 0).float()).data[0]]\n\n    valid_loss = np.mean(losses)  # type: float\n\n    valid_jaccard = np.mean(jaccard)\n\n    print('Valid loss: {:.5f}, jaccard: {:.5f}'.format(valid_loss, valid_jaccard))\n    metrics = {'valid_loss': valid_loss, 'jaccard_loss': valid_jaccard}\n    return metrics\n\n\ndef get_jaccard(y_true, y_pred):\n    epsilon = 1e-15\n    intersection = (y_pred * y_true).sum(dim=-2).sum(dim=-1).sum(dim = -1)\n    union = y_true.sum(dim=-2).sum(dim=-1).sum(dim=-1) + y_pred.sum(dim=-2).sum(dim=-1).sum(dim = -1)\n\n    return (intersection / (union - intersection + epsilon)).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a624fa2767c4da4f400d1ccbcc6225ffad3335c","collapsed":true},"cell_type":"code","source":"# sume helper functions\ndef variable(x, volatile=False):\n    if isinstance(x, (list, tuple)):\n        return [variable(y, volatile=volatile) for y in x]\n    return cuda(Variable(x, volatile=volatile))\n\ndef cuda(x):\n    return x.cuda(async=True) if torch.cuda.is_available() else x\n\ndef write_event(log, step: int, **data):\n    data['step'] = step\n    data['dt'] = datetime.now().isoformat()\n    log.write(json.dumps(data, sort_keys=True))\n    log.write('\\n')\n    log.flush()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"19f4a8e0394b099302fe5b27d0e40ac188759f89"},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":true,"_uuid":"c89c02e2183c787abe4cb5f4cfe58fdc4651843e","collapsed":true},"cell_type":"code","source":"# main train routine\n# Implementation from  https://github.com/ternaus/robot-surgery-segmentation\ndef train(lr, model, criterion, train_loader, valid_loader, validation, init_optimizer, n_epochs=1, fold=1):\n    optimizer = init_optimizer(lr)\n    #model = nn.DataParallel(model, device_ids=None)\n    if torch.cuda.is_available():\n        model.cuda()\n       \n    model_path = Path('model_{fold}.pt'.format(fold=fold))\n    if model_path.exists():\n        state = torch.load(str(model_path))\n        epoch = state['epoch']\n        step = state['step']\n        model.load_state_dict(state['model'])\n        print('Restored model, epoch {}, step {:,}'.format(epoch, step))\n    else:\n        epoch = 1\n        step = 0\n\n    save = lambda ep: torch.save({\n        'model': model.state_dict(),\n        'epoch': ep,\n        'step': step,\n    }, str(model_path))\n\n\n    report_each = 50\n    log = open('train_{fold}.log'.format(fold=fold),'at', encoding='utf8')\n    valid_losses = []\n    for epoch in range(epoch, n_epochs + 1):\n        model.train()\n        random.seed()\n        tq = tqdm(total=len(train_loader) *  BATCH_SIZE)\n        tq.set_description('Epoch {}, lr {}'.format(epoch, lr))\n        losses = []\n        tl = train_loader\n        try:\n            mean_loss = 0\n            for i, (inputs, targets) in enumerate(tl):\n                inputs, targets = variable(inputs), variable(targets)\n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                optimizer.zero_grad()\n                batch_size = inputs.size(0)\n                loss.backward()\n                optimizer.step()\n                step += 1\n                tq.update(batch_size)\n                losses.append(loss.data[0])\n                mean_loss = np.mean(losses[-report_each:])\n                tq.set_postfix(loss='{:.5f}'.format(mean_loss))\n                if i and i % report_each == 0:\n                    write_event(log, step, loss=mean_loss)\n            write_event(log, step, loss=mean_loss)\n            tq.close()\n            save(epoch + 1)\n            valid_metrics = validation(model, criterion, valid_loader)\n            write_event(log, step, **valid_metrics)\n            valid_loss = valid_metrics['valid_loss']\n            valid_losses.append(valid_loss)\n        except KeyboardInterrupt:\n            tq.close()\n            print('Ctrl+C, saving snapshot')\n            save(epoch)\n            print('done.')\n            return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fb484e1bb365c81757f33d96ca13a5cdedeafa4","collapsed":true},"cell_type":"code","source":"def make_loader(in_df, batch_size, shuffle=False, transform=None):\n        return DataLoader(\n            dataset=ShipDataset(in_df, transform=transform),\n            shuffle=shuffle,\n            num_workers = 0,\n            batch_size = batch_size,\n            pin_memory=torch.cuda.is_available()\n        )\n\ntrain_loader = make_loader(train_df, batch_size =  BATCH_SIZE, shuffle=True, transform=train_transform)\nvalid_loader = make_loader(valid_df, batch_size = BATCH_SIZE / 2, transform=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66ef01b93576c49d53d96418814cd9706f8e9909","collapsed":true},"cell_type":"code","source":"# run just 3 epoch\nmodel = UNet()\ntrain(init_optimizer=lambda lr: Adam(model.parameters(), lr=lr),\n        lr = 1e-4,\n        n_epochs = 3,\n        model=model,\n        criterion=LossBinary(jaccard_weight=5),\n        train_loader=train_loader,\n        valid_loader=valid_loader,\n        validation=validation,\n        )\n        ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"79d438d4c4fa90f730be8706ed6e4368ef1e3e30"},"cell_type":"markdown","source":"## Log visualization"},{"metadata":{"trusted":true,"_uuid":"8ec8240a9adb85ce5442a83bf4652142c7a7d490","collapsed":true},"cell_type":"code","source":"log_file = 'train_1.log'\nlogs = pd.read_json(log_file, lines=True)\n\nplt.figure(figsize=(26,6))\nplt.subplot(1, 2, 1)\nplt.plot(logs.step[logs.loss.notnull()],\n            logs.loss[logs.loss.notnull()],\n            label=\"on training set\")\n \n#plt.plot(logs.step[logs.valid_loss.notnull()],\n#            logs.valid_loss[logs.valid_loss.notnull()],\n#            label = \"on validation set\")\n         \nplt.xlabel('step')\nplt.legend(loc='center left')\nplt.tight_layout()\nplt.show();\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a4952cf01846db60d8eb451381aec104207fa4f8"},"cell_type":"markdown","source":"## Prediction for one image "},{"metadata":{"trusted":true,"_uuid":"20b560f910d65a8c434659751090908472f2b66e","collapsed":true},"cell_type":"code","source":"model = UNet()\nmodel_path ='model_1.pt'\nstate = torch.load(str(model_path))\nstate = {key.replace('module.', ''): value for key, value in state['model'].items()}\nmodel.load_state_dict(state)\nif torch.cuda.is_available():\n    model.cuda()\n\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"feaf680644709ac85bdb97700a491ef3a26a8692","collapsed":true},"cell_type":"code","source":"valid_ds  = ShipDataset(valid_df)\n# to show gt image\nimshow(*valid_ds[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44d9aae53ea2e437520fc467e4eaacd837ca8c93","collapsed":true},"cell_type":"code","source":"# predict and show prediction \nimg,_ = valid_ds[0]\ninput_img = torch.unsqueeze(variable(img, volatile=True), dim=0)\nmask = F.sigmoid(model(input_img))\nout_mask = torch.squeeze(mask.data.cpu(), dim = 0)\nimshow(img,out_mask)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8cf612760a0adc391ff99dbe931a155b839b0169"},"cell_type":"markdown","source":"## Prediction for Test images\n\nFor test purposes predict only for 5000 images. Prediction of the full set of images (88500 images) takes more than 6hr !!!"},{"metadata":{"trusted":true,"_uuid":"b10eb3058e18264f2c652799a2b416ffb71c2d61","collapsed":true},"cell_type":"code","source":"test_paths = os.listdir(test_image_dir)\nprint(len(test_paths), 'test images found')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b9f6762c7dcf88ce8d076b8698e905166e1ba3a","collapsed":true},"cell_type":"code","source":"test_df = pd.DataFrame({'ImageId': test_paths, 'EncodedPixels':None})\nfrom skimage.morphology import binary_opening, disk\ntest_df=test_df[:5000]\nloader = DataLoader(\n        dataset=ShipDataset(test_df, transform=None, mode='predict'),\n        shuffle=False,\n        batch_size=BATCH_SIZE,\n        num_workers=0,\n        pin_memory=torch.cuda.is_available()\n    ) \n    \nout_pred_rows = []\nfor batch_num, (inputs, paths) in enumerate(tqdm(loader, desc='Predict')):\n    inputs = variable(inputs, volatile=True)\n    outputs = model(inputs)\n    for i, image_name in enumerate(paths):\n        mask = F.sigmoid(outputs[i,0]).data.cpu().numpy()\n        cur_seg = binary_opening(mask>0.5, disk(2))\n        cur_rles = multi_rle_encode(cur_seg)\n        if len(cur_rles)>0:\n            for c_rle in cur_rles:\n                out_pred_rows += [{'ImageId': image_name, 'EncodedPixels': c_rle}]\n        else:\n            out_pred_rows += [{'ImageId': image_name, 'EncodedPixels': None}]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d3c30371412af263b7de56ae1c352c6732aab35","collapsed":true},"cell_type":"code","source":"submission_df = pd.DataFrame(out_pred_rows)[['ImageId', 'EncodedPixels']]\nsubmission_df.to_csv('submission.csv', index=False)\nsubmission_df.sample(10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}