{"cells":[{"metadata":{},"cell_type":"markdown","source":"Starter Kernel for ensembling multiple models. In this kernel I use Catalyst + Mlcomp. Most of the code for helper functions and model definition is taken from this great kernel by Andrew (https://www.kaggle.com/artgor/segmentation-in-pytorch-using-convenient-tools).\n\nIn this kernel I try to present two model ensembling approach.\n1. Simple Average Ensembling\n2. Voting Ensemble"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true,"collapsed":true},"cell_type":"code","source":"!pip install catalyst\n!pip install pretrainedmodels\n!pip install git+https://github.com/qubvel/segmentation_models.pytorch\n!pip install pytorch_toolbelt\n!pip install torchvision==0.4\n!pip install albumentations==0.3.2\n!pip install mlcomp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Import Required Packages"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport collections\nimport time \nimport tqdm\nfrom PIL import Image\nfrom functools import partial\ntrain_on_gpu = True\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\n\nimport albumentations as albu\nfrom albumentations import torch as AT\n\nfrom catalyst.data import Augmentor\nfrom catalyst.dl import utils\nfrom catalyst.data.reader import ImageReader, ScalarReader, ReaderCompose, LambdaReader\nfrom catalyst.dl.runner import SupervisedRunner\nfrom catalyst.contrib.models.segmentation import Unet\nfrom catalyst.dl.callbacks import DiceCallback, EarlyStoppingCallback, InferCallback, CheckpointCallback\n\nimport segmentation_models_pytorch as smp\nfrom tqdm import tqdm, tqdm_notebook\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom torch.jit import load\n\nfrom mlcomp.contrib.transform.albumentations import ChannelTranspose\nfrom mlcomp.contrib.dataset.classify import ImageDataset\nfrom mlcomp.contrib.transform.rle import rle2mask, mask2rle\nfrom mlcomp.contrib.transform.tta import TtaWrap","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Helper Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img(x, folder: str='train_images'):\n    \"\"\"\n    Return image based on image name and folder.\n    \"\"\"\n    data_folder = f\"{path}/{folder}\"\n    image_path = os.path.join(data_folder, x)\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    return img\n\n\ndef rle_decode(mask_rle: str = '', shape: tuple = (1400, 2100)):\n    '''\n    Decode rle encoded mask.\n    \n    :param mask_rle: run-length as string formatted (start length)\n    :param shape: (height, width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape, order='F')\n\n\ndef make_mask(df: pd.DataFrame, image_name: str='img.jpg', shape: tuple = (1400, 2100)):\n    \"\"\"\n    Create mask based on df, image name and shape.\n    \"\"\"\n    encoded_masks = df.loc[df['im_id'] == image_name, 'EncodedPixels']\n    masks = np.zeros((shape[0], shape[1], 4), dtype=np.float32)\n\n    for idx, label in enumerate(encoded_masks.values):\n        if label is not np.nan:\n            mask = rle_decode(label)\n            masks[:, :, idx] = mask\n            \n    return masks\n\n\ndef to_tensor(x, **kwargs):\n    \"\"\"\n    Convert image or mask.\n    \"\"\"\n    return x.transpose(2, 0, 1).astype('float32')\n\n\ndef mask2rle(img):\n    '''\n    Convert mask to rle.\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels= img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\n\ndef visualize(image, mask, original_image=None, original_mask=None):\n    \"\"\"\n    Plot image and masks.\n    If two pairs of images and masks are passes, show both.\n    \"\"\"\n    fontsize = 14\n    class_dict = {0: 'Fish', 1: 'Flower', 2: 'Gravel', 3: 'Sugar'}\n    \n    if original_image is None and original_mask is None:\n        f, ax = plt.subplots(1, 5, figsize=(24, 24))\n\n        ax[0].imshow(image)\n        for i in range(4):\n            ax[i + 1].imshow(mask[:, :, i])\n            ax[i + 1].set_title(f'Mask {class_dict[i]}', fontsize=fontsize)\n    else:\n        f, ax = plt.subplots(2, 5, figsize=(24, 12))\n\n        ax[0, 0].imshow(original_image)\n        ax[0, 0].set_title('Original image', fontsize=fontsize)\n                \n        for i in range(4):\n            ax[0, i + 1].imshow(original_mask[:, :, i])\n            ax[0, i + 1].set_title(f'Original mask {class_dict[i]}', fontsize=fontsize)\n        \n        ax[1, 0].imshow(image)\n        ax[1, 0].set_title('Transformed image', fontsize=fontsize)\n        \n        \n        for i in range(4):\n            ax[1, i + 1].imshow(mask[:, :, i])\n            ax[1, i + 1].set_title(f'Transformed mask {class_dict[i]}', fontsize=fontsize)\n            \n            \ndef visualize_with_raw(image, mask, original_image=None, original_mask=None, raw_image=None, raw_mask=None):\n    \"\"\"\n    Plot image and masks.\n    If two pairs of images and masks are passes, show both.\n    \"\"\"\n    fontsize = 14\n    class_dict = {0: 'Fish', 1: 'Flower', 2: 'Gravel', 3: 'Sugar'}\n\n    f, ax = plt.subplots(3, 5, figsize=(24, 12))\n\n    ax[0, 0].imshow(original_image)\n    ax[0, 0].set_title('Original image', fontsize=fontsize)\n\n    for i in range(4):\n        ax[0, i + 1].imshow(original_mask[:, :, i])\n        ax[0, i + 1].set_title(f'Original mask {class_dict[i]}', fontsize=fontsize)\n\n\n    ax[1, 0].imshow(raw_image)\n    ax[1, 0].set_title('Original image', fontsize=fontsize)\n\n    for i in range(4):\n        ax[1, i + 1].imshow(raw_mask[:, :, i])\n        ax[1, i + 1].set_title(f'Raw predicted mask {class_dict[i]}', fontsize=fontsize)\n        \n    ax[2, 0].imshow(image)\n    ax[2, 0].set_title('Transformed image', fontsize=fontsize)\n\n\n    for i in range(4):\n        ax[2, i + 1].imshow(mask[:, :, i])\n        ax[2, i + 1].set_title(f'Predicted mask with processing {class_dict[i]}', fontsize=fontsize)\n            \n            \ndef plot_with_augmentation(image, mask, augment):\n    \"\"\"\n    Wrapper for `visualize` function.\n    \"\"\"\n    augmented = augment(image=image, mask=mask)\n    image_flipped = augmented['image']\n    mask_flipped = augmented['mask']\n    visualize(image_flipped, mask_flipped, original_image=image, original_mask=mask)\n    \n    \nsigmoid = lambda x: 1 / (1 + np.exp(-x))\n\n\ndef post_process(probability, threshold, min_size):\n    \"\"\"\n    Post processing of each predicted mask, components with lesser number of pixels\n    than `min_size` are ignored\n    \"\"\"\n    # don't remember where I saw it\n    mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]\n    num_component, component = cv2.connectedComponents(mask.astype(np.uint8))\n    predictions = np.zeros((350, 525), np.float32)\n    num = 0\n    for c in range(1, num_component):\n        p = (component == c)\n        if p.sum() > min_size:\n            predictions[p] = 1\n            num += 1\n    return predictions, num\n\n\ndef get_training_augmentation():\n    train_transform = [\n        albu.Resize(704, 1056),\n        albu.HorizontalFlip(p=0.5),\n        albu.VerticalFlip(p=0.5),\n        albu.ShiftScaleRotate(scale_limit=0.1, rotate_limit=0, shift_limit=0.1, p=0.5, border_mode=0),\n        # albu.GridDistortion(p=0.2),\n        # albu.OpticalDistortion(p=0.2, distort_limit=2, shift_limit=0.5),\n    ]\n    return albu.Compose(train_transform)\n\n\ndef get_validation_augmentation():\n    \"\"\"Add paddings to make image shape divisible by 32\"\"\"\n    test_transform = [\n        albu.Resize(704, 1056)\n    ]\n    return albu.Compose(test_transform)\n\n\ndef get_preprocessing(preprocessing_fn):\n    \"\"\"Construct preprocessing transform\n    \n    Args:\n        preprocessing_fn (callbale): data normalization function \n            (can be specific for each pretrained neural network)\n    Return:\n        transform: albumentations.Compose\n    \n    \"\"\"\n    \n    _transform = [\n        albu.Lambda(image=preprocessing_fn),\n        albu.Lambda(image=to_tensor, mask=to_tensor),\n    ]\n    return albu.Compose(_transform)\n\n\ndef dice(img1, img2):\n    img1 = np.asarray(img1).astype(np.bool)\n    img2 = np.asarray(img2).astype(np.bool)\n\n    intersection = np.logical_and(img1, img2)\n\n    return 2. * intersection.sum() / (img1.sum() + img2.sum())\n\n\ndef ensemble_voting(p_channel, threshold, min_size):\n    mask = cv2.threshold(p_channel, threshold, 1, cv2.THRESH_BINARY)[1]\n    num_component, component = cv2.connectedComponents(mask.astype(np.uint8))\n    predictions = np.zeros((350, 525), np.float32)\n    num = 0\n    for c in range(1, num_component):\n        p = (component == c)\n        if p.sum() > min_size:\n            predictions[p] = 1\n            num += 1\n    return predictions, num","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define Models"},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Andrews resnet50 from [here](https://www.kaggle.com/artgor/segmentation-in-pytorch-using-convenient-tools)"},{"metadata":{"trusted":true},"cell_type":"code","source":"ENCODER = 'resnet50'\nENCODER_WEIGHTS = 'imagenet'\n\nACTIVATION = None\nunet_resnet50 = smp.Unet(\n    encoder_name=ENCODER, \n    encoder_weights=ENCODER_WEIGHTS, \n    classes=4, \n    activation=ACTIVATION,\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Ryches resnet18 from [here](https://www.kaggle.com/artgor/segmentation-in-pytorch-using-convenient-tools)"},{"metadata":{"trusted":true},"cell_type":"code","source":"ENCODER = 'resnet18'\nENCODER_WEIGHTS = 'imagenet'\n\nACTIVATION = None\nunet_resnet18 = smp.Unet(\n    encoder_name=ENCODER, \n    encoder_weights=ENCODER_WEIGHTS, \n    classes=4, \n    activation=ACTIVATION,\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Loading model weights"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_meta = torch.load('../input/segmentation-in-pytorch-using-convenient-tools/logs/segmentation/checkpoints/best.pth')\nunet_resnet50.load_state_dict(model_meta[\"model_state_dict\"])\n\nmodel_meta = torch.load('../input/turbo-charging-andrew-s-pytorch/logs/segmentation_unet/checkpoints/best.pth')\nunet_resnet18.load_state_dict(model_meta[\"model_state_dict\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unet_resnet50.to(DEVICE);\nunet_resnet18.to(DEVICE);\n\nunet_resnet50.eval();\nunet_resnet18.eval();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Ensembling"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model:\n    def __init__(self, models, voting=False):\n        self.models = models\n        self.voting = voting\n    \n    def __call__(self, x):\n        res = []\n        x = x.to(DEVICE)\n        with torch.no_grad():\n            for m in self.models:\n                res.append(m(x))\n        res = torch.stack(res)\n        if (self.voting):\n            return res\n        return torch.mean(res, dim=0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Define list of models"},{"metadata":{"trusted":true},"cell_type":"code","source":"models = [unet_resnet50, unet_resnet18, unet_resnet18]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '../input/understanding_cloud_organization'\n\ndef create_transforms(additional):\n    res = list(additional)\n    # add necessary transformations\n    res.extend([\n        albu.Normalize(\n            mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)\n        ),\n        ChannelTranspose()\n    ])\n    res = albu.Compose(res)\n    return res\n\nimg_folder = f'{path}/test_images'\nbatch_size = 2\nnum_workers = 0\n\n# Different transforms for TTA wrapper\ntransforms = [\n    [albu.Resize(352, 576)],\n    [albu.Resize(352, 576), albu.HorizontalFlip(p=1.0)],\n    [albu.Resize(352, 576), albu.VerticalFlip(p=1.0)]\n]\n\ntransforms = [create_transforms(t) for t in transforms]\ndatasets = [TtaWrap(ImageDataset(img_folder=img_folder, transforms=t), tfms=t) for t in transforms]\nloaders = [DataLoader(d, num_workers=num_workers, batch_size=batch_size, shuffle=False) for d in datasets]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Average Ensemble Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"# set ensemble mode to simple average\nmodel = Model(models, voting=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# I chose the thresholds randomly, but a better way would be to decide based on validation set\n\nthresholds = [0.5, 0.5, 0.5, 0.5]\nmin_area = [20000, 20000, 20000, 15000]\n\nclass_dict = {0: 'Fish', 1: 'Flower', 2: 'Gravel', 3: 'Sugar'}\ncount_flag = 0 # Jung, counting how many we have fixed\n\n\nres = []\n# Iterate over all TTA loaders\ntotal = len(datasets[0])//batch_size\nfor loaders_batch in tqdm_notebook(zip(*loaders), total=total):\n    preds = []\n    # preds_sig = []\n    image_file = []\n    for i, batch in enumerate(loaders_batch):\n        features = batch['features'].cuda()\n        p = torch.sigmoid(model(features))\n        # inverse operations for TTA\n        p = datasets[i].inverse(p)\n        preds.append(p)\n        image_file = batch['image_file']\n    \n    # TTA mean\n    preds = torch.stack(preds)\n    preds = torch.mean(preds, dim=0)\n    preds = preds.detach().cpu().numpy()\n    \n    # Batch post processing\n    for p, file in zip(preds, image_file):\n        file = os.path.basename(file)\n        # Image postprocessing\n        for i in range(4):\n            imageid_classid = file + '_' + str(class_dict[i])\n            p_channel = p[i]\n            if p_channel.shape != (350, 525):\n                p_channel = cv2.resize(p_channel, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)\n\n            predict, num_predict = post_process(p_channel, thresholds[i], min_area[i])\n            rle_mask = ''\n            if num_predict != 0:\n                rle_mask = mask2rle(predict)\n                flag=True\n            res.append({\n                'Image_Label': imageid_classid,\n                'EncodedPixels': rle_mask\n            })","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualize results"},{"metadata":{"trusted":true},"cell_type":"code","source":"image_vis = features[0].detach().cpu().numpy().transpose(1, 2, 0)\nmask = preds[0].astype('uint8').transpose(1, 2, 0)\npr_mask = np.zeros((350, 525, 4))\nfor j in range(4):\n    probability = cv2.resize(preds[0].transpose(1, 2, 0)[:, :, j], dsize=(525, 350), interpolation=cv2.INTER_LINEAR)\n    pr_mask[:, :, j], _ = post_process(probability, thresholds[j], min_area[j])\n\nvisualize_with_raw(image=image_vis, mask=pr_mask, original_image=image_vis, original_mask=mask, raw_image=image_vis, raw_mask=preds[0].transpose(1, 2, 0))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generate Submission File"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame(res)\ndf.to_csv('submission_avg_ensemble.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Voting Ensemble Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"# set ensemble mode to voting\nmodel_voting = Model(models, voting=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"thresholds = [0.5, 0.5, 0.5, 0.5]\nmin_area = [20000, 20000, 20000, 15000]\n\nclass_dict = {0: 'Fish', 1: 'Flower', 2: 'Gravel', 3: 'Sugar'}\n\nres = []\n# Iterate over all TTA loaders\ntotal = len(datasets[0])//batch_size\nfor loaders_batch in tqdm_notebook(zip(*loaders), total=total):\n    preds = []\n    # preds_sig = []\n    image_file = []\n    for i, batch in enumerate(loaders_batch):\n        features = batch['features'].cuda()\n        p = torch.sigmoid(model_voting(features))\n        # inverse operations for TTA\n        p = datasets[i].inverse(p)\n        preds.append(p)\n        image_file = batch['image_file']\n    \n    # TTA mean\n    preds = torch.stack(preds)\n    preds = torch.mean(preds, dim=0)\n    preds = preds.detach().cpu().numpy()\n    preds = np.transpose(preds, [1, 0, 2, 3, 4])\n    \n    # Batch post processing\n    for p, file in zip(preds, image_file):\n        file = os.path.basename(file)\n        # for all models\n        for i in range(4):\n            # Image postprocessing\n            pred_masks = []\n            for m in range(len(models)):\n                imageid_classid = file + '_' + str(class_dict[i])\n                p_channel = p[m][i]\n                if p_channel.shape != (350, 525):\n                    p_channel = cv2.resize(p_channel, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)\n\n                mask = cv2.threshold(p_channel, thresholds[i], 1, cv2.THRESH_BINARY)[1]\n                pred_masks.append(torch.from_numpy(mask))\n\n            voting_mask = torch.mean(torch.stack(pred_masks), dim=0)\n            \n            # based on number of models choose the ensemble voting threshold, in my case i have 3 models if any 2 vote \n            # for 1 pixel we choose that pixel in answer\n            predict, num_predict = ensemble_voting(voting_mask.numpy(), 0.5, min_area[i])\n          \n            rle_mask = ''\n            if num_predict != 0:\n                rle_mask = mask2rle(predict)\n\n            res.append({\n              'Image_Label': imageid_classid,\n              'EncodedPixels': rle_mask\n            })","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualize results"},{"metadata":{"trusted":true},"cell_type":"code","source":"image_vis = features[0].detach().cpu().numpy().transpose(1, 2, 0)\nmaskk = preds[0].astype('uint8').transpose(0, 2, 3, 1)\npr_mask = np.zeros((350, 525, 4))\nfor i in range(4):\n    # Image postprocessing\n    pred_masks = []\n    for m in range(len(models)):\n        p_channel = preds[0][m][i]\n        if p_channel.shape != (350, 525):\n            p_channel = cv2.resize(p_channel, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)\n\n      # predict, num_predict = post_process(p_channel, thresholds[i], min_area[i])\n        mask = cv2.threshold(p_channel, thresholds[i], 1, cv2.THRESH_BINARY)[1]\n        pred_masks.append(torch.from_numpy(mask))\n\n    voting_mask = torch.mean(torch.stack(pred_masks), dim=0)\n    pr_mask[:, :, i], _ = ensemble_voting(voting_mask.numpy(), 0.5, min_area[i])\n\n\nvisualize_with_raw(image=image_vis, mask=pr_mask, original_image=image_vis,\n                   original_mask=np.mean(maskk, axis=0), raw_image=image_vis, raw_mask=np.mean(preds[0], axis=0).transpose(1, 2, 0))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generate Submission File"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame(res)\ndf.to_csv('submission_voting_ensemble.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Please upvote if this kernel was helpful."},{"metadata":{},"cell_type":"markdown","source":"I made sure I didn't reveal any models which should break public LB in last few days, just the code for how to ensemble."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}