{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this kernel we visualize Mixed Sample Data Augmentations (MSDAs) like Mixup, Cutmix, Fmix. We first visualize them individually and then altogether aswell.\nAs mentioned by The Zoo in their writeup, they have used FMix. They found FMix to work better than Cutmix and the images were also more natural than Cutmix as quoted by them.\n\nPlease upvote the kernel if you find the visaulizations helpful.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from os.path import exists\nif not exists('fmix.zip'):\n    !wget -O fmix.zip https://github.com/ecs-vlc/fmix/archive/master.zip\n    !unzip -qq fmix.zip\n    !mv FMix-master/* ./\n    !rm -r FMix-master","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path.append('../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import glob\nimport math\nimport time\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport PIL.Image\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom tqdm import tqdm_notebook as tqdm\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport albumentations\nfrom fmix import  sample_mask\n\n\ndevice = torch.device('cpu')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"HEIGHT = 137\nWIDTH = 236\n\ndata_dir = '../input/bengaliaicv19feather'\nbatch_size = 512\nnum_workers = 4\n\ndf_train = pd.read_csv(os.path.join('../input/bengaliai-cv19', f'train.csv'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data & Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_data(files):\n    tmp = []\n    for f in files:\n        F = os.path.join(data_dir, f)\n        data = pd.read_feather(F)\n        res = data.iloc[:, 1:].values\n        imgs = []\n        for i in tqdm(range(res.shape[0])):\n            img = res[i].squeeze().reshape(HEIGHT, WIDTH)\n#             img = cv2.resize(img, (137, 236))\n            imgs.append(img)\n        imgs = np.asarray(imgs)\n        \n        tmp.append(imgs)\n    tmp = np.concatenate(tmp, 0)\n    return tmp\n\n\nclass BengaliDataset(Dataset):\n    def __init__(self, csv, data, idx, split, mode, transform=None):\n\n        self.csv = csv.reset_index()\n        self.data = data\n        self.idx = np.asarray(idx)\n        self.split = split\n        self.mode = mode\n        self.transform = transform\n\n    def __len__(self):\n        return self.idx.shape[0]\n\n    def __getitem__(self, index):\n        index = self.idx[index]\n        this_img_id = self.csv.iloc[index].image_id\n        \n        image = self.data[index]\n        image = 255 - image\n\n        if self.transform is not None:\n            image_origin = image.astype(np.float32).copy()\n            res = self.transform(image=image)\n            image = res['image'].astype(np.float32)\n        else:\n            image_origin = image.astype(np.float32).copy()\n            image = image.astype(np.float32)\n\n        image /= 255\n        image = image[np.newaxis, :, :]\n        image = np.repeat(image, 3, 0)  # 1ch to 3ch\n        ###\n        image_origin /= 255\n        image_origin = image_origin[np.newaxis, :, :]\n        image_origin = np.repeat(image_origin, 3, 0)  # 1ch to 3ch\n        ###\n\n        if self.mode == 'test':\n            return torch.tensor(image)\n        else:\n            label_1 = self.csv.iloc[index].grapheme_root\n            label_2 = self.csv.iloc[index].vowel_diacritic\n            label_3 = self.csv.iloc[index].consonant_diacritic\n            label = [label_1, label_2, label_3]\n            return torch.tensor(image), torch.tensor(image_origin), torch.tensor(label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_data = read_data(['train_image_data_0.feather'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\ndef cutmix(data, target, alpha):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_target = target[indices]\n\n    lam = np.clip(np.random.beta(alpha, alpha),0.3,0.4)\n    bbx1, bby1, bbx2, bby2 = rand_bbox(data.size(), lam)\n    new_data = data.clone()\n    new_data[:, :, bby1:bby2, bbx1:bbx2] = data[indices, :, bby1:bby2, bbx1:bbx2]\n    # adjust lambda to exactly match pixel ratio\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (data.size()[-1] * data.size()[-2]))\n    targets = (target, shuffled_target, lam)\n\n    return new_data, targets\n\ndef mixup(data, target, alpha):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_target = target[indices]\n\n    lam = np.clip(np.random.beta(alpha, alpha),0.3,0.7)\n    data = lam*data + (1-lam)*shuffled_data\n    targets = (target, shuffled_target, lam)\n\n    return data, targets\n\n\ndef fmix(data, targets, alpha, decay_power, shape, max_soft=0.0):\n    lam, mask = sample_mask(alpha, decay_power, shape, max_soft)\n    indices = torch.randperm(data.size(0)).to(device)\n    shuffled_data = data[indices]\n    shuffled_targets = targets[indices]\n    x1 = torch.from_numpy(mask)*data\n    x2 = torch.from_numpy(1-mask)*shuffled_data\n    targets=(targets, shuffled_targets, lam)\n    \n    return (x1+x2), targets","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Mixup"},{"metadata":{},"cell_type":"markdown","source":"Two images are mixed wiht weights: lambda and 1-lambda. lambda is generated from symmetric beta distribution with parameter alpha. There is no cutting and pasting involved in Mixup. \nBelow are some images generated from Mixup. 1st and 3rd column contain original image. 2nd and 4th column contain their respective mixedup versions.\n\nHere is the link to the paper for details: https://arxiv.org/abs/1710.09412 "},{"metadata":{"trusted":true},"cell_type":"code","source":"df_show = df_train.iloc[:1000]\ndataset_show = BengaliDataset(df_show, img_data, list(range(df_show.shape[0])), 'train', 'train', transform=None)\ndataloader = torch.utils.data.DataLoader(dataset_show, batch_size=batch_size,num_workers=num_workers, shuffle=False)\niter_data = iter(dataloader)\ndata, data_org, target = next(iter_data)\ndata, target = mixup(data_org, target, 1.)\n\n\nfrom pylab import rcParams\nrcParams['figure.figsize'] = 20,10\nfor i in range(3):\n    f, axarr = plt.subplots(1,4)\n    for p in range(0,3,2):\n        idx = np.random.randint(0, len(data))\n        img_org = data_org[idx]\n        new_img = data[idx]\n        axarr[p].imshow(img_org.permute(1,2,0))\n        axarr[p+1].imshow(new_img.permute(1,2,0))\n        axarr[p].set_title('original')\n        axarr[p+1].set_title('mixup image')\n        axarr[p].axis('off')\n        axarr[p+1].axis('off')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Cutmix"},{"metadata":{},"cell_type":"markdown","source":"1. Cutmix involves cutting a **rectangular** portion of a random image and then pasting it onto the concerned image at the same spot from where the portion was cut. How is the size of the portion decided? \n2. For that, lambda is generated from symmetric beta distribution with parameter alpha. Cut fraction - denoted by 'cut_rat' in code - is then calulated as square root of 1-lambda. \n3. Lengths of sides of rectangles are then calculated by multiplying that fraction with the height and weight of an image. A random (x,y) coordinate is generated from uniform distribution with higher limit as width and height. This (x,y) coordinate is then the centre of the rectangular portion to be cut. \n4. The boundary coordinates are then obtained by subtracting and adding length/2 to the centre 'x' coordinate and also breadth/2 to the centre 'y' coordinate. Thus we have 4 coordinates namely, (bbx1,cy), (bbx2,cy), (bby1,cx), (bby2,cx); each of them is on the centre of the rectangular sides. Thus, a rectangular portion to be cut is generated.\n\nHere is the link to the paper for details: https://arxiv.org/abs/1905.04899"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_show = df_train.iloc[:1000]\ndataset_show = BengaliDataset(df_show, img_data, list(range(df_show.shape[0])), 'train', 'train', transform=None)\ndataloader = torch.utils.data.DataLoader(dataset_show, batch_size=batch_size,num_workers=num_workers, shuffle=False)\niter_data = iter(dataloader)\ndata, data_org, target = next(iter_data)\ndata, target = cutmix(data_org, target, 1.)\n\n\nfrom pylab import rcParams\nrcParams['figure.figsize'] = 20,40\nfor i in range(3):\n    f, axarr = plt.subplots(1,4)\n    for p in range(0,3,2):\n        idx = np.random.randint(0, len(data))\n        img_org = data_org[idx]\n        new_img = data[idx]\n        axarr[p].imshow(img_org.permute(1,2,0))\n        axarr[p+1].imshow(new_img.permute(1,2,0))\n        axarr[p].set_title('original')\n        axarr[p+1].set_title('cutmix image')\n        axarr[p].axis('off')\n        axarr[p+1].axis('off')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# FMix"},{"metadata":{},"cell_type":"markdown","source":"1. FMix involves cutting **random shaped** portion from a random images and pasting it onto the concerned image.\n2. It doesn't involve explicit cutting and pasting, but there is generation of masks which define which portions of the image to take into consideration. \n3. Masks are obtained by applying threshold to low frequency images sampled from Fourier space. \n\nHere is the link to the paper for further details: https://arxiv.org/abs/2002.12047"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_show = df_train.iloc[:1000]\ndataset_show = BengaliDataset(df_show, img_data, list(range(df_show.shape[0])), 'train', 'train', transform=None)\ndataloader = torch.utils.data.DataLoader(dataset_show, batch_size=batch_size,num_workers=num_workers, shuffle=False)\niter_data = iter(dataloader)\ndata, data_org, target = next(iter_data)\ndata, target = fmix(data_org, target,alpha=1., decay_power=3., shape=(137,236))\n\n\nfrom pylab import rcParams\nrcParams['figure.figsize'] = 20,10\nfor i in range(3):\n    f, axarr = plt.subplots(1,4)\n    for p in range(0,3,2):\n        idx = np.random.randint(0, len(data))\n        img_org = data_org[idx]\n        new_img = data[idx]\n        axarr[p].imshow(img_org.permute(1,2,0))\n        axarr[p+1].imshow(new_img.permute(1,2,0))\n        axarr[p].set_title('original')\n        axarr[p+1].set_title('fmix image')\n        axarr[p].axis('off')\n        axarr[p+1].axis('off')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# All together"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_show = df_train.iloc[:1000]\ndataset_show = BengaliDataset(df_show, img_data, list(range(df_show.shape[0])), 'train', 'train', transform=None)\ndataloader = torch.utils.data.DataLoader(dataset_show, batch_size=batch_size,num_workers=num_workers, shuffle=False)\niter_data = iter(dataloader)\ndata, data_org, target_org = next(iter_data)\nfmix_data, target = fmix(data_org, target_org,alpha=1., decay_power=3., shape=(137,236))\ncutmix_data, target = cutmix(data_org, target_org, 1.)\nmixup_data, target = mixup(data_org, target_org, 1.)\n\nfrom pylab import rcParams\nrcParams['figure.figsize'] = 20,10\nfor i in range(5):\n    f, axarr = plt.subplots(1,4)\n    idx = np.random.randint(0, len(data))\n    img_org = data_org[idx]\n    mix_img = mixup_data[idx]\n    cut_img = cutmix_data[idx]\n    fmix_img = fmix_data[idx] \n    axarr[0].imshow(img_org.permute(1,2,0))\n    axarr[1].imshow(mix_img.permute(1,2,0))\n    axarr[2].imshow(cut_img.permute(1,2,0))\n    axarr[3].imshow(fmix_img.permute(1,2,0))\n    axarr[0].set_title('original')\n    axarr[1].set_title('mixup image')\n    axarr[2].set_title('cutmix image')\n    axarr[3].set_title('fmix image')\n    axarr[0].axis('off')\n    axarr[1].axis('off')\n    axarr[2].axis('off')\n    axarr[3].axis('off')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}