{"cells":[{"metadata":{},"cell_type":"markdown","source":"# FMix\n\n### FMix is a variant of MixUp, CutMix, etc. introduced in the paper 'FMix: Enhancing Mixed Sampled Data Augmentation'. It uses masks sampled from Fourier space to mix training examples. \n\n### In short, it is like cutmix, but with a irregular shaped mask instead of rectangular. In this kernel, let's see how FMix and Cutmix looks like in our dataset"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"from IPython.display import Image, display\n\ndisplay(Image('../input/image-fmix/FMix-master/fmix_example.png'))  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"display(Image('../input/image-fmix/FMix-master/fmix_3d.gif'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"package_path = '../input/image-fmix/FMix-master' #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\nimport sys; sys.path.append(package_path)\nfrom fmix import sample_mask","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom  torch.cuda.amp import autocast, GradScaler\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\n#from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataloader"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(\n        self, df, data_root, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        \n        img  = get_img(path)/255.\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_default_transforms():\n    return Compose([\n            CenterCrop(128, 128, p=1.),\n            Resize(128, 128),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = CassavaDataset(train, '../input/cassava-leaf-disease-classification/train_images/', transforms=get_default_transforms(), output_label=True)\n\ntrain_loader = torch.utils.data.DataLoader(\n    train_ds, \n    batch_size=32,\n    num_workers=1,\n    shuffle=False,\n    pin_memory=False,\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Reference Kernel: https://www.kaggle.com/virajbagal/mixup-cutmix-fmix-visualisations"},{"metadata":{},"cell_type":"markdown","source":"## Augmentation APIs"},{"metadata":{"trusted":true},"cell_type":"code","source":"from pylab import rcParams\nrcParams['figure.figsize'] = 20,40","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\ndef cutmix(data, target, alpha):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_target = target[indices]\n\n    lam = np.clip(np.random.beta(alpha, alpha),0.3,0.4)\n    bbx1, bby1, bbx2, bby2 = rand_bbox(data.size(), lam)\n    new_data = data.clone()\n    new_data[:, :, bby1:bby2, bbx1:bbx2] = data[indices, :, bby1:bby2, bbx1:bbx2]\n    # adjust lambda to exactly match pixel ratio\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (data.size()[-1] * data.size()[-2]))\n    targets = (target, shuffled_target, lam)\n\n    return new_data, targets\n\ndef fmix(data, targets, alpha, decay_power, shape, max_soft=0.0, reformulate=False):\n    lam, mask = sample_mask(alpha, decay_power, shape, max_soft, reformulate)\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_targets = targets[indices]\n    x1 = torch.from_numpy(mask)*data\n    x2 = torch.from_numpy(1-mask)*shuffled_data\n    targets=(targets, shuffled_targets, lam)\n    \n    return (x1+x2), targets","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Cutmix "},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_data = iter(train_loader)\ndata, target = next(iter_data)\ndata_aug, target = cutmix(data, target, 1.)\n\n\nfor i in range(3):\n    f, axarr = plt.subplots(1,4)\n    for p in range(0,3,2):\n        idx = np.random.randint(0, len(data))\n        img_org = data[idx]\n        new_img = data_aug[idx]\n        axarr[p].imshow(img_org.permute(1,2,0))\n        axarr[p+1].imshow(new_img.permute(1,2,0))\n        axarr[p].set_title('original')\n        axarr[p+1].set_title('cutmix image')\n        axarr[p].axis('off')\n        axarr[p+1].axis('off')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> As we could see, there would be a 'rectangle' part that is replaced by another image"},{"metadata":{},"cell_type":"markdown","source":"## FMix"},{"metadata":{},"cell_type":"markdown","source":"1. FMix is a variant of MixUp, CutMix, Paper: [FMix: Enhancing Mixed Sampled Data Augmentation](https://arxiv.org/abs/2002.12047)\n2. It uses masks sampled from Fourier space to mix training examples\n3. [Parameters](https://github.com/ecs-vlc/FMix/blob/master/fmix.py#L124) for sample_mask:\n    * alpha: Alpha value for beta distribution from which to sample mean of mask\n    * decay_power: Decay power for frequency decay prop 1/f**d\n    * shape: Shape of desired mask, list up to 3 dims\n    * max_soft: Softening value between 0 and 0.5 which smooths hard edges in the mask."},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_data = iter(train_loader)\ndata, target = next(iter_data)\ndata_aug, target = fmix(data, target, alpha=1., decay_power=3., shape=(128,128))\n#print(target[2])\nfor i in range(3):\n    f, axarr = plt.subplots(1,4)\n    for p in range(0,3,2):\n        idx = np.random.randint(0, len(data))\n        img_org = data[idx]\n        new_img = data_aug[idx]\n        axarr[p].imshow(img_org.permute(1,2,0))\n        axarr[p+1].imshow(new_img.permute(1,2,0))\n        axarr[p].set_title('original')\n        axarr[p+1].set_title('fmix image')\n        axarr[p].axis('off')\n        axarr[p+1].axis('off')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_data = iter(train_loader)\ndata, target = next(iter_data)\ndata_aug, target = fmix(data, target, alpha=1., decay_power=10., shape=(128,128))\n\nfor i in range(3):\n    f, axarr = plt.subplots(1,4)\n    for p in range(0,3,2):\n        idx = np.random.randint(0, len(data))\n        img_org = data[idx]\n        new_img = data_aug[idx]\n        axarr[p].imshow(img_org.permute(1,2,0))\n        axarr[p+1].imshow(new_img.permute(1,2,0))\n        axarr[p].set_title('original')\n        axarr[p+1].set_title('fmix image')\n        axarr[p].axis('off')\n        axarr[p+1].axis('off')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> In short: Fmix provides a irregularly shaped mask to do the cutmix "},{"metadata":{},"cell_type":"markdown","source":"## What's next\n1. Apply cutmix and fmix in the training data loader and check validation performance.\n2. Further improvement to create diversity within a batch of samples: Instead of doing cutmix and fmix from the same batch of samples and use the same mask, we could do cutmix and fmix from images in the same set of data and have different mask for every image within the batch!"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}