{"cells":[{"metadata":{"_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"markdown","source":"# Description\n\nThis notebook contains visualizations of Mixup, Cutmix, Augmix and GridMask. Feel free to change the parameters and play around to see what works best for you. "},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline\n\nimport fastai\nfrom fastai.vision import *\nfrom fastai.callbacks import SaveModelCallback\nfrom torch.nn.modules.normalization import GroupNorm\nimport os\nfrom torch.nn.modules import Conv2d\nfrom sklearn.model_selection import KFold\nfrom over9000 import *\nfrom csvlogger import *\nimport pretrainedmodels\nfrom mish_activation import *\nimport warnings\nfrom fastai.vision import Image as Img\nwarnings.filterwarnings(\"ignore\")\n\nfastai.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sz = 128\nbs = 16\nnfolds = 4 #keep the same split as the initial dataset\nfold = 0\nSEED = 2019\nTRAIN = '../input/grapheme-imgs-128x128/'\nLABELS = '../input/bengaliai-cv19/train.csv'\narch = pretrainedmodels.__dict__['se_resnext50_32x4d']\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(LABELS)\nnunique = list(df.nunique())[1:-1]\nprint(nunique)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stats = ([0.0692], [0.2051])\ndata = (ImageList.from_df(df, path='.', folder=TRAIN, suffix='.png', \n        cols='image_id', convert_mode='L')\n        .split_by_idx(range(fold*len(df)//nfolds,(fold+1)*len(df)//nfolds))\n        .label_from_df(cols=['grapheme_root','vowel_diacritic','consonant_diacritic'])\n        .transform(size=sz, padding_mode='zeros')\n        .databunch(bs=bs)).normalize(stats)\n\ndata.show_batch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Head(nn.Module):\n    def __init__(self, nc, n, ps=0.5):\n        super().__init__()\n        layers = [AdaptiveConcatPool2d(), Mish(), Flatten()] + \\\n            bn_drop_lin(nc*2, 512, True, ps, Mish()) + \\\n            bn_drop_lin(512, n, True, ps)\n        self.fc = nn.Sequential(*layers)\n        self._init_weight()\n        \n    def _init_weight(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                torch.nn.init.kaiming_normal_(m.weight)\n            elif isinstance(m, nn.BatchNorm2d):\n                m.weight.data.fill_(1.0)\n                m.bias.data.zero_()\n        \n    def forward(self, x):\n        return self.fc(x)\n\n#change the first conv to accept 1 chanel input\nclass Dnet_1ch(nn.Module):\n    def __init__(self, arch=arch, n=nunique, pre=True, ps=0.5):\n        super().__init__()\n        m = arch(pretrained='imagenet') if pre else arch(pretrained=None)\n#         convert_to_gem(m)\n#         convert_to_conv2d(m)\n#         convert_to_groupnorm(m)        \n        conv = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)\n        w = (m.layer0.conv1.weight.sum(1)).unsqueeze(1)\n        conv.weight = nn.Parameter(w)\n        \n        self.layer0 = nn.Sequential(conv, m.layer0.bn1, m.layer0.relu1, m.layer0.pool)\n        self.layer1 = m.layer1\n        self.layer2 = m.layer2\n        self.layer3 = m.layer3\n        self.layer4 = nn.Sequential(m.layer4[0], m.layer4[1], m.layer4[2])\n\n        \n        nc = self.layer4[-1].se_module.fc2.out_channels #changes as per architecture\n        self.head1 = Head(nc,n[0])\n        self.head2 = Head(nc,n[1])\n        self.head3 = Head(nc,n[2])\n        #to_Mish(self.layer0), to_Mish(self.layer1), to_Mish(self.layer2)\n        #to_Mish(self.layer3), to_Mish(self.layer4)\n        \n    def forward(self, x):    \n        x = self.layer0(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        \n        x1 = self.head1(x)\n        x2 = self.head2(x)\n        x3 = self.head3(x)\n        \n        return x1,x2,x3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss"},{"metadata":{},"cell_type":"markdown","source":"Cross entropy loss is applied independently to each part of the prediction and the result is summed with the corresponding weight."},{"metadata":{"trusted":true},"cell_type":"code","source":"class Loss_combine(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n    def forward(self, input, target,reduction='mean'):\n        x1,x2,x3 = input\n        x1,x2,x3 = x1.float(),x2.float(),x3.float()\n        y = target.long()\n        return 0.7*F.cross_entropy(x1,y[:,0],reduction=reduction) + 0.1*F.cross_entropy(x2,y[:,1],reduction=reduction) + \\\n          0.2*F.cross_entropy(x3,y[:,2],reduction=reduction)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The code below computes the competition metric and recall macro metrics for individual components of the prediction. The code is partially borrowed from fast.ai."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class Metric_idx(Callback):\n    def __init__(self, idx, average='macro'):\n        super().__init__()\n        self.idx = idx\n        self.n_classes = 0\n        self.average = average\n        self.cm = None\n        self.eps = 1e-9\n        \n    def on_epoch_begin(self, **kwargs):\n        self.tp = 0\n        self.fp = 0\n        self.cm = None\n    \n    def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):\n        last_output = last_output[self.idx]\n        last_target = last_target[:,self.idx]\n        preds = last_output.argmax(-1).view(-1).cpu()\n        targs = last_target.long().cpu()\n        \n        if self.n_classes == 0:\n            self.n_classes = last_output.shape[-1]\n            self.x = torch.arange(0, self.n_classes)\n        cm = ((preds==self.x[:, None]) & (targs==self.x[:, None, None])) \\\n          .sum(dim=2, dtype=torch.float32)\n        if self.cm is None: self.cm =  cm\n        else:               self.cm += cm\n\n    def _weights(self, avg:str):\n        if self.n_classes != 2 and avg == \"binary\":\n            avg = self.average = \"macro\"\n            warn(\"average=`binary` was selected for a non binary case. \\\n                 Value for average has now been set to `macro` instead.\")\n        if avg == \"binary\":\n            if self.pos_label not in (0, 1):\n                self.pos_label = 1\n                warn(\"Invalid value for pos_label. It has now been set to 1.\")\n            if self.pos_label == 1: return Tensor([0,1])\n            else: return Tensor([1,0])\n        elif avg == \"micro\": return self.cm.sum(dim=0) / self.cm.sum()\n        elif avg == \"macro\": return torch.ones((self.n_classes,)) / self.n_classes\n        elif avg == \"weighted\": return self.cm.sum(dim=1) / self.cm.sum()\n        \n    def _recall(self):\n        rec = torch.diag(self.cm) / (self.cm.sum(dim=1) + self.eps)\n        if self.average is None: return rec\n        else:\n            if self.average == \"micro\": weights = self._weights(avg=\"weighted\")\n            else: weights = self._weights(avg=self.average)\n            return (rec * weights).sum()\n    \n    def on_epoch_end(self, last_metrics, **kwargs): \n        return add_metrics(last_metrics, self._recall())\n    \nMetric_grapheme = partial(Metric_idx,0)\nMetric_vowel = partial(Metric_idx,1)\nMetric_consonant = partial(Metric_idx,2)\n\nclass Metric_tot(Callback):\n    def __init__(self):\n        super().__init__()\n        self.grapheme = Metric_idx(0)\n        self.vowel = Metric_idx(1)\n        self.consonant = Metric_idx(2)\n        \n    def on_epoch_begin(self, **kwargs):\n        self.grapheme.on_epoch_begin(**kwargs)\n        self.vowel.on_epoch_begin(**kwargs)\n        self.consonant.on_epoch_begin(**kwargs)\n    \n    def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):\n        self.grapheme.on_batch_end(last_output, last_target, **kwargs)\n        self.vowel.on_batch_end(last_output, last_target, **kwargs)\n        self.consonant.on_batch_end(last_output, last_target, **kwargs)\n        \n    def on_epoch_end(self, last_metrics, **kwargs): \n        return add_metrics(last_metrics, 0.5*self.grapheme._recall() +\n                0.25*self.vowel._recall() + 0.25*self.consonant._recall())","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"#fix the issue in fast.ai of saving gradients along with weights\n#so only weights are written, and files are ~4 times smaller\n\nclass SaveModelCallback(TrackerCallback):\n    \"A `TrackerCallback` that saves the model when monitored quantity is best.\"\n    def __init__(self, learn:Learner, monitor:str='valid_loss', mode:str='auto',\n                 every:str='improvement', name:str='bestmodel'):\n        super().__init__(learn, monitor=monitor, mode=mode)\n        self.every,self.name = every,name\n        if self.every not in ['improvement', 'epoch']:\n            warn(f'SaveModel every {self.every} is invalid, falling back to \"improvement\".')\n            self.every = 'improvement'\n                 \n    def jump_to_epoch(self, epoch:int)->None:\n        try: \n            self.learn.load(f'{self.name}_{epoch-1}', purge=False)\n            print(f\"Loaded {self.name}_{epoch-1}\")\n        except: print(f'Model {self.name}_{epoch-1} not found.')\n\n    def on_epoch_end(self, epoch:int, **kwargs:Any)->None:\n        \"Compare the value monitored to its best score and maybe save the model.\"\n        if self.every==\"epoch\":\n            if epoch==31:\n                self.learn.save(f'{self.name}_{epoch}')\n#             torch.save(learn.model.state_dict(),f'{self.name}_{epoch}.pth')\n        else: #every=\"improvement\"\n            current = self.get_monitor_value()\n            if current is not None and self.operator(current, self.best):\n                #print(f'Better model found at epoch {epoch} \\\n                #  with {self.monitor} value: {current}.')\n                self.best = current\n                self.learn.save(f'{self.name}')\n#                 torch.save(learn.model.state_dict(),f'{self.name}.pth')\n\n    def on_train_end(self, **kwargs):\n        \"Load the best model.\"\n        if self.every==\"improvement\" and os.path.isfile(f'{self.name}.pth'):\n            #self.learn.load(f'{self.name}', purge=False)\n            self.model.load_state_dict(torch.load(f'{self.name}.pth'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EpochCallback(TrackerCallback):\n    \"A `TrackerCallback` that stops training after specified epochs\"\n    def __init__(self, learn:Learner):\n        super().__init__(learn)\n\n    def on_epoch_end(self, epoch:int, **kwargs:Any)->None:\n        \"Stop training\"\n        if epoch==31:\n            torch.save(learn.opt.state_dict(),f'latest_optimizer_{epoch}.pth')\n            torch.save(learn.model.state_dict(),f'latest_model_{epoch}.pth')\n            return {\"stop_training\":True}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LookAhead"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"from collections import defaultdict\n\nimport torch\nfrom torch.optim.optimizer import Optimizer\n\n\nclass Lookahead(Optimizer):\n    r\"\"\"PyTorch implementation of the lookahead wrapper.\n    Lookahead Optimizer: https://arxiv.org/abs/1907.08610\n    \"\"\"\n\n    def __init__(self, optimizer, la_steps=5, la_alpha=0.8, pullback_momentum=\"none\"):\n        \"\"\"optimizer: inner optimizer\n        la_steps (int): number of lookahead steps\n        la_alpha (float): linear interpolation factor. 1.0 recovers the inner optimizer.\n        pullback_momentum (str): change to inner optimizer momentum on interpolation update\n        \"\"\"\n        self.optimizer = optimizer\n        self._la_step = 0  # counter for inner optimizer\n        self.la_alpha = la_alpha\n        self._total_la_steps = la_steps\n        pullback_momentum = pullback_momentum.lower()\n        assert pullback_momentum in [\"reset\", \"pullback\", \"none\"]\n        self.pullback_momentum = pullback_momentum\n\n        self.state = defaultdict(dict)\n\n        # Cache the current optimizer parameters\n        for group in optimizer.param_groups:\n            for p in group['params']:\n                param_state = self.state[p]\n                param_state['cached_params'] = torch.zeros_like(p.data)\n                param_state['cached_params'].copy_(p.data)\n                if self.pullback_momentum == \"pullback\":\n                    param_state['cached_mom'] = torch.zeros_like(p.data)\n\n    def __getstate__(self):\n        return {\n            'state': self.state,\n            'optimizer': self.optimizer,\n            'la_alpha': self.la_alpha,\n            '_la_step': self._la_step,\n            '_total_la_steps': self._total_la_steps,\n            'pullback_momentum': self.pullback_momentum\n        }\n\n    def zero_grad(self):\n        self.optimizer.zero_grad()\n\n    def get_la_step(self):\n        return self._la_step\n\n    def state_dict(self):\n        return self.optimizer.state_dict()\n\n    def load_state_dict(self, state_dict):\n        self.optimizer.load_state_dict(state_dict)\n\n    def _backup_and_load_cache(self):\n        \"\"\"Useful for performing evaluation on the slow weights (which typically generalize better)\n        \"\"\"\n        for group in self.optimizer.param_groups:\n            for p in group['params']:\n                param_state = self.state[p]\n                param_state['backup_params'] = torch.zeros_like(p.data)\n                param_state['backup_params'].copy_(p.data)\n                p.data.copy_(param_state['cached_params'])\n\n    def _clear_and_load_backup(self):\n        for group in self.optimizer.param_groups:\n            for p in group['params']:\n                param_state = self.state[p]\n                p.data.copy_(param_state['backup_params'])\n                del param_state['backup_params']\n\n    @property\n    def param_groups(self):\n        return self.optimizer.param_groups\n\n    def step(self, closure=None):\n        \"\"\"Performs a single Lookahead optimization step.\n        Arguments:\n            closure (callable, optional): A closure that reevaluates the model\n                and returns the loss.\n        \"\"\"\n        loss = self.optimizer.step(closure)\n        self._la_step += 1\n\n        if self._la_step >= self._total_la_steps:\n            self._la_step = 0\n            # Lookahead and cache the current optimizer parameters\n            for group in self.optimizer.param_groups:\n                for p in group['params']:\n                    param_state = self.state[p]\n                    p.data.mul_(self.la_alpha).add_(1.0 - self.la_alpha, param_state['cached_params'])  # crucial line\n                    param_state['cached_params'].copy_(p.data)\n                    if self.pullback_momentum == \"pullback\":\n                        internal_momentum = self.optimizer.state[p][\"momentum_buffer\"]\n                        self.optimizer.state[p][\"momentum_buffer\"] = internal_momentum.mul_(self.la_alpha).add_(\n                            1.0 - self.la_alpha, param_state[\"cached_mom\"])\n                        param_state[\"cached_mom\"] = self.optimizer.state[p][\"momentum_buffer\"]\n                    elif self.pullback_momentum == \"reset\":\n                        self.optimizer.state[p][\"momentum_buffer\"] = torch.zeros_like(p.data)\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmix"},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(image):\n    return torch.from_numpy(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (Rotate, GaussNoise, GaussianBlur, MedianBlur, RandomScale, Compose, OneOf, DualTransform, RandomBrightness,\n                            RandomContrast, MotionBlur, Solarize, Equalize, Posterize, ShiftScaleRotate,\n                           IAASharpen, IAAAffine)\n\n\naug_list = [GaussNoise(p=1), GaussianBlur(p=1),\n           RandomBrightness(p=1), RandomContrast(p=1), ShiftScaleRotate(p=1, rotate_limit=20)]\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# GridMask"},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations.augmentations import functional as Func\n\nclass GridMask(DualTransform):\n    \"\"\"GridMask augmentation for image classification and object detection.\n    \n    Author: Qishen Ha\n    Email: haqishen@gmail.com\n    2020/01/29\n\n    Args:\n        num_grid (int): number of grid in a row or column.\n        fill_value (int, float, lisf of int, list of float): value for dropped pixels.\n        rotate ((int, int) or int): range from which a random angle is picked. If rotate is a single int\n            an angle is picked from (-rotate, rotate). Default: (-90, 90)\n        mode (int):\n            0 - cropout a quarter of the square of each grid (left top)\n            1 - reserve a quarter of the square of each grid (left top)\n            2 - cropout 2 quarter of the square of each grid (left top & right bottom)\n\n    Targets:\n        image, mask\n\n    Image types:\n        uint8, float32\n\n    Reference:\n    |  https://arxiv.org/abs/2001.04086\n    |  https://github.com/akuxcw/GridMask\n    \"\"\"\n\n    def __init__(self, num_grid=3, fill_value=0, rotate=0, mode=0, always_apply=False, p=0.5):\n        super(GridMask, self).__init__(always_apply, p)\n        if isinstance(num_grid, int):\n            num_grid = (num_grid, num_grid)\n        if isinstance(rotate, int):\n            rotate = (-rotate, rotate)\n        self.num_grid = num_grid\n        self.fill_value = fill_value\n        self.rotate = rotate\n        self.mode = mode\n        self.masks = None\n        self.rand_h_max = []\n        self.rand_w_max = []\n\n    def init_masks(self, height, width):\n        if self.masks is None:\n            self.masks = []\n            n_masks = self.num_grid[1] - self.num_grid[0] + 1\n            for n, n_g in enumerate(range(self.num_grid[0], self.num_grid[1] + 1, 1)):\n                grid_h = height / n_g\n                grid_w = width / n_g\n                this_mask = np.ones((int((n_g + 1) * grid_h), int((n_g + 1) * grid_w))).astype(np.uint8)\n                for i in range(n_g + 1):\n                    for j in range(n_g + 1):\n                        this_mask[\n                             int(i * grid_h) : int(i * grid_h + grid_h / 2),\n                             int(j * grid_w) : int(j * grid_w + grid_w / 2)\n                        ] = self.fill_value\n                        if self.mode == 2:\n                            this_mask[\n                                 int(i * grid_h + grid_h / 2) : int(i * grid_h + grid_h),\n                                 int(j * grid_w + grid_w / 2) : int(j * grid_w + grid_w)\n                            ] = self.fill_value\n                \n                if self.mode == 1:\n                    this_mask = 1 - this_mask\n\n                self.masks.append(this_mask)\n                self.rand_h_max.append(grid_h)\n                self.rand_w_max.append(grid_w)\n\n    def apply(self, image, mask, rand_h, rand_w, angle, **params):\n        h, w = image.shape[:2]\n        mask = Func.rotate(mask, angle) if self.rotate[1] > 0 else mask\n        mask = mask[:,:,np.newaxis] if image.ndim == 3 else mask\n        image *= mask[rand_h:rand_h+h, rand_w:rand_w+w].astype(image.dtype)\n        return image\n\n    def get_params_dependent_on_targets(self, params):\n        img = params['image']\n        height, width = img.shape[:2]\n        self.init_masks(height, width)\n\n        mid = np.random.randint(len(self.masks))\n        mask = self.masks[mid]\n        rand_h = np.random.randint(self.rand_h_max[mid])\n        rand_w = np.random.randint(self.rand_w_max[mid])\n        angle = np.random.randint(self.rotate[0], self.rotate[1]) if self.rotate[1] > 0 else 0\n\n        return {'mask': mask, 'rand_h': rand_h, 'rand_w': rand_w, 'angle': angle}\n\n    @property\n    def targets_as_params(self):\n        return ['image']\n\n    def get_transform_init_args_names(self):\n        return ('num_grid', 'fill_value', 'rotate', 'mode')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MixUp"},{"metadata":{},"cell_type":"markdown","source":"The code below modifies fast.ai MixUp calback to make it compatible with the current data."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class MixUpLoss(Module):\n    \"Adapt the loss function `crit` to go with mixup.\"\n    \n    def __init__(self, crit, reduction='mean'):\n        super().__init__()\n        if hasattr(crit, 'reduction'): \n            self.crit = crit\n            self.old_red = crit.reduction\n            setattr(self.crit, 'reduction', 'none')\n        else: \n            self.crit = partial(crit, reduction='none')\n            self.old_crit = crit\n        self.reduction = reduction\n        \n    def forward(self, output, target):\n        if len(target.shape) == 2 and target.shape[1] == 7:\n            loss1, loss2 = self.crit(output,target[:,0:3].long()), self.crit(output,target[:,3:6].long())\n            d = loss1 * target[:,-1] + loss2 * (1-target[:,-1])\n        else:  d = self.crit(output, target)\n        if self.reduction == 'mean':    return d.mean()\n        elif self.reduction == 'sum':   return d.sum()\n        return d\n    \n    def get_old(self):\n        if hasattr(self, 'old_crit'):  return self.old_crit\n        elif hasattr(self, 'old_red'): \n            setattr(self.crit, 'reduction', self.old_red)\n            return self.crit","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CutMix"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# https://github.com/oguiza/DataAugmentation/blob/master/ImageDataAugmentation.py\n\ndef rand_bbox(last_input_size, λ):\n    '''lambd is always between .5 and 1'''\n\n    W = last_input_size[-1]\n    H = last_input_size[-2]\n    cut_rat = np.sqrt(1. - λ) # 0. - .707\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n    return bbx1, bby1, bbx2, bby2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cutout(x, n_holes:uniform_int=1, length:uniform_int=40):\n    \"Cut out `n_holes` number of square holes of size `length` in image at random locations.\"\n    h,w = x.shape[1:]\n    for n in range(n_holes):\n        h_y = np.random.randint(0, h)\n        h_x = np.random.randint(0, w)\n        y1 = int(np.clip(h_y - length / 2, 0, h))\n        y2 = int(np.clip(h_y + length / 2, 0, h))\n        x1 = int(np.clip(h_x - length / 2, 0, w))\n        x2 = int(np.clip(h_x + length / 2, 0, w))\n        x[:, y1:y2, x1:x2] = 1\n    return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Choice CallBack"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class Choice(LearnerCallback):\n    def __init__(self, learn:Learner, auglist, mixup_alpha:float=0.4, stack_x:bool=False, stack_y:bool=True, prob=0.5, cutmix_alpha:float=0.4,\n                aug_alpha:float=1, mix_depth:float=-1, mixture_width:int=3):\n        super().__init__(learn)\n        self.cut_alpha,self.alpha,self.stack_x,self.stack_y = cutmix_alpha, mixup_alpha,stack_x,stack_y\n        self.aug_alpha,self.mix_depth, self.mixture_width, self.aug_list = aug_alpha, mix_depth, mixture_width, auglist\n        \n    def on_train_begin(self, **kwargs):\n        if self.stack_y: self.learn.loss_func = MixUpLoss(self.learn.loss_func)\n            \n    def aug(self,image,aug_list, alpha=1, mixture_width=3, mix_depth=3):\n        ws = np.float32(np.random.dirichlet([alpha] * mixture_width))\n        m = np.float32(np.random.beta(alpha, alpha))\n\n        mix = torch.zeros_like(preprocess(image))\n        for i in range(mixture_width):\n            image_aug = image.copy()\n            depth = mix_depth if mix_depth > 0 else np.random.randint(1, 4)\n            \n            for _ in range(depth):\n                op = np.random.choice(aug_list)\n                image_aug = op(image = image_aug)['image']\n    # Preprocessing commutes since all coefficients are convex\n            mix += ws[i] * preprocess(image_aug)\n\n        mixed = (1 - m) * preprocess(image) + m * mix\n        return mixed\n            \n    def on_batch_begin(self, last_input, last_target, train, epoch,**kwargs):\n        if ((epoch in range(0,10)) or (epoch in range(40,50)) or (epoch in range(80,90))):\n            name='Mixup'\n        if ((epoch in range(10,20)) or (epoch in range(50,60)) or (epoch in range(90,100))):\n            name='Cutmix'\n        if ((epoch in range(20,30)) or (epoch in range(60,70))):\n            name='Augmix'\n        if ((epoch in range(30,40)) or (epoch in range(70,80))):\n            name='GridMask'\n        \n        print(name)\n        if name=='Mixup':\n            if not train: return\n            self.learn.loss_func = MixUpLoss(self.learn.loss_func)\n            lambd = np.random.beta(self.alpha, self.alpha, last_target.size(0))\n            lambd = np.concatenate([lambd[:,None], 1-lambd[:,None]], 1).max(1)\n            lambd = last_input.new(lambd)\n            shuffle = torch.randperm(last_target.size(0)).to(last_input.device)\n            x1, y1 = last_input[shuffle], last_target[shuffle]\n            if self.stack_x:\n                new_input = [last_input, last_input[shuffle], lambd]\n            else: \n                out_shape = [lambd.size(0)] + [1 for _ in range(len(x1.shape) - 1)]\n                new_input = (last_input * lambd.view(out_shape) + x1 * (1-lambd).view(out_shape))\n            if self.stack_y:\n                new_target = torch.cat([last_target.float(), y1.float(), lambd[:,None].float()], 1)\n            else:\n                if len(last_target.shape) == 2:\n                    lambd = lambd.unsqueeze(1).float()\n                new_target = last_target.float() * lambd + y1.float() * (1-lambd)\n            return {'last_input': new_input, 'last_target': new_target}\n        \n        elif name=='Cutmix':\n            if not train: return\n            self.learn.loss_func = MixUpLoss(self.learn.loss_func)\n            lambd = np.random.beta(self.cut_alpha, self.cut_alpha)\n            lambd = max(lambd, 1- lambd)\n            shuffle = torch.randperm(last_target.size(0)).to(last_input.device)\n            \n            x1, y1 = last_input[shuffle], last_target[shuffle]\n            #Get new input\n            last_input_size = last_input.shape\n            bbx1, bby1, bbx2, bby2 = rand_bbox(last_input.size(), lambd)\n            new_input = last_input.clone()\n            new_input[:, ..., bby1:bby2, bbx1:bbx2] = last_input[shuffle, ..., bby1:bby2, bbx1:bbx2]\n            lambd = last_input.new([lambd])\n            if self.stack_x:\n                lambd = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (last_input_size[-1] * last_input_size[-2]))\n                lambd = last_input.new([lambd])\n            if self.stack_y:\n                new_target = torch.cat([last_target.float(), y1.float(),\n                                    lambd.repeat(last_input_size[0]).unsqueeze(1).float()], 1)\n            else:\n                if len(last_target.shape) == 2:\n                    lambd = lambd.unsqueeze(1).float()\n                new_target = last_target.float() * lambd + y1.float() * (1-lambd)\n            return {'last_input': new_input, 'last_target': new_target}\n        \n        elif name=='Augmix':\n            \"Applies mixup to `last_input` and `last_target` if `train`.\"\n            if not train: return\n            self.learn.loss_func = Loss_combine()\n            num_images = last_input.size(0)\n            new_input = last_input.clone()\n            for i in range(num_images):\n                image = new_input[i,:,:,:].permute(1,2,0).cpu().numpy()\n                image = self.aug(image, self.aug_list, alpha=self.aug_alpha, mixture_width = self.mixture_width, mix_depth=self.mix_depth)\n                new_input[i,:,:,:] = image.permute(2,0,1)\n            return {'last_input': new_input ,'last_target': last_target}\n        \n        elif name=='GridMask':\n            if not train: return\n            self.learn.loss_func = Loss_combine()\n            tfms = Compose([OneOf([GridMask(num_grid=(10,15), rotate=10, mode=0, fill_value=0),\n                                   GridMask(num_grid=(10,15), rotate=10, mode=2, fill_value=0),\n                                   GridMask(num_grid=(10,15), rotate=0, mode=0, fill_value=0),\n                                   GridMask(num_grid=(10,15), rotate=0, mode=2, fill_value=0)], p=1)])\n\n            num_images = last_input.size(0)\n            new_input = last_input.clone()\n            for i in range(num_images):\n                image = new_input[i,:,:,:].permute(1,2,0).cpu().numpy()\n                image = tfms(image = image)['image']\n                new_input[i,:,:,:] = torch.from_numpy(image).permute(2,0,1)\n            \n            return {'last_input': new_input, 'last_target': last_target}\n        \n        elif name=='Cutout':\n            if not train: return\n            self.learn.loss_func = Loss_combine()\n            new_input = last_input.clone()\n            for i in range(last_input.size(0)):\n                hole = np.random.choice([4,8])\n                new_input[i,:,:,:] = cutout(new_input[i,:,:,:], n_holes=hole, length=10)\n            \n            return {'last_input': new_input, 'last_target': last_target}\n        \n    def on_train_end(self, **kwargs):\n        if self.stack_y: \n            try:\n                self.learn.loss_func = self.learn.loss_func.get_old()\n            except:\n                self.learn.loss_func = MixUpLoss(self.learn.loss_func)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{},"cell_type":"markdown","source":"I have performed a check of different optimizers and schedules on a [similar task](https://www.kaggle.com/c/Kannada-MNIST/discussion/122430), and [Over9000 optimizer](https://github.com/mgrankin/over9000) cosine annealing **without warm-up** worked the best. Freezing the backbone at the initial stage of training didn't give me any advantage in that test, so here I perform the training straight a way with discriminative learning rate (smaller lr for backbone)."},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.optim import Adam\n\ndef lookahead_adam(params, la_alpha=0.8, la_steps=5, *args, **kwargs):\n     adam = Adam(params, *args, **kwargs)\n     \n     return Lookahead(adam, la_alpha, la_steps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def choice(learn:Learner, auglist, mixup_alpha:float=0.4, stack_x:bool=False, stack_y:bool=True, prob=0.5, cutmix_alpha:float=1.0,\n                aug_alpha:float=1, mix_depth:float=-1, mixture_width:int=3) -> Learner:\n    \"Add cutmix https://arxiv.org/pdf/1905.04899.pdf to `learn`.\"\n    learn.callback_fns.append(partial(Choice, auglist, mixup_alpha=mixup_alpha, stack_x=stack_x, stack_y=stack_y, prob=prob, cutmix_alpha=cutmix_alpha,\n                aug_alpha=aug_alpha, mix_depth=mix_depth, mixture_width=mixture_width))\n    return learn\n\nsetattr(choice, 'cb_fn', Choice)\nLearner.choice = choice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"model = Dnet_1ch(pre=False)\nlearn = Learner(data, model, loss_func=Loss_combine(),opt_func=lookahead_adam,\n        metrics=[Metric_grapheme(),Metric_vowel(),Metric_consonant(),Metric_tot()]).choice(aug_list)\nlogger = CSVLogger(learn,f'log{fold}')\nlearn.clip_grad = 1.0\nlearn.split([model.head1])\nlearn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"learn.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.fit_one_cycle(100,max_lr=slice(0.2e-2,1e-2),wd=[1e-3,0.1e-1], pct_start=0.0, \n#                     div_factor=100, callbacks=[logger, SaveModelCallback(learn,monitor='metric_tot',\n#     mode='max',name=f'seresnext', every='epoch'), Choice(learn,auglist=aug_list),EpochCallback(learn)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xb, yb = learn.data.one_batch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cb = learn.callback_fns[1]\ncb_fn = partial(cb.func, **cb.keywords)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[Img(xb[0]).show(ax=ax) for i, ax in enumerate(plt.subplots(4, 4, figsize=(15,15))[1].flatten()) ]\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[Img(cb_fn(learn, aug_list).on_batch_begin(xb, yb, True,5)['last_input'][0]).show(ax=ax)\n for i, ax in enumerate(plt.subplots(4, 4, figsize=(15,15))[1].flatten())]\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[Img(cb_fn(learn, aug_list).on_batch_begin(xb, yb, True,15)['last_input'][0]).show(ax=ax) for i, ax in enumerate(plt.subplots(4, 4, figsize=(15,15))[1].flatten())]\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[Img(cb_fn(learn, aug_list).on_batch_begin(xb, yb, True,25)['last_input'][0]).show(ax=ax) for i, ax in enumerate(plt.subplots(4, 4, figsize=(15,15))[1].flatten())]\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[Img(cb_fn(learn, aug_list).on_batch_begin(xb, yb, True,35)['last_input'][0]).show(ax=ax) for i, ax in enumerate(plt.subplots(4, 4, figsize=(15,15))[1].flatten())]\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xb, yb = learn.data.one_batch()\n[Img(xb[0]).show(ax=ax) for i, ax in enumerate(plt.subplots(4, 4, figsize=(15,15))[1].flatten()) ]\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[Img(cb_fn(learn, aug_list).on_batch_begin(xb, yb, True,45)['last_input'][0]).show(ax=ax)\n for i, ax in enumerate(plt.subplots(4, 4, figsize=(15,15))[1].flatten())]\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[Img(cb_fn(learn, aug_list).on_batch_begin(xb, yb, True,55)['last_input'][0]).show(ax=ax)\n for i, ax in enumerate(plt.subplots(4, 4, figsize=(15,15))[1].flatten())]\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[Img(cb_fn(learn, aug_list).on_batch_begin(xb, yb, True,65)['last_input'][0]).show(ax=ax)\n for i, ax in enumerate(plt.subplots(4, 4, figsize=(15,15))[1].flatten())]\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[Img(cb_fn(learn, aug_list).on_batch_begin(xb, yb, True,75)['last_input'][0]).show(ax=ax)\n for i, ax in enumerate(plt.subplots(4, 4, figsize=(15,15))[1].flatten())]\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[Img(cb_fn(learn, aug_list).on_batch_begin(xb, yb, True,85)['last_input'][0]).show(ax=ax)\n for i, ax in enumerate(plt.subplots(4, 4, figsize=(15,15))[1].flatten())]\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[Img(cb_fn(learn, aug_list).on_batch_begin(xb, yb, True,95)['last_input'][0]).show(ax=ax)\n for i, ax in enumerate(plt.subplots(4, 4, figsize=(15,15))[1].flatten())]\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}