{"cells":[{"metadata":{},"cell_type":"markdown","source":"It is another implementation of the GridMask transform. It combines ideas of the version authored by [@haqishen](https://www.kaggle.com/haqishen) and [paper](https://arxiv.org/abs/2001.04086) authors but with some minor improvements and interface changes. The ratio \"r\" of filling units is variable here.\n\n**Links:**\n* [https://www.kaggle.com/haqishen/gridmask](https://www.kaggle.com/haqishen/gridmask)\n* [https://github.com/akuxcw/GridMask/blob/master/imagenet_grid/utils/grid.py](https://github.com/akuxcw/GridMask/blob/master/imagenet_grid/utils/grid.py)"},{"metadata":{"id":"GrjSWI04Grya","colab_type":"text"},"cell_type":"markdown","source":"# Imports"},{"metadata":{"id":"sfplqOHLFw0e","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport gc\n\n%matplotlib inline \n\nrng = np.random.default_rng()","execution_count":null,"outputs":[]},{"metadata":{"id":"pvv1_vPqq5n9","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"TRAIN_PARQS = ['train_image_data_0.parquet']","execution_count":null,"outputs":[]},{"metadata":{"id":"ii9xfeItGtsC","colab_type":"text"},"cell_type":"markdown","source":"# Data Load"},{"metadata":{"id":"XI3WCUbxC71Q","colab_type":"code","outputId":"3c1c1dc1-8535-42e7-ab3a-d836a27daadf","colab":{"base_uri":"https://localhost:8080/","height":204},"trusted":true},"cell_type":"code","source":"DATA_FOLDER = '../input/bengaliai-cv19'\n\ntrain_df = pd.read_csv(os.path.join(DATA_FOLDER, 'train.csv'))\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"_sbxYeR3Ob7I","colab_type":"text"},"cell_type":"markdown","source":"# Preprocessing"},{"metadata":{"id":"OItvu-NfVbit","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nimport torch\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"id":"odg_zOZANji8","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"IMG_H = 137\nIMG_W = 236\nIMG_SIZE = 128\nBATCH_SIZE = 64\nDEBUG = True\nRAND_SEED = 605\nLABEL_CLS = ['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']","execution_count":null,"outputs":[]},{"metadata":{"id":"jdOiHafijvKl","colab_type":"code","outputId":"c602f52b-79be-486f-8986-8aa59503e3c5","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"np.random.seed(RAND_SEED)\ntorch.manual_seed(RAND_SEED)","execution_count":null,"outputs":[]},{"metadata":{"id":"S7rl4tN7Njeh","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def prepare_data(img_path, \n               label_df=None, \n               debug=DEBUG, \n               dbg_max_count=300,\n               label_cls=LABEL_CLS):\n    img_path = [img_path] if isinstance(img_path, str) else img_path\n    parq_list = []\n\n    for i_path in img_path:\n      parq = pd.read_parquet(os.path.join(DATA_FOLDER, i_path))\n      parq_list.append(parq.iloc[:, 1:].values.astype(np.uint8).reshape(-1, IMG_H, IMG_W))\n\n      if debug: break\n\n    images = np.concatenate(parq_list, axis=0)\n    if debug:\n      images = images[:dbg_max_count]\n    \n    labels = label_df if label_df is None else label_df[label_cls].values[:images.shape[0]]\n\n    return images, labels\n","execution_count":null,"outputs":[]},{"metadata":{"id":"NOfZp9-ZY7jy","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"class SmartCrop:\n  \"\"\"Crop the image by light pixels edges\"\"\"\n\n  def __init__(self, size, padding=15, x_marg=10, y_marg=10, mask_threshold=80, noise_threshold=30):\n    self.size = size\n    self.padding = padding\n    self.x_marg = x_marg\n    self.y_marg = y_marg\n    self.noise_threshold = noise_threshold\n    self.mask_threshold = mask_threshold\n\n  def __call__(self, image):\n    img_mask = (image > self.mask_threshold).astype(np.int)\n    rows = np.any(img_mask, axis=1)\n    cols = np.any(img_mask, axis=0)\n    ymin, ymax = np.where(rows)[0][[0, -1]]\n    xmin, xmax = np.where(cols)[0][[0, -1]]\n\n    #cropping may cut too much, so we need to add it back\n    xmin = xmin - self.x_marg if (xmin > self.x_marg) else 0\n    ymin = ymin - self.y_marg if (ymin > self.y_marg) else 0\n    xmax = xmax + self.x_marg if (xmax < IMG_W - self.x_marg) else IMG_W\n    ymax = ymax + self.y_marg if (ymax < IMG_H - self.y_marg) else IMG_H\n    image = image[ymin:ymax, xmin:xmax]\n    image[image < self.noise_threshold] = 0 #remove low intensity pixels as noise\n\n    lx, ly = xmax-xmin,ymax-ymin\n    l = max(lx,ly) + self.padding\n\n    #make sure that the aspect ratio is kept in rescaling\n    image = np.pad(image, [((l-ly)//2,), ((l-lx)//2,)], mode='constant')\n    return cv2.resize(image, (self.size, self.size))\n\nclass InverseMaxNorm8bit:\n  \"\"\"Inverse np.uint8 pixels and scale with max\"\"\"\n  def __call__(self, x):\n    x_inv = 255 - x\n    return (x_inv*(255/x_inv.max())).astype(np.uint8)\n\nclass Inverse8bit:\n  \"\"\"Inverse np.uint8 pixels\"\"\"\n  def __call__(self, x):\n    return (255 - x).astype(np.uint8)","execution_count":null,"outputs":[]},{"metadata":{"id":"1_y35tDENjTU","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"class BengaliData(Dataset):\n  \"\"\"\n    Bengali graphics data\n  \"\"\"\n\n  def __init__(self, images, labels=None, transform=None):\n    \n    assert isinstance(images, np.ndarray)\n    assert (images.shape[1], images.shape[2]) == (IMG_H, IMG_W)\n\n    if labels is not None:\n      assert isinstance(labels, np.ndarray)\n      assert images.shape[0] == labels.shape[0]\n\n    self.images = images\n    self.labels = labels\n    self.transform = transform\n    self.inverse = InverseMaxNorm8bit()\n    self.smart_crop = SmartCrop(IMG_SIZE)\n\n  def __len__(self):\n    return len(self.images)\n\n  def __getitem__(self, idx):\n        x = self.inverse(self.images[idx])\n        x = self.smart_crop(x)\n\n        if self.transform is not None:\n            x = self.transform(image=x)['image']        \n\n        x = torch.tensor(x).unsqueeze(0)\n\n        if self.labels is not None:\n            y = self.labels[idx]\n            return x, torch.tensor(y)\n        else:\n            return x","execution_count":null,"outputs":[]},{"metadata":{"id":"fiGcDq6ZrDOs","colab_type":"code","outputId":"4153b877-42cf-49b8-f7a4-200d9bff661f","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"\nimg_avr = 0.07247582 #0.100065514 #0.06922848809290576\nimg_std = 0.20951288996105613 #0.2403564531227429 #0.20515700083327537","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"oAIlBoUqTDcD"},"cell_type":"markdown","source":"# Apply GridMask"},{"metadata":{"id":"sIVw4Yyz7Zi2","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import albumentations as A\nfrom albumentations import Normalize \nfrom albumentations.augmentations import functional as F\nfrom albumentations.core import transforms_interface as TI","execution_count":null,"outputs":[]},{"metadata":{"id":"vMkZSdrWHoFa","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"class GridMask(TI.ImageOnlyTransform):\n    \"\"\"GridMask augmentation for image classification and object detection.\n    \n    Author: @artur.k.space\n    \n    Args:\n        ratio (int or (int, int)): ratio which define \"l\" size of grid units\n        num_grid (int): number of grid in a row or column.\n        fill_value (int, float, lisf of int, list of float): value for dropped pixels.\n        rotate ((int, int) or int): range from which a random angle is picked. If rotate is a single int\n            an angle is picked from (-rotate, rotate). Default: (-90, 90)\n        mode (int):\n            0 - cropout a quarter of the square of each grid (left top)\n            1 - reserve a quarter of the square of each grid (left top)\n            2 - cropout 2 quarter of the square of each grid (left top & right bottom)\n\n    Targets:\n        image\n\n    Image types:\n        uint8, float32\n\n    Reference:\n    |  https://arxiv.org/abs/2001.04086\n    |  https://github.com/akuxcw/GridMask\n    |  https://www.kaggle.com/haqishen/gridmask\n    \"\"\"\n\n    def __init__(self, ratio=(0.4, 0.7), num_grid=3, fill_value=0, rotate=90, mode=0, always_apply=False, p=0.5):\n        super(GridMask, self).__init__(always_apply, p)\n\n        if isinstance(num_grid, int):\n            num_grid = (num_grid, num_grid)\n        if isinstance(rotate, int):\n            rotate = (-rotate, rotate)\n\n        self.num_grid = num_grid\n        self.fill_value = fill_value\n        self.rotate = rotate\n        self.mode = mode\n        self.masks = None\n        self.ratio = 0.5 if mode == 2 else ratio\n        self.hh = None # diagonal\n        self.height, self.width = None, None\n\n    def init_masks(self, height, width):\n        self.masks = []\n        self.height, self.width = height, width\n        self.hh = int(np.ceil(np.sqrt(height**2 + width**2)))\n\n        for n, n_grid in enumerate(range(self.num_grid[0], self.num_grid[1] + 1, 1)):\n            self.masks.append(self.make_grid(n_grid))\n\n    def make_grid(self, n_grid):\n        assert self.hh is not None\n\n        d_h = self.height / n_grid\n        d_w = self.width / n_grid\n                \n        mask = np.ones((self.hh, self.hh), np.float32)\n        r = np.random.uniform(self.ratio[0], self.ratio[1]) if isinstance(self.ratio, tuple) else self.ratio\n\n        l_h = int(np.ceil(d_h*r))\n        l_w = int(np.ceil(d_w*r))\n        \n        for i in range(-1, self.hh//int(d_h)+1):\n            s = int(d_h*i + d_h)\n            t = s+l_h\n            s = max(min(s, self.hh), 0)\n            t = max(min(t, self.hh), 0)\n\n            if self.mode == 2:\n                mask[s:t,:] = 1 - mask[s:t] # invert\n            else:\n                mask[s:t,:] = self.fill_value\n\n        for i in range(-1, self.hh//int(d_w)+1):\n            s = int(d_w*i + d_w)\n            t = s+l_w\n            s = max(min(s, self.hh), 0)\n            t = max(min(t, self.hh), 0)\n\n            if self.mode == 2:\n                mask[:,s:t] = 1 - mask[:,s:t] # invert\n            else:\n                mask[:,s:t] = self.fill_value\n\n        if self.mode == 1:\n            mask = 1 - mask\n\n        return mask \n\n    def apply(self, image, **params):\n        h, w = image.shape[:2]\n\n        if self.masks is None: self.init_masks(h, w)\n\n        mask = rng.choice(self.masks)\n        rand_h = np.random.randint(self.hh-h)\n        rand_w = np.random.randint(self.hh-w)\n        angle = np.random.randint(self.rotate[0], self.rotate[1]) if self.rotate[1] > 0 else 0\n\n        mask = F.rotate(mask, angle) if self.rotate[1] > 0 else mask\n        mask = mask[:,:,np.newaxis] if image.ndim == 3 else mask\n        image *= mask[rand_h:rand_h+h, rand_w:rand_w+w].astype(image.dtype)\n        return image\n\n    def get_transform_init_args_names(self):\n        return (\"ratio\", \"num_grid\", \"fill_value\", \"rotate\", \"mode\")","execution_count":null,"outputs":[]},{"metadata":{"id":"D37jEFZrlKMo","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"x_train, y_train = prepare_data(TRAIN_PARQS, train_df, debug=DEBUG)","execution_count":null,"outputs":[]},{"metadata":{"id":"ziepfTg3mPu3","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"gm1 = GridMask(num_grid=(3, 8), ratio=(0.1, 0.2), mode=0, p=1, fill_value=0)\ngm2 = GridMask(num_grid=(5, 7), mode=2, p=1)\ngm3 = GridMask(num_grid=(3, 5), ratio=(0.5, 0.8), mode=1, p=1, fill_value=0)\n\ntransform = A.Compose([\n   \n    A.OneOf([gm1, gm2, gm3], p=1),\n\n    Normalize(img_avr, img_std),\n])","execution_count":null,"outputs":[]},{"metadata":{"id":"XGsbeuWGLxp-","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"train_ds = BengaliData(x_train, y_train, transform=transform)","execution_count":null,"outputs":[]},{"metadata":{"id":"X2aTG7T2mJ8N","colab_type":"code","outputId":"f689955f-fe2c-49e4-e259-7dc581e4f779","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"x_0, y_0 = train_ds[0]\nprint(\"Train DS len:\", len(train_ds), type(x_0), 'device:', x_0.device,  x_0.dtype, x_0.shape, y_0)\n","execution_count":null,"outputs":[]},{"metadata":{"id":"BZXNXXNnZtGy","colab_type":"text"},"cell_type":"markdown","source":"### Plot transformed"},{"metadata":{"id":"97W7a2y8c7zP","colab_type":"code","outputId":"3a14f153-7b4e-412e-c908-fd8d1f2f8dd3","colab":{"base_uri":"https://localhost:8080/","height":284},"trusted":true},"cell_type":"code","source":"nrow, ncol = 1, 5\n\nfig, axes = plt.subplots(nrow, ncol, figsize=(18, 8))\naxes = axes.flatten()\nfor i, ax in enumerate(axes):\n    image, label = train_ds[0]\n    ax.imshow(image[0], cmap='Greys')\n    ax.set_title(f'label: {label}')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"0yiKfRiH76Lf","colab_type":"code","outputId":"a1b20792-7525-4290-c399-42e1978371c1","colab":{"base_uri":"https://localhost:8080/","height":585},"trusted":true},"cell_type":"code","source":"nrow, ncol = 3, 6\n\nfig, axes = plt.subplots(nrow, ncol, figsize=(18, 8))\naxes = axes.flatten()\nfor i, ax in enumerate(axes):\n    image, label = train_ds[i]\n    ax.imshow(image[0], cmap='Greys')\n    ax.set_title(f'label: {label}')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"yjW2Ak1vlWA3","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":229},"outputId":"dbccdd00-f7a5-49fa-8e2a-f256df8ba4ee","trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, len(gm1.masks), figsize=(18, 8))\naxes = axes.flatten()\n\nfor i, mask in enumerate(gm1.masks):\n    axes[i].imshow(mask, cmap='Greys')\n\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"apfnimK5zenB","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":442},"outputId":"ab5e80be-37ba-4350-cd00-7bf69db076fa","trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, len(gm2.masks), figsize=(18, 8))\naxes = axes.flatten()\n\nfor i, mask in enumerate(gm2.masks):\n    axes[i].imshow(mask, cmap='Greys')\n\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"RghuPio5zfS4","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":442},"outputId":"78251a46-f710-4351-c330-26ee6f7c5580","trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, len(gm3.masks), figsize=(18, 8))\naxes = axes.flatten()\n\nfor i, mask in enumerate(gm3.masks):\n    axes[i].imshow(mask, cmap='Greys')\n\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dynamic Probability Transform"},{"metadata":{},"cell_type":"markdown","source":"It's been said in the [paper](https://arxiv.org/abs/2001.04086) that encreasing probability of transformer calls during the model training had better results. I've created this transformer wrapper to try out this hypotethis. Also it can be applied with other transformers (tested only with GridMask)."},{"metadata":{"trusted":true},"cell_type":"code","source":"class DynamicProb(TI.BasicTransform):\n    \"\"\"DynamicProb improvement\n    \n    Author: @artur.k.space\n    \n    Args:\n        transform (BasicTransform): Albumentations transformer instance\n        final_cnt (int): final instance calls count of the last step\n        p_steps (int): progression steps number\n        p (tuple(float)): min prob, max prob\n    \"\"\"\n    def __init__(self, transform: TI.BasicTransform, final_cnt: int, p_steps: int = 5, p: tuple = (0, 0.8), always_apply = False):\n        super().__init__(always_apply, p=1)\n\n        transform.always_apply = True\n        self.transform = transform\n\n        self._cnt = 0\n        self._stepid = 0\n        self._prob = p[0]\n        self.p_steps = p_steps\n        self.c_range = np.linspace(0, final_cnt, num=p_steps, dtype=np.int32)\n        self.p_range = np.linspace(p[0], p[1], num=p_steps, dtype=np.float32)\n\n    def _update_p(self):\n        self._stepid += 1\n        self._prob = self.p_range[self._stepid]\n\n    def __call__(self, force_apply=False, **kwargs):\n        result = kwargs \n        \n        if self._prob > np.random.rand():\n            result = self.transform(**kwargs)\n\n        self._cnt += 1\n\n        if self.p_steps - 1 > self._stepid and self._cnt >= self.c_range[self._stepid+1]:\n            self._update_p()\n\n        return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"steps_num = 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gm = DynamicProb(GridMask(num_grid=(5, 7), mode=2, p=1), final_cnt=int(BATCH_SIZE*(steps_num-1)), p_steps=steps_num, p=(0, 1))\ntransform = A.Compose([\n    gm,\n    A.Normalize(img_avr, img_std),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = BengaliData(x_train, y_train, transform=transform)\ntrain_ds = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=0)\ntrain_iter = iter(train_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nrow, ncol = 3, 6\n\nfor i in range(steps_num):\n    print(\"Batch #%s GM prob %s\" % (i, gm._prob))\n\n    batch, labels = next(train_iter)\n    \n    fig, axes = plt.subplots(nrow, ncol, figsize=(16, 5))\n    axes = axes.flatten()\n    for i, ax in enumerate(axes):\n        image, label = batch[i], labels[i]\n        ax.imshow(image[0], cmap='Greys')\n        ax.set_title(f'label: {label}')\n    plt.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For using with other transformers you can do smthng like this:\n\nEPOCHS = 50\ngm_final_cnt = int((EPOCHS-1)*len(x_train)*0.33)\n\ngm1 = DynamicProb(GridMask(num_grid=(3, 8), ratio=(0.1, 0.2), mode=0, p=1, fill_value=0), final_cnt=gm_final_cnt, p_steps=steps_num, p=(0, 0.9))\ngm2 = DynamicProb(GridMask(num_grid=(5, 7), mode=2, p=1),                                 final_cnt=gm_final_cnt, p_steps=steps_num, p=(0, 0.8))\ngm3 = DynamicProb(GridMask(num_grid=(3, 5), ratio=(0.5, 0.8), mode=1, p=1, fill_value=0), final_cnt=gm_final_cnt, p_steps=steps_num, p=(0, 0.7))\nA.OneOf([gm1, gm2, gm3], p=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"Bengali-ai-gridmask.ipynb","provenance":[],"collapsed_sections":[],"toc_visible":true,"machine_shape":"hm"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"nbformat":4,"nbformat_minor":4}