{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n\n# Abstraction \n\n- Dataset: [Bengali.AI Handwritten Grapheme Classification](https://www.kaggle.com/c/bengaliai-cv19)\n- Task: **Multi-TasK and Multi-Class Classification**. ","metadata":{}},{"cell_type":"code","source":"# general packages\nimport os\nimport cv2\nimport gc\nimport math\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom glob import glob\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\n\n#sklearns \nfrom sklearn.metrics import cohen_kappa_score, accuracy_score\nfrom sklearn.model_selection import train_test_split \n\n# keras modules \nimport tensorflow as tf\nimport keras\nfrom keras.applications.densenet import DenseNet121, DenseNet169, DenseNet201\nfrom keras.optimizers import Adam, Nadam, SGD\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Model, load_model, Sequential\nfrom keras.layers import Dense, GlobalAveragePooling2D, Dropout, Conv2D, GlobalMaxPooling2D, concatenate\nfrom keras.layers import (MaxPooling2D, Input, Average, Activation, MaxPool2D,\n Flatten, LeakyReLU, BatchNormalization)\nfrom keras import models\nfrom keras import layers\nfrom keras.preprocessing.image import load_img\nfrom keras.preprocessing.image import img_to_array\n\nfrom keras.utils import Sequence\nfrom keras import utils as np_utils\nfrom keras.callbacks import (Callback, ModelCheckpoint,\n LearningRateScheduler,EarlyStopping, \n ReduceLROnPlateau,CSVLogger)\n\nwarnings.simplefilter('ignore')\nsns.set_style('whitegrid')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-18T05:40:46.350337Z","iopub.execute_input":"2022-01-18T05:40:46.350637Z","iopub.status.idle":"2022-01-18T05:40:53.491121Z","shell.execute_reply.started":"2022-01-18T05:40:46.350582Z","shell.execute_reply":"2022-01-18T05:40:53.490217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# declare some parameter\nSEED = 2020\nbatch_size = 12 \ndim = (128, 128)\nSIZE = 128\nstats = (0.0692, 0.2051)\nHEIGHT = 137 \nWIDTH = 236\n\ndef seed_all(SEED):\n random.seed(SEED)\n np.random.seed(SEED)\n os.environ['PYTHONHASHSEED'] = str(SEED)\n \n# seed all\nseed_all(SEED)\n\n# load files\nim_path = '../input/grapheme-imgs-128x128/'\ntrain = pd.read_csv('../input/bengaliai-cv19/train.csv')\ntest = pd.read_csv('../input/bengaliai-cv19/test.csv')\n\ntrain = train.sample(frac=1).reset_index(drop=True) # shuffling \ntrain['filename'] = train.image_id.apply(lambda filename: im_path + filename + '.png')\n\n# top 5 samples\ntrain.head()","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-01-18T05:40:53.493173Z","iopub.execute_input":"2022-01-18T05:40:53.493676Z","iopub.status.idle":"2022-01-18T05:40:53.888739Z","shell.execute_reply.started":"2022-01-18T05:40:53.493625Z","shell.execute_reply":"2022-01-18T05:40:53.888104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Augmentation \nWe will be using following two augmentation method. The program will choose randomly one of them while training. \n\n- [GridMask](https://arxiv.org/abs/2001.04086): It utilizes information removal to achieve state-of-the-art results in a variety of computer vision tasks. It is based on the deletion of regions of the input image.\n- [AugMix](https://arxiv.org/abs/1912.02781): A data processing technique that is simple to implement, adds limited computational overhead, and helps models withstand unforeseen corruptions. \n\n### GridMask From [Qishen Ha](https://www.kaggle.com/haqishen/gridmask) ","metadata":{}},{"cell_type":"code","source":"## Grid Mask\n# code takesn from https://www.kaggle.com/haqishen/gridmask\nimport albumentations\nfrom albumentations.core.transforms_interface import DualTransform, ImageOnlyTransform\nfrom albumentations.augmentations import functional as F\n\nclass GridMask(DualTransform):\n \"\"\"GridMask augmentation for image classification and object detection.\n\n Args:\n num_grid (int): number of grid in a row or column.\n fill_value (int, float, lisf of int, list of float): value for dropped pixels.\n rotate ((int, int) or int): range from which a random angle is picked. If rotate is a single int\n an angle is picked from (-rotate, rotate). Default: (-90, 90)\n mode (int):\n 0 - cropout a quarter of the square of each grid (left top)\n 1 - reserve a quarter of the square of each grid (left top)\n 2 - cropout 2 quarter of the square of each grid (left top & right bottom)\n\n Targets:\n image, mask\n\n Image types:\n uint8, float32\n\n Reference:\n | https://arxiv.org/abs/2001.04086\n | https://github.com/akuxcw/GridMask\n \"\"\"\n\n def __init__(self, num_grid=3, fill_value=0, rotate=0, mode=0, always_apply=False, p=0.5):\n super(GridMask, self).__init__(always_apply, p)\n if isinstance(num_grid, int):\n num_grid = (num_grid, num_grid)\n if isinstance(rotate, int):\n rotate = (-rotate, rotate)\n self.num_grid = num_grid\n self.fill_value = fill_value\n self.rotate = rotate\n self.mode = mode\n self.masks = None\n self.rand_h_max = []\n self.rand_w_max = []\n\n def init_masks(self, height, width):\n if self.masks is None:\n self.masks = []\n n_masks = self.num_grid[1] - self.num_grid[0] + 1\n for n, n_g in enumerate(range(self.num_grid[0], self.num_grid[1] + 1, 1)):\n grid_h = height / n_g\n grid_w = width / n_g\n this_mask = np.ones((int((n_g + 1) * grid_h), int((n_g + 1) * grid_w))).astype(np.uint8)\n for i in range(n_g + 1):\n for j in range(n_g + 1):\n this_mask[\n int(i * grid_h) : int(i * grid_h + grid_h / 2),\n int(j * grid_w) : int(j * grid_w + grid_w / 2)\n ] = self.fill_value\n if self.mode == 2:\n this_mask[\n int(i * grid_h + grid_h / 2) : int(i * grid_h + grid_h),\n int(j * grid_w + grid_w / 2) : int(j * grid_w + grid_w)\n ] = self.fill_value\n \n if self.mode == 1:\n this_mask = 1 - this_mask\n\n self.masks.append(this_mask)\n self.rand_h_max.append(grid_h)\n self.rand_w_max.append(grid_w)\n\n def apply(self, image, mask, rand_h, rand_w, angle, **params):\n h, w = image.shape[:2]\n mask = F.rotate(mask, angle) if self.rotate[1] > 0 else mask\n mask = mask[:,:,np.newaxis] if image.ndim == 3 else mask\n image *= mask[rand_h:rand_h+h, rand_w:rand_w+w].astype(image.dtype)\n return image\n\n def get_params_dependent_on_targets(self, params):\n img = params['image']\n height, width = img.shape[:2]\n self.init_masks(height, width)\n\n mid = np.random.randint(len(self.masks))\n mask = self.masks[mid]\n rand_h = np.random.randint(self.rand_h_max[mid])\n rand_w = np.random.randint(self.rand_w_max[mid])\n angle = np.random.randint(self.rotate[0], self.rotate[1]) if self.rotate[1] > 0 else 0\n\n return {'mask': mask, 'rand_h': rand_h, 'rand_w': rand_w, 'angle': angle}\n\n @property\n def targets_as_params(self):\n return ['image']\n\n def get_transform_init_args_names(self):\n return ('num_grid', 'fill_value', 'rotate', 'mode')","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:40:53.890466Z","iopub.execute_input":"2022-01-18T05:40:53.890905Z","iopub.status.idle":"2022-01-18T05:40:54.407755Z","shell.execute_reply.started":"2022-01-18T05:40:53.890734Z","shell.execute_reply":"2022-01-18T05:40:54.406992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# augmix : https://github.com/google-research/augmix\n\nfrom PIL import Image\nfrom PIL import ImageOps\nimport numpy as np\n\ndef int_parameter(level, maxval):\n \"\"\"Helper function to scale `val` between 0 and maxval .\n Args:\n level: Level of the operation that will be between [0, `PARAMETER_MAX`].\n maxval: Maximum value that the operation can have. This will be scaled to\n level/PARAMETER_MAX.\n Returns:\n An int that results from scaling `maxval` according to `level`.\n \"\"\"\n return int(level * maxval / 10)\n\n\ndef float_parameter(level, maxval):\n \"\"\"Helper function to scale `val` between 0 and maxval.\n Args:\n level: Level of the operation that will be between [0, `PARAMETER_MAX`].\n maxval: Maximum value that the operation can have. This will be scaled to\n level/PARAMETER_MAX.\n Returns:\n A float that results from scaling `maxval` according to `level`.\n \"\"\"\n return float(level) * maxval / 10.\n\ndef sample_level(n):\n return np.random.uniform(low=0.1, high=n)\n\ndef autocontrast(pil_img, _):\n return ImageOps.autocontrast(pil_img)\n\ndef equalize(pil_img, _):\n return ImageOps.equalize(pil_img)\n\ndef posterize(pil_img, level):\n level = int_parameter(sample_level(level), 4)\n return ImageOps.posterize(pil_img, 4 - level)\n\ndef rotate(pil_img, level):\n degrees = int_parameter(sample_level(level), 30)\n if np.random.uniform() > 0.5:\n degrees = -degrees\n return pil_img.rotate(degrees, resample=Image.BILINEAR)\n\ndef solarize(pil_img, level):\n level = int_parameter(sample_level(level), 256)\n return ImageOps.solarize(pil_img, 256 - level)\n\ndef shear_x(pil_img, level):\n level = float_parameter(sample_level(level), 0.3)\n if np.random.uniform() > 0.5:\n level = -level\n return pil_img.transform((SIZE, SIZE),\n Image.AFFINE, (1, level, 0, 0, 1, 0),\n resample=Image.BILINEAR)\n\ndef shear_y(pil_img, level):\n level = float_parameter(sample_level(level), 0.3)\n if np.random.uniform() > 0.5:\n level = -level\n return pil_img.transform((SIZE, SIZE),\n Image.AFFINE, (1, 0, 0, level, 1, 0),\n resample=Image.BILINEAR)\n\ndef translate_x(pil_img, level):\n level = int_parameter(sample_level(level), SIZE / 3)\n if np.random.random() > 0.5:\n level = -level\n return pil_img.transform((SIZE, SIZE),\n Image.AFFINE, (1, 0, level, 0, 1, 0),\n resample=Image.BILINEAR)\n\n\ndef translate_y(pil_img, level):\n level = int_parameter(sample_level(level), SIZE / 3)\n if np.random.random() > 0.5:\n level = -level\n return pil_img.transform((SIZE, SIZE),\n Image.AFFINE, (1, 0, 0, 0, 1, level),\n resample=Image.BILINEAR)\n\naugmentations = [\n autocontrast, \n equalize,\n posterize,\n rotate,\n solarize, \n shear_x, \n shear_y,\n translate_x,\n translate_y\n]\n\n# taken from https://www.kaggle.com/iafoss/image-preprocessing-128x128\nMEAN = [ 0.06922848809290576, 0.06922848809290576, 0.06922848809290576]\nSTD = [ 0.20515700083327537, 0.20515700083327537, 0.20515700083327537]\n\ndef normalize(image):\n \"\"\"Normalize input image channel-wise to zero mean and unit variance.\"\"\"\n image = image.transpose(2, 0, 1) # Switch to channel-first\n mean, std = np.array(MEAN), np.array(STD)\n image = (image - mean[:, None, None]) / std[:, None, None]\n return image.transpose(1, 2, 0)\n\n\ndef apply_op(image, op, severity):\n image = np.clip(image * 255., 0, 255).astype(np.uint8)\n pil_img = Image.fromarray(image) # Convert to PIL.Image\n pil_img = op(pil_img, severity)\n return np.asarray(pil_img) / 255.\n\n\ndef augment_and_mix(image, severity=1, width=3, depth=1, alpha=1.):\n \"\"\"Perform AugMix augmentations and compute mixture.\n Args:\n image: Raw input image as float32 np.ndarray of shape (h, w, c)\n severity: Severity of underlying augmentation operators (between 1 to 10).\n width: Width of augmentation chain\n depth: Depth of augmentation chain. -1 enables stochastic depth uniformly\n from [1, 3]\n alpha: Probability coefficient for Beta and Dirichlet distributions.\n Returns:\n mixed: Augmented and mixed image.\n \"\"\"\n ws = np.float32(\n np.random.dirichlet([alpha] * width))\n m = np.float32(np.random.beta(alpha, alpha))\n\n mix = np.zeros_like(image)\n for i in range(width):\n image_aug = image.copy()\n depth = depth if depth > 0 else np.random.randint(1, 4)\n \n for _ in range(depth):\n op = np.random.choice(augmentations)\n image_aug = apply_op(image_aug, op, severity)\n mix = np.add(mix, ws[i] * normalize(image_aug), out=mix, \n casting=\"unsafe\")\n\n mixed = (1 - m) * normalize(image) + m * mix\n return mixed","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:40:54.410924Z","iopub.execute_input":"2022-01-18T05:40:54.411361Z","iopub.status.idle":"2022-01-18T05:40:54.443467Z","shell.execute_reply.started":"2022-01-18T05:40:54.411310Z","shell.execute_reply":"2022-01-18T05:40:54.442748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Grapheme Data Generator","metadata":{}},{"cell_type":"code","source":"class GraphemeGenerator(keras.utils.Sequence):\n def __init__(self, data, batch_size, dim, shuffle=True, transform=None):\n self._data = data\n self._label_1 = pd.get_dummies(self._data['grapheme_root'], \n columns = ['grapheme_root'])\n self._label_2 = pd.get_dummies(self._data['vowel_diacritic'], \n columns = ['vowel_diacritic'])\n self._label_3 = pd.get_dummies(self._data['consonant_diacritic'], \n columns = ['consonant_diacritic'])\n self._list_idx = data.index.values\n self._batch_size = batch_size\n self._dim = dim\n self._shuffle = shuffle\n self.transform = transform\n self.on_epoch_end()\n \n def __len__(self):\n return int(np.floor(len(self._data)/self._batch_size))\n \n def __getitem__(self, index):\n batch_idx = self._indices[index*self._batch_size:(index+1)*self._batch_size]\n _idx = [self._list_idx[k] for k in batch_idx]\n\n Data = np.empty((self._batch_size, *self._dim, 1))\n Target_1 = np.empty((self._batch_size, 168), dtype = int)\n Target_2 = np.empty((self._batch_size, 11 ), dtype = int)\n Target_3 = np.empty((self._batch_size, 7 ), dtype = int)\n \n for i, k in enumerate(_idx):\n # load the image file using cv2\n image = cv2.imread(im_path + self._data['image_id'][k] + '.png')\n image = cv2.resize(image, self._dim) \n \n if self.transform is not None:\n if np.random.rand() > 0.7:\n # albumentation : grid mask\n res = self.transform(image=image)\n image = res['image']\n else:\n # augmix augmentation\n image = augment_and_mix(image)\n \n # scaling \n image = (image.astype(np.float32)/255.0 - stats[0])/stats[1]\n \n # gray scaling \n gray = lambda rgb : np.dot(rgb[... , :3] , [0.299 , 0.587, 0.114]) \n image = gray(image) \n \n # expand the axises \n image = image[:, :, np.newaxis]\n Data[i,:, :, :] = image\n \n Target_1[i,:] = self._label_1.loc[k, :].values\n Target_2[i,:] = self._label_2.loc[k, :].values\n Target_3[i,:] = self._label_3.loc[k, :].values\n \n return Data, [Target_1, Target_2, Target_3]\n \n \n def on_epoch_end(self):\n self._indices = np.arange(len(self._list_idx))\n if self._shuffle:\n np.random.shuffle(self._indices)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:40:54.446931Z","iopub.execute_input":"2022-01-18T05:40:54.447338Z","iopub.status.idle":"2022-01-18T05:40:54.466847Z","shell.execute_reply.started":"2022-01-18T05:40:54.447188Z","shell.execute_reply":"2022-01-18T05:40:54.465817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"!pip install ../input/efficientnet-keras-source-code/ -q","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-18T05:42:37.454203Z","iopub.execute_input":"2022-01-18T05:42:37.454492Z","iopub.status.idle":"2022-01-18T05:43:05.428807Z","shell.execute_reply.started":"2022-01-18T05:42:37.454445Z","shell.execute_reply":"2022-01-18T05:43:05.427995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import efficientnet.keras as efn \n\n# we will be using EfficientNetB0\nwg = '../input/efficientnet-keras-weights-b0b5/efficientnet-b0_imagenet_1000_notop.h5'\nefnet = efn.EfficientNetB0(weights=wg, \n include_top = False, \n input_shape=(128, 128, 3))","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:07.475330Z","iopub.execute_input":"2022-01-18T05:43:07.475636Z","iopub.status.idle":"2022-01-18T05:43:13.296248Z","shell.execute_reply.started":"2022-01-18T05:43:07.475579Z","shell.execute_reply":"2022-01-18T05:43:13.295443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Group Normalization\n\nBN's error increases rapidly when the batch size becomes smaller, caused by inaccurate batch statistics estimation. [Group Normalization](https://arxiv.org/pdf/1803.08494.pdf) (GN) as a simple alternative to BN. GN divides the channels into groups and computes within each group the mean and variance for normalization. GN's computation is independent of batch sizes, and its accuracy is stable in a wide range of batch sizes.\n\nCode from [Somshubra Majumdar](https://github.com/titu1994/Keras-Group-Normalization) ","metadata":{}},{"cell_type":"code","source":"from keras.engine import Layer\n\nclass GroupNormalization(Layer):\n \"\"\"Group normalization layer\n Group Normalization divides the channels into groups and computes within each group\n the mean and variance for normalization. GN's computation is independent of batch sizes,\n and its accuracy is stable in a wide range of batch sizes\n # Arguments\n groups: Integer, the number of groups for Group Normalization.\n axis: Integer, the axis that should be normalized\n (typically the features axis).\n For instance, after a `Conv2D` layer with\n `data_format=\"channels_first\"`,\n set `axis=1` in `BatchNormalization`.\n epsilon: Small float added to variance to avoid dividing by zero.\n center: If True, add offset of `beta` to normalized tensor.\n If False, `beta` is ignored.\n scale: If True, multiply by `gamma`.\n If False, `gamma` is not used.\n When the next layer is linear (also e.g. `nn.relu`),\n this can be disabled since the scaling\n will be done by the next layer.\n beta_initializer: Initializer for the beta weight.\n gamma_initializer: Initializer for the gamma weight.\n beta_regularizer: Optional regularizer for the beta weight.\n gamma_regularizer: Optional regularizer for the gamma weight.\n beta_constraint: Optional constraint for the beta weight.\n gamma_constraint: Optional constraint for the gamma weight.\n # Input shape\n Arbitrary. Use the keyword argument `input_shape`\n (tuple of integers, does not include the samples axis)\n when using this layer as the first layer in a model.\n # Output shape\n Same shape as input.\n # References\n - [Group Normalization](https://arxiv.org/abs/1803.08494)\n \"\"\"\n\n def __init__(self,\n groups=32,\n axis=-1,\n epsilon=1e-5,\n center=True,\n scale=True,\n beta_initializer='zeros',\n gamma_initializer='ones',\n beta_regularizer=None,\n gamma_regularizer=None,\n beta_constraint=None,\n gamma_constraint=None,\n **kwargs):\n super(GroupNormalization, self).__init__(**kwargs)\n self.supports_masking = True\n self.groups = groups\n self.axis = axis\n self.epsilon = epsilon\n self.center = center\n self.scale = scale\n self.beta_initializer = initializers.get(beta_initializer)\n self.gamma_initializer = initializers.get(gamma_initializer)\n self.beta_regularizer = regularizers.get(beta_regularizer)\n self.gamma_regularizer = regularizers.get(gamma_regularizer)\n self.beta_constraint = constraints.get(beta_constraint)\n self.gamma_constraint = constraints.get(gamma_constraint)\n\n def build(self, input_shape):\n dim = input_shape[self.axis]\n\n if dim is None:\n raise ValueError('Axis ' + str(self.axis) + ' of '\n 'input tensor should have a defined dimension '\n 'but the layer received an input with shape ' +\n str(input_shape) + '.')\n\n if dim < self.groups:\n raise ValueError('Number of groups (' + str(self.groups) + ') cannot be '\n 'more than the number of channels (' +\n str(dim) + ').')\n\n if dim % self.groups != 0:\n raise ValueError('Number of groups (' + str(self.groups) + ') must be a '\n 'multiple of the number of channels (' +\n str(dim) + ').')\n\n self.input_spec = InputSpec(ndim=len(input_shape),\n axes={self.axis: dim})\n shape = (dim,)\n\n if self.scale:\n self.gamma = self.add_weight(shape=shape,\n name='gamma',\n initializer=self.gamma_initializer,\n regularizer=self.gamma_regularizer,\n constraint=self.gamma_constraint)\n else:\n self.gamma = None\n if self.center:\n self.beta = self.add_weight(shape=shape,\n name='beta',\n initializer=self.beta_initializer,\n regularizer=self.beta_regularizer,\n constraint=self.beta_constraint)\n else:\n self.beta = None\n self.built = True\n\n def call(self, inputs, **kwargs):\n input_shape = K.int_shape(inputs)\n tensor_input_shape = K.shape(inputs)\n\n # Prepare broadcasting shape.\n reduction_axes = list(range(len(input_shape)))\n del reduction_axes[self.axis]\n broadcast_shape = [1] * len(input_shape)\n broadcast_shape[self.axis] = input_shape[self.axis] // self.groups\n broadcast_shape.insert(1, self.groups)\n\n reshape_group_shape = K.shape(inputs)\n group_axes = [reshape_group_shape[i] for i in range(len(input_shape))]\n group_axes[self.axis] = input_shape[self.axis] // self.groups\n group_axes.insert(1, self.groups)\n\n # reshape inputs to new group shape\n group_shape = [group_axes[0], self.groups] + group_axes[2:]\n group_shape = K.stack(group_shape)\n inputs = K.reshape(inputs, group_shape)\n\n group_reduction_axes = list(range(len(group_axes)))\n group_reduction_axes = group_reduction_axes[2:]\n\n mean = K.mean(inputs, axis=group_reduction_axes, keepdims=True)\n variance = K.var(inputs, axis=group_reduction_axes, keepdims=True)\n\n inputs = (inputs - mean) / (K.sqrt(variance + self.epsilon))\n\n # prepare broadcast shape\n inputs = K.reshape(inputs, group_shape)\n outputs = inputs\n\n # In this case we must explicitly broadcast all parameters.\n if self.scale:\n broadcast_gamma = K.reshape(self.gamma, broadcast_shape)\n outputs = outputs * broadcast_gamma\n\n if self.center:\n broadcast_beta = K.reshape(self.beta, broadcast_shape)\n outputs = outputs + broadcast_beta\n\n outputs = K.reshape(outputs, tensor_input_shape)\n\n return outputs\n\n def get_config(self):\n config = {\n 'groups': self.groups,\n 'axis': self.axis,\n 'epsilon': self.epsilon,\n 'center': self.center,\n 'scale': self.scale,\n 'beta_initializer': initializers.serialize(self.beta_initializer),\n 'gamma_initializer': initializers.serialize(self.gamma_initializer),\n 'beta_regularizer': regularizers.serialize(self.beta_regularizer),\n 'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),\n 'beta_constraint': constraints.serialize(self.beta_constraint),\n 'gamma_constraint': constraints.serialize(self.gamma_constraint)\n }\n base_config = super(GroupNormalization, self).get_config()\n return dict(list(base_config.items()) + list(config.items()))\n\n def compute_output_shape(self, input_shape):\n return input_shap","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:16.708039Z","iopub.execute_input":"2022-01-18T05:43:16.708347Z","iopub.status.idle":"2022-01-18T05:43:16.745543Z","shell.execute_reply.started":"2022-01-18T05:43:16.708296Z","shell.execute_reply":"2022-01-18T05:43:16.744829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Replace all Batch Normalization layers by Group Normalization layers\nfor i, layer in enumerate(efnet.layers):\n if \"batch_normalization\" in layer.name:\n efnet.layers[i] = GroupNormalization(groups=32, \n axis=-1, \n epsilon=0.00001)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:16.747997Z","iopub.execute_input":"2022-01-18T05:43:16.748524Z","iopub.status.idle":"2022-01-18T05:43:16.773361Z","shell.execute_reply.started":"2022-01-18T05:43:16.748474Z","shell.execute_reply":"2022-01-18T05:43:16.772428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Complete Model","metadata":{}},{"cell_type":"code","source":"def E0(input_dim, output_dim, base_model):\n \n input_tensor = Input(input_dim)\n \n x = Conv2D(3, (3, 3), padding='same', kernel_initializer='he_uniform', \n bias_initializer='zeros')(input_tensor)\n curr_output = base_model(x)\n curr_output = GlobalAveragePooling2D()(curr_output)\n curr_output = Dropout(0.5)(curr_output)\n curr_output = Dense(512, activation='elu')(curr_output)\n curr_output = Dropout(0.5)(curr_output)\n \n oputput1 = Dense(168, activation='softmax', name='gra') (curr_output)\n oputput2 = Dense(11, activation='softmax', name='vow') (curr_output)\n oputput3 = Dense(7, activation='softmax', name='cons') (curr_output)\n output_tensor = [oputput1, oputput2, oputput3]\n\n model = Model(input_tensor, output_tensor)\n \n return model\n\n# building the complete model\nmodel_E0 = E0(input_dim=(128,128,1),\n output_dim=(168,11,7), base_model = efnet)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:16.800460Z","iopub.execute_input":"2022-01-18T05:43:16.800781Z","iopub.status.idle":"2022-01-18T05:43:18.945838Z","shell.execute_reply.started":"2022-01-18T05:43:16.800730Z","shell.execute_reply":"2022-01-18T05:43:18.945057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def D169(input_dim, output_dim, base_model):\n \n input_tensor = Input(input_dim)\n \n x = Conv2D(3, (3, 3), padding='same', kernel_initializer='he_uniform', \n bias_initializer='zeros')(input_tensor)\n curr_output = base_model(x)\n curr_output = BatchNormalization()(curr_output)\n curr_output = GlobalAveragePooling2D()(curr_output)\n curr_output = Dense(512, activation='relu')(curr_output)\n curr_output = Dropout(0.5)(curr_output)\n curr_output = Dense(1024, activation='relu')(curr_output)\n \n oputput1 = Dense(168, activation='softmax', name='gra') (curr_output)\n oputput2 = Dense(11, activation='softmax', name='vow') (curr_output)\n oputput3 = Dense(7, activation='softmax', name='cons') (curr_output)\n \n output_tensor = [oputput1, oputput2, oputput3]\n \n model = Model(input_tensor, output_tensor)\n \n return model\n\n# define next model\ndensenet = DenseNet169(include_top=False, weights=None,\n input_shape=(128, 128, 3))\n\nmodel_D169 = D169(input_dim=(128,128,1),\n output_dim=(168,11,7), base_model = densenet)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:18.948134Z","iopub.execute_input":"2022-01-18T05:43:18.948590Z","iopub.status.idle":"2022-01-18T05:43:34.080455Z","shell.execute_reply.started":"2022-01-18T05:43:18.948542Z","shell.execute_reply":"2022-01-18T05:43:34.079677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Optimizer \nFor the Optmizer, we will be using **AdaBound**. It trains as fast as Adam and as good as SGD, for developing state-of-the-art deep learning models on a wide variety of popular tasks in the field of CV, NLP, and etc.\n\nRead More: [Paper](https://arxiv.org/abs/1902.09843) | [Code](https://github.com/titu1994/keras-adabound) ","metadata":{}},{"cell_type":"code","source":"# code: https://github.com/titu1994/keras-adabound \nclass AdaBound(keras.optimizers.Optimizer):\n \"\"\"AdaBound optimizer.\n Default parameters follow those provided in the original paper.\n # Arguments\n lr: float >= 0. Learning rate.\n final_lr: float >= 0. Final learning rate.\n beta_1: float, 0 < beta < 1. Generally close to 1.\n beta_2: float, 0 < beta < 1. Generally close to 1.\n gamma: float >= 0. Convergence speed of the bound function.\n epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`.\n decay: float >= 0. Learning rate decay over each update.\n weight_decay: Weight decay weight.\n amsbound: boolean. Whether to apply the AMSBound variant of this\n algorithm.\n # References\n - [Adaptive Gradient Methods with Dynamic Bound of Learning Rate]\n (https://openreview.net/forum?id=Bkg3g2R9FX)\n - [Adam - A Method for Stochastic Optimization]\n (https://arxiv.org/abs/1412.6980v8)\n - [On the Convergence of Adam and Beyond]\n (https://openreview.net/forum?id=ryQu7f-RZ)\n \"\"\"\n\n def __init__(self, learning_rate=0.001, final_lr=0.1, beta_1=0.9, beta_2=0.999, gamma=1e-3,\n epsilon=None, decay=0., amsbound=False, weight_decay=0.0, **kwargs):\n super(AdaBound, self).__init__(**kwargs)\n\n if not 0. <= gamma <= 1.:\n raise ValueError(\"Invalid `gamma` parameter. Must lie in [0, 1] range.\")\n\n with K.name_scope(self.__class__.__name__):\n self.iterations = K.variable(0, dtype='int64', name='iterations')\n self.learning_rate = K.variable(learning_rate, name='learning_rate')\n self.beta_1 = K.variable(beta_1, name='beta_1')\n self.beta_2 = K.variable(beta_2, name='beta_2')\n self.decay = K.variable(decay, name='decay')\n\n self.final_lr = final_lr\n self.gamma = gamma\n\n if epsilon is None:\n epsilon = K.epsilon()\n self.epsilon = epsilon\n self.initial_decay = decay\n self.amsbound = amsbound\n\n self.weight_decay = float(weight_decay)\n self.base_lr = float(learning_rate)\n\n def get_updates(self, loss, params):\n grads = self.get_gradients(loss, params)\n self.updates = [K.update_add(self.iterations, 1)]\n\n learning_rate = self.learning_rate\n if self.initial_decay > 0:\n learning_rate = learning_rate * (1. / (1. + self.decay * K.cast(self.iterations,\n K.dtype(self.decay))))\n\n t = K.cast(self.iterations, K.floatx()) + 1\n\n # Applies bounds on actual learning rate\n step_size = learning_rate * (K.sqrt(1. - K.pow(self.beta_2, t)) /\n (1. - K.pow(self.beta_1, t)))\n\n final_lr = self.final_lr * learning_rate / self.base_lr\n lower_bound = final_lr * (1. - 1. / (self.gamma * t + 1.))\n upper_bound = final_lr * (1. + 1. / (self.gamma * t))\n\n ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n if self.amsbound:\n vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n else:\n vhats = [K.zeros(1) for _ in params]\n self.weights = [self.iterations] + ms + vs + vhats\n\n for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):\n # apply weight decay\n if self.weight_decay != 0.:\n g += self.weight_decay * K.stop_gradient(p)\n\n m_t = (self.beta_1 * m) + (1. - self.beta_1) * g\n v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)\n\n if self.amsbound:\n vhat_t = K.maximum(vhat, v_t)\n denom = (K.sqrt(vhat_t) + self.epsilon)\n self.updates.append(K.update(vhat, vhat_t))\n else:\n denom = (K.sqrt(v_t) + self.epsilon)\n\n # Compute the bounds\n step_size_p = step_size * K.ones_like(denom)\n step_size_p_bound = step_size_p / denom\n bounded_lr_t = m_t * K.minimum(K.maximum(step_size_p_bound,\n lower_bound), upper_bound)\n\n p_t = p - bounded_lr_t\n\n self.updates.append(K.update(m, m_t))\n self.updates.append(K.update(v, v_t))\n new_p = p_t\n\n # Apply constraints.\n if getattr(p, 'constraint', None) is not None:\n new_p = p.constraint(new_p)\n\n self.updates.append(K.update(p, new_p))\n return self.updates\n\n def get_config(self):\n config = {'learning_rate': float(K.get_value(self.learning_rate)),\n 'final_lr': float(self.final_lr),\n 'beta_1': float(K.get_value(self.beta_1)),\n 'beta_2': float(K.get_value(self.beta_2)),\n 'gamma': float(self.gamma),\n 'decay': float(K.get_value(self.decay)),\n 'epsilon': self.epsilon,\n 'weight_decay': self.weight_decay,\n 'amsbound': self.amsbound}\n base_config = super(AdaBound, self).get_config()\n return dict(list(base_config.items()) + list(config.items())) ","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:34.081928Z","iopub.execute_input":"2022-01-18T05:43:34.082197Z","iopub.status.idle":"2022-01-18T05:43:34.115577Z","shell.execute_reply.started":"2022-01-18T05:43:34.082153Z","shell.execute_reply":"2022-01-18T05:43:34.114659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Compiling**","metadata":{}},{"cell_type":"code","source":"from keras import backend as K\n\nfor model in [model_E0, model_D169]:\n # compiling \n model.compile(\n\n optimizer = AdaBound(learning_rate=0.001,\n final_lr=.1,\n gamma=1e-3,\n weight_decay=5e-4,\n amsbound=False), \n\n loss = {'gra' : 'categorical_crossentropy', \n 'vow' : 'categorical_crossentropy', \n 'cons': 'categorical_crossentropy'},\n\n loss_weights = {'gra' : 1.0,\n 'vow' : 1.0,\n 'cons': 1.0},\n\n metrics={'gra' : 'accuracy', \n 'vow' : 'accuracy', \n 'cons': 'accuracy'}\n )\n","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-18T05:43:34.116931Z","iopub.execute_input":"2022-01-18T05:43:34.117358Z","iopub.status.idle":"2022-01-18T05:43:34.369995Z","shell.execute_reply.started":"2022-01-18T05:43:34.117308Z","shell.execute_reply":"2022-01-18T05:43:34.368770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize the model layouts\nfrom keras.utils import plot_model\n\nplot_model(model_E0, to_file='model.png')","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:34.372608Z","iopub.execute_input":"2022-01-18T05:43:34.372902Z","iopub.status.idle":"2022-01-18T05:43:35.560588Z","shell.execute_reply.started":"2022-01-18T05:43:34.372854Z","shell.execute_reply":"2022-01-18T05:43:35.559637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(model_D169, to_file='model.png') ","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:35.563348Z","iopub.execute_input":"2022-01-18T05:43:35.563932Z","iopub.status.idle":"2022-01-18T05:43:35.693619Z","shell.execute_reply.started":"2022-01-18T05:43:35.563877Z","shell.execute_reply":"2022-01-18T05:43:35.692749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generating Training Set and Validation Set\n\nHere we will use **GridMask** and **AugMix** augmentation on the data set randomly. In GridMask, we can pass various argument to get various type of augmentation results. Please, visit to [this](https://www.kaggle.com/haqishen/gridmask) kernel for more detials. I stick with the simplest form. ","metadata":{}},{"cell_type":"code","source":"# grid mask augmentation\ntransforms_train = albumentations.Compose([\n GridMask(num_grid=3, rotate=15, p=1),\n])\n\n# for way one - data generator\ntrain_labels, val_labels = train_test_split(train, test_size = 0.10, random_state = SEED,\n stratify = train[['grapheme_root', \n 'vowel_diacritic', \n 'consonant_diacritic']])\n\n# training generator\ntrain_generator = GraphemeGenerator(train_labels, batch_size, dim, \n shuffle = True, transform=None)\n\n# validation generator: no shuffle , not augmentation\nval_generator = GraphemeGenerator(val_labels, batch_size, dim, \n shuffle = False)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:35.695508Z","iopub.execute_input":"2022-01-18T05:43:35.695828Z","iopub.status.idle":"2022-01-18T05:43:37.891385Z","shell.execute_reply.started":"2022-01-18T05:43:35.695779Z","shell.execute_reply":"2022-01-18T05:43:37.890653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, item in enumerate(train_generator):\n x,y = item\n print(x.shape)\n if i == 0: break","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:37.892727Z","iopub.execute_input":"2022-01-18T05:43:37.893001Z","iopub.status.idle":"2022-01-18T05:43:37.999583Z","shell.execute_reply.started":"2022-01-18T05:43:37.892957Z","shell.execute_reply":"2022-01-18T05:43:37.998736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize the training samples ","metadata":{}},{"cell_type":"code","source":"from pylab import rcParams\n\n# helper function to plot sample \ndef plot_imgs(dataset_show):\n '''\n code: method from - https://www.kaggle.com/haqishen/gridmask\n '''\n rcParams['figure.figsize'] = 20,10\n for i in range(2):\n f, ax = plt.subplots(1,5)\n for p in range(5):\n idx = np.random.randint(0, len(dataset_show))\n img, label = dataset_show[idx]\n ax[p].grid(False)\n ax[p].imshow(img[0][:,:,0], cmap=plt.get_cmap('gray'))\n ax[p].set_title(idx)\n\n# calling the above function \nplot_imgs(val_generator) # non-augmented ","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:38.000963Z","iopub.execute_input":"2022-01-18T05:43:38.001426Z","iopub.status.idle":"2022-01-18T05:43:40.210098Z","shell.execute_reply.started":"2022-01-18T05:43:38.001372Z","shell.execute_reply":"2022-01-18T05:43:40.209294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_imgs(train_generator) # augmented ","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:40.211425Z","iopub.execute_input":"2022-01-18T05:43:40.211886Z","iopub.status.idle":"2022-01-18T05:43:42.409839Z","shell.execute_reply.started":"2022-01-18T05:43:40.211834Z","shell.execute_reply":"2022-01-18T05:43:42.408885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.callbacks import (ModelCheckpoint, LearningRateScheduler,\n EarlyStopping, ReduceLROnPlateau, TensorBoard, CSVLogger)\n\n# some call back function; feel free to add more for experiment\ndef Call_Back():\n # model check point\n checkpoint = ModelCheckpoint('E0Train_B0.h5', \n monitor = 'val_loss', \n verbose = 0, save_best_only=True, \n mode = 'min',\n save_weights_only = True)\n \n csv_logger = CSVLogger('E0.csv')\n early = EarlyStopping(monitor='val_loss', \n mode='min', patience=5)\n \n return [checkpoint, csv_logger, early]\n\n# epoch size \nepochs = 60 # increase the number, ex.: 100/200\ntraining = False # setting it true for training the model\n\n# calling all callbacks \ncallbacks = Call_Back()\n\nif training:\n # acatual training (fitting)\n train_history = model.fit_generator(\n train_generator,\n steps_per_epoch=int(len(train_labels)/batch_size), \n validation_data=val_generator,\n validation_steps = int(len(val_labels)/batch_size),\n epochs=epochs,\n callbacks=callbacks\n )\nelse: \n model_E0.load_weights('../input/0909e0/0.909.h5')\n model_D169.load_weights('../input/newdensenet/newde69.h5')","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:42.414786Z","iopub.execute_input":"2022-01-18T05:43:42.415317Z","iopub.status.idle":"2022-01-18T05:43:44.750301Z","shell.execute_reply.started":"2022-01-18T05:43:42.415250Z","shell.execute_reply":"2022-01-18T05:43:44.749527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction ","metadata":{}},{"cell_type":"code","source":"# helper function\ndef bbox(img):\n rows = np.any(img, axis=1)\n cols = np.any(img, axis=0)\n rmin, rmax = np.where(rows)[0][[0, -1]]\n cmin, cmax = np.where(cols)[0][[0, -1]]\n return rmin, rmax, cmin, cmax\n\ndef crop_resize(img0, size=SIZE, pad=16):\n #crop a box around pixels large than the threshold \n #some images contain line at the sides\n ymin,ymax,xmin,xmax = bbox(img0[5:-5,5:-5] > 80)\n #cropping may cut too much, so we need to add it back\n xmin = xmin - 13 if (xmin > 13) else 0\n ymin = ymin - 10 if (ymin > 10) else 0\n xmax = xmax + 13 if (xmax < WIDTH - 13) else WIDTH\n ymax = ymax + 10 if (ymax < HEIGHT - 10) else HEIGHT\n img = img0[ymin:ymax,xmin:xmax]\n #remove lo intensity pixels as noise\n img[img < 28] = 0\n lx, ly = xmax-xmin,ymax-ymin\n l = max(lx,ly) + pad\n #make sure that the aspect ratio is kept in rescaling\n img = np.pad(img, [((l-ly)//2,), ((l-lx)//2,)], mode='constant')\n \n return cv2.resize(img,(size,size))","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:44.751735Z","iopub.execute_input":"2022-01-18T05:43:44.752020Z","iopub.status.idle":"2022-01-18T05:43:44.765024Z","shell.execute_reply.started":"2022-01-18T05:43:44.751976Z","shell.execute_reply":"2022-01-18T05:43:44.763016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test data generator\ndef test_batch_generator(df, batch_size):\n num_imgs = len(df)\n\n for batch_start in range(0, num_imgs, batch_size):\n curr_batch_size = min(num_imgs, batch_start + batch_size) - batch_start\n idx = np.arange(batch_start, batch_start + curr_batch_size)\n\n names_batch = df.iloc[idx, 0].values\n imgs_batch = 255 - df.iloc[idx, 1:].values.reshape(-1, HEIGHT, WIDTH).astype(np.uint8)\n X_batch = np.zeros((curr_batch_size, SIZE, SIZE, 1))\n \n for j in range(curr_batch_size):\n img = (imgs_batch[j,]*(255.0/imgs_batch[j,].max())).astype(np.uint8)\n img = crop_resize(img, size=SIZE)\n img = (img.astype(np.float32)/255.0 - stats[0])/stats[1]\n img = img[:, :, np.newaxis]\n X_batch[j,] = img\n\n yield X_batch, names_batch\n\n\n# load the parquet files \nTEST = [\n \"../input/bengaliai-cv19/test_image_data_0.parquet\",\n \"../input/bengaliai-cv19/test_image_data_1.parquet\",\n \"../input/bengaliai-cv19/test_image_data_2.parquet\",\n \"../input/bengaliai-cv19/test_image_data_3.parquet\",\n]\n\n# placeholders \nrow_id = []\ntarget = []\n\n# iterative over the test sets\nfor fname in tqdm(TEST):\n test_ = pd.read_parquet(fname)\n test_gen = test_batch_generator(test_, batch_size=batch_size)\n\n for batch_x, batch_name in test_gen:\n # prediction\n batch_predict_1 = model_E0.predict(batch_x, batch_size = 128)\n batch_predict_2 = model_D169.predict(batch_x, batch_size = 128)\n \n for idx, name in enumerate(batch_name):\n row_id += [\n f\"{name}_consonant_diacritic\",\n f\"{name}_grapheme_root\",\n f\"{name}_vowel_diacritic\",\n ]\n target += [\n np.argmax((batch_predict_1[2] + batch_predict_2[2])/2, axis=1)[idx],\n np.argmax((batch_predict_1[0] + batch_predict_2[0])/2, axis=1)[idx],\n np.argmax((batch_predict_1[1] + batch_predict_2[1])/2, axis=1)[idx],\n ]\n\n del test_\n gc.collect()\n \n \ndf_sample = pd.DataFrame(\n {\n 'row_id': row_id,\n 'target':target\n },\n columns = ['row_id','target'] \n)\n\ndf_sample.to_csv('submission.csv',index=False)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:43:44.766805Z","iopub.execute_input":"2022-01-18T05:43:44.767222Z","iopub.status.idle":"2022-01-18T05:50:14.706014Z","shell.execute_reply.started":"2022-01-18T05:43:44.767058Z","shell.execute_reply":"2022-01-18T05:50:14.705215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample","metadata":{"execution":{"iopub.status.busy":"2022-01-18T05:50:14.707483Z","iopub.execute_input":"2022-01-18T05:50:14.707999Z","iopub.status.idle":"2022-01-18T05:50:14.721959Z","shell.execute_reply.started":"2022-01-18T05:50:14.707949Z","shell.execute_reply":"2022-01-18T05:50:14.721232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"That's it. Feel free to experiment on various stuff, like model architecture, image size, augmentation methods etc. You might want to train longer and tweak the parameters. However, here are some my experimental observation, hope it may give you some insight.\n\n```\n> Exp1 | LB: 0.9410\ninput : 100 px, batch size : 32, split: 80/20\nmodel : EfficientB4 + BN + GAP + Dropout\noptimizer : Adam(1e-3)\nepoch : 20\ndata-aug : False\n\n> Exp2 | LB: 0.9470\ninput : 125 px, batch size : 32, split: 80/20\nmodel : EfficientB0 + Group Normalization + GAP + Dropout\noptimizer : AdaBound(1e-3)\nepoch : 20\ndata-aug : AugMix\n\n> Exp3 | LB: 0.9497\ninput : 138 px, batch size : 32, split: 80/20\nmodel : DenseNet121 + Filter Response Normalization + GAP + Dropout \nsched : WarmUpCosineDecayScheduler\noptimizer : Adam(1e-3)\nepoch : 40\ndata-aug : AugMix + CutOut\n\n> Exp4 | LB: 0.9535\ninput : 128 px, batch size : 32, split: 80/20\nmodel : EfficientB5 + BN + GAP + Dropout\noptimizer : AdaBound(1e-3)\nepoch : 30\ndata-aug : AugMix + GridMask\n\n> Exp5 | LB: 0.9620\ninput : 150 px, batch size : 40, split: 80/20 (stratify)\nmodel : EfficientB5 + BN + [GAP + GMP] + BN + Dense + Dropout\noptimizer : Adam(1e-3)\nepoch : 60\ndata-aug : AugMix + GridMask (70%)\n```","metadata":{}}]}