{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nimport math\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\nfrom tensorflow.keras import layers, optimizers\nfrom sklearn.model_selection import train_test_split\nimport cv2\nfrom albumentations.augmentations import functional as F\nimport random\nimport albumentations\nfrom keras.utils import plot_model\nfrom PIL import Image\nfrom PIL import ImageOps\nfrom keras.callbacks import (Callback, ModelCheckpoint,\n                                        LearningRateScheduler,EarlyStopping, \n                                        ReduceLROnPlateau,CSVLogger)\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#We already went through spiritual and holistic EDA in many kernels including mine, let's get started with network","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Mobilenet_V1:"},{"metadata":{},"cell_type":"markdown","source":"![](https://miro.medium.com/max/863/1*Voah8cvrs7gnTDf6acRvDw.png)"},{"metadata":{},"cell_type":"markdown","source":"Mobile net architecture is introduced to reduce the computational cost of convolution operation. The convolution is performed in two steps\n\n1. Depthwise - Independent of channels\n2. Pointwise - Changes the channel dimension"},{"metadata":{},"cell_type":"markdown","source":"![](https://miro.medium.com/max/1385/1*0tqgajmb-M6VBAbvQKQY6Q.png)"},{"metadata":{},"cell_type":"markdown","source":"Depthwise followed by 1x1 conv for expanding the channel dimension "},{"metadata":{},"cell_type":"markdown","source":"## Mobilenet_V2"},{"metadata":{},"cell_type":"markdown","source":"![](https://machinethink.net/images/mobilenet-v2/ResidualBlock.png)"},{"metadata":{},"cell_type":"markdown","source":"MobileNet-v2 utilizes a module architecture similar to the residual unit with bottleneck architecture of ResNet; the modified version of the residual unit where conv3x3 is replaced by depthwise convolution.\n\nHere first conv 1x1 is a expansion layer followed by depth wise convolution followed by projection layer.\n\nThis is also called **Inverted residuals**"},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH='../input/bengaliai-cv19/'\nSEED=2019\nHEIGHT=137\nWIDTH=236\nbatch_size=64\n\nresizedimgs='../input/grapheme-imgs-128x128/'\nmobilenetpath='../input/mobilenet-v2-128'\n\nclass_map = pd.read_csv(PATH+\"class_map.csv\")\nsample_submission = pd.read_csv(PATH+\"sample_submission.csv\")\ntest = pd.read_csv(PATH+\"test.csv\")\ntrain = pd.read_csv(PATH+\"train.csv\")\n\ntrain['filename'] = train.image_id.apply(lambda filename: resizedimgs + filename + '.png')\n\ndef seed_all(SEED):\n    random.seed(SEED)\n    np.random.seed(SEED)\n    \n# seed all\nseed_all(SEED)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_files, valid_files, y_train, y_valid = train_test_split(\n    train.filename.values, \n    train[['grapheme_root','vowel_diacritic', 'consonant_diacritic']].values, \n    test_size=0.25, \n    random_state=2019\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Augmentation using albumentation"},{"metadata":{},"cell_type":"markdown","source":"Credits: https://www.kaggle.com/haqishen/augmix-based-on-albumentations"},{"metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def int_parameter(level, maxval):\n    \"\"\"Helper function to scale `val` between 0 and maxval .\n    Args:\n    level: Level of the operation that will be between [0, `PARAMETER_MAX`].\n    maxval: Maximum value that the operation can have. This will be scaled to\n      level/PARAMETER_MAX.\n    Returns:\n    An int that results from scaling `maxval` according to `level`.\n    \"\"\"\n    return int(level * maxval / 10)\n\n\ndef float_parameter(level, maxval):\n    \"\"\"Helper function to scale `val` between 0 and maxval.\n    Args:\n    level: Level of the operation that will be between [0, `PARAMETER_MAX`].\n    maxval: Maximum value that the operation can have. This will be scaled to\n      level/PARAMETER_MAX.\n    Returns:\n    A float that results from scaling `maxval` according to `level`.\n    \"\"\"\n    return float(level) * maxval / 10.\n\n\ndef sample_level(n):\n    return np.random.uniform(low=0.1, high=n)\n\n\ndef autocontrast(pil_img, _):\n    return ImageOps.autocontrast(pil_img)\n\n\ndef equalize(pil_img, _):\n    return ImageOps.equalize(pil_img)\n\n\ndef posterize(pil_img, level):\n    level = int_parameter(sample_level(level), 4)\n    return ImageOps.posterize(pil_img, 4 - level)\n\n\ndef rotate(pil_img, level):\n    degrees = int_parameter(sample_level(level), 30)\n    if np.random.uniform() > 0.5:\n        degrees = -degrees\n    return pil_img.rotate(degrees, resample=Image.BILINEAR)\n\n\ndef solarize(pil_img, level):\n    level = int_parameter(sample_level(level), 256)\n    return ImageOps.solarize(pil_img, 256 - level)\n\n\ndef shear_x(pil_img, level):\n    level = float_parameter(sample_level(level), 0.3)\n    if np.random.uniform() > 0.5:\n        level = -level\n    return pil_img.transform(pil_img.size,\n                           Image.AFFINE, (1, level, 0, 0, 1, 0),\n                           resample=Image.BILINEAR)\n\n\ndef shear_y(pil_img, level):\n    level = float_parameter(sample_level(level), 0.3)\n    if np.random.uniform() > 0.5:\n        level = -level\n    return pil_img.transform(pil_img.size,\n                           Image.AFFINE, (1, 0, 0, level, 1, 0),\n                           resample=Image.BILINEAR)\n\n\ndef translate_x(pil_img, level):\n    level = int_parameter(sample_level(level), pil_img.size[0] / 3)\n    if np.random.random() > 0.5:\n        level = -level\n    return pil_img.transform(pil_img.size,\n                           Image.AFFINE, (1, 0, level, 0, 1, 0),\n                           resample=Image.BILINEAR)\n\n\ndef translate_y(pil_img, level):\n    level = int_parameter(sample_level(level), pil_img.size[0] / 3)\n    if np.random.random() > 0.5:\n        level = -level\n    return pil_img.transform(pil_img.size,\n                           Image.AFFINE, (1, 0, 0, 0, 1, level),\n                           resample=Image.BILINEAR)\n\n\n# operation that overlaps with ImageNet-C's test set\ndef color(pil_img, level):\n    level = float_parameter(sample_level(level), 1.8) + 0.1\n    return ImageEnhance.Color(pil_img).enhance(level)\n\n\n# operation that overlaps with ImageNet-C's test set\ndef contrast(pil_img, level):\n    level = float_parameter(sample_level(level), 1.8) + 0.1\n    return ImageEnhance.Contrast(pil_img).enhance(level)\n\n\n# operation that overlaps with ImageNet-C's test set\ndef brightness(pil_img, level):\n    level = float_parameter(sample_level(level), 1.8) + 0.1\n    return ImageEnhance.Brightness(pil_img).enhance(level)\n\n\n# operation that overlaps with ImageNet-C's test set\ndef sharpness(pil_img, level):\n    level = float_parameter(sample_level(level), 1.8) + 0.1\n    return ImageEnhance.Sharpness(pil_img).enhance(level)\n\n\naugmentations = [\n    autocontrast, equalize, posterize, rotate, solarize, shear_x, shear_y,\n    translate_x, translate_y\n]\n\naugmentations_all = [\n    autocontrast, equalize, posterize, rotate, solarize, shear_x, shear_y,\n    translate_x, translate_y, color, contrast, brightness, sharpness\n]\n\ndef normalize(image):\n    \"\"\"Normalize input image channel-wise to zero mean and unit variance.\"\"\"\n    return image - 127\n\ndef apply_op(image, op, severity):\n    #   image = np.clip(image, 0, 255)\n    pil_img = Image.fromarray(image)  # Convert to PIL.Image\n    pil_img = op(pil_img, severity)\n    return np.asarray(pil_img)\n\ndef augment_and_mix(image, severity=3, width=3, depth=-1, alpha=1.):\n    \"\"\"Perform AugMix augmentations and compute mixture.\n    Args:\n    image: Raw input image as float32 np.ndarray of shape (h, w, c)\n    severity: Severity of underlying augmentation operators (between 1 to 10).\n    width: Width of augmentation chain\n    depth: Depth of augmentation chain. -1 enables stochastic depth uniformly\n      from [1, 3]\n    alpha: Probability coefficient for Beta and Dirichlet distributions.\n    Returns:\n    mixed: Augmented and mixed image.\n    \"\"\"\n    ws = np.float32(\n      np.random.dirichlet([alpha] * width))\n    m = np.float32(np.random.beta(alpha, alpha))\n\n    mix = np.zeros_like(image).astype(np.float32)\n    for i in range(width):\n        image_aug = image.copy()\n        depth = depth if depth > 0 else np.random.randint(1, 4)\n        for _ in range(depth):\n            op = np.random.choice(augmentations)\n            image_aug = apply_op(image_aug, op, severity)\n        # Preprocessing commutes since all coefficients are convex\n        mix += ws[i] * image_aug\n#         mix += ws[i] * normalize(image_aug)\n\n    mixed = (1 - m) * image + m * mix\n#     mixed = (1 - m) * normalize(image) + m * mix\n    return mixed\n\n\nclass RandomAugMix(ImageOnlyTransform):\n\n    def __init__(self, severity=3, width=3, depth=-1, alpha=1., always_apply=False, p=0.5):\n        super().__init__(always_apply, p)\n        self.severity = severity\n        self.width = width\n        self.depth = depth\n        self.alpha = alpha\n\n    def apply(self, image, **params):\n        image = augment_and_mix(\n            image,\n            self.severity,\n            self.width,\n            self.depth,\n            self.alpha\n        )\n        return image\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_generator(filenames, y,type='train', batch_size=64, shape=(128, 128, 1), random_state=2019,transform=None):\n    \n    indices = np.arange(len(filenames))\n    \n    while True:\n        np.random.shuffle(indices)\n        \n        for i in range(0, len(indices), batch_size):\n            batch_idx = indices[i:i+batch_size]\n            size = len(batch_idx)\n            \n            batch_files = filenames[batch_idx]\n            X_batch = np.zeros((size, *shape))\n            y_batch = y[batch_idx]\n            \n            for i, file in enumerate(batch_files):\n                img = cv2.imread(file)\n                img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n                \n                if transform is not None:\n                    res = transform(image=img)\n                    img = res['image']\n                        #img = augment_and_mix(img)\n                        \n                if type!='train':\n                    img = cv2.resize(img, shape[:2])\n                    \n                X_batch[i, :, :, 0] = img / 255.\n            \n            yield X_batch, [y_batch[:, i] for i in range(y_batch.shape[1])]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_train = albumentations.Compose([\n    RandomAugMix(severity=3, width=3, alpha=1., p=1.),\n])\n\ntrain_gen = data_generator(train_files, y_train,type='train',transform=transforms_train)\nvalid_gen = data_generator(valid_files, y_valid,type='test')\n\ntrain_steps = round(len(train_files) / batch_size) + 1\nvalid_steps = round(len(valid_files) / batch_size) + 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Mobile_net model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(mobilenet):\n    x_in = layers.Input(shape=(128, 128, 1))\n    x = layers.Conv2D(3, (3, 3), padding='same')(x_in)\n    x = mobilenet(x)\n    \n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.5)(x)\n    x = layers.Dense(256, activation='relu')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.5)(x)\n    \n    out_grapheme = layers.Dense(168, activation='softmax', name='grapheme')(x)\n    out_vowel = layers.Dense(11, activation='softmax', name='vowel')(x)\n    out_consonant = layers.Dense(7, activation='softmax', name='consonant')(x)\n    \n    model = Model(inputs=x_in, outputs=[out_grapheme, out_vowel, out_consonant])\n    \n    model.compile(\n        optimizers.Adam(lr=0.0001), \n        metrics=['accuracy'], \n        loss='sparse_categorical_crossentropy'\n    )\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mobilenet = MobileNetV2(include_top=False, weights=mobilenetpath+'/mobilenet_v2_weights_tf_dim_ordering_tf_kernels_1.0_128_no_top.h5', input_shape=(128, 128, 3))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_model(mobilenet)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Learn the language"},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks = [tf.keras.callbacks.ModelCheckpoint('model.h5', save_best_only=True)]\n\ntrain_history = model.fit_generator(\n    train_gen,\n    steps_per_epoch=train_steps,\n    epochs=5,\n    validation_data=valid_gen,\n    validation_steps=valid_steps,\n    callbacks=callbacks\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dig the history"},{"metadata":{"trusted":true},"cell_type":"code","source":"histories=pd.DataFrame(train_history.history)\n\nplt.style.context(\"fivethirtyeight\")\n\ndef plot_log(data, show=True):\n\n    fig = plt.figure(figsize=(8,10))\n    fig.subplots_adjust(top=0.95, bottom=0.05, right=0.95)\n    fig.add_subplot(211)\n    \n    for key in data.keys():\n        if key.find('loss') >= 0:  # training loss\n            plt.plot(data[key].values, label=key)\n    plt.legend()\n    plt.title('Training and Validtion Loss')\n\n    fig.add_subplot(212)\n    for key in data.keys():\n        if key.find('acc') >= 0:  # acc\n            plt.plot(data[key].values, label=key)\n    plt.legend()\n    plt.title('Training and Validation Accuracy')\n\n    if show:\n        plt.show()\n        \nplot_log(histories)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Inference Soon!**"},{"metadata":{},"cell_type":"markdown","source":"## References:\n\n* https://www.kaggle.com/xhlulu/bengali-ai-simple-densenet-in-keras\n* https://www.kaggle.com/nandhuelan/densernet-pytorch-bengali-v2\n* https://towardsdatascience.com/review-mobilenetv1-depthwise-separable-convolution-light-weight-model-a382df364b69\n* https://www.kaggle.com/ipythonx/keras-grapheme-gridmask-augmix-in-efficientnet/data"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}