{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# Introduction\n\nThe purpose of this kernel is to show how to use the class ImageDataGenerator while loading the images from a folder.\n\nI heavily inspired from this kernel:\nhttps://www.kaggle.com/kaushal2896/bengali-graphemes-starter-eda-multi-output-cnn\n\nWe will use the images dataset provided by iafoss:\nhttps://www.kaggle.com/iafoss/image-preprocessing-128x128"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"## Load packages"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport warnings\nimport cv2\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.metrics import Recall\nfrom tensorflow.keras.layers import (\n    Input,\n    Conv2D,\n    MaxPooling2D,\n    Flatten,\n    Dense,\n    BatchNormalization,\n    Dropout,\n    LeakyReLU,\n)\nfrom tensorflow.keras.optimizers import SGD\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.utils import Sequence, plot_model\n\nfrom sklearn.preprocessing import LabelBinarizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, recall_score\n\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\nstats = (0.0692, 0.2051)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Basic data exploration"},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = '/kaggle/input/bengaliai-cv19/'\nIMG_PATH = '/kaggle/input/grapheme-imgs-128x128/'\n\ntrain = pd.read_csv(f'{DATA_PATH}train.csv')\ntrain['filename'] = train['image_id'] + '.png'  # This column will be used by the ImageDataGenerator\ntest = pd.read_csv(f'{DATA_PATH}test.csv')\nclass_map = pd.read_csv(f'{DATA_PATH}class_map.csv')\nsample_submission = pd.read_csv(f'{DATA_PATH}sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape, test.shape, sample_submission.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image Loaders\n\nWe will use two classes:\n\n- **MultiOutputDataGenerator**: based on Keras ImageDataGenerator. Used for data augmentation.\n- **ImageGenerator**: based on Keras Sequence. Used for loading and preprocessing images in batches."},{"metadata":{"trusted":true},"cell_type":"code","source":"HEIGHT = 137\nWIDTH = 236\nIMG_SIZE = 128\nN_CHANNELS = 1\n\nBATCH_SIZE = 128\ninput_shape = (IMG_SIZE, IMG_SIZE, N_CHANNELS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiOutputDataGenerator(ImageDataGenerator):\n    def flow_from_dataframe(\n        self,\n        dataframe,\n        directory=None,\n        x_col='filename',\n        y_col='class',\n        weight_col=None,\n        target_size=(256, 256),\n        color_mode='rgb',\n        classes=None,\n        class_mode='categorical',\n        batch_size=32,\n        shuffle=True,\n        seed=None,\n        save_to_dir=None,\n        save_prefix='',\n        save_format='png',\n        subset=None,\n        interpolation='nearest',\n        validate_filenames=True,\n        **kwargs\n    ):\n\n        for flow_x, flow_y in super().flow_from_dataframe(\n            dataframe,\n            directory=directory,\n            x_col=x_col,\n            y_col=y_col,\n            weight_col=weight_col,\n            target_size=target_size,\n            color_mode=color_mode,\n            classes=classes,\n            class_mode=class_mode,\n            batch_size=batch_size,\n            shuffle=shuffle,\n            seed=seed,\n            save_to_dir=save_to_dir,\n            save_prefix=save_prefix,\n            save_format=save_format,\n            subset=subset,\n            interpolation=interpolation,\n            validate_filenames=validate_filenames\n        ):\n            # The flow_y will have shape 128 * 3. We want it to be a list of 3 numpy arrays\n            # with the following shapes [128 * 168, 128 * 11, 128 * 7]\n            Y_root = kwargs.get('le_root').transform(flow_y[:,0])\n            Y_vowel = kwargs.get('le_vowel').transform(flow_y[:,1])\n            Y_consonant = kwargs.get('le_consonant').transform(flow_y[:,2])\n\n            yield flow_x, [Y_root, Y_vowel, Y_consonant]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageGenerator(Sequence):\n    def __init__(self, data, batch_size, dim, shuffle=True, **kwargs):\n        self.data = data\n        self.list_ids = data.index.values\n        self.batch_size = batch_size\n        self.dim = dim\n        self.shuffle = shuffle\n        self.le_root = kwargs.get('le_root')\n        self.le_vowel = kwargs.get('le_vowel')\n        self.le_consonant = kwargs.get('le_consonant')\n        self.on_epoch_end()\n\n    def __len__(self):\n        return int(len(self.data) // self.batch_size)\n\n    def __getitem__(self, index):\n        batch_ids = self.indexes[index * self.batch_size: (index + 1) * self.batch_size]\n        valid_ids  = [self.list_ids[i] for i in batch_ids]\n\n        X = np.empty((self.batch_size, *self.dim, 1))\n        Y_root = self.le_root.transform(self.data.loc[valid_ids, 'grapheme_root'].values)\n        Y_vowel = self.le_vowel.transform(self.data.loc[valid_ids, 'vowel_diacritic'].values)\n        Y_consonant = self.le_consonant.transform(self.data.loc[valid_ids, 'consonant_diacritic'].values)\n        \n        for i, k in enumerate(valid_ids):\n            img_path = f'{IMG_PATH}{self.data[\"image_id\"][k]}.png'\n            img = cv2.imread(img_path, cv2.COLOR_BGR2GRAY)\n            img = img[:, :, np.newaxis]\n            X[i, :, :, :] = img\n\n        return X, [Y_root, Y_vowel, Y_consonant]\n\n    def on_epoch_end(self):\n        self.indexes = np.arange(len(self.list_ids))\n        if self.shuffle:\n            np.random.shuffle(self.indexes)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build Model\n\nWe use a miniVGG architecture."},{"metadata":{"trusted":true},"cell_type":"code","source":"input_shape = (IMG_SIZE, IMG_SIZE, N_CHANNELS)\n\n# TODO: replace this with a better model\ndef build_model():\n    inputs = Input(shape=input_shape)\n\n    chan_dim = -1\n    # first CONV => RELU => CONV => RELU => POOL layer set\n    model = Conv2D(\n        32, (3, 3), padding=\"same\", input_shape=input_shape, activation=\"relu\"\n    )(inputs)\n    model = BatchNormalization(axis=chan_dim)(model)\n    model = Conv2D(32, (3, 3), padding=\"same\", activation=\"relu\")(model)\n    model = BatchNormalization(axis=chan_dim)(model)\n    model = MaxPooling2D(pool_size=(2, 2))(model)\n    model = Dropout(0.25)(model)\n\n    # second CONV => RELU => CONV => RELU => POOL layer set\n    model = Conv2D(64, (3, 3), padding=\"same\", activation=\"relu\")(model)\n    model = BatchNormalization(axis=chan_dim)(model)\n    model = Conv2D(64, (3, 3), padding=\"same\", activation=\"relu\")(model)\n    model = BatchNormalization(axis=chan_dim)(model)\n    model = MaxPooling2D(pool_size=(2, 2))(model)\n    model = Dropout(0.25)(model)\n\n    # first (and only) set of FC => RELU layers\n    model = Flatten()(model)\n    model = Dense(512, activation=\"relu\")(model)\n    model = BatchNormalization()(model)\n    model = Dropout(0.5)(model)\n\n    # softmax classifier\n    head_root = Dense(168, activation=\"softmax\")(model)\n    head_vowel = Dense(11, activation=\"softmax\")(model)\n    head_consonant = Dense(7, activation=\"softmax\")(model)\n\n    model = Model(inputs=inputs, outputs=[head_root, head_vowel, head_consonant])\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_model(model, to_file='model.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt = SGD(lr=0.01, decay=0.01 / 40, momentum=0.9, nesterov=True)\nmodel.compile(\n    optimizer=opt, loss=\"categorical_crossentropy\", metrics=[\"accuracy\", Recall()]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le_root = LabelBinarizer()\n_ = le_root.fit_transform(train['grapheme_root'].values)\n\nle_vowel = LabelBinarizer()\n_ = le_vowel.fit_transform(train['vowel_diacritic'].values)\n\nle_consonant = LabelBinarizer()\n_ = le_consonant.fit_transform(train['consonant_diacritic'].values)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will use the **MultiOutputDataGenerator** for the training set and the **ImageGenerator** for the validation set."},{"metadata":{"trusted":true},"cell_type":"code","source":"trainX, valX = train_test_split(train, test_size=0.15, random_state=SEED)\ntrain_generator = MultiOutputDataGenerator(\n    featurewise_center=False,  # set input mean to 0 over the dataset\n    samplewise_center=False,  # set each sample mean to 0\n    featurewise_std_normalization=False,  # divide inputs by std of the dataset\n    samplewise_std_normalization=False,  # divide each input by its std\n    zca_whitening=False,  # apply ZCA whitening\n    rotation_range=8,  # randomly rotate images in the range (degrees, 0 to 180)\n    zoom_range = 0.15, # Randomly zoom image \n    width_shift_range=0.15,  # randomly shift images horizontally (fraction of total width)\n    height_shift_range=0.15,  # randomly shift images vertically (fraction of total height)\n    horizontal_flip=False,  # randomly flip images\n    vertical_flip=False   # randomly flip images,\n)\nval_generator = ImageGenerator(\n    data=valX,\n    batch_size=BATCH_SIZE,\n    dim=(IMG_SIZE, IMG_SIZE),\n    **{'le_root': le_root, 'le_vowel': le_vowel, 'le_consonant': le_consonant}\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# TODO: run this with more epochs\nEPOCHS = 10\nhistory = model.fit_generator(\n    train_generator.flow_from_dataframe(\n        dataframe=train,\n        directory=IMG_PATH,\n        x_col='filename',\n        y_col=['grapheme_root', 'vowel_diacritic', 'consonant_diacritic'],\n        class_mode='other',\n        batch_size=BATCH_SIZE,\n        target_size=(IMG_SIZE, IMG_SIZE), # Default value is 256 x 256\n        color_mode='grayscale',\n        shuffle=False,\n        **{'le_root': le_root, 'le_vowel': le_vowel, 'le_consonant': le_consonant}\n    ),\n    steps_per_epoch=int(trainX.shape[0] / BATCH_SIZE),\n    validation_data=val_generator,\n    validation_steps=int(valX.shape[0] / BATCH_SIZE),\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Evaluate the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"root_score = np.mean(history.history['val_dense_1_recall'])\nvowel_score = np.mean(history.history['val_dense_2_recall'])\nconsonant_score = np.mean(history.history['val_dense_3_recall'])\nprint(root_score, vowel_score, consonant_score, 0.5 * root_score + 0.25 * vowel_score + 0.25 * consonant_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_loss(his, prefix, epoch, title):\n    plt.style.use('ggplot')\n    plt.figure()\n    plt.plot(np.arange(0, epoch), his.history[f'{prefix}_loss'], label='train_loss')\n    plt.plot(np.arange(0, epoch), his.history[f'val_{prefix}_loss'], label='val_loss')\n    plt.title(title)\n    plt.xlabel('Epoch #')\n    plt.ylabel('Loss')\n    plt.legend(loc='upper right')\n    plt.show()\n\ndef plot_acc(his, prefix, epoch, title):\n    plt.style.use('ggplot')\n    plt.figure()\n    plt.plot(np.arange(0, epoch), his.history[f'{prefix}_accuracy'], label='train_acc')\n    plt.plot(np.arange(0, epoch), his.history[f'val_{prefix}_accuracy'], label='val_accuracy')\n    plt.title(title)\n    plt.xlabel('Epoch #')\n    plt.ylabel('Accuracy')\n    plt.legend(loc='upper right')\n    plt.show()\n\ndef plot_results():\n    m = {'dense_1': 'root', 'dense_2': 'vowel', 'dense_3': 'consonant'}\n    for ol in ['dense_1', 'dense_2', 'dense_3']:\n        plot_loss(history, ol, EPOCHS, f'Training on: {m[ol]}')\n        plot_acc(history, ol, EPOCHS, f'Training on: {m[ol]}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_results()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## TODOS\n\n- Replace miniVGG net with a better model\n- Use more epochs in your training"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}