{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm.auto import tqdm\nfrom glob import glob\nimport time, gc\nimport cv2\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n# from tensorflow import keras\nimport matplotlib.image as mpimg\nfrom keras.preprocessing.image import ImageDataGenerator\n\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ReduceLROnPlateau\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport PIL.Image as Image, PIL.ImageDraw as ImageDraw, PIL.ImageFont as ImageFont\nfrom matplotlib import pyplot as plt\nimport tensorflow as tf\nimport seaborn as sns\n\n\nfrom keras.layers import Input\nfrom keras.layers import Conv2D, MaxPool2D, Dense, BatchNormalization, Activation, add, GlobalAvgPool2D, GlobalAveragePooling2D, Dropout\nfrom keras.models import Model\nfrom keras import regularizers\nfrom keras.utils import plot_model\nfrom keras import backend as K\nfrom keras import models\n\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df_ = pd.read_csv('/kaggle/input/bengaliai-cv19/train.csv')\ntrain_df_ = train_df_.drop(['grapheme'], axis=1, inplace=False)\ntrain_df_[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']] = train_df_[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].astype('uint8')\nIMG_SIZE=64\nN_CHANNELS=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_regularizers(model,\n                     kernel_regularization_factor=1e-4,\n                     bias_regularization_factor=1e-4,\n                     gamma_regularization_factor=1e-4,\n                     beta_regularization_factor=1e-4):\n    update_model_required = False\n    for layer in model.layers:\n        if not np.isclose(kernel_regularization_factor, 0.0) and hasattr(\n                layer, \"kernel_regularizer\"):\n            update_model_required = True\n            layer.kernel_regularizer = regularizers.l2(kernel_regularization_factor)\n        if not np.isclose(bias_regularization_factor, 0.0) and hasattr(\n                layer, \"bias_regularizer\"):\n            if layer.use_bias:\n                update_model_required = True\n                layer.bias_regularizer = regularizers.l2(bias_regularization_factor)\n        if not np.isclose(gamma_regularization_factor, 0.0) and hasattr(\n                layer, \"gamma_regularizer\"):\n            if layer.scale:\n                update_model_required = True\n                layer.gamma_regularizer = regularizers.l2(gamma_regularization_factor)\n        if not np.isclose(beta_regularization_factor, 0.0) and hasattr(\n                layer, \"beta_regularizer\"):\n            if layer.center:\n                update_model_required = True\n                layer.beta_regularizer = regularizers.l2(beta_regularization_factor)\n    if update_model_required:\n        print(\"Adding regularizers ...\")\n        # https://github.com/keras-team/keras/issues/2717#issuecomment-447570737\n        vanilla_weights = model.get_weights()\n        model = models.model_from_json(model.to_json())\n        model.set_weights(vanilla_weights)\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Resnet Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# def conv2d_bn(x, nb_filter, kernel_size, strides=(1, 1), padding='same'):\n#     x = Conv2D(nb_filter, kernel_size=kernel_size, strides=strides, padding=padding)(x)\n# #     x = Conv2D(nb_filter, kernel_size=kernel_size, strides=strides, padding=padding)(x)\n#     x = BatchNormalization()(x)\n#     x = Activation('relu')(x)\n\n#     return x\n \n\n# def shortcut(input, residual):\n \n#     input_shape = K.int_shape(input)\n#     residual_shape = K.int_shape(residual)\n#     stride_height = int(round(input_shape[1] / residual_shape[1]))\n#     stride_width = int(round(input_shape[2] / residual_shape[2]))\n#     equal_channels = input_shape[3] == residual_shape[3]\n \n#     identity = input\n#     if stride_width > 1 or stride_height > 1 or not equal_channels:\n#         identity = Conv2D(filters=residual_shape[3],\n#                            kernel_size=(1, 1),\n#                            strides=(stride_width, stride_height),\n#                            padding=\"valid\",\n#                            kernel_regularizer=regularizers.l2(0.0001))(input)\n \n#     return add([identity, residual])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def basic_block(nb_filter, strides=(1, 1)):\n#     def f(input):\n \n#         conv1 = conv2d_bn(input, nb_filter, kernel_size=(3, 3), strides=strides)\n#         residual = conv2d_bn(conv1, nb_filter, kernel_size=(3, 3))\n\n#         return shortcut(input, residual)\n \n#     return f\n \n    \n# def residual_block(nb_filter, repetitions, is_first_layer=False):\n#     def f(input):\n#         for i in range(repetitions):\n#             strides = (1, 1)\n#             if i == 0 and not is_first_layer:\n#                 strides = (2, 2)\n#             input = basic_block(nb_filter, strides)(input)\n#         return input\n \n#     return f","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def resnet_18(input_shape=(64,64,1), nclass=1024):\n\n#     input_ = Input(shape=input_shape)\n \n#     conv1 = conv2d_bn(input_, 16, kernel_size=(5, 5), strides=(2, 2))\n# #     conv1 = MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding='same')(conv1)\n#     conv1 = BatchNormalization()(conv1)\n#     conv1 = Activation('relu')(conv1)\n# #     conv1 = Dropout(rate=0.3)(conv1)\n    \n#     conv1 = residual_block(32, 2, is_first_layer=True)(conv1)\n#     conv1 = MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding='same')(conv1)\n#     conv1 = residual_block(64, 2, is_first_layer=True)(conv1)\n#     conv1 = MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding='same')(conv1)\n#     conv1 = residual_block(96, 2, is_first_layer=True)(conv1)\n#     conv1 = residual_block(128, 2, is_first_layer=True)(conv1)\n#     conv1 = MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding='same')(conv1)\n#     conv1 = residual_block(160, 2, is_first_layer=True)(conv1)\n#     conv1 = residual_block(192, 2, is_first_layer=True)(conv1)\n#     conv1 = residual_block(224, 2, is_first_layer=True)(conv1)\n#     conv1 = MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding='same')(conv1)\n\n# #     conv6 = residual_block(512, 2, is_first_layer=True)(conv5)\n \n#     pool2 = GlobalAvgPool2D()(conv1)\n# #     pool2 = Dense(nclass, activation='softmax')(pool2)\n# #     dense = GlobalAveragePooling2D()(pool2)\n# #     dense = Flatten()(dense)\n#     dense = Dense(1024, activation = \"relu\")(pool2)\n#     dense = Dropout(rate=0.3)(dense)\n#     dense = Dense(512, activation = \"relu\")(dense)\n    \n#     head_root = Dense(256, activation = \"relu\")(dense)\n#     head_root = Dense(168, activation = 'softmax')(head_root)\n    \n#     head_vowel = Dense(128, activation = \"relu\")(dense)\n#     head_vowel = Dense(11, activation = 'softmax')(head_vowel)\n    \n#     head_consonant = Dense(7, activation = 'softmax')(dense)\n    \n#     model = Model(inputs=input_, outputs=[head_root, head_vowel, head_consonant])\n#     model.summary()\n#     model = add_regularizers(model)\n\n#     return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def resblock(inputs,filter_num):\n#     out = keras.layers.Conv2D(filter_num, (3, 3), padding='SAME')(inputs)\n#     out = keras.layers.Conv2D(filter_num, (3, 3), padding='SAME')(out)\n#     out = keras.layers.Conv2D(filter_num, (3, 3), padding='SAME')(out)\n#     out = keras.layers.Conv2D(filter_num, (3, 3), padding='SAME')(out)\n#     out = keras.layers.BatchNormalization(momentum=0.15)(out)\n#     out = keras.layers.Activation('relu')(out)\n#     inputs = keras.layers.MaxPool2D(pool_size=(2, 2))(inputs)\n#     out = keras.layers.MaxPool2D(pool_size=(2, 2))(out)\n    \n#     out = keras.layers.Conv2D(filter_num,(5, 5),strides=1,padding='SAME')(out)\n#     out = keras.layers.BatchNormalization(momentum=0.15)(out)\n#     out = keras.layers.Activation('relu')(out)\n#     out = keras.layers.Dropout(rate=0.3)(out)\n    \n#     identity = keras.layers.Dense(filter_num)(inputs)\n#     output = keras.layers.add([out,identity])\n#     output = keras.layers.Activation('relu')(output)\n#     return output\n\n# def my_resnet(inputs):\n#     model = resblock(inputs,32)\n#     model = resblock(model,64)\n#     model = resblock(model,128)\n#     model = resblock(model,256)\n    \n#     identity = keras.layers.Conv2D(256, (3, 3), padding='SAME', activation='relu', input_shape=(2, 2, 1))(inputs)\n#     identity = keras.layers.BatchNormalization(momentum=0.15)(identity)\n#     identity = keras.layers.Activation('relu')(identity)\n#     identity = keras.layers.Dropout(rate=0.3)(identity)\n#     output = keras.layers.add([model,identity])\n#     output = keras.layers.Activation('relu')(output)\n#     return output\n# #     return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # preprocessing\n# inputs = keras.layers.Input(shape = (IMG_SIZE, IMG_SIZE, 1))\n# model = keras.layers.Conv2D(16, (3, 3), padding='SAME', activation='relu', input_shape=(IMG_SIZE, IMG_SIZE, 1))(inputs)\n# model = keras.layers.Conv2D(16, (3, 3), padding='SAME', activation='relu')(model)\n# model = keras.layers.Conv2D(16, (3, 3), padding='SAME', activation='relu')(model)\n# model = keras.layers.BatchNormalization(momentum=0.15)(model)\n# model = keras.layers.MaxPool2D(pool_size=(2, 2))(model)\n# model = keras.layers.Conv2D(filters=32, kernel_size=(5, 5), padding='SAME', activation='relu')(model)\n# model = keras.layers.Dropout(rate=0.3)(model)\n\n# # iterate the resblock\n# model = my_resnet(model)\n\n# # only for root\n# # dense0 = resblock(model,320)\n# # dense0 = keras.layers.GlobalAveragePooling2D()(dense0)\n# # dense0 = keras.layers.Flatten()(dense0)\n# # dense0 = keras.layers.Dense(1024, activation = \"relu\")(dense0)\n# # dense0 = keras.layers.Dropout(rate=0.3)(dense0)\n# # dense0 = keras.layers.Dense(512, activation = \"relu\")(dense0)\n# # head_root = keras.layers.Dense(168, activation = 'softmax')(dense0)\n\n# # adaptive other two\n# dense1 = keras.layers.GlobalAveragePooling2D()(model)\n# dense1 = keras.layers.Flatten()(dense1)\n# dense1 = keras.layers.Dense(1024, activation = \"relu\")(dense1)\n# dense1 = keras.layers.Dropout(rate=0.3)(dense1)\n# dense1 = keras.layers.Dense(512, activation = \"relu\")(dense1)\n# head_root = keras.layers.Dense(168, activation = 'softmax')(dense1)\n# head_vowel = keras.layers.Dense(11, activation = 'softmax')(dense1)\n# head_consonant = keras.layers.Dense(7, activation = 'softmax')(dense1)\n\n# model = keras.models.Model(inputs=inputs, outputs=[head_root, head_vowel, head_consonant])\n\n# model.summary()\n# model = add_regularizers(model)\n# model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = resnet_18()\n# adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)\n# model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_model(model, to_file='model.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize(df, size=64, need_progress_bar=True):\n    resized = {}\n    resize_size=64\n    if need_progress_bar:\n        l = tqdm(range(df.shape[0]))\n    else:\n        l = range(df.shape[0])\n        \n    for i in l:\n        image=df.loc[df.index[i]].values.reshape(137,236)\n        _, thresh = cv2.threshold(image, 30, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n        contours, _ = cv2.findContours(thresh,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)[-2:]\n\n        idx = 0 \n        ls_xmin = []\n        ls_ymin = []\n        ls_xmax = []\n        ls_ymax = []\n        for cnt in contours:\n            idx += 1\n            x,y,w,h = cv2.boundingRect(cnt)\n            ls_xmin.append(x)\n            ls_ymin.append(y)\n            ls_xmax.append(x + w)\n            ls_ymax.append(y + h)\n        xmin = min(ls_xmin)\n        ymin = min(ls_ymin)\n        xmax = max(ls_xmax)\n        ymax = max(ls_ymax)\n\n        roi = image[ymin:ymax,xmin:xmax]\n        resized_roi = cv2.resize(roi, (resize_size, resize_size),interpolation=cv2.INTER_AREA)\n        resized[df.index[i]] = resized_roi.reshape(-1)\n    \n    del l\n    resized = pd.DataFrame(resized).T\n#     return cv2.normalize(resized, None, 0, 1, cv2.NORM_MINMAX)\n    return resized\n\n\ndef get_dummies(df):\n    cols = []\n    for col in df:\n        cols.append(pd.get_dummies(df[col].astype(str)))\n    return pd.concat(cols, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 256\nepochs = 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiOutputDataGenerator(ImageDataGenerator):\n\n    def flow(self,\n             x,\n             y=None,\n             batch_size=32,\n             shuffle=True,\n             sample_weight=None,\n             seed=None,\n             save_to_dir=None,\n             save_prefix='',\n             save_format='png',\n             subset=None):\n\n        targets = None\n        target_lengths = {}\n        ordered_outputs = []\n        for output, target in y.items():\n            if targets is None:\n                targets = target\n            else:\n                targets = np.concatenate((targets, target), axis=1)\n            target_lengths[output] = target.shape[1]\n            ordered_outputs.append(output)\n\n\n        for flowx, flowy in super().flow(x, targets, batch_size=batch_size,\n                                         shuffle=shuffle):\n            target_dict = {}\n            i = 0\n            for output in ordered_outputs:\n                target_length = target_lengths[output]\n                target_dict[output] = flowy[:, i: i + target_length]\n                i += target_length\n\n            yield flowx, target_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"HEIGHT = 137\nWIDTH = 236","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# os.listdir('/kaggle/input')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"factors = [0.75,0.5,0.45,0.4,0.35,0.3,0.25,0.2,0.1]\n# Set a learning rate annealer. Learning rate will be half after 3 epochs if accuracy is not increased\nlearning_rate_reduction_root = ReduceLROnPlateau(monitor='dense_3_accuracy', \n                                            factor=factors[0],\n                                            patience=1,\n                                            cooldown=1,\n                                            verbose=1,\n                                            min_lr=0.000001)\n# learning_rate_reduction_vowel = ReduceLROnPlateau(monitor='dense_6_accuracy', \n#                                             factor=factors[1],\n#                                             patience=2, \n#                                             cooldown=1,\n#                                             verbose=1,\n#                                             min_lr=0.00001)\n# learning_rate_reduction_consonant = ReduceLROnPlateau(monitor='dense_7_accuracy', \n#                                             factor=factors[0],\n#                                             patience=1, \n#                                             cooldown=1,\n#                                             verbose=1,\n#                                             min_lr=0.00005)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # FROM HERE!!!!!!!!!!!\n# df_a = pd.read_parquet(f'/kaggle/input/bengaliai-cv19/train_image_data_0.parquet')\n# df_c = pd.read_parquet(f'/kaggle/input/bengaliai-cv19/train_image_data_2.parquet')\n# df_ac = pd.concat([df_a,df_c])\n# del df_a, df_c\n\n# train_df = pd.merge(df_ac, train_df_, on='image_id').drop(['image_id'], axis=1)\n# del df_ac\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# X_train = train_df.drop(['grapheme_root', 'vowel_diacritic', 'consonant_diacritic'], axis=1)\n# Y_train_root = pd.get_dummies(train_df['grapheme_root']).values\n# Y_train_vowel = pd.get_dummies(train_df['vowel_diacritic']).values\n# Y_train_consonant = pd.get_dummies(train_df['consonant_diacritic']).values\n# del train_df\n# gc.collect()\n# X_train = resize(X_train).astype(np.float32)/255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get the model\nmodel = models.load_model('/kaggle/input/10thmodel/my_model10_12.h5')\nadam = Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.000002)\nmodel.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # CNN takes images in shape `(batch_size, h, w, channels)`, so reshape the images\n# X_train = X_train.values.reshape(-1, IMG_SIZE, IMG_SIZE, N_CHANNELS)\n# gc.collect()\n\n# print(f'Training images: {X_train.shape}')\n# print(f'Training labels root: {Y_train_root.shape}')\n# print(f'Training labels vowel: {Y_train_vowel.shape}')\n# print(f'Training labels consonants: {Y_train_consonant.shape}')\n\n# # Divide the data into training and validation set\n# x_train, x_test, y_train_root, y_test_root, y_train_vowel, y_test_vowel, y_train_consonant, y_test_consonant = train_test_split(X_train, Y_train_root, Y_train_vowel, Y_train_consonant, test_size=0.08, random_state=666)\n# # del X_train\n# # del Y_train_root, Y_train_vowel, Y_train_consonant\n\n# # Data augmentation for creating more training data\n# datagen = MultiOutputDataGenerator(\n#     featurewise_center=False,  # set input mean to 0 over the dataset\n#     samplewise_center=False,  # set each sample mean to 0\n#     featurewise_std_normalization=False,  # divide inputs by std of the dataset\n#     samplewise_std_normalization=False,  # divide each input by its std\n#     zca_whitening=False,  # apply ZCA whitening\n#     rotation_range=8,  # randomly rotate images in the range (degrees, 0 to 180)\n#     zoom_range = 0.15, # Randomly zoom image \n#     width_shift_range=0.15,  # randomly shift images horizontally (fraction of total width)\n#     height_shift_range=0.15,  # randomly shift images vertically (fraction of total height)\n#     horizontal_flip=False,  # randomly flip images\n#     vertical_flip=False)  # randomly flip images\n\n\n# # This will just calculate parameters required to augment the given data. This won't perform any augmentations\n# datagen.fit(x_train)\n\n# # Fit the model\n# history = model.fit_generator(datagen.flow(x_train, {'dense_3': y_train_root, 'dense_4': y_train_vowel, 'dense_5': y_train_consonant}, batch_size=batch_size),\n#                           epochs = epochs, validation_data = (x_test, [y_test_root, y_test_vowel, y_test_consonant]), \n#                           steps_per_epoch=x_train.shape[0] // batch_size, \n#                           callbacks=[learning_rate_reduction_root])\n\n# # Delete to reduce memory usage\n# del x_train\n# del x_test\n# del y_train_root\n# del y_test_root\n# del y_train_vowel\n# del y_test_vowel\n# del y_train_consonant\n# del y_test_consonant\n# del datagen\n\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# os.makedirs('./save_weights')\n# # history = load_parquet(train_df, IMG_SIZE, N_CHANNELS, 0)\n# model.save('./save_weights/my_model10_9.h5')\n# histories = []\n# histories.append(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def load_parquet(train_df_, IMG_SIZE, N_CHANNELS, i):\n#     train_df = pd.merge(pd.read_parquet(f'/kaggle/input/bengaliai-cv19/train_image_data_{i}.parquet'), train_df_, on='image_id').drop(['image_id'], axis=1)\n#     X_train = train_df.drop(['grapheme_root', 'vowel_diacritic', 'consonant_diacritic'], axis=1)\n#     X_train = resize(X_train).astype(np.float64)/255\n# #     X_train = cv2.normalize(X_train, None, 0, 1, cv2.NORM_MINMAX)\n# #     X_train = resize(X_train)\n#     gc.collect()\n    \n#     # CNN takes images in shape `(batch_size, h, w, channels)`, so reshape the images\n#     X_train = X_train.values.reshape(-1, IMG_SIZE, IMG_SIZE, N_CHANNELS)\n    \n#     Y_train_root = pd.get_dummies(train_df['grapheme_root']).values\n#     Y_train_vowel = pd.get_dummies(train_df['vowel_diacritic']).values\n#     Y_train_consonant = pd.get_dummies(train_df['consonant_diacritic']).values\n#     del train_df\n\n#     print(f'Training images: {X_train.shape}')\n#     print(f'Training labels root: {Y_train_root.shape}')\n#     print(f'Training labels vowel: {Y_train_vowel.shape}')\n#     print(f'Training labels consonants: {Y_train_consonant.shape}')\n\n#     # Divide the data into training and validation set\n#     x_train, x_test, y_train_root, y_test_root, y_train_vowel, y_test_vowel, y_train_consonant, y_test_consonant = train_test_split(X_train, Y_train_root, Y_train_vowel, Y_train_consonant, test_size=0.08, random_state=666)\n#     del X_train\n#     del Y_train_root, Y_train_vowel, Y_train_consonant\n\n#     # Data augmentation for creating more training data\n#     datagen = MultiOutputDataGenerator(\n#         featurewise_center=False,  # set input mean to 0 over the dataset\n#         samplewise_center=False,  # set each sample mean to 0\n#         featurewise_std_normalization=False,  # divide inputs by std of the dataset\n#         samplewise_std_normalization=False,  # divide each input by its std\n#         zca_whitening=False,  # apply ZCA whitening\n#         rotation_range=8,  # randomly rotate images in the range (degrees, 0 to 180)\n#         zoom_range = 0.15, # Randomly zoom image \n#         width_shift_range=0.15,  # randomly shift images horizontally (fraction of total width)\n#         height_shift_range=0.15,  # randomly shift images vertically (fraction of total height)\n#         horizontal_flip=False,  # randomly flip images\n#         vertical_flip=False)  # randomly flip images\n\n\n#     # This will just calculate parameters required to augment the given data. This won't perform any augmentations\n#     datagen.fit(x_train)\n\n#     # Fit the model\n#     history = model.fit_generator(datagen.flow(x_train, {'dense_4': y_train_root, 'dense_6': y_train_vowel, 'dense_7': y_train_consonant}, batch_size=batch_size),\n#                               epochs = epochs, validation_data = (x_test, [y_test_root, y_test_vowel, y_test_consonant]), \n#                               steps_per_epoch=x_train.shape[0] // batch_size, \n#                               callbacks=[learning_rate_reduction_root])\n    \n#     # Delete to reduce memory usage\n#     del x_train\n#     del x_test\n#     del y_train_root\n#     del y_test_root\n#     del y_train_vowel\n#     del y_test_vowel\n#     del y_train_consonant\n#     del y_test_consonant\n#     del datagen\n    \n#     gc.collect()\n#     return history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# os.makedirs('./save_weights')\n# history = load_parquet(train_df_, IMG_SIZE, N_CHANNELS, 0)\n# model.save('./save_weights/my_model9_0.h5')\n# histories = []\n# histories.append(history)\n# history = load_parquet(train_df_, IMG_SIZE, N_CHANNELS, 1)\n# model.save('./save_weights/my_model8_9.h5')\n# histories.append(history)\n# # # history = load_parquet(train_df_, IMG_SIZE, N_CHANNELS, 2)\n# # # model.save('./save_weights/my_model1_2.h5')\n# # # history = load_parquet(train_df_, IMG_SIZE, N_CHANNELS, 3)\n# # # model.save('./save_weights/my_model1_3.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# adam = Adam(lr=0.0005, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)\n# model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# history = load_parquet(train_df_, IMG_SIZE, N_CHANNELS, 1)\n# model.save('./save_weights/my_model9_1.h5')\n# histories.append(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# histories = []\n# model0 = keras.models.load_model('/kaggle/input/pretrained-model/my_model1_0.h5')\n# model1 = keras.models.load_model('/kaggle/input/pretrained-model/my_model1_1.h5')\n# model2 = keras.models.load_model('/kaggle/input/pretrained-model/my_model1_2.h5')\n# model3 = keras.models.load_model('/kaggle/input/pretrained-model/my_model1_3.h5')\n# histories.append(model0)\n# histories.append(model1)\n# histories.append(model2)\n# histories.append(model3)\n# del model0\n# del model1\n# del model2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\ndef plot_loss(his, epoch, title):\n    plt.style.use('ggplot')\n    plt.figure()\n    plt.plot(np.arange(0, epoch), his.history['loss'], label='train_loss')\n    plt.plot(np.arange(0, epoch), his.history['dense_4_loss'], label='train_root_loss')\n    plt.plot(np.arange(0, epoch), his.history['dense_6_loss'], label='train_vowel_loss')\n    plt.plot(np.arange(0, epoch), his.history['dense_7_loss'], label='train_consonant_loss')\n    \n    plt.plot(np.arange(0, epoch), his.history['val_dense_4_loss'], label='val_train_root_loss')\n    plt.plot(np.arange(0, epoch), his.history['val_dense_6_loss'], label='val_train_vowel_loss')\n    plt.plot(np.arange(0, epoch), his.history['val_dense_7_loss'], label='val_train_consonant_loss')\n    \n    plt.title(title)\n    plt.xlabel('Epoch #')\n    plt.ylabel('Loss')\n    plt.legend(loc='upper right')\n    plt.show()\n\ndef plot_acc(his, epoch, title):\n    plt.style.use('ggplot')\n    plt.figure()\n    plt.plot(np.arange(0, epoch), his.history['dense_4_accuracy'], label='train_root_acc')\n    plt.plot(np.arange(0, epoch), his.history['dense_6_accuracy'], label='train_vowel_accuracy')\n    plt.plot(np.arange(0, epoch), his.history['dense_7_accuracy'], label='train_consonant_accuracy')\n    \n    plt.plot(np.arange(0, epoch), his.history['val_dense_4_accuracy'], label='val_root_acc')\n    plt.plot(np.arange(0, epoch), his.history['val_dense_6_accuracy'], label='val_vowel_accuracy')\n    plt.plot(np.arange(0, epoch), his.history['val_dense_7_accuracy'], label='val_consonant_accuracy')\n    plt.title(title)\n    plt.xlabel('Epoch #')\n    plt.ylabel('Accuracy')\n    plt.legend(loc='upper right')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for dataset in range(1):\n#     plot_loss(histories[dataset], epochs, f'Training Dataset: {dataset}')\n#     plot_acc(histories[dataset], epochs, f'Training Dataset: {dataset}')\n# # plot_loss(histories[0], epochs, f'Training Dataset: {0}')\n# # plot_acc(histories[0], epochs, f'Training Dataset: {0}')\n# del histories\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_dict = {\n    'grapheme_root': [],\n    'vowel_diacritic': [],\n    'consonant_diacritic': []\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = keras.models.load_model('/kaggle/input/pretrained-model/my_model1_3.h5')\ncomponents = ['consonant_diacritic', 'grapheme_root', 'vowel_diacritic']\ntarget=[] # model predictions placeholder\nrow_id=[] # row_id place holder\nfor i in range(4):\n    df_test_img = pd.read_parquet('/kaggle/input/bengaliai-cv19/test_image_data_{}.parquet'.format(i)) \n    df_test_img.set_index('image_id', inplace=True)\n\n    X_test = resize(df_test_img, need_progress_bar=False)/255\n    X_test = X_test.values.reshape(-1, IMG_SIZE, IMG_SIZE, N_CHANNELS)\n    \n    preds = model.predict(X_test)\n\n    for i, p in enumerate(preds_dict):\n        preds_dict[p] = np.argmax(preds[i], axis=1)\n\n    for k,id in enumerate(df_test_img.index.values):  \n        for i,comp in enumerate(components):\n            id_sample=id+'_'+comp\n            row_id.append(id_sample)\n            target.append(preds_dict[comp][k])\n    del df_test_img\n    del X_test\n    gc.collect()\n\ndf_sample = pd.DataFrame(\n    {\n        'row_id': row_id,\n        'target':target\n    },\n    columns = ['row_id','target'] \n)\ndf_sample.to_csv('submission.csv',index=False)\ndf_sample.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}