{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We used the notebook: https://www.kaggle.com/kaushal2896/bengali-graphemes-starter-eda-multi-output-cnn as a baseline for this notebook."},{"metadata":{},"cell_type":"markdown","source":"# Import the libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib import pyplot as plt\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport PIL.Image as Image, PIL.ImageDraw as ImageDraw, PIL.ImageFont as ImageFont\nimport random \nimport os\nimport cv2\nimport gc\nfrom tqdm.auto import tqdm\nimport numpy as np\nfrom tensorflow import keras\nimport matplotlib.image as mpimg\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Model\nfrom keras.models import clone_model\nfrom keras.layers import Dense,Conv2D,Flatten,MaxPool2D,Dropout,BatchNormalization, Input\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ReduceLROnPlateau\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport datetime as dt\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"project_dir = \"/kaggle/input/bengaliai-cv19/\"\n\ntrain_data_pre = pd.read_csv(project_dir+\"/train.csv\")\ntrain_data_pre.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setting up the data"},{"metadata":{},"cell_type":"markdown","source":"we took this method from the notebook: https://www.kaggle.com/shawon10/bangla-graphemes-image-processing-deep-cnn"},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize(df, size=64, need_progress_bar=True):\n    resized = {}\n    resize_size=64\n    if need_progress_bar:\n        for i in tqdm(range(df.shape[0])):\n            image=df.loc[df.index[i]].values.reshape(137,236)\n            _, thresh = cv2.threshold(image, 30, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n            contours, _ = cv2.findContours(thresh,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)[-2:]\n\n            idx = 0 \n            ls_xmin = []\n            ls_ymin = []\n            ls_xmax = []\n            ls_ymax = []\n            for cnt in contours:\n                idx += 1\n                x,y,w,h = cv2.boundingRect(cnt)\n                ls_xmin.append(x)\n                ls_ymin.append(y)\n                ls_xmax.append(x + w)\n                ls_ymax.append(y + h)\n            xmin = min(ls_xmin)\n            ymin = min(ls_ymin)\n            xmax = max(ls_xmax)\n            ymax = max(ls_ymax)\n\n            roi = image[ymin:ymax,xmin:xmax]\n            resized_roi = cv2.resize(roi, (resize_size, resize_size),interpolation=cv2.INTER_AREA)\n            resized[df.index[i]] = resized_roi.reshape(-1)\n    else:\n        for i in range(df.shape[0]):\n            #image = cv2.resize(df.loc[df.index[i]].values.reshape(137,236),(size,size),None,fx=0.5,fy=0.5,interpolation=cv2.INTER_AREA)\n            image=df.loc[df.index[i]].values.reshape(137,236)\n            _, thresh = cv2.threshold(image, 30, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n            contours, _ = cv2.findContours(thresh,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)[-2:]\n\n            idx = 0 \n            ls_xmin = []\n            ls_ymin = []\n            ls_xmax = []\n            ls_ymax = []\n            for cnt in contours:\n                idx += 1\n                x,y,w,h = cv2.boundingRect(cnt)\n                ls_xmin.append(x)\n                ls_ymin.append(y)\n                ls_xmax.append(x + w)\n                ls_ymax.append(y + h)\n            xmin = min(ls_xmin)\n            ymin = min(ls_ymin)\n            xmax = max(ls_xmax)\n            ymax = max(ls_ymax)\n\n            roi = image[ymin:ymax,xmin:xmax]\n            resized_roi = cv2.resize(roi, (resize_size, resize_size),interpolation=cv2.INTER_AREA)\n            resized[df.index[i]] = resized_roi.reshape(-1)\n    resized = pd.DataFrame(resized).T\n    return resized","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# some analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = train_data_pre[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic','grapheme']]\ny_train = np.array(train_labels.iloc[1:,0:3])\nroot_dist = np.bincount(y_train[:,0])\nvowel_dist = np.bincount(y_train[:,1])\nconsonant_dist = np.bincount(y_train[:,2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\n\nplt.rcParams[\"patch.force_edgecolor\"] = True\n#ax = sns.distplot(root_dist, kde=False, rug=False, vertical = False)\n#plt.xlabel=\"Class count\"\n#ax.set_title(\"Histogram of the distribution of the grapheme root classes\")\n#ax.set(xlabel='Nr. of samples in a class', ylabel='Nr. of classes')\nimport matplotlib.pyplot as plt\n\nhist1 = plt.hist(root_dist, color=\"skyblue\")\nplt.xlabel('Nr. of samples in a class')\nplt.ylabel('Nr. of classes')\nplt.title('Histogram of the distribution of the grapheme root classes')\nplt.xticks(hist1[1])\n\nplt.show()\n\ndf_vowel = pd.DataFrame(\n    [[i+1, vowel_dist[i]] for i in range(len(vowel_dist))]\n)\ndf_vowel.columns = ['Class', 'Nr. of samples in a class']\n\n\nax = sns.barplot(x=\"Class\", y=\"Nr. of samples in a class\", data=df_vowel, palette=\"Blues_d\")\nax.set_title(\"Bar chart of the number of samples in the vowel diacritics classes\")\nplt.show()\n\ndf_consonant = pd.DataFrame(\n    [[i+1, consonant_dist[i]] for i in range(len(consonant_dist))]\n)\ndf_consonant.columns = ['Class', 'Nr. of samples in a class']\n\nax = sns.barplot(x=\"Class\", y=\"Nr. of samples in a class\", data=df_consonant,  palette=\"Blues_d\")\nax.set_title(\"Bar chart of the number of samples in the consonant diacritics classes\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## The model"},{"metadata":{},"cell_type":"markdown","source":"We switched from three separate models to one multi-output model because of the following reasons:\n* Storing such a deep convolutional network needs a lot of RAM, especially with 3 separate models since we had to store three models instead of one. Because of this, our Kaggle notebook exceeded its maximum amount of RAM usage.\n* With one network we were able to make the network 'deeper' --we could add more layers with more channels; therefore our accuracy score increased heavily. "},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_SIZE = 64\n\ninputs = Input(shape = (IMG_SIZE, IMG_SIZE, 1))\n\nmodel = Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu', input_shape=(IMG_SIZE, IMG_SIZE, 1))(inputs)\nmodel = Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = BatchNormalization(momentum=0.15)(model)\nmodel = MaxPool2D(pool_size=(2, 2))(model)\nmodel = Conv2D(filters=32, kernel_size=(5, 5), padding='SAME', activation='relu')(model)\nmodel = Dropout(rate=0.3)(model)\n\nmodel = Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = BatchNormalization(momentum=0.15)(model)\nmodel = MaxPool2D(pool_size=(2, 2))(model)\nmodel = Conv2D(filters=64, kernel_size=(5, 5), padding='SAME', activation='relu')(model)\nmodel = BatchNormalization(momentum=0.15)(model)\nmodel = Dropout(rate=0.3)(model)\n\nmodel = Conv2D(filters=128, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=128, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=128, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=128, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = BatchNormalization(momentum=0.15)(model)\nmodel = MaxPool2D(pool_size=(2, 2))(model)\nmodel = Conv2D(filters=128, kernel_size=(5, 5), padding='SAME', activation='relu')(model)\nmodel = BatchNormalization(momentum=0.15)(model)\nmodel = Dropout(rate=0.3)(model)\n\nmodel = Conv2D(filters=256, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=256, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=256, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = Conv2D(filters=256, kernel_size=(3, 3), padding='SAME', activation='relu')(model)\nmodel = BatchNormalization(momentum=0.15)(model)\nmodel = MaxPool2D(pool_size=(2, 2))(model)\nmodel = Conv2D(filters=256, kernel_size=(5, 5), padding='SAME', activation='relu')(model)\nmodel = BatchNormalization(momentum=0.15)(model)\nmodel = Dropout(rate=0.3)(model)\n\nmodel = Flatten()(model)\nmodel = Dense(1024, activation = \"relu\")(model)\nmodel = Dropout(rate=0.3)(model)\ndense = Dense(512, activation = \"relu\")(model)\n\nhead_root = Dense(168, activation = 'softmax', name='root')(dense)\nhead_vowel = Dense(11, activation = 'softmax', name = 'vowel')(dense)\nhead_consonant = Dense(7, activation = 'softmax', name='consonant')(dense)\n\nmodel = Model(inputs=inputs, outputs=[head_root, head_vowel, head_consonant])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = keras.optimizers.Adam(learning_rate=0.001)\nmodel.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiOutputDataGenerator(keras.preprocessing.image.ImageDataGenerator):\n\n    def flow(self,\n             x,\n             y=None,\n             batch_size=32,\n             shuffle=True,\n             sample_weight=None,\n             seed=None,\n             save_to_dir=None,\n             save_prefix='',\n             save_format='png',\n             subset=None):\n\n        targets = None\n        target_lengths = {}\n        ordered_outputs = []\n        for output, target in y.items():\n            if targets is None:\n                targets = target\n            else:\n                targets = np.concatenate((targets, target), axis=1)\n            target_lengths[output] = target.shape[1]\n            ordered_outputs.append(output)\n\n\n        for flowx, flowy in super().flow(x, targets, batch_size=batch_size,\n                                         shuffle=shuffle):\n            target_dict = {}\n            i = 0\n            for output in ordered_outputs:\n                target_length = target_lengths[output]\n                target_dict[output] = flowy[:, i: i + target_length]\n                i += target_length\n\n            yield flowx, target_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils import class_weight\nbatch_size = 256\nepochs = 25\ntarget_dict = {}\nclass_weight_dict = {}\nmodel_types = ['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']\n\nhistory_dict = []\n\nfor i in range(0,4):\n    train_data =  pd.merge(pd.read_feather(f'/kaggle/input/bengaliaicv19feather/train_image_data_{i}.feather') , train_data_pre, on='image_id').drop(['image_id'], axis=1)\n    train_labels = train_data[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic','grapheme']]\n    train_data = train_data.drop(['grapheme_root', 'vowel_diacritic', 'consonant_diacritic','grapheme'], axis=1)\n    train_data = resize(train_data)/255\n    train_data = train_data.values.reshape(-1, 64, 64, 1)\n\n\n    for target in model_types:\n        y_train_one_hot = train_labels[target]\n        y_train_one_hot = pd.get_dummies(y_train_one_hot).values\n        target_dict[target] = y_train_one_hot\n        class_weight_dict[target] = class_weight.compute_class_weight('balanced',\n                                                     np.unique(train_labels[target]),\n                                                     train_labels[target])\n\n    #splitting the data into a train and validation set\n    x_train, x_test, y_train_root, y_test_root, y_train_vowel, y_test_vowel, y_train_consonant, y_test_consonant = train_test_split(train_data, target_dict['grapheme_root'], target_dict['vowel_diacritic'], target_dict['consonant_diacritic'], test_size=0.1, random_state=123)\n\n\n    datagen = MultiOutputDataGenerator(\n        rotation_range=10,  \n        zoom_range = 0.2, \n        width_shift_range=0.12,  \n        height_shift_range=0.12,\n        horizontal_flip=True)\n    \n    datagen.fit(x_train)\n    history = model.fit_generator(datagen.flow(x_train, {'root': y_train_root, 'vowel': y_train_vowel, 'consonant': y_train_consonant}, batch_size=batch_size), \n                                                 epochs = epochs, validation_data = (x_test, [y_test_root, y_test_vowel, y_test_consonant]), \n                                  steps_per_epoch=x_train.shape[0] // batch_size, class_weight={'root': class_weight_dict['grapheme_root'], \n                                                                                                'vowel': class_weight_dict['vowel_diacritic'], \n                                                                                                'consonant': class_weight_dict['consonant_diacritic']})\n    history_dict.append(history)\n    model.save('model.h5')\n    del x_train, x_test, y_train_root, y_test_root, y_train_vowel, y_test_vowel, y_train_consonant, y_test_consonant, train_data, train_labels\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot the training process"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(history.history.keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = 0\nfor history in history_dict:\n    \n    #accuracy\n    output_names = ['root', 'vowel', 'consonant']\n    plt.figure()\n    for i in range(3):\n        plt.plot(np.arange(0, epochs), history.history[output_names[i]+'_accuracy'], label='train_accuracy')\n        plt.plot(np.arange(0, epochs), history.history['val_'+output_names[i]+'_accuracy'], label='val_accuracy')\n        plt.title('Accuracy of '+model_types[i]+' for training dataset '+ str(dataset))\n        plt.xlabel('Epoch #')\n        plt.ylabel('Accuracy')\n        plt.legend(loc='lower right')\n        plt.show()\n        print('Validation accuracy for {} is: {}'.format(model_types[i], max(history.history['val_'+output_names[i]+'_accuracy'])))\n    #loss\n    plt.figure()\n    for i in range(3):\n        plt.plot(np.arange(0, epochs), history.history[output_names[i]+'_loss'], label='train_accuracy')\n        plt.plot(np.arange(0, epochs), history.history['val_'+output_names[i]+'_loss'], label='val_accuracy')\n        plt.title('Loss of '+model_types[i]+' for training dataset '+ str(dataset))\n        plt.xlabel('Epoch #')\n        plt.ylabel('Loss')\n        plt.legend(loc='upper right')\n        plt.show()\n        print('Validation loss for {} is: {}'.format(model_types[i], max(history.history['val_'+output_names[i]+'_loss'])))\n    dataset += 1\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict the test files for submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_SIZE=64\nN_CHANNELS=1\npreds_dict = {\n    'grapheme_root': [],\n    'vowel_diacritic': [],\n    'consonant_diacritic': []\n}\ncomponents = ['consonant_diacritic', 'grapheme_root', 'vowel_diacritic']\ntarget=[] # model predictions placeholder\nrow_id=[] # row_id place holder\nfor i in range(4):\n    df_test_img = pd.read_feather('/kaggle/input/bengaliaicv19feather/test_image_data_{}.feather'.format(i)) \n    df_test_img.set_index('image_id', inplace=True)\n\n    X_test = resize(df_test_img, need_progress_bar=False)/255\n    X_test = X_test.values.reshape(-1, IMG_SIZE, IMG_SIZE, N_CHANNELS)\n    \n    preds = model.predict(X_test)\n\n    for i, p in enumerate(preds_dict):\n        preds_dict[p] = np.argmax(preds[i], axis=1)\n\n    for k,id in enumerate(df_test_img.index.values):  \n        for i,comp in enumerate(components):\n            id_sample=id+'_'+comp\n            row_id.append(id_sample)\n            target.append(preds_dict[comp][k])\n    del df_test_img\n    del X_test\n    gc.collect()\n\ndf_sample = pd.DataFrame(\n    {\n        'row_id': row_id,\n        'target':target\n    },\n    columns = ['row_id','target'] \n)\ndf_sample.to_csv('submission.csv',index=False)\ndf_sample.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}