{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport time\nfrom pathlib import Path# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"train=pd.read_csv('/kaggle/input/bengaliai-cv19/train.csv')\ntest=pd.read_csv('/kaggle/input/bengaliai-cv19/test.csv')\nsample_sub=pd.read_csv('/kaggle/input/bengaliai-cv19/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# from pathlib import Path\n# featherdir = Path('/kaggle/input/bengaliaicv19feather')\n# start=time.time()\n# train0 = pd.read_feather(featherdir/'train_image_data_0.feather')\n# train1 = pd.read_feather(featherdir/'train_image_data_1.feather')\n# train2 = pd.read_feather(featherdir/'train_image_data_2.feather')\n# train3 = pd.read_feather(featherdir/'train_image_data_3.feather')\n# print(f\"Time to load {time.time()-start} seconds\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from tqdm import tqdm\nimport cv2\n# IMG_SIZE=128\n# def resize(df, size=IMG_SIZE, need_progress_bar=True):\n#     resized = {}\n#     if need_progress_bar:\n#         for i in tqdm(range(df.shape[0])):\n#             img0 = 255 - df.loc[df.index[i]].values.reshape(HEIGHT, WIDTH).astype(np.uint8)\n#             #normalize each image by its max val\n#             img = (img0*(255.0/img0.max())).astype(np.uint8)\n#             img = crop_resize(img0, size=size)\n#             img = ((img.astype(np.float32)/255.0) - 0.0692)/0.2051\n#             resized[df.index[i]] = img.reshape(-1)\n# #             resized = cv2.cvtColor(resized, cv2.COLOR_GRAY2RGB)\n#     else:\n#         for i in range(df.shape[0]):\n#             img0 = 255 - df.loc[df.index[i]].values.reshape(HEIGHT, WIDTH).astype(np.uint8)\n#             #normalize each image by its max val\n# #             img = (img0[i]).astype(np.uint8)\n#             img = (img0*(255.0/img0.max())).astype(np.uint8)\n#             img = crop_resize(img0, size=size)\n#             img = ((img.astype(np.float32)/255.0) - 0.0692)/0.2051\n#             resized[df.index[i]] = img.reshape(-1)\n#     return pd.DataFrame(resized).T\ndef resize(df, size=64, need_progress_bar=True):\n    resized = {}\n    if need_progress_bar:\n        for i in tqdm(range(df.shape[0])):\n            image = cv2.resize(df.loc[df.index[i]].values.reshape(137,236),(size,size))\n            resized[df.index[i]] = image.reshape(-1)\n            \n    else:\n        for i in range(df.shape[0]):\n            image = cv2.resize(df.loc[df.index[i]].values.reshape(137,236),(size,size))\n            resized[df.index[i]] = image.reshape(-1)\n            \n    resized = pd.DataFrame(resized).T\n    resized.columns=resized.columns.astype(str)\n    return resized","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#### train0\n# x_tot,x2_tot = [],[]\n\n# featherdir = Path('/kaggle/input/bengaliaicv19feather')\nstart=time.time()\ntrain0=pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_0.parquet')\n# train0=pd.read_feather(featherdir/'train_image_data_0.feather')\nprint(f\"TRAIN0 SHAPE: {train0.shape}  and it took {time.time()-start} seconds\")\ntrain0=train0.iloc[:,1:]\ntrain0_r=resize(train0)/255\n\n# data = 255 - df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH).astype(np.uint8)\n# for idx in tqdm(range(len(df))):\n#     name = df.iloc[idx,0]\n#     #normalize each image by its max val\n#     img = (data[idx]*(255.0/data[idx].max())).astype(np.uint8)\n#     img = crop_resize(img)\n\n#     x_tot.append((img/255.0))\n#     x2_tot.append(((img/255.0)**2)) \n# df = pd.read_parquet(fname)\n        #the input is inverted\n\nimport gc\ndel train0\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"###### train1\ncurrent=time.time()\ntrain1=pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_1.parquet')\n# train1=pd.read_feather(featherdir/'train_image_data_1.feather')\nprint(f\"TRAIN1 SHAPE: {train1.shape}  and it took {time.time()-current} seconds\")\ntrain1=train1.iloc[:,1:]\ntrain1_r=resize(train1)/255\n# train1_r=resize(train1)\n# train1_r=(train1_r-np.mean(train1_r))/(train1_r.max()-train1_r.min())\ndel train1\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train0_r.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"################################ CODE TO CHECK MEMORY USAGE################&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&\n# gl.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"\n# del train1","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"######## train2\ncurrent=time.time()\ntrain2=pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_2.parquet')\n# train2=pd.read_feather(featherdir/'train_image_data_2.feather')\nprint(f\"TRAIN2 SHAPE: {train2.shape}  and it took {time.time()-current} seconds\")\ntrain2=train2.iloc[:,1:]\ntrain2_r=resize(train2)/255\n# train2_r=resize(train2)\n# train2_r=(train2_r-np.mean(train2_r))/(train2_r.max()-train2_r.min())\ndel train2\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"\n# del train2\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"########### train3\ncurrent=time.time()\ntrain3=pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_3.parquet')\n# train3=pd.read_feather(featherdir/'train_image_data_3.feather')\nprint(f\"TRAIN3 SHAPE: {train3.shape}  and it took {time.time()-current} seconds\")\ntrain3=train3.iloc[:,1:]\ntrain3_r=resize(train3)/255\n# train3_r=resize(train3)\n# train3_r=(train3_r-np.mean(train3_r))/(train3_r.max()-train3_r.min())\ndel train3\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"\n# del train3\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"resized train\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"xtrain0=(train0_r).to_numpy()\ndel train0_r\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"xtrain1=train1_r.to_numpy()\ndel train1_r\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"xtrain2=train2_r.to_numpy()\ndel train2_r\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"xtrain3=train3_r.to_numpy()\ndel train3_r\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"xtrain=np.concatenate((xtrain0,xtrain1,xtrain2,xtrain3),axis=0)\ndel xtrain1\ndel xtrain2\ndel xtrain3\ndel xtrain0\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"xtrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_root=train['grapheme_root'].values\ny_vowel=train['vowel_diacritic'].values\ny_consonant=train['consonant_diacritic'].values\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"xtrain=xtrain.reshape(-1,64,64,1).astype(np.float32)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_vowel.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"############################# IMPORTING LIBRARIES CNN ##########################\nimport tensorflow as tf\nfrom keras.models import Sequential\nfrom keras.layers import Dense,Dropout,Activation,BatchNormalization,Flatten,Conv2D,MaxPool2D\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\nfrom keras import layers,regularizers\nfrom keras.optimizers import Adam,RMSprop\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model = Sequential()\nmodel.add(layers.Conv2D(64, (3,3), padding='same', input_shape=(64, 64, 1)))\nmodel.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"))\nmodel.add(layers.LeakyReLU(alpha=0.1))\nmodel.add(layers.Conv2D(64,  (3,3), padding='same'))\nmodel.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"))\nmodel.add(layers.LeakyReLU(alpha=0.1))\nmodel.add(layers.Conv2D(64,  (3,3), padding='same'))\nmodel.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"))\nmodel.add(layers.LeakyReLU(alpha=0.1))\n\nmodel.add(layers.MaxPooling2D(2, 2))\nmodel.add(layers.Dropout(0.2))\n\nmodel.add(layers.Conv2D(128, (3,3), padding='same'))\nmodel.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"))\nmodel.add(layers.LeakyReLU(alpha=0.1))\nmodel.add(layers.Conv2D(128, (3,3), padding='same'))\nmodel.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"))\nmodel.add(layers.LeakyReLU(alpha=0.1))\nmodel.add(layers.Conv2D(128, (3,3), padding='same'))\nmodel.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"))\nmodel.add(layers.LeakyReLU(alpha=0.1))\n\nmodel.add(layers.MaxPooling2D(2,2))\nmodel.add(layers.Dropout(0.2)) \n\nmodel.add(layers.Conv2D(256, (3,3), padding='same'))\nmodel.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"))\nmodel.add(layers.LeakyReLU(alpha=0.1))\nmodel.add(layers.Conv2D(256, (3,3), padding='same'))\nmodel.add(layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"))\nmodel.add(layers.LeakyReLU(alpha=0.1))\n\nmodel.add(layers.MaxPooling2D(2,2))\nmodel.add(layers.Dropout(0.2))\n\n\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(1024))\nmodel.add(layers.LeakyReLU(alpha=0.1))\n\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.Dense(256))\nmodel.add(layers.LeakyReLU(alpha=0.1))\n\nmodel.add(layers.BatchNormalization())\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from keras.models import clone_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model_root=clone_model(model)\nmodel_consonant=clone_model(model)\nmodel_vowel=clone_model(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model_root.add(Dense(168, activation = 'softmax'))\nmodel_vowel.add(Dense(11, activation = 'softmax'))\nmodel_consonant.add(Dense(7, activation = 'softmax'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# from keras.optimizers import Adam,RMSprop\n# optimizer = RMSprop(learning_rate=0.002,rho=0.9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model_root.compile(optimizer='adam', loss=['categorical_crossentropy'], metrics=['accuracy'])\nmodel_vowel.compile(optimizer='adam', loss=['categorical_crossentropy'], metrics=['accuracy'])\nmodel_consonant.compile(optimizer='adam', loss=['categorical_crossentropy'], metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model_root.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"model created\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"######################### ROOT #########################","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from tensorflow.keras.utils import to_categorical","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_root.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_root = pd.get_dummies(y_root)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# xx = to_categorical(y_root, 168)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_root=y_root.to_numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"type(y_root)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"200840/3","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def shuffle(matrix, target, test_proportion):\n    ratio = int(matrix.shape[0]/test_proportion) #should be int\n    X_train = matrix[ratio:,:]\n    X_test =  matrix[:ratio,:]\n    Y_train = target[ratio:,:]\n    Y_test =  target[:ratio,:]\n    return X_train, X_test, Y_train, Y_test\n\nX_train, X_val, Y_train, Y_val = shuffle(xtrain, y_root, 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print('reached')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"\n# from sklearn.model_selection import train_test_split\n# X_train, X_val, Y_train, Y_val = train_test_split(xtrain,y_root, test_size=0.08, random_state=42)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\ndatagen = ImageDataGenerator(\n    rotation_range=8, \n    zoom_range=0.15, \n    width_shift_range=0.15, \n    height_shift_range=0.15,\n#     rescale=1./255\n)\ndatagen.fit(X_train)\nlearning_rate_reduction = tf.keras.callbacks.ReduceLROnPlateau( \n    monitor='loss',    # Quantity to be monitored.\n    factor=0.5,       # Factor by which the learning rate will be reduced. new_lr = lr * factor\n    patience=3,        # The number of epochs with no improvement after which learning rate will be reduced.\n    verbose=1,         # 0: quiet - 1: update messages.\n    mode=\"auto\",       # {auto, min, max}. In min mode, lr will be reduced when the quantity monitored has stopped decreasing; \n                       # in the max mode it will be reduced when the quantity monitored has stopped increasing; \n                       # in auto mode, the direction is automatically inferred from the name of the monitored quantity.\n    min_delta=0.0001,  # threshold for measuring the new optimum, to only focus on significant changes.\n    cooldown=0,        # number of epochs to wait before resuming normal operation after learning rate (lr) has been reduced.\n    min_lr=0     # lower bound on the learning rate.\n    )\n\nes = EarlyStopping(monitor='loss', mode='auto', verbose=1, patience=300, restore_best_weights=True)\nmodel_root.fit_generator(datagen.flow(X_train, Y_train, batch_size=32),\n                              steps_per_epoch=len(X_train)//32,\n                              epochs=50,\n                              validation_data=(np.array(X_val),np.array(Y_val)),\n                              validation_steps=50,\n                              callbacks=[learning_rate_reduction, es])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# model_root.fit(X_train, Y_train, batch_size=1024, epochs=20, validation_data=(X_val, Y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"del X_train\ndel X_val\ndel Y_train\ndel Y_val\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"del y_root\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"root done !\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"##################### CONSONANT ##################################","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"del xtrain\ndel train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import pickle","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"Pkl_root = \"model_root.pkl\" \n# Pkl_consonant = \"model_consonant.pkl\"  \n# Pkl_vowel = \"model_vowel.pkl\"  \n\n\nwith open(Pkl_root, 'wb') as file:  \n    pickle.dump(model_root, file)\n# with open(Pkl_consonant, 'wb') as file:  \n#     pickle.dump(model_consonant, file)\n# with open(Pkl_vowel, 'wb') as file:  \n#     pickle.dump(model_vowel, file)    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"Model DONE !\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}