{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#Load libraries\nfrom matplotlib import pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport PIL.Image as Image, PIL.ImageDraw as ImageDraw, PIL.ImageFont as ImageFont\nimport random\nimport os\nimport cv2\nimport gc\nfrom tqdm.auto import tqdm\nimport sys\nimport random\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import regularizers\nfrom tensorflow.keras.utils import plot_model  \nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Network parameters in head if the notebook:"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# NN paramz:\nnetwork_deepth = 5\n\n# train paramz:\nepochs = 3\ntraintestsplit = 0\nbatch_size = 88\nshape_base = (236, 137)\nshape_scale_fuctor = 3 # is used to calcualte shape for NN input layer\noptimizer = 'adam'\n\n# where infomation is stored:\ninput_dir = '/kaggle/input/bengaliai-cv19/'\n\n# compile shape:\nshape = (shape_base[0] / shape_scale_fuctor, shape_base[1] / shape_scale_fuctor)\nshape = (int(shape[0]), int(shape[1]))\nprint('image size: %i x %i' % (shape[0], shape[1])) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Loading data memory effective:\n1. load only train dataset\n2. change data type for Int8"},{"metadata":{"trusted":true},"cell_type":"code","source":"# loading train data and optimizing for memory economy\nprint(\"data loading\")\ntrain_data  = pd.read_csv(input_dir + 'train.csv')\ntrain_data['grapheme_root'] = train_data['grapheme_root'].astype('uint8')\ntrain_data['vowel_diacritic'] = train_data['vowel_diacritic'].astype('uint8')\ntrain_data['consonant_diacritic'] = train_data['consonant_diacritic'].astype('uint8')\n\ntrain_data.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Memory effective resize function:\n1. changing np.array type to np.float32\n2. collecting memory every 500 iterations"},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize(df, shape):\n    resized_dic = {}\n    for i in tqdm(range(df.shape[0])):\n        resized_dic[df.index[i]] = cv2.resize(df.loc[df.index[i]].values.reshape(shape_base[1],shape_base[0]),shape, interpolation = cv2.INTER_LINEAR).reshape(-1).astype(np.float32) / 255\n        if i%500 == 0: # memory clearing\n            gc.collect()\n    resized = pd.DataFrame(resized_dic).T\n    del resized_dic\n    gc.collect()\n    return resized","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Creating Neural Network based on ResNet arcitecture:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def res_net_block_1(input_data, filters):\n    x1 = layers.Conv2D(filters, 3, activation='relu', padding='same')(input_data)\n    x1 = layers.LeakyReLU(alpha=0.01)(x1)\n    x2 = layers.BatchNormalization()(x1)\n    x2 = layers.Dropout(0.1)(x2)\n\n    x3 = layers.Conv2D(filters, 5, activation=None, padding='same')(x2)\n    x3 = layers.LeakyReLU(alpha=0.01)(x3)\n    x4 = layers.BatchNormalization()(x3)\n    x4 = layers.Dropout(0.1)(x4)\n\n    x5 = layers.Conv2D(filters, 1, activation=None, padding='same')(input_data)\n    x5 = layers.LeakyReLU(alpha=0.01)(x5)\n\n    x = layers.Add()([x4, x5])\n    x = layers.Activation('relu')(x)\n    return x\n\ndef res_net_block_2(input_data, filters):\n    x1 = layers.Conv2D(filters, 3, activation='relu', padding='same')(input_data)\n    x1 = layers.LeakyReLU(alpha=0.01)(x1)\n    x2 = layers.BatchNormalization()(x1)\n    x2 = layers.Dropout(0.1)(x2)\n\n    x3 = layers.Conv2D(filters, 5, activation=None, padding='same')(input_data)\n    x3 = layers.LeakyReLU(alpha=0.01)(x3)\n    x4 = layers.BatchNormalization()(x3)\n    x4 = layers.Dropout(0.1)(x4)\n\n    x5 = layers.Conv2D(filters, 1, activation=None, padding='same')(input_data)\n    x5 = layers.LeakyReLU(alpha=0.01)(x5)\n\n    x = layers.Add()([x2, x4, x5])\n    x = layers.Activation('relu')(x)\n    return x\n\n# multy output\ndef resnet_multiOutput(input_shape, outputsizes, num_res_net_blocks):\n    inputs = layers.Input(shape=(input_shape[1],input_shape[0],1))\n    x = layers.Conv2D(32, (3,3), activation='relu')(inputs)\n    x = layers.LeakyReLU(alpha=0.01, name='Leaky_ReLU_1')(x)\n    x = layers.Conv2D(64, (3,3), activation='relu')(x)\n    x = layers.LeakyReLU(alpha=0.01, name='Leaky_ReLU_2')(x)\n    x = layers.MaxPooling2D(3)(x)\n    x = layers.Dropout(0.1)(x)\n\n    for i in range(num_res_net_blocks):\n        x = res_net_block_1(x, 64)\n        x = res_net_block_2(x, 64)\n        \n    x = layers.Conv2D(64, 3, activation='relu')(x)\n    x = layers.LeakyReLU(alpha=0.01, name='Leaky_ReLU_3')(x)\n    x = layers.GlobalAveragePooling2D()(x)\n    \n    # dence layers\n    dense = layers.Dense(1024, activation='relu')(x)\n    dense = layers.Dropout(0.5)(dense)\n    dense = layers.Dense(512, activation='relu')(x)\n    dense = layers.Dropout(0.5)(dense)\n    \n    # output layers\n    head_root = layers.Dense(outputsizes[0], activation = 'softmax', name='dense_grapheme_root')(dense)\n    head_vowel = layers.Dense(outputsizes[1], activation = 'softmax', name='dense_vowel_diacritic')(dense)\n    head_consonant = layers.Dense(outputsizes[2], activation = 'softmax', name='dense_consonant_diacritic')(dense)\n    \n    model = tf.keras.Model(inputs, [head_root, head_vowel, head_consonant])\n    model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Generator to fit the multioutput network:"},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiOutputDataGenerator(ImageDataGenerator):\n    def flow(self,\n             x,\n             y=None,\n             batch_size=32,\n             shuffle=True,\n             sample_weight=None,\n             seed=None,\n             save_to_dir=None,\n             save_prefix='',\n             save_format='png',\n             subset=None):\n\n        targets = None\n        target_lengths = {}\n        ordered_outputs = []\n        for output, target in y.items():\n            if targets is None:\n                targets = target\n            else:\n                targets = np.concatenate((targets, target), axis=1)\n            target_lengths[output] = target.shape[1]\n            ordered_outputs.append(output)\n\n\n        for flowx, flowy in super().flow(x, targets, batch_size=batch_size,\n                                         shuffle=shuffle):\n            target_dict = {}\n            i = 0\n            for output in ordered_outputs:\n                target_length = target_lengths[output]\n                target_dict[output] = flowy[:, i: i + target_length]\n                i += target_length\n\n            yield flowx, target_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Training function:"},{"metadata":{"trusted":true},"cell_type":"code","source":"histories = []\ndef trainMultiOutput(ds_num, batch_size, epochs, model):\n    print('loading dataset %i' %  ds_num)\n    b_train_data = pd.merge(pd.read_parquet(input_dir + f'train_image_data_{ds_num}.parquet'), train_data, on='image_id').drop(['image_id'], axis=1)\n    gc.collect()\n    train_image = resize(b_train_data.drop(['grapheme_root', 'vowel_diacritic', 'consonant_diacritic', 'grapheme'], axis=1), shape)\n    train_image = train_image.values.reshape(-1, shape[1], shape[0], 1)\n    gc.collect()\n    \n    datagen = MultiOutputDataGenerator(featurewise_center=False,  # set input mean to 0 over the dataset\n                            samplewise_center=False,  # set each sample mean to 0\n                            featurewise_std_normalization=False,  # divide inputs by std of the dataset\n                            samplewise_std_normalization=False,  # divide each input by its std\n                            zca_whitening=False,  # apply ZCA whitening\n                            rotation_range=8,  # randomly rotate images in the range (degrees, 0 to 180)\n                            zoom_range=0.15,  # Randomly zoom image\n                            width_shift_range=0.15,  # randomly shift images horizontally (fraction of total width)\n                            height_shift_range=0.15,  # randomly shift images vertically (fraction of total height)\n                            horizontal_flip=False,  # randomly flip images\n                            vertical_flip=False)  # randomly flip images\n\n    # This will just calculate parameters required to augment the given data. This won't perform any augmentations\n    datagen.fit(train_image)\n\n    # traintest split\n    x_train = train_image\n    y_train_root = pd.get_dummies(b_train_data['grapheme_root']).values\n    y_train_vowel = pd.get_dummies(b_train_data['vowel_diacritic']).values\n    y_train_consonant = pd.get_dummies(b_train_data['consonant_diacritic']).values\n    \n    if traintestsplit > 0:\n        x_train, x_test, y_train_root, y_test_root, y_train_vowel, y_test_vowel, y_train_consonant, y_test_consonant = train_test_split(\n            train_image, y_train_root, y_train_vowel, y_train_consonant, test_size=traintestsplit, random_state=999)\n        del train_image\n        del b_train_data\n    \n    # fit\n    gc.collect()\n    history = model.fit_generator(datagen.flow(x_train, {'dense_grapheme_root': y_train_root, 'dense_vowel_diacritic': y_train_vowel, 'dense_consonant_diacritic': y_train_consonant},\n                                                   batch_size=batch_size),\n                                                   epochs=epochs, validation_data=(x_test, [y_test_root, y_test_vowel, y_test_consonant]) if traintestsplit > 0 else None,\n                                                   steps_per_epoch = x_train.shape[0] // batch_size,\n                                                   #callbacks=[learning_rate_reduction], \n                                                   verbose=2)\n    histories.append(history)\n    \n    del datagen\n    del y_train_root\n    del y_train_vowel\n    del y_train_consonant\n\n    if traintestsplit == 0:\n        del train_image\n        del b_train_data\n    else:\n        del x_train\n        del x_test\n        del y_test_root\n        del y_test_vowel\n        del y_test_consonant\n        \n    gc.collect()\n    print('trained')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train NN here:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TRAIN HERE\nmodel = resnet_multiOutput(shape, [168, 11, 7], network_deepth)\n\n# fit\ntrainMultiOutput(0, batch_size, epochs, model)\ntrainMultiOutput(1, batch_size, epochs, model)\ntrainMultiOutput(2, batch_size, epochs, model)\ntrainMultiOutput(3, batch_size, epochs, model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Showing train results:"},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\ndef plot_loss(his, epoch, title):\n    plt.style.use('ggplot')\n    plt.figure()\n    plt.plot(np.arange(0, epoch), his.history['loss'], label='train_loss')\n    plt.plot(np.arange(0, epoch), his.history['dense_grapheme_root_loss'], label='train_root_loss')\n    plt.plot(np.arange(0, epoch), his.history['dense_vowel_diacritic_loss'], label='train_vowel_loss')\n    plt.plot(np.arange(0, epoch), his.history['dense_consonant_diacritic_loss'], label='train_consonant_loss')\n\n    plt.plot(np.arange(0, epoch), his.history['dense_grapheme_root_loss'], label='val_train_root_loss')\n    plt.plot(np.arange(0, epoch), his.history['dense_vowel_diacritic_loss'], label='val_train_vowel_loss')\n    plt.plot(np.arange(0, epoch), his.history['dense_consonant_diacritic_loss'], label='val_train_consonant_loss')\n\n    plt.title(title)\n    plt.xlabel('Epoch #')\n    plt.ylabel('Loss')\n    plt.legend(loc='upper right')\n    plt.show()\n\n\ndef plot_acc(his, epoch, title):\n    plt.style.use('ggplot')\n    plt.figure()\n    plt.plot(np.arange(0, epoch), his.history['dense_grapheme_root_accuracy'], label='train_root_accuracy')\n    plt.plot(np.arange(0, epoch), his.history['dense_vowel_diacritic_accuracy'], label='train_vowel_accuracy')\n    plt.plot(np.arange(0, epoch), his.history['dense_consonant_diacritic_accuracy'], label='train_consonant_accuracy')\n\n    plt.plot(np.arange(0, epoch), his.history['dense_grapheme_root_accuracy'], label='val_root_acc')\n    plt.plot(np.arange(0, epoch), his.history['dense_vowel_diacritic_accuracy'], label='val_vowel_accuracy')\n    plt.plot(np.arange(0, epoch), his.history['dense_consonant_diacritic_accuracy'], label='val_consonant_accuracy')\n    plt.title(title)\n    plt.xlabel('Epoch #')\n    plt.ylabel('Accuracy')\n    plt.legend(loc='upper right')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for dataset in range(len(histories)):\n    plot_loss(histories[dataset], epochs, f'Training Dataset: {dataset}')\n    plot_acc(histories[dataset], epochs, f'Training Dataset: {dataset}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Loading test data:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# load test data\ntest_data = pd.read_csv(input_dir + 'test.csv')\nclass_map_df = pd.read_csv(input_dir + 'class_map.csv')\nsample_sub_data = pd.read_csv(input_dir + 'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Preparing forecast and submission:"},{"metadata":{"trusted":true},"cell_type":"code","source":"perdict = {\n    'grapheme_root': [],\n    'vowel_diacritic': [],\n    'consonant_diacritic': []\n}\n\ncomponents = ['consonant_diacritic', 'grapheme_root', 'vowel_diacritic']\ntarget=[] # model predictions placeholder\nrow_id=[] # row_id place holder\nn_cls = [7,168,11] # number of classes in each of the 3 targets\nfor i in range(4):\n    print('загружаем тестовые данные %i' % i)\n    df_test_img = pd.read_parquet(input_dir + 'test_image_data_{}.parquet'.format(i)) \n    df_test_img.set_index('image_id', inplace=True)\n\n    X_test = resize(df_test_img, shape)\n    X_test = X_test.values.reshape(-1, shape[1], shape[0], 1)\n\n    preds = model.predict(X_test)\n    for i, p in enumerate(perdict):\n        perdict[p] = np.argmax(preds[i], axis=1)\n\n    for k,id in enumerate(df_test_img.index.values):  \n        for i,comp in enumerate(components):\n            id_sample=id+'_'+comp\n            row_id.append(id_sample)\n            target.append(perdict[comp][k])\n\ndf_sample = pd.DataFrame(\n    {'row_id': row_id,\n    'target':target\n    },\n    columns =['row_id','target'] \n)\ndf_sample.to_csv('submission.csv',index=False)\nprint('submission saved !!!')\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sample.head(20)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}