{"cells":[{"metadata":{"_uuid":"4d1e3bfb-2e12-4de3-ba73-3afcb1817ce6","_cell_guid":"dfcf9537-08c5-4872-92d9-a7e245b9305e","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nfrom keras.models import Model\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization, Input, GlobalAveragePooling2D\nfrom keras.optimizers import RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\nimport keras_applications\nfrom keras.initializers import glorot_uniform\nimport gc\nimport os\nimport cv2\nimport keras\nresnext = keras_applications.resnext.ResNeXt50(\n    backend=keras.backend,\n    layers=keras.layers,\n    models=keras.models,\n    utils=keras.utils,\n    include_top=False, \n    weights='imagenet',\n    input_shape=(64,64,3)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CUTMIX_ALPHA = 1\nCUTMIX_PROB = 0.25\n\ndimension = 64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = pd.read_feather(\"/kaggle/input/bengali64pp/train64.feather\")\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = x_train.drop([\"image_id\",\"index\"],axis=1)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = x_train.values.reshape(-1,dimension,dimension,1)/255.\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = pd.read_feather(\"/kaggle/input/bengali64pp/test64.feather\").drop([\"image_id\",\"index\"],axis=1)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = x_test.values.reshape(-1,dimension,dimension,1)/255.\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiOutputDataGenerator(tf.keras.preprocessing.image.ImageDataGenerator):\n    \n    def cutmix(self, X1, y1, X2, y2, ordered_outputs, target_lengths):\n        lam = np.random.beta(CUTMIX_ALPHA, CUTMIX_ALPHA)\n        bx1, by1, bx2, by2 = rand_bbox(X1.shape, lam)\n        \n        X1[:, bx1:bx2, by1:by2, :] = X2[:, bx1:bx2, by1:by2, :]\n        X = X1\n        target_dict = {}\n        i = 0\n        for output in ordered_outputs:\n            target_length = target_lengths[output]\n            target_dict[output] = y1[:, i: i + target_length] * lam + y2[:, i: i + target_length] * (1 - lam)\n            i += target_length\n        y = None\n        for output, target in target_dict.items():\n            if y is None:\n                y = target\n            else:\n                y = np.concatenate((y, target), axis=1)\n        return X, y\n\n    def flow(self,\n             x,\n             y=None,\n             batch_size=32,\n             shuffle=True,\n             sample_weight=None,\n             seed=None,\n             save_to_dir=None,\n             save_prefix='',\n             save_format='png',\n             subset=None):\n\n        targets = None\n        target_lengths = {}\n        ordered_outputs = []\n        for output, target in y.items():\n            if targets is None:\n                targets = target\n            else:\n                targets = np.concatenate((targets, target), axis=1)\n            target_lengths[output] = target.shape[1]\n            ordered_outputs.append(output)\n\n        batches = super().flow(x, targets, batch_size=batch_size, shuffle=shuffle)\n        \n        while True:\n            batch_x, batch_y = next(batches)\n            \n            while True:\n                batch_x_2, batch_y_2 = next(batches)\n                m1, m2 = batch_x.shape[0], batch_x_2.shape[0]\n                if m1 < m2:\n                    batch_x_2 = batch_x_2[:m1]\n                    batch_y_2 = batch_y_2[:m1]\n                    break\n                elif m1 == m2:\n                    break\n            if np.random.rand() < CUTMIX_PROB:\n                batch_x, batch_y = self.cutmix(batch_x, batch_y, batch_x_2, batch_y_2, ordered_outputs, target_lengths)\n                \n            target_dict = {}\n            i = 0\n            for output in ordered_outputs:\n                target_length = target_lengths[output]\n                target_dict[output] = batch_y[:, i: i + target_length]\n                i += target_length\n\n            yield batch_x, target_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n    return bbx1, bby1, bbx2, bby2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cdaa43f4-9d44-4d18-9a97-62f51349ab22","_cell_guid":"f5831245-b38f-4f49-a9b3-4012904e8c93","trusted":true},"cell_type":"code","source":"datagen = MultiOutputDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=5,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0, # Randomly zoom image \n        shear_range = 5,\n        width_shift_range=0,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_labels(labels):\n    labels = labels[['grapheme_root','vowel_diacritic','consonant_diacritic']].astype(\"uint8\")\n    Y_train_root = pd.get_dummies(labels['grapheme_root']).values\n    Y_train_vowel = pd.get_dummies(labels['vowel_diacritic']).values\n    Y_train_consonant = pd.get_dummies(labels['consonant_diacritic']).values\n    return Y_train_root,Y_train_vowel,Y_train_consonant","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_Y = pd.read_feather(\"/kaggle/input/bengali64pp/trainLabels64.feather\")\ntest_Y = pd.read_feather(\"/kaggle/input/bengali64pp/testLabels64.feather\")\n\ny_train_root,y_train_vowel,y_train_consonant = to_labels(train_Y)\ny_test_root,y_test_vowel,y_test_consonant = to_labels(test_Y)\n\nprint('fit datagen')\ndatagen.fit(x_train[:1000,:,:,:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set a learning rate annealer. Learning rate will be half after 3 epochs if accuracy is not increased\nlearning_rate_reduction_root = ReduceLROnPlateau(monitor='root_accuracy', \n                                            patience=14, \n                                            verbose=1,\n                                            factor=0.5, \n                                            min_lr=0.00001)\nlearning_rate_reduction_vowel = ReduceLROnPlateau(monitor='vowel_accuracy', \n                                            patience=10, \n                                            verbose=1,\n                                            factor=0.5, \n                                            min_lr=0.00001)\nlearning_rate_reduction_consonant = ReduceLROnPlateau(monitor='consonant_accuracy', \n                                            patience=10, \n                                            verbose=1,\n                                            factor=0.5, \n                                            min_lr=0.00001)\n\n# Save the best model (the most accurate model on validation data)\ncheckpoint_cb = keras.callbacks.ModelCheckpoint(\"resnextMultiFull.h5\", save_best_only=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#define model\nX_input = Input((dimension,dimension,1))\n\nx = Conv2D(filters=3,kernel_size=(3,3),padding='same',activation='relu')(X_input)\nx = resnext(x)\nprint('premade Loaded')\nx = GlobalAveragePooling2D()(x)\n\nhead_root = Dense(168, activation = 'softmax', kernel_initializer=glorot_uniform(seed=0), name=\"root\")(x)\nhead_vowel = Dense(11, activation = 'softmax', kernel_initializer=glorot_uniform(seed=0), name=\"vowel\")(x)\nhead_consonant = Dense(7, activation = 'softmax', kernel_initializer=glorot_uniform(seed=0), name=\"consonant\")(x)\n\nmodel = Model(inputs=X_input, outputs=[head_root, head_vowel, head_consonant], name='DenseRetrain')\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lossWeights={\"root\": 2, \"vowel\": 1, \"consonant\": 1}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer='rmsprop',\n              loss='categorical_crossentropy',\n              metrics=['accuracy'],\n             loss_weights=lossWeights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\ntrain_steps = x_train.shape[0] // batch_size\nvalid_steps = x_test.shape[0] // batch_size\n\ntrain_history = model.fit_generator(\n    datagen.flow(x_train, {'root': y_train_root, 'vowel': y_train_vowel, 'consonant': y_train_consonant},batch_size=batch_size),\n    epochs=50,\n    callbacks=[learning_rate_reduction_root,learning_rate_reduction_vowel,learning_rate_reduction_consonant,checkpoint_cb],\n    validation_data=(x_test,[y_test_root, y_test_vowel, y_test_consonant]),\n    steps_per_epoch = train_steps,\n    validation_steps = valid_steps\n)\n\npd.DataFrame(train_history.history).to_csv('resnextHistory.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}