{"cells":[{"metadata":{"trusted":true,"_kg_hide-output":false,"_kg_hide-input":false},"cell_type":"code","source":"!pip install prefetch_generator\nfrom prefetch_generator import background","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Environment Setup"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport gc\nimport os\n# define constants\nORIGINAL_HEIGHT = 137\nORIGINAL_WIDTH = 236\nPROCESSED_HEIGHT = 128\nPROCESSED_WIDTH = 128\n#for generator batching\nROWS_PER_FILE = 20084+1\nNUM_FILES = 200840 // ROWS_PER_FILE\nVALID_SIZE = 200840 % ROWS_PER_FILE\n\"\"\"Set manually depending on chosen preprocessing\"\"\"\nTRAIN_BATCH_SIZE = 195\nVALID_BATCH_SIZE = 365\n\"\"\"End of choose manually\"\"\"\nassert ROWS_PER_FILE % TRAIN_BATCH_SIZE == 0, \"TRAIN_BATCH_SIZE is not a divisor of ROWS_PER_FILE\"\nassert VALID_SIZE % VALID_BATCH_SIZE == 0, \"VALID_BATCH_SIZE is not a divisor of VALID_SIZE\"\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"## Grid Mask\n# code takesn from https://www.kaggle.com/haqishen/gridmask\n\nimport albumentations\nfrom albumentations.core.transforms_interface import DualTransform, ImageOnlyTransform\nfrom albumentations.augmentations import functional as F\n\nclass GridMask(DualTransform):\n    \"\"\"GridMask augmentation for image classification and object detection.\n\n    Args:\n        num_grid (int): number of grid in a row or column.\n        fill_value (int, float, lisf of int, list of float): value for dropped pixels.\n        rotate ((int, int) or int): range from which a random angle is picked. If rotate is a single int\n            an angle is picked from (-rotate, rotate). Default: (-90, 90)\n        mode (int):\n            0 - cropout a quarter of the square of each grid (left top)\n            1 - reserve a quarter of the square of each grid (left top)\n            2 - cropout 2 quarter of the square of each grid (left top & right bottom)\n\n    Targets:\n        image, mask\n\n    Image types:\n        uint8, float32\n\n    Reference:\n    |  https://arxiv.org/abs/2001.04086\n    |  https://github.com/akuxcw/GridMask\n    \"\"\"\n\n    def __init__(self, num_grid=3, fill_value=0, rotate=0, mode=0, always_apply=False, p=0.5):\n        super(GridMask, self).__init__(always_apply, p)\n        if isinstance(num_grid, int):\n            num_grid = (num_grid, num_grid)\n        if isinstance(rotate, int):\n            rotate = (-rotate, rotate)\n        self.num_grid = num_grid\n        self.fill_value = fill_value\n        self.rotate = rotate\n        self.mode = mode\n        self.masks = None\n        self.rand_h_max = []\n        self.rand_w_max = []\n\n    def init_masks(self, height, width):\n        if self.masks is None:\n            self.masks = []\n            n_masks = self.num_grid[1] - self.num_grid[0] + 1\n            for n, n_g in enumerate(range(self.num_grid[0], self.num_grid[1] + 1, 1)):\n                grid_h = height / n_g\n                grid_w = width / n_g\n                this_mask = np.ones((int((n_g + 1) * grid_h), int((n_g + 1) * grid_w))).astype(np.uint8)\n                for i in range(n_g + 1):\n                    for j in range(n_g + 1):\n                        this_mask[\n                             int(i * grid_h) : int(i * grid_h + grid_h / 2),\n                             int(j * grid_w) : int(j * grid_w + grid_w / 2)\n                        ] = self.fill_value\n                        if self.mode == 2:\n                            this_mask[\n                                 int(i * grid_h + grid_h / 2) : int(i * grid_h + grid_h),\n                                 int(j * grid_w + grid_w / 2) : int(j * grid_w + grid_w)\n                            ] = self.fill_value\n                \n                if self.mode == 1:\n                    this_mask = 1 - this_mask\n\n                self.masks.append(this_mask)\n                self.rand_h_max.append(grid_h)\n                self.rand_w_max.append(grid_w)\n\n    def apply(self, image, mask, rand_h, rand_w, angle, **params):\n        h, w = image.shape[:2]\n        mask = F.rotate(mask, angle) if self.rotate[1] > 0 else mask\n        mask = mask[:,:,np.newaxis] if image.ndim == 3 else mask\n        image *= mask[rand_h:rand_h+h, rand_w:rand_w+w].astype(image.dtype)\n        return image\n\n    def get_params_dependent_on_targets(self, params):\n        img = params['image']\n        height, width = img.shape[:2]\n        self.init_masks(height, width)\n\n        mid = np.random.randint(len(self.masks))\n        mask = self.masks[mid]\n        rand_h = np.random.randint(self.rand_h_max[mid])\n        rand_w = np.random.randint(self.rand_w_max[mid])\n        angle = np.random.randint(self.rotate[0], self.rotate[1]) if self.rotate[1] > 0 else 0\n\n        return {'mask': mask, 'rand_h': rand_h, 'rand_w': rand_w, 'angle': angle}\n\n    @property\n    def targets_as_params(self):\n        return ['image']\n\n    def get_transform_init_args_names(self):\n        return ('num_grid', 'fill_value', 'rotate', 'mode')\n    \ngridMaskTransform = albumentations.Compose([\n    GridMask(num_grid=3, rotate=15, p=1),\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data loading and preparation"},{"metadata":{},"cell_type":"markdown","source":"I am loading the preprocessed data to avoid having to wait ~10min whenever I start the Kernel. Preprocessing Kernel available [here](https://www.kaggle.com/larswigger/bengali-preprocessing). Since I do not have enough RAM to load all images in 128x128 Pixels I have to use generators."},{"metadata":{"trusted":true},"cell_type":"code","source":"@background(max_prefetch=1)\ndef train_data_generator_function():\n    while True:\n        for file_index in range(NUM_FILES):\n            X = np.load(f\"/kaggle/input/bengali-preprocessing/processed_20085_128_{file_index}.npy\").reshape(-1, PROCESSED_HEIGHT, PROCESSED_WIDTH, 1)\n            root = np.load(f\"/kaggle/input/bengali-preprocessing/root_20085_label_{file_index}.npy\")\n            vowel = np.load(f\"/kaggle/input/bengali-preprocessing/vowel_20085_label_{file_index}.npy\")\n            consonant = np.load(f\"/kaggle/input/bengali-preprocessing/consonant_20085_label_{file_index}.npy\")\n            for batch_index in range(ROWS_PER_FILE // TRAIN_BATCH_SIZE):             \n                #yprocess images in batch range\n                for i in range(batch_index*TRAIN_BATCH_SIZE,(batch_index+1)*TRAIN_BATCH_SIZE):\n                        X[i] = gridMaskTransform(image=X[i])[\"image\"]\n                yield (X[batch_index*TRAIN_BATCH_SIZE:(batch_index+1)*TRAIN_BATCH_SIZE], \n                       [root[batch_index*TRAIN_BATCH_SIZE:(batch_index+1)*TRAIN_BATCH_SIZE],\n                        vowel[batch_index*TRAIN_BATCH_SIZE:(batch_index+1)*TRAIN_BATCH_SIZE],\n                        consonant[batch_index*TRAIN_BATCH_SIZE:(batch_index+1)*TRAIN_BATCH_SIZE]])              \n            del X, root, vowel, consonant\n            gc.collect()\n\ntrain_data_generator = train_data_generator_function()        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#generator does not provide advantage for this one\nX = np.load(\"/kaggle/input/bengali-preprocessing/processed_20085_128_valid.npy\").reshape(-1, PROCESSED_HEIGHT, PROCESSED_WIDTH, 1)\nroot = np.load(\"/kaggle/input/bengali-preprocessing/root_20085_label_valid.npy\")\nvowel = np.load(\"/kaggle/input/bengali-preprocessing/vowel_20085_label_valid.npy\")\nconsonant = np.load(\"/kaggle/input/bengali-preprocessing/consonant_20085_label_valid.npy\")\nvalid_data = (X, [root, vowel, consonant])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model definition"},{"metadata":{},"cell_type":"markdown","source":"Model based on [this one](https://www.kaggle.com/kaushal2896/bengali-graphemes-starter-eda-multi-output-cnn/notebook#Basic-Model)"},{"metadata":{"trusted":true},"cell_type":"code","source":"inputs = tf.keras.layers.Input(shape=(128,128,1))\nmodel = tf.keras.layers.Lambda(lambda x: x / 255.0)(inputs)#rescaling before passing into main model\nmodel = tf.keras.layers.Conv2D(3, (3, 3), padding='same')(model)\n\nresnet50 = tf.keras.applications.ResNet50(include_top=False, weights=\"imagenet\", input_shape=(128,128,3), pooling=\"max\")(model)\nresnet50.trainable = False\nmodel = tf.keras.layers.Flatten()(resnet50)\nmodel = tf.keras.layers.BatchNormalization()(model)\nmodel = tf.keras.layers.Dropout(0.3)(model)\nmodel = tf.keras.layers.Dense(256, activation=\"relu\")(model)\nmodel = tf.keras.layers.BatchNormalization()(model)\nmodel = tf.keras.layers.Dropout(0.3)(model)\n\nhead_root = tf.keras.layers.Dense(168, activation=\"softmax\", name=\"Root\")(model)\nhead_vowel = tf.keras.layers.Dense(11, activation=\"softmax\", name=\"Vowel\")(model)\nhead_consonant = tf.keras.layers.Dense(7, activation=\"softmax\", name=\"Consonant\")(model)\n\nmodel = tf.keras.Model(inputs=inputs, outputs=[head_root, head_vowel, head_consonant])\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.utils.plot_model(model, to_file='model.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate_reduction_root = tf.keras.callbacks.ReduceLROnPlateau(monitor='Root_accuracy', \n                                            patience=3, \n                                            verbose=1,\n                                            factor=0.5, \n                                            min_lr=0.00001)\nlearning_rate_reduction_vowel = tf.keras.callbacks.ReduceLROnPlateau(monitor='Vowel_accuracy', \n                                            patience=3, \n                                            verbose=1,\n                                            factor=0.5, \n                                            min_lr=0.00001)\nlearning_rate_reduction_consonant = tf.keras.callbacks.ReduceLROnPlateau(monitor='Consonant_accuracy', \n                                            patience=3, \n                                            verbose=1,\n                                            factor=0.5, \n                                            min_lr=0.00001)\nbackup_callback = tf.keras.callbacks.ModelCheckpoint(filepath=\"backup_{epoch}.h5\",\n                                                     save_weights_only=False,\n                                                     period=5,\n                                                     verbosity=1)\n\nlast_model_callback = tf.keras.callbacks.ModelCheckpoint(filepath=\"Last_Model.h5\",\n                                                        save_weights_only=False,\n                                                        verbosity=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 20 #for reuse later on\nhistory = model.fit(train_data_generator,\n                    validation_data=valid_data,\n                    epochs=EPOCHS,\n                    steps_per_epoch=NUM_FILES*(ROWS_PER_FILE // TRAIN_BATCH_SIZE),\n                    #validation_steps=(VALID_SIZE // VALID_BATCH_SIZE),\n                    callbacks=[learning_rate_reduction_root , learning_rate_reduction_vowel, learning_rate_reduction_consonant, backup_callback, last_model_callback])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Analysis"},{"metadata":{},"cell_type":"markdown","source":"Analyzing Training History"},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_df = pd.DataFrame(history.history)\nhist_df[\"Epoch\"] = np.arange(len(hist_df))+1\nhist_df.set_index(\"Epoch\", inplace=True)\nwith open('history.csv', mode='w') as f:\n    hist_df.to_csv(f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plot loss\nselection = [col for col in hist_df.columns if \"loss\" in col]\nhist_df[selection].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plot accuracy\nselection = [col for col in hist_df.columns if \"accuracy\" in col]\nhist_df[selection].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Freed after analysis: \" + str(gc.collect()))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}