{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Bengali.AI Training Notebook\n\nBy Team RausNaus, consisting of Edwin Wenink, Freek van den Bergh and Jordy Naus"},{"metadata":{},"cell_type":"markdown","source":"## About this notebook\n\n- This notebook offers an adjusted (pretrained) ResNet-50 network with the ability to freeze layers\n- Features: conversion to 224x224 RGB input, Cropping, Cutout, Adam optimizer, learning rate scheduler.\n- If PRETRAINED=True, this notebook requires internet connection\n- Otherwise, set PRETRAINED=False and optionally load in a model located at MODEL_PATH\n- Logging using \"Weights and Biases\" requires an API key\n- This committed notebook performs only one training epoch for illustration purposes. Adjust in the \"Training parameters\" section."},{"metadata":{},"cell_type":"markdown","source":"## Global Variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_HEIGHT = 137\nIMG_WIDTH = 236\n\nDATA_PATH = \"../input/bengaliai-cv19/\"\nFEATHER_PATH = \"../input/bengaliaicv19feather/\"\n\n# SET THESE BEFORE RUNNING\n\n# If USE_LOCAL_MODEL, then the model weights at MODEL_PATH will be loaded into the network\nUSE_LOCAL_MODEL = False\nMODEL_PATH = \"/kaggle/input/pretrainednet/rausnaus_resnet50_1584356517.pth\"\n# If PRETRAINED, weights pretrained on ImageNet will be loaded (USE_LOCAL_MODEL overrides this)\nPRETRAINED = True\n# Logging uses Weights and Biases (https://www.wandb.com/)\n# If MONITOR, provide your own API key\nMONITOR = False\nWANDB_ID = '[ID GOES HERE]'\nTRAINING = True\nFREEZE_FROM_LAYER = 0 #Choose 9 to only freeze \"classifier\" layers to ResNet50","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Imports\n\nAll packages required to run this notebook"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install torchtoolbox \n\nimport numpy as np\nimport pandas as pd \n\nimport random\nimport gc\nimport wandb\nimport time\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torchvision import transforms\nfrom torchvision import models\nfrom torchtoolbox.transform import Cutout\nfrom sklearn import metrics as metric\nimport cv2\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Install Weights and Biases in current session and login (Freek: idk of die hash mijn log-in is of dat ie algemeen is)\nif (MONITOR):\n    !pip install --upgrade wandb\n    !wandb login WANDB_ID\n    # Initialize wand logging\n    wandb.init(project=\"bengali-ai\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data"},{"metadata":{},"cell_type":"markdown","source":"## Data Transformation Classes"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomCropResize(object):\n\n    def __call__(self, image):\n        image_array = np.array(image)\n        cropped_image_array = self.crop_image(image_array)\n        return cv2.resize(self.remove_low_values(cropped_image_array), (224,224))\n\n    def __repr__(self):\n        return self.__class__.__name__ + '()'\n    \n    def remove_low_values(self, img, threshold=20):\n        \"\"\"\n        Sets low values to 0 to save memory and reduce noise\n        \"\"\"\n        return np.where(img < threshold, 0, img )\n\n    def boundary_box(self, img, original):\n        \"\"\"\n        Returns the x and y-values of the top, bottom, left, and right of the first non-zero entries in an array\n        Source: https://www.kaggle.com/iafoss/image-preprocessing-128x128\n        \"\"\" \n        # For any row/column containing >=1 True values, np.any returns True\n        rows = np.any(img, axis = 1)\n        cols = np.any(img, axis = 0)\n        # Select indices of the first and last row to be \"True\", i.e. have a non-zero element. \n        row_top, row_bottom = np.where(rows)[0][[0,-1]] \n        column_left, column_right = np.where(cols)[0][[0,-1]]   \n        return row_top, row_bottom, column_left, column_right\n\n    def crop_image(self, image, threshold = 40):\n        image = image[5:-5, 5:-5] \n        row_top, row_bottom, column_left, column_right = self.boundary_box(image > threshold, image)\n        image = image[row_top:row_bottom, column_left:column_right]\n        n_rows, n_cols = image.shape\n        diff = int(abs(n_rows - n_cols)/2)\n        # If dimension is odd, introduce padding to make it even (2*0.5)\n        fix = (n_rows+n_cols) % 2\n        if (n_rows > n_cols):           \n            padded_image = np.pad(image, [(0,0),(diff, diff+fix)], mode = 'constant')\n        else:\n            padded_image = np.pad(image, [(diff, diff+fix),(0,0)], mode = 'constant')\n        return padded_image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ToRGBArray(object):\n    \"\"\"\n    Converts a 1D array of shape (H,W) to a shape (3,H,W) by duplicating the original image\n    \"\"\"\n\n    def __call__(self, image):\n        rgb_image = np.repeat(np.expand_dims(image, axis=0), repeats=3, axis=0)\n        # The shape is now (W, 3, H) for some reason, but we want (3, H, W) so we move the axis\n        return np.moveaxis(rgb_image, 0, -1)\n\n    def __repr__(self):\n        return self.__class__.__name__ + '()'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset Class"},{"metadata":{"trusted":true},"cell_type":"code","source":"class BengaliDataset(Dataset):\n    \n    def __init__(self, file_nr, labels, transform=None):\n        self.transform = transform\n        \n        self.df = labels.merge(pd.read_feather(f'{FEATHER_PATH}train_image_data_{file_nr}.feather'), on=\"image_id\")\n        self.df = self.df.drop(['image_id'], axis=1)\n        self.data = self.df.iloc[:, 3:].values.reshape(-1, IMG_HEIGHT, IMG_WIDTH).astype(np.uint8)\n        \n    def __len__(self):\n        return self.data.shape[0]\n    \n    def __getitem__(self, index):\n        root, vowel, cons = self.df.iloc[index,:3]\n        img = self.data[index]\n        # Normalization (such that max value is always 255) and reducing memory fingerprint\n        img = ((255-img)*(255.0/img.max())).astype(np.uint8)\n        if self.transform:\n            img = self.transform(img)\n        return root, vowel, cons, img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Function for Creating Train/Validation DataLoaders"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_val_loaders(dataset, batch_size=64, shuffle=True, val_percentage=0.1):\n    dataset_size = len(dataset)\n    indices = list(range(dataset_size))\n    split = int(val_percentage * dataset_size)\n    if shuffle:\n        np.random.shuffle(indices)\n    train_indices, val_indices = indices[split:], indices[:split]\n\n    # Creating PT data samplers and loaders:\n    train_sampler = SubsetRandomSampler(train_indices)\n    val_sampler = SubsetRandomSampler(val_indices)\n\n    train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, \n                                               sampler=train_sampler)\n    val_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size,\n                                                sampler=val_sampler)\n    \n    return train_loader, val_loader","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{},"cell_type":"markdown","source":"## Model Class"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Net(nn.Module):\n    \n    def __init__(self, model):\n        super(Net, self).__init__()\n        self.model = model\n        fc_in = model.fc.in_features\n        \n        # Remove last layer of original model\n        self.model = nn.Sequential(*list(model.children())[:-1])\n        \n        # Define layers to be added\n        self.bn1  = nn.BatchNorm1d(fc_in)\n        self.drop1 = nn.Dropout(0.25)\n        self.lin1  = nn.Linear(fc_in, 512)\n        self.relu = nn.ReLU(inplace=False)\n        \n        self.bn2 = nn.BatchNorm1d(512)\n        self.drop2 = nn.Dropout(0.5)\n        \n        # Final layers\n        self.fc1 = nn.Linear(512, 11)\n        self.fc2 = nn.Linear(512,168)\n        self.fc3 = nn.Linear(512,7)\n    \n    def forward(self, x):\n        # Run input through ResNet50\n        x = self.model(x)\n        \n        # Turn x into the right shape\n        x = x.view(x.size(0), -1)\n        \n        # Put output x through our self defined layers\n        x = self.bn1(x)\n        x = self.drop1(x)\n        x = self.lin1(x)\n        x = self.relu(x)\n        \n        x = self.bn2(x)\n        x = self.drop2(x)\n        vowel_preds = self.fc1(x)\n        root_preds = self.fc2(x)\n        cons_preds = self.fc3(x)\n        \n        return vowel_preds, root_preds, cons_preds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load model and move to GPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"if USE_LOCAL_MODEL:\n    model.load_state_dict(torch.load(MODEL_PATH))\nelif PRETRAINED:\n    model = models.resnet50(pretrained=True)\nelse:\n    model = models.resnet50()\n\n# Freeze layers (depending on FREEZE_FROM_LAYER)\nlayer_count = 0\nfor child in model.children():\n    if layer_count < FREEZE_FROM_LAYER:\n        for name, param in child.named_parameters():\n            param.requires_grad = False\n            print(f\"Child {name} frozen\")\n    layer_count += 1\n    \n# Change the final layers of our model:\nmodel = Net(model)\n\nprint('-'*50)\n\nfor name, param in model.named_parameters():\n    if param.requires_grad:\n        print(f\"{name} is selected for finetuning\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Updated model:\")\nprint(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use GPU if GPU available, else use CPU\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\n\n# Tell model to use hardware available (CPU/GPU)\nmodel = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{},"cell_type":"markdown","source":"## Validation Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_weighted_metric(metric, preds, labels, weights, **kwargs):\n    metric_scores = []\n    for i in range(3):\n        metric_scores.append(metric(labels[i], preds[i], **kwargs))               \n    return np.average(metric_scores, weights=weights)\n\ndef validate(model, loss, scheduler, val_loader):\n    # put model in evaluation mode\n    model.eval()\n    \n    val_losses = []\n    r_preds, r_labels = [], []\n    v_preds, v_labels = [], []\n    c_preds, c_labels = [], []\n    \n    start_time = time.time()\n\n    with torch.no_grad():\n        for batch_idx, (roots, vowels, cons, imgs) in enumerate(val_loader):\n            # Each 50 batches print progress and how long those 50 batches took\n            if batch_idx%50 == 0:\n                print(f\"Batch {batch_idx+1}/{len(val_loader)} in {time.time()-start_time} sec.\")\n                start_time = time.time()\n                \n            # get predictions\n            roots = roots.to(device); vowels = vowels.to(device); cons = cons.to(device); imgs = imgs.to(device, dtype=torch.float32)\n            vowel_pred, root_pred, consonant_pred = model(imgs)\n\n            _, vowel_diacritic = torch.max(vowel_pred, 1)\n            _, grapheme_root = torch.max(root_pred, 1)\n            _, consonant_diacritic = torch.max(consonant_pred, 1)\n\n            # compute validation loss\n            vowel_loss = loss(vowel_pred, vowels)\n            root_loss = loss(root_pred, roots)\n            consonant_loss = loss(consonant_pred, cons)\n\n            # Append to history, to be averaged later\n            val_losses.append([vowel_loss, root_loss, consonant_loss])\n\n            # Keep track of predictions of accuracy and recall computations             \n            v_preds.extend(vowel_diacritic.cpu().numpy());     v_labels.extend(vowels.cpu().numpy())\n            r_preds.extend(grapheme_root.cpu().numpy());       r_labels.extend(roots.cpu().numpy())\n            c_preds.extend(consonant_diacritic.cpu().numpy()); c_labels.extend(cons.cpu().numpy())\n\n    weighted_accuracy = compute_weighted_metric(metric.accuracy_score, [v_preds, r_preds, c_preds], [v_labels, r_labels, c_labels], [1,2,1])\n    weighted_recall = compute_weighted_metric(metric.recall_score, [v_preds, r_preds, c_preds], [v_labels, r_labels, c_labels], [1,2,1], average=\"macro\")\n\n    # Compute loss\n    avg_val_loss = np.array(val_losses).sum(axis=0)/len(val_losses)\n    weighted_val_loss = np.sum(avg_val_loss)\n\n    # Scheduler volgt de validation loss\n    scheduler.step(weighted_val_loss)\n\n    if MONITOR:\n        wandb.log({\"Validation accuracy (weighted)\":weighted_accuracy})\n        wandb.log({\"Validation recall (weighted)\":weighted_recall})    \n        wandb.log({\"Validation vowel loss\":avg_val_loss[0], \"Validation root loss\":avg_val_loss[1], \"Validation consonant loss\":avg_val_loss[2], \"Validation weighted loss\":weighted_val_loss })","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, loss_function, optimizer, train_loader):\n    \n    model.train()\n    \n    start_time = time.time()\n\n    # Train on batch\n    for batch_idx, (roots, vowels, cons, imgs) in enumerate(train_loader):\n\n        # Each 50 batches print progress and how long those 50 batches took\n        if batch_idx%50 == 0:\n            print(f\"Batch {batch_idx+1}/{len(train_loader)} in {time.time()-start_time} sec.\")\n            start_time = time.time()\n\n        # In PyTorch you have to manually reset gradients before each mini_batch\n        optimizer.zero_grad()\n\n        # Get predictions\n        roots = roots.to(device); vowels = vowels.to(device); cons = cons.to(device); imgs = imgs.to(device, dtype=torch.float32)\n        vowel_pred, root_pred, consonant_pred = model(imgs)  \n\n        # Compute loss\n        vowel_loss = loss_function(vowel_pred, vowels)\n        root_loss = loss_function(root_pred, roots)\n        consonant_loss = loss_function(consonant_pred, cons)\n\n        # Update gradients\n        weighted_total_loss = vowel_loss + root_loss + consonant_loss\n        weighted_total_loss.backward()\n        optimizer.step()\n\n        weighted_loss = weighted_total_loss/3 \n\n        # Visualise losses after every batch in Weights & Biases\n        if (MONITOR):\n            wandb.log({\"Train vowel loss\":vowel_loss, \"Train root loss\":root_loss, \"Train consonant loss\":consonant_loss, \"Train total loss\":weighted_total_loss, \"Train weighted loss\":weighted_loss})\n\n    # Save model after training\n    torch.save(model.state_dict(), f'/kaggle/working/models/rausnaus_resnet50_{int(time.time())}.pth')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 1\nlearning_rate = 0.001\nbatch_size = 64\n\n# Filter on requires_grad: do not optimze frozen layers\noptimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=learning_rate)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=5, min_lr=1e-08)\nloss = nn.CrossEntropyLoss()\n\n# Preprocessing pipeline\npreprocess = transforms.Compose([\n    CustomCropResize(), # Crop and resize\n    ToRGBArray(), # Convert 1D image to 3D\n    Cutout(),\n    transforms.ToTensor(), # Convert to range [0,1]\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), # Normalise according to ImageNet\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create directory for saving models\n!mkdir /kaggle/working/models/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Monitor model with Weights & Biases\nif (MONITOR):\n    wandb.watch(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Actual training"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(os.path.join(DATA_PATH, \"train.csv\"))\ntrain_df.drop(['grapheme'], axis=1, inplace=True)\ntrain_df[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']] = train_df[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].astype('uint8')\n\n# If-statement for faster commiting\nif TRAINING:\n    for epoch in range(epochs):\n        print(f\"Epoch {epoch}/{epochs}\")\n        for datafile_index in random.sample([0,1,2,3],4):\n            print(f\"Loading parquet file {datafile_index}\")\n            dataset = BengaliDataset(datafile_index, train_df, transform=preprocess)\n            train_loader, val_loader = get_train_val_loaders(dataset, batch_size=batch_size, shuffle=True, val_percentage=0.1)\n\n            print(\"Training\")\n            train(model, loss, optimizer, train_loader)\n            print(\"Validation\")\n            validate(model, loss, scheduler, val_loader)\n            \n            del dataset\n            gc.collect()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}