{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Make npy Files"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport cv2\nfrom tqdm import tqdm\nimport os\n\nDIR = \"../input/bengaliai-cv19/\"\nfiles = [\"train_image_data_0.parquet\",\"train_image_data_1.parquet\",\"train_image_data_2.parquet\",\"train_image_data_3.parquet\"]\n\nif not os.path.isfile('train_full_128.npy'):\n    all_image_list = []\n    for f in files:\n        path = DIR + f\n        df = pd.read_parquet(path)\n        values = 255 - df.iloc[:, 1:].values.reshape(-1, 137, 236).astype(np.uint8)\n        img_list = []\n        for i in tqdm(range(len(values))):\n            img = cv2.resize(values[i],(128,128))\n            img_list.append(img)\n\n        img_list = np.array(img_list)\n        all_image_list.append(img_list)\n\n    all_image_list = np.concatenate(all_image_list)\n    np.save(\"train_full_128\",all_image_list)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Clean and clear memory"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import gc\ndel all_image_list\ndel img_list\ndel df\ndel img\ndel values\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset\nimport pandas as pd\nimport torch\nimport numpy as np\nclass BengaliDataset2(Dataset):\n    def __init__(self,npy_file,label_csv,aug=None,norm=None):\n        self.npy_file = np.load(npy_file)\n        self.norm = norm\n        df = pd.read_csv(label_csv)\n        # for faster access i think\n        self.grapheme_root = df[\"grapheme_root\"].values\n        self.vowel_diacritic = df[\"vowel_diacritic\"].values\n        self.consonant_diacritic = df[\"consonant_diacritic\"].values\n\n        self.aug = aug\n\n    def __getitem__(self, index):\n        image_arr = self.npy_file[index]\n        # only do this on training\n        #use albumentations library\n        if self.aug != None:\n            image_arr = self.aug(image=image_arr)[\"image\"]\n\n        image_arr = (image_arr/255).astype(np.float32)\n        image_arr = torch.from_numpy(image_arr)\n\n        if self.norm != None:\n            mean = self.norm['mean']\n            std = self.norm['std']\n            image_arr = (image_arr -  mean)/std\n\n        grapheme_root = torch.Tensor([self.grapheme_root[index]]).long()\n        vowel_diacritic = torch.Tensor([self.vowel_diacritic[index]]).long()\n        consonant_diacritic = torch.Tensor([self.consonant_diacritic[index]]).long()\n        \n        return {\"image\":image_arr.unsqueeze(0).repeat(3, 1, 1),\"grapheme_root\":grapheme_root,\"vowel_diacritic\":vowel_diacritic,\"consonant_diacritic\":consonant_diacritic}\n\n    def __len__(self):\n        return self.npy_file.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations\n\nmean = 13.4/255\nstd = 40.8/255\n\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\nfrom typing import Tuple, List, Dict\nclass ImageTransformer:\n    \"\"\"\n    DataAugmentor for Image Classification.\n    Args:\n        data_augmentations: List of tuple(method: str, params :dict), each elems pass to albumentations\n    \"\"\"\n\n    def __init__(self, data_augmentations: List[Tuple[str, Dict]]):\n        \"\"\"Initialize.\"\"\"\n        augmentations_list = [\n            self._get_augmentation(aug_name)(**params)\n            for aug_name, params in data_augmentations]\n        self.data_aug = albumentations.Compose(augmentations_list)\n    \n    def __call__(self,image):\n        return self.data_aug(image=image)\n    \n    def __call2__(self, pair: Tuple[np.ndarray]) -> Tuple[np.ndarray]:\n        \"\"\"Forward\"\"\"\n        img_arr, label = pair\n        return self.data_aug(image=img_arr)[\"image\"], label\n\n    def _get_augmentation(self, aug_name: str) -> ImageOnlyTransform:\n        \"\"\"Get augmentations from albumentations\"\"\"\n        if hasattr(albumentations, aug_name):\n            return getattr(albumentations, aug_name)\n        else:\n            return eval(aug_name)\n        \nclass RandomErasing(ImageOnlyTransform):\n    \"\"\"Class of RandomErase for Albumentations.\"\"\"\n\n    def __init__(\n        self, s: Tuple[float]=(0.02, 0.4), r: Tuple[float]=(0.3, 2.7),\n        mask_value_min: int=0, mask_value_max: int=255,\n        always_apply: bool=False, p: float=1.0\n    ) -> None:\n        \"\"\"Initialize.\"\"\"\n        super().__init__(always_apply, p)\n        self.s = s\n        self.r = r\n        self.mask_value_min = mask_value_min\n        self.mask_value_max = mask_value_max\n\n    def apply(self, image: np.ndarray, **params):\n        \"\"\"\n        Apply transform.\n        Note: Input image shape is (Height, Width, Channel).\n        \"\"\"\n        image_copy = np.copy(image)\n\n        # # decide mask value randomly\n        mask_value = np.random.randint(self.mask_value_min, self.mask_value_max + 1)\n\n        h, w = image.shape\n        # # decide num of pixcels for mask.\n        mask_area_pixel = np.random.randint(h * w * self.s[0], h * w * self.s[1])\n\n        # # decide aspect ratio for mask.\n        mask_aspect_ratio = np.random.rand() * self.r[1] + self.r[0]\n\n        # # decide mask hight and width\n        mask_height = int(np.sqrt(mask_area_pixel / mask_aspect_ratio))\n        if mask_height > h - 1:\n            mask_height = h - 1\n        mask_width = int(mask_aspect_ratio * mask_height)\n        if mask_width > w - 1:\n            mask_width = w - 1\n\n        # # decide position of mask.\n        top = np.random.randint(0, h - mask_height)\n        left = np.random.randint(0, w - mask_width)\n        bottom = top + mask_height\n        right = left + mask_width\n        image_copy[top:bottom, left:right].fill(mask_value)\n\n        return image_copy\n    \naugment = ImageTransformer([('RandomErasing',{'p':0.5})])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = BengaliDataset2(\"train_full_128.npy\",\"../input/bengaliai-cv19/train.csv\",aug =augment,norm={'mean':mean,'std':std})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import DataLoader\nimport torchvision.transforms as transforms\nimport torchvision.models as models\nimport time\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport copy\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import recall_score,confusion_matrix,ConfusionMatrixDisplay,classification_report","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, criterion, optimizer, device, dataloaders, scheduler=None, num_epochs=25):\n    since = time.time()\n\n    best_recall = 0.0\n    \n    dataset_sizes = {'train': len(dataloaders['train'].dataset)}\n\n    train_acc_list = []; train_loss_list= []; val_acc_list = []; val_loss_list = []; unseen_acc_list = []; unseen_loss_list = []\n\n    for epoch in range(num_epochs):\n        start = time.time()\n        print('Epoch {}/{}'.format(epoch+1, num_epochs))\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train']:\n            \n            #used for calculating recall per epoch\n            grapheme_output = []\n            vowel_output = []\n            consonant_output = []\n            grapheme_label = []\n            vowel_label = []\n            consonant_label = []\n\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            grapheme_corrects = 0\n            vowel_corrects = 0\n            consonant_corrects = 0\n\n            # Iterate over data.\n            for i,data in enumerate(dataloaders[phase]):\n\n                inputs = data['image']\n                grapheme_root_label = data['grapheme_root']\n                vowel_diacritic_label = data['vowel_diacritic']\n                consonant_diacritic_label = data['consonant_diacritic']\n                inputs = inputs.to(device)\n\n                grapheme_root_label =  grapheme_root_label.to(device)\n                vowel_diacritic_label = vowel_diacritic_label.to(device)\n                consonant_diacritic_label =  consonant_diacritic_label.to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    g,v,c = model(inputs)\n                    \n                    grapheme_preds = g.argmax(dim=1)\n                    vowel_preds = v.argmax(dim=1) \n                    consonant_preds = c.argmax(dim=1)\n\n                    loss = criterion(g,v,c, grapheme_root_label.squeeze(1),vowel_diacritic_label.squeeze(1),consonant_diacritic_label.squeeze(1))\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n                  \n                # statistics\n                running_loss += loss.item() * inputs.size(0)\n                #For accuracy\n                grapheme_corrects += torch.sum(grapheme_preds == grapheme_root_label.data.squeeze(1))\n                vowel_corrects += torch.sum(vowel_preds == vowel_diacritic_label.data.squeeze(1))\n                consonant_corrects += torch.sum(consonant_preds== consonant_diacritic_label.data.squeeze(1))\n\n                if phase == 'train':\n                  scheduler.step(epoch+i/dataset_sizes['train'])\n                \n            if phase == 'val' or phase == 'unseen':\n              grapheme_final_output = torch.cat(grapheme_output)    \n              grapheme_final_label =  torch.cat(grapheme_label)\n              \n              vowel_final_output = torch.cat(vowel_output)    \n              vowel_final_label =  torch.cat(vowel_label)\n              \n              consonant_final_output = torch.cat(consonant_output)    \n              consonant_final_label =  torch.cat(consonant_label)\n\n              grapheme_recall = recall_score(grapheme_final_output,grapheme_final_label,average='macro')\n              vowel_recall = recall_score(vowel_final_output,vowel_final_label,average='macro')\n              consonant_recall = recall_score(consonant_final_output,consonant_final_label,average='macro')\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            running_corrects = 0.5*grapheme_corrects.double() + 0.25*vowel_corrects.double() + 0.25*consonant_corrects.double()\n\n            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n            epoch_acc = running_corrects / dataset_sizes[phase]\n\n            \n          \n            if phase == \"train\":\n                # Note this are running values (calculated per batch) rather than actual values at the end of each epoch\n                # Decreases training time\n                # Not accurate especially at first few epochs\n                train_acc_list.append(epoch_acc)\n                train_loss_list.append(epoch_loss)\n          \n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n            \n                \n        end = time.time()\n        print(f\"time per epoch:{end-start}s\")\n\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(\n        time_elapsed // 60, time_elapsed % 60))\n    print('Best val recall: {:4f}'.format(best_recall))\n\n    plots = (train_acc_list,train_loss_list,val_acc_list,val_loss_list,unseen_acc_list,unseen_loss_list)\n\n    return model, plots\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def loss(grapheme_root_output,vowel_diacritic_output,consonant_diacritic_output,grapheme_root_label,vowel_diacritic_label,consonant_diacritic_label):\n    gloss = nn.CrossEntropyLoss()(grapheme_root_output,grapheme_root_label)\n    vloss = nn.CrossEntropyLoss()(vowel_diacritic_output,vowel_diacritic_label)\n    closs = nn.CrossEntropyLoss()(consonant_diacritic_output,consonant_diacritic_label)\n\n    return 0.5*gloss + 0.25*vloss + 0.25*closs\n\ndef evaluate_test(model,criterion,dataloader,device):\n    model.eval()\n    running_loss = 0.0\n    grapheme_corrects = 0.0\n    vowel_corrects = 0.0\n    consonant_corrects = 0.0\n    \n    grapheme_output = []\n    vowel_output = []\n    consonant_output = []\n\n    grapheme_label = []\n    vowel_label = []\n    consonant_label = []\n\n\n    for data in dataloader:\n\n        inputs = data['image']\n\n        grapheme_root_label = data['grapheme_root']\n        vowel_diacritic_label = data['vowel_diacritic']\n        consonant_diacritic_label = data['consonant_diacritic']\n\n        inputs = inputs.to(device)\n\n        grapheme_root_label =  grapheme_root_label.to(device)\n        vowel_diacritic_label = vowel_diacritic_label.to(device)\n        consonant_diacritic_label =  consonant_diacritic_label.to(device)\n\n        with torch.no_grad():\n            g,v,c = model(inputs)\n\n            loss = criterion(g,v,c, grapheme_root_label.squeeze(1),vowel_diacritic_label.squeeze(1),consonant_diacritic_label.squeeze(1))\n            grapheme_preds = g.argmax(dim=1)\n            vowel_preds = v.argmax(dim=1)\n            consonant_preds = c.argmax(dim=1)\n\n        # statistics\n        running_loss += loss.item() * inputs.size(0)\n        \n\n        grapheme_corrects += torch.sum(grapheme_preds == grapheme_root_label.data.squeeze(1))\n        vowel_corrects += torch.sum(vowel_preds == vowel_diacritic_label.data.squeeze(1))\n        consonant_corrects += torch.sum(consonant_preds== consonant_diacritic_label.data.squeeze(1))\n        \n\n        grapheme_output.append(grapheme_preds.cpu())\n        grapheme_label.append(grapheme_root_label.data.squeeze(1).cpu())\n        vowel_output.append(vowel_preds.cpu())\n        vowel_label.append(vowel_diacritic_label.data.squeeze(1).cpu())\n        consonant_output.append(consonant_preds.cpu())\n        consonant_label.append(consonant_diacritic_label.data.squeeze(1).cpu())\n\n    grapheme_final_output = torch.cat(grapheme_output)    \n    grapheme_final_label =  torch.cat(grapheme_label)\n    \n    vowel_final_output = torch.cat(vowel_output)    \n    vowel_final_label =  torch.cat(vowel_label)\n    \n    consonant_final_output = torch.cat(consonant_output)    \n    consonant_final_label =  torch.cat(consonant_label)\n  \n\n    grapheme_recall = recall_score(grapheme_final_output,grapheme_final_label,average='macro')\n    vowel_recall = recall_score(vowel_final_output,vowel_final_label,average='macro')\n    consonant_recall = recall_score(consonant_final_output,consonant_final_label,average='macro')\n\n    print(\"grapheme recall:\",grapheme_recall)\n    print(\"vowel_recall:\",vowel_recall)\n    print(\"consonant_recall:\",consonant_recall)\n\n    print(\"final recall:\",0.5*grapheme_recall+0.25*vowel_recall+0.25*consonant_recall)\n\n    # print(classification_report(grapheme_final_label,grapheme_final_output))\n    # fig, axs = plt.subplots()\n    # fig.set_figheight(15)\n    # fig.set_figwidth(15)\n    # cm_vowel = confusion_matrix(grapheme_final_label,grapheme_final_output,normalize='true')\n    # cm = ConfusionMatrixDisplay(cm_vowel,[x for x in range(168)])\n    # cm.plot(ax=axs)\n\n\n    loss = running_loss / len(dataloader.dataset)\n\n    running_corrects = 0.5*grapheme_corrects.double() + 0.25*vowel_corrects.double() + 0.25*consonant_corrects.double()\n\n    # epoch_acc = running_corrects.double() / dataset_sizes[phase]\n    acc = running_corrects / len(dataloader.dataset)\n\n    print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n        \"Final Test Accuracy\", loss, acc))\n    return grapheme_final_output,grapheme_final_label,vowel_final_output,vowel_final_label,consonant_final_output,consonant_final_label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_model_metrics(plots,name):\n    train_acc_list,train_loss_list,val_acc_list,val_loss_list,test_acc_list,test_loss_list = plots\n    plot(train_acc_list,val_acc_list,test_acc_list,\"accuracy\",name)\n    plot(train_loss_list,val_loss_list,test_loss_list,\"loss\",name)\n\n\ndef plot(train,val,test,metric,name):\n    plt.title(name)\n    plt.plot(train,label=\"train {}\".format(metric))\n    plt.plot(val,label=\"val {}\".format(metric))\n    plt.plot(test,label=\"test {}\".format(metric))\n    plt.legend(loc=\"best\")\n    plt.savefig(\"{}-{}\".format(name,metric))\n    plt.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.insert(0,\"../input/pretrainedmodels/pretrainedmodels-0.7.4\")\nimport pretrainedmodels\n\nclass SEModule(nn.Module):\n    def __init__(self, channels=2048, reduction=16):\n        super(SEModule, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,\n                             padding=0)\n        self.relu = nn.ReLU(inplace=True)\n        self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,\n                             padding=0)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        module_input = x\n        x = self.avg_pool(x)\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.fc2(x)\n        x = self.sigmoid(x)\n        return module_input * x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Easier to split stuff up and backpropagate\nclass MyModel(nn.Module):\n  def __init__(self,pretrained=True):\n    super().__init__()\n    if pretrained:\n        self.model = pretrainedmodels.__dict__[\"se_resnext50_32x4d\"](pretrained=\"imagenet\")\n    else:\n        self.model = pretrainedmodels.__dict__[\"se_resnext50_32x4d\"](pretrained=None)\n    self.model = nn.Sequential(*list(self.model.children())[:-2])\n    \n    self.se_g = SEModule()\n    self.se_v = SEModule()\n    self.se_c = SEModule()\n    \n    self.avg_pool = nn.AdaptiveAvgPool2d(1)\n    \n    self.fc_g = nn.Sequential(nn.Linear(2048,512), nn.ReLU(), nn.Dropout(p=0.2), nn.Linear(512,168))\n    self.fc_v = nn.Sequential(nn.Linear(2048,512), nn.ReLU(), nn.Dropout(p=0.2), nn.Linear(512,11))\n    self.fc_c = nn.Sequential(nn.Linear(2048,512), nn.ReLU(), nn.Dropout(p=0.2), nn.Linear(512,7))\n    \n  def forward(self,x):\n    x = self.model(x)\n    \n    g = self.se_g(x)\n    v = self.se_v(x)\n    c = self.se_c(x)\n    \n    g = torch.flatten(self.avg_pool(g),1)\n    v = torch.flatten(self.avg_pool(v),1)\n    c = torch.flatten(self.avg_pool(c),1)\n    \n    g = self.fc_g(g)\n    v = self.fc_v(v)\n    c = self.fc_c(c)\n    return g,v,c","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel = MyModel()\nmodel.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = DataLoader(train_data, batch_size=128, num_workers=4,shuffle=True)\ndataloaders = {'train': train_loader}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = loss\noptimizer = optim.SGD(model.parameters(),lr=1.5e-02, momentum=0.9, weight_decay=1e-04, nesterov=True)\nnew_scheduler = lr_scheduler.CosineAnnealingLR(optimizer,30)\n\nmodel,plots = train_model(model, criterion, optimizer,\n            device, dataloaders,scheduler=new_scheduler, num_epochs=30)\n\ntorch.save(model.state_dict(),\"seresnext_50.pth\")\n\n\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}