{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Part 1: build dataset and visualize"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nimport time\nimport math\nimport gc\nimport cv2\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision\nimport torchvision.transforms as T \nimport torchvision.models as models\nimport matplotlib.pyplot as plt\n\nimport sklearn.metrics\n\ndef timeSince(since):\n    now = time.time()\n    s = now - since\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\n\nroot_train = '/kaggle/input/bengaliaicv19feather'\nroot_test = '/kaggle/input/bengaliai-cv19'\n\n#pretrained = '/kaggle/input/trained-model/new_resnet2.pth'\n# resnet.pth -->  T.Compose([T.ToPILImage(), T.CenterCrop(size), T.ToTensor()])\n# new_resnet.pth --> T.Compose([T.ToPILImage(), T.CenterCrop(150), T.Resize((size,size)), T.RandomAffine(45), T.ToTensor()])\n\npath = 'new_resnet34.pth'\n# keeping training by adding noise in dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start = time.time()\nimg_arrs = np.concatenate((pd.read_feather(os.path.join(root_train,'train_image_data_0.feather')).drop(columns = ['image_id']).values, \n                           pd.read_feather(os.path.join(root_train,'train_image_data_1.feather')).drop(columns = ['image_id']).values, \n                           pd.read_feather(os.path.join(root_train,'train_image_data_2.feather')).drop(columns = ['image_id']).values, \n                           pd.read_feather(os.path.join(root_train,'train_image_data_3.feather')).drop(columns = ['image_id']).values), \n                          axis=0)\nprint(timeSince(start))\nprint(img_arrs.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#idx = 38\n#threshold = 0.5\n#size = 128\n\n#img_arr = img_arrs[idx]/255.\n#fig = plt.figure(figsize=(20, 6))\n\n#ax = fig.add_subplot(2, 3, 1, xticks=[], yticks=[])\n#original_img = img_arr.reshape(137,236)\n#ax.imshow(original_img, cmap='gray')\n\n#ax = fig.add_subplot(2, 3, 2, xticks=[], yticks=[])\n#cropped_img = original_img[:, 30:190]\n#ax.imshow(cropped_img, cmap='gray')\n\n#ax = fig.add_subplot(2, 3, 3, xticks=[], yticks=[])\n#resized = cv2.resize(cropped_img,(size,size))\n#ax.imshow(resized, cmap='gray')\n\n\n#ax = fig.add_subplot(2, 3, 4, xticks=[], yticks=[])\n#rotate = cv2.rotate(resized, cv2.ROTATE_90_CLOCKWISE)\n#ax.imshow(rotate, cmap='gray')\n\n#ax = fig.add_subplot(2, 3, 5, xticks=[], yticks=[])\n#rotate = cv2.rotate(resized, cv2.ROTATE_90_COUNTERCLOCKWISE)\n#ax.imshow(rotate, cmap='gray')\n\n#ax = fig.add_subplot(2, 3, 6, xticks=[], yticks=[])\n#rotate = cv2.rotate(resized, cv2.ROTATE_180)\n#ax.imshow(rotate, cmap='gray')\n\n#plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#idx = 653\n#size = 128\n#threshold = 0.5\n\n#img_arr = 255 - img_arrs[idx] # flip black and white, so the default padding value (0) could match\n#original_img = img_arr.reshape(137,236,1)\n\n#fig = plt.figure(figsize=(20, 6))\n#transforms = T.Compose([T.ToPILImage(), T.ToTensor()])\n#img_tensor = transforms(original_img)\n#ax = fig.add_subplot(2, 3, 1, xticks=[], yticks=[])\n#ax.imshow(img_tensor[0], cmap='gray')\n\n#transforms = T.Compose([T.ToPILImage(),T.RandomAffine(90),T.CenterCrop(150), T.Resize((size,size)),T.ToTensor()])\n#img_tensor = transforms(original_img)\n#ax = fig.add_subplot(2, 3, 2, xticks=[], yticks=[])\n#ax.imshow(img_tensor[0], cmap='gray')\n\n#img_tensor = transforms(original_img)\n#ax = fig.add_subplot(2, 3, 3, xticks=[], yticks=[])\n#ax.imshow(img_tensor[0], cmap='gray')\n\n#img_tensor = transforms(original_img)\n#ax = fig.add_subplot(2, 3, 4, xticks=[], yticks=[])\n#ax.imshow(img_tensor[0], cmap='gray')\n\n#img_tensor = transforms(original_img)\n#ax = fig.add_subplot(2, 3, 5, xticks=[], yticks=[])\n#ax.imshow(img_tensor[0], cmap='gray')\n\n#img_tensor = transforms(original_img)\n#ax = fig.add_subplot(2, 3, 6, xticks=[], yticks=[])\n#ax.imshow(img_tensor[0], cmap='gray')\n\n##new_tensor = torch.where(img_tensor <= threshold, torch.zeros_like(img_tensor), img_tensor)\n##plt.imshow(new_tensor[0], cmap='gray')\n#plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class graphemeDataset(Dataset):\n    def __init__(self, img_arrs, target_file = None):\n        self.img_arrs = img_arrs\n        self.target_file = target_file\n        \n        if target_file is None:\n            self.transforms = T.Compose([T.ToPILImage(), T.CenterCrop(150), T.Resize((128,128)),T.ToTensor()])\n        else:\n            self.transforms = T.Compose([T.ToPILImage(),T.RandomAffine(90),T.CenterCrop(150), T.Resize((128,128)),T.ToTensor()])\n            # add targets for training\n            target_df = pd.read_csv(target_file)\n            self.grapheme = target_df['grapheme_root'].values\n            self.vowel = target_df['vowel_diacritic'].values\n            self.consonant = target_df['consonant_diacritic'].values\n            del target_df\n            gc.collect()\n               \n    def __getitem__(self, idx):\n        img_arr = 255 - self.img_arrs[idx] # flip black and white, so the default padding value (0) could match\n        new_tensor = self.transforms(img_arr.reshape(137, 236, 1))\n        \n        if self.target_file is None:\n            return new_tensor\n        else:\n            grapheme_tensor = torch.tensor(self.grapheme[idx], dtype=torch.long)\n            vowel_tensor = torch.tensor(self.vowel[idx], dtype=torch.long)\n            consonant_tensor = torch.tensor(self.consonant[idx], dtype=torch.long)\n            return new_tensor, grapheme_tensor, vowel_tensor, consonant_tensor\n    \n    def __len__(self):\n        return len(self.img_arrs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = graphemeDataset(img_arrs, target_file = '/kaggle/input/bengaliai-cv19/train.csv')\nprint(dataset.__len__())\n\nbatch_size = 128\ndata_loader = DataLoader(dataset, batch_size = batch_size, shuffle = True, drop_last = True)\ndataiter = iter(data_loader)\nprint(round(dataset.__len__()/batch_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_tensor, grapheme_tensor, vowel_tensor, consonant_tensor = next(dataiter)\n\nprint(img_tensor.size())\nprint(grapheme_tensor.size())\nprint(vowel_tensor.size())\nprint(consonant_tensor.size())\n\nfig = plt.figure(figsize=(25, 8))\nplot_size = 32\nfor idx in np.arange(plot_size):\n    ax = fig.add_subplot(4, plot_size/4, idx+1, xticks=[], yticks=[])\n    ax.imshow(img_tensor[idx][0], cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Part 2: design model architecture"},{"metadata":{"trusted":true},"cell_type":"code","source":"class simpleCNN(nn.Module):\n    def __init__(self, hidden_dim = 32):\n        super(simpleCNN, self).__init__()\n        self.hidden_dim = hidden_dim\n\n        self.features = nn.Sequential(\n            nn.Conv2d(1, hidden_dim, kernel_size = 3),\n            nn.ReLU(inplace = True),\n            nn.Conv2d(hidden_dim, hidden_dim, kernel_size = 3),\n            nn.ReLU(inplace = True),\n            nn.BatchNorm2d(hidden_dim),\n            nn.MaxPool2d(2),\n            nn.Conv2d(hidden_dim, hidden_dim, kernel_size = 5),\n            nn.ReLU(inplace = True),\n            nn.Dropout(p = 0.3),\n            \n            nn.Conv2d(hidden_dim, hidden_dim*2, kernel_size = 3),\n            nn.ReLU(inplace = True),\n            nn.Conv2d(hidden_dim*2, hidden_dim*2, kernel_size = 3),\n            nn.ReLU(inplace = True),\n            nn.BatchNorm2d(hidden_dim*2),\n            nn.MaxPool2d(2),\n            nn.Conv2d(hidden_dim*2, hidden_dim*2, kernel_size = 5),\n            nn.BatchNorm2d(hidden_dim*2),\n            nn.Dropout(p = 0.3),\n            \n            nn.Conv2d(hidden_dim*2, hidden_dim*4, kernel_size = 3),\n            nn.ReLU(inplace = True),\n            nn.Conv2d(hidden_dim*4, hidden_dim*4, kernel_size = 3),\n            nn.ReLU(inplace = True),\n            nn.BatchNorm2d(hidden_dim*4),\n            nn.MaxPool2d(2),\n            nn.Conv2d(hidden_dim*4, hidden_dim*4, kernel_size = 5),\n            nn.BatchNorm2d(hidden_dim*4),\n            nn.Dropout(p = 0.3),\n            \n            nn.Conv2d(hidden_dim*4, hidden_dim*8, kernel_size = 3),\n            nn.ReLU(inplace = True),\n            nn.Conv2d(hidden_dim*8, hidden_dim*8, kernel_size = 3),\n            nn.ReLU(inplace = True),\n            nn.BatchNorm2d(hidden_dim*8),\n            nn.Dropout(p = 0.3),\n            \n            nn.Flatten()\n        )\n        \n        self.grapheme_classifier = nn.Sequential(\n            nn.Linear(self.hidden_dim*8, 168),\n            nn.LogSoftmax(dim=1)\n        )\n\n        self.vowel_classifier = nn.Sequential(\n            nn.Linear(self.hidden_dim*8, self.hidden_dim*4),\n            nn.ReLU(inplace=True),\n            nn.Linear(self.hidden_dim*4, 11),\n            nn.LogSoftmax(dim=1)\n        )\n        \n        self.consonant_classifier = nn.Sequential(\n            nn.Linear(self.hidden_dim*8, self.hidden_dim*4),\n            nn.ReLU(inplace=True),\n            nn.Linear(self.hidden_dim*4, 7),\n            nn.LogSoftmax(dim=1)\n        )\n        \n    def forward(self, x):\n        x = self.features(x)\n        c1 = self.grapheme_classifier(x)\n        c2 = self.vowel_classifier(x)\n        c3 = self.consonant_classifier(x)\n        return c1, c2, c3\n        #return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class modified_resnet50(nn.Module):\n    def __init__(self):\n        super(modified_resnet50, self).__init__()\n        \n        resnet50 = models.resnet50()\n        resnet50.conv1 =  nn.Conv2d(1, 64, kernel_size = 7, stride = 2, padding = 3, bias = False)\n        layers = list(resnet50.children())[:-1] + [nn.Flatten()]\n        self.features= nn.Sequential(*layers)\n        \n        self.grapheme_classifier = nn.Sequential(\n            nn.Linear(2048, 168),\n            nn.LogSoftmax(dim=1)\n        )\n\n        self.vowel_classifier = nn.Sequential(\n            nn.Linear(2048, 11),\n            nn.LogSoftmax(dim=1)\n        )\n        \n        self.consonant_classifier = nn.Sequential(\n            nn.Linear(2048, 7),\n            nn.LogSoftmax(dim=1)\n        )\n        \n    def forward(self, x):\n        x = self.features(x)\n        c1 = self.grapheme_classifier(x)\n        c2 = self.vowel_classifier(x)\n        c3 = self.consonant_classifier(x)\n        return c1, c2, c3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class modified_resnet34(nn.Module):\n    def __init__(self):\n        super(modified_resnet34, self).__init__()\n        \n        resnet34 = models.resnet34()\n        resnet34.conv1 =  nn.Conv2d(1, 64, kernel_size = 5, stride = 1, padding = 2, bias = False)\n        layers = list(resnet34.children())[:-1] + [nn.Flatten()]\n        self.features= nn.Sequential(*layers)\n        \n        self.grapheme_classifier = nn.Sequential(\n            nn.Linear(512, 168),\n            nn.LogSoftmax(dim=1)\n        )\n\n        self.vowel_classifier = nn.Sequential(\n            nn.Linear(512, 11),\n            nn.LogSoftmax(dim=1)\n        )\n        \n        self.consonant_classifier = nn.Sequential(\n            nn.Linear(512, 7),\n            nn.LogSoftmax(dim=1)\n        )\n        \n    def forward(self, x):\n        x = self.features(x)\n        c1 = self.grapheme_classifier(x)\n        c2 = self.vowel_classifier(x)\n        c3 = self.consonant_classifier(x)\n        return c1, c2, c3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model = modified_resnet34()\n#c1, c2, c3 = model(img_tensor)\n#print(c1.size())\n#print(c2.size())\n#print(c3.size())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Part 3: train model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_training(model, dataset, path, batch_size, epoches, print_every):\n    start = time.time()\n    \n    criterion = nn.NLLLoss()\n    data_loader = DataLoader(dataset, batch_size = batch_size, shuffle = True, drop_last = True)\n    \n    model.train()\n    optimizer = torch.optim.Adam(model.parameters(), lr = 0.0005)\n    exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size = 7, gamma = 0.1)\n    \n    losses = []\n    loss_hold = 999\n    \n    for e in range(epoches):\n        for counter, (img_tensor, grapheme_tensor, vowel_tensor, consonant_tensor) in enumerate(data_loader):\n            img_tensor = img_tensor.to(device)\n            grapheme_tensor = grapheme_tensor.to(device)\n            vowel_tensor = vowel_tensor.to(device)\n            consonant_tensor = consonant_tensor.to(device)\n            c1, c2, c3 = model(img_tensor)\n            \n            l1 = criterion(c1, grapheme_tensor)\n            l2 = criterion(c2, vowel_tensor)\n            l3 = criterion(c3, consonant_tensor)\n            loss = 0.5* l1 + 0.25*l2 + 0.25*l3\n            \n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            \n            if counter % print_every == 0:\n                print('Epoch {}/{}({}) {}...loss {:6.4f}...grapheme {:6.4f}...vowel {:6.4f}...consonant {:6.4f}'.format(\n                    e+1, epoches, counter, timeSince(start), loss.item(), l1.item(),l2.item(),l3.item()))\n                losses.append((loss.item(), l1.item(),l2.item(),l3.item()))\n                \n                if loss.item() < loss_hold:\n                    torch.save(model.state_dict(), path)\n                    loss_hold = loss.item()\n                    print('Check point saved...')\n         \n        exp_lr_scheduler.step()\n    return losses","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = modified_resnet34().to(device)\n#model.load_state_dict(torch.load(pretrained))\n#model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"losses = model_training(model, dataset, path, batch_size = 128, epoches = 21, print_every = 500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"losses = np.array(losses)\nplt.figure(figsize=(10,5))\nplt.plot(losses.T[0], label='weighted loss')\nplt.plot(losses.T[1], label='grapheme')\nplt.plot(losses.T[2], label='vowel')\nplt.plot(losses.T[3], label='consonant')\nplt.xlabel(\"iterations\")\nplt.ylabel(\"Loss\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sample part of the training data to check training accuracy\n#start = time.time()\n#loader = DataLoader(dataset, batch_size = 1024, shuffle = True)\n#dataiter = iter(loader)\n#img_tensor, grapheme_tensor, vowel_tensor, consonant_tensor = dataiter.next()\n\n#model = modified_resnet()\n#model.load_state_dict(torch.load(path))\n#model.to(device)\n#with torch.no_grad():\n#    img_tensor = img_tensor.to(device)\n#    c1, c2, c3 = model(img_tensor)\n    \n#grapheme_pred = c1.argmax(1).cpu().tolist()\n#vowel_pred = c2.argmax(1).cpu().tolist()\n#consonant_pred = c3.argmax(1).cpu().tolist()\n\n#grapheme_true =grapheme_tensor.tolist()\n#vowel_true = vowel_tensor.tolist()\n#consonant_true = consonant_tensor.tolist()\n        \n#scores = [sklearn.metrics.recall_score(grapheme_true, grapheme_pred, average='macro'),\n#          sklearn.metrics.recall_score(vowel_true, vowel_pred, average='macro'),\n#          sklearn.metrics.recall_score(consonant_true, consonant_pred, average='macro')]\n#final_score = np.average(scores, weights=[2,1,1])\n#print('train acc: {:6.2f} %'.format(100*final_score))\n#print(timeSince(start))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cleanup to release memory\ndel img_arrs\ngc.collect()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}