{"cells":[{"metadata":{},"cell_type":"markdown","source":"\"\"\"Wide ResNet-50-2 model from\n    `\"Wide Residual Networks\" <https://arxiv.org/pdf/1605.07146.pdf>`_\n    The model is the same as ResNet except for the bottleneck number of channels\n    which is twice larger in every block. The number of channels in outer 1x1\n    convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048\n    channels, and in Wide ResNet-50-2 has 2048-1024-2048.\n \"\"\"\n# TRAINING PART"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport os\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torchvision import transforms,models\nfrom tqdm import tqdm_notebook as tqdm\nimport math\nimport torch.nn.functional as F\nfrom torch.nn import init\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Load feather file for training"},{"metadata":{"trusted":false},"cell_type":"code","source":"train = pd.read_csv('train.csv')\ndata0 = pd.read_feather('train_data_0.feather')\ndata1 = pd.read_feather('train_data_1.feather')\ndata2 = pd.read_feather('train_data_2.feather')\ndata3 = pd.read_feather('train_data_3.feather')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"data_full = pd.concat([data0,data1,data2,data3],ignore_index=True)\nlen(data_full)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GraphemeDataset(Dataset):\n    def __init__(self,df,label,_type='train'):\n        self.df = df\n        self.label = label\n    def __len__(self):\n        return len(self.df)\n    def __getitem__(self,idx):\n        label1 = self.label.vowel_diacritic.values[idx]\n        label2 = self.label.grapheme_root.values[idx]\n        label3 = self.label.consonant_diacritic.values[idx]\n        image = self.df.iloc[idx][1:].values.reshape(64,64).astype(np.float)    #128 was 64\n        return image,label1,label2,label3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**** Model code"},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):\n    \"\"\"3x3 convolution with padding\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n                     padding=dilation, groups=groups, bias=False, dilation=dilation)\n\ndef conv1x1(in_planes, out_planes, stride=1):\n    \"\"\"1x1 convolution\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)\n\n\nclass Bottleneck(nn.Module):\n    expansion = 4\n    __constants__ = ['downsample']\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        super(Bottleneck, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        width = int(planes * (base_width / 64.)) * groups\n        # Both self.conv2 and self.downsample layers downsample the input when stride != 1\n        self.conv1 = conv1x1(inplanes, width)\n        self.bn1 = norm_layer(width)\n        self.conv2 = conv3x3(width, width, stride, groups, dilation)\n        self.bn2 = norm_layer(width)\n        self.conv3 = conv1x1(width, planes * self.expansion)\n        self.bn3 = norm_layer(planes * self.expansion)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        identity = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass ResNet(nn.Module):\n\n    def __init__(self, block, layers, num_classes=1000, zero_init_residual=False,\n                 groups=1, width_per_group=64, replace_stride_with_dilation=None,\n                 norm_layer=None):\n        super(ResNet, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        self._norm_layer = norm_layer\n\n        self.inplanes = 64\n        self.dilation = 1\n        if replace_stride_with_dilation is None:\n            # each element in the tuple indicates if we should replace\n            # the 2x2 stride with a dilated convolution instead\n            replace_stride_with_dilation = [False, False, False]\n        if len(replace_stride_with_dilation) != 3:\n            raise ValueError(\"replace_stride_with_dilation should be None \"\n                             \"or a 3-element tuple, got {}\".format(replace_stride_with_dilation))\n        self.groups = groups\n        self.base_width = width_per_group\n        self.conv1 = nn.Conv2d(1, self.inplanes, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        self.bn1 = norm_layer(self.inplanes)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0])\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2,\n                                       dilate=replace_stride_with_dilation[0])\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2,\n                                       dilate=replace_stride_with_dilation[1])\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2,\n                                       dilate=replace_stride_with_dilation[2])\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        # vowel_diacritic\n        self.fc1 = nn.Linear(512 * block.expansion,11)\n        # grapheme_root\n        self.fc2 = nn.Linear(512 * block.expansion,168)\n        # consonant_diacritic\n        self.fc3 = nn.Linear(512 * block.expansion,7)\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n        # Zero-initialize the last BN in each residual branch,\n        # so that the residual branch starts with zeros, and each residual block behaves like an identity.\n        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677\n        if zero_init_residual:\n            for m in self.modules():\n                if isinstance(m, Bottleneck):\n                    nn.init.constant_(m.bn3.weight, 0)\n                elif isinstance(m, BasicBlock):\n                    nn.init.constant_(m.bn2.weight, 0)\n\n    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):\n        norm_layer = self._norm_layer\n        downsample = None\n        previous_dilation = self.dilation\n        if dilate:\n            self.dilation *= stride\n            stride = 1\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                conv1x1(self.inplanes, planes * block.expansion, stride),\n                norm_layer(planes * block.expansion),\n            )\n        layers = []\n        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,\n                            self.base_width, previous_dilation, norm_layer))\n        self.inplanes = planes * block.expansion\n        for _ in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups=self.groups,\n                                base_width=self.base_width, dilation=self.dilation,\n                                norm_layer=norm_layer))\n\n        return nn.Sequential(*layers)\n\n    def _forward_impl(self, x):\n        # See note [TorchScript super()]\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        x = torch.flatten(x, 1)\n        x1 = self.fc1(x)\n        x2 = self.fc2(x)\n        x3 = self.fc3(x)\n        return x1,x2,x3\n\n    def forward(self, x):\n        return self._forward_impl(x)\n\n\ndef _resnet(arch, block, layers, pretrained, progress, **kwargs):\n    model = ResNet(block, layers, **kwargs)\n    if pretrained:\n        state_dict = load_state_dict_from_url(model_urls[arch],\n                                              progress=progress)\n        model.load_state_dict(state_dict)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def wide_resnet50_2(pretrained=False, progress=True, **kwargs):\n    \"\"\"\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    kwargs['width_per_group'] = 64 * 2\n    return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3],\n                   pretrained, progress, **kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model=wide_resnet50_2().to(device)\nmodel.load_state_dict(torch.load('G:/Bengal Ai Hand Written Grapheme  Reconition/densenet169Weights/wideres50moreAcc99.158.pth'))\noptimizer = torch.optim.Adam(model.parameters(), lr=4e-4)\n#scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=1e-4, max_lr=0.05)\ncriterion = nn.CrossEntropyLoss()\nbatch_size=32   #increasing batch size degrades accuracy(in my case)whereas it decrease runtime also","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#training + validation\nepochs = 100\nmodel.train()\nlosses = []\naccs = []\nval_losses = []\nval_accs = []\nfor epoch in range(epochs):\n    train_index =train.groupby(['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']).apply(lambda x: x.sample(80)).image_id.values\n    reduced_train = train.loc[train.image_id.isin(train_index)]\n    train_data = data_full.loc[data_full.image_id.isin(train_index)]\n    \n    train_image = GraphemeDataset(train_data,reduced_train)\n    \n    ##data for training\n    train_loader = torch.utils.data.DataLoader(train_image,batch_size=batch_size,shuffle=True)\n    \n    test_index =train.groupby(['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']).apply(lambda x: x.sample(20)).image_id.values\n    reduced_test = train.loc[train.image_id.isin(test_index)]\n    test_data = data_full.loc[data_full.image_id.isin(test_index)]\n    \n    test_image = GraphemeDataset(test_data,reduced_test)\n    \n    ##data for test\n    test_loader = torch.utils.data.DataLoader(test_image,batch_size=batch_size,shuffle=True)\n    \n    print('epochs {}/{} '.format(epoch+1,epochs))\n    running_loss = 0.0\n    running_acc = 0.0\n    for idx, (inputs,labels1,labels2,labels3) in tqdm(enumerate(train_loader),total=len(train_loader)):\n        inputs = inputs.to(device)\n        labels1 = labels1.to(device)\n        labels2 = labels2.to(device)\n        labels3 = labels3.to(device)\n        \n        optimizer.zero_grad()\n        outputs1,outputs2,outputs3 = model(inputs.unsqueeze(1).float())\n        loss1 = criterion(outputs1,labels1)\n        loss2 = criterion(outputs2,labels2)\n        loss3 = criterion(outputs3,labels3)\n        running_loss += loss1+loss2+loss3\n        running_acc += (outputs1.argmax(1)==labels1).float().mean()\n        running_acc += (outputs2.argmax(1)==labels2).float().mean()\n        running_acc += (outputs3.argmax(1)==labels3).float().mean()\n        (loss1+loss2+loss3).backward()\n        optimizer.step()\n        del(inputs)\n        del(labels1)\n        del(labels2)\n        del(labels3)\n        del(outputs1)\n        del(outputs2)\n        del(outputs3)\n        del(loss1)\n        del(loss2)\n        del(loss3)\n    #scheduler.step()\n    losses.append(running_loss/len(train_loader))\n    accs.append(running_acc/(len(train_loader)*3))\n    print('acc : {:.4f}%'.format(running_acc*100/(len(train_loader)*3)))\n    print('loss : {:.4f}'.format(running_loss/len(train_loader)))\n    \n    \n    ## data feed for validation\n    with torch.no_grad():\n        running_loss = 0.0\n        running_acc = 0.0\n        for idx, (inputs,labels1,labels2,labels3) in tqdm(enumerate(test_loader),total=len(test_loader)):  #here tqdm is used for progressbar and total=len(t..) means prpgress bar highest value is len(t..)\n            \n            inputs = inputs.to(device)\n            labels1 = labels1.to(device)\n            labels2 = labels2.to(device)\n            labels3 = labels3.to(device)\n            \n            outputs1,outputs2,outputs3 = model(inputs.unsqueeze(1).float())\n            \n            loss1 = criterion(outputs1,labels1)\n            loss2 = criterion(outputs2,labels2)\n            loss3 = criterion(outputs3,labels3)\n            running_loss += loss1+loss2+loss3\n            running_acc += (outputs1.argmax(1)==labels1).float().mean()\n            running_acc += (outputs2.argmax(1)==labels2).float().mean()\n            running_acc += (outputs3.argmax(1)==labels3).float().mean()\n            del(inputs)\n            del(labels1)\n            del(labels2)\n            del(labels3)\n            del(outputs1)\n            del(outputs2)\n            del(outputs3)\n            del(loss1)\n            del(loss2)\n            del(loss3)\n        val_losses.append(running_loss/len(test_loader))\n        val_accs.append(running_acc/(len(test_loader)*3))\n        print('val_acc : {:.4f}%'.format(running_acc*100/(len(test_loader)*3)))\n        path=os.path.join('G:/Bengal Ai Hand Written Grapheme  Reconition/densenet169Weights','wideres50moreAcc{0:.3f}.pth'.format(running_acc*100/(len(test_loader)*3)))\n        if(running_acc*100/(len(test_loader)*3)>96.5):\n            torch.save(model.state_dict(),path)     \n        print('val_loss : {:.4f}'.format(running_loss/len(test_loader)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfig,ax = plt.subplots(1,2,figsize=(15,5))\nax[0].plot(losses,label='train')\nax[0].plot(val_losses,label='valid')\nax[0].set_title('loss')\nax[0].legend()\nax[1].plot(accs,label='train')\nax[1].plot(val_accs,label='valid')\nax[1].set_title('acc')\nax[0].legend()\nplt.show()\nfig.savefig('WideResnt50_8020split.png', dpi=100)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# INFERENCE PART"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/bengaliai-cv19/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GraphemeDataset(Dataset):\n    def __init__(self,df,_type='train'):\n        self.df = df\n    def __len__(self):\n        return len(self.df)\n    def __getitem__(self,idx):\n        image = self.df.iloc[idx][1:].values.reshape(64,64).astype(float)\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel = wide_resnet50_2().to(device)  #execute **Model code in the TRAINING section\nmodel.load_state_dict(torch.load('/kaggle/input/wideres50/wideres50moreAcc99.428.pth'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Resize(df,size=64):\n    resized = {} \n    df = df.set_index('image_id')\n    for i in tqdm(range(df.shape[0])):\n        image = cv2.resize(df.loc[df.index[i]].values.reshape(137,236),(size,size))\n        resized[df.index[i]] = image.reshape(-1)\n    resized = pd.DataFrame(resized).T.reset_index()\n    resized.columns = resized.columns.astype(str)\n    resized.rename(columns={'index':'image_id'},inplace=True)\n    return resized","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nmodel.eval()\ntest_data = ['test_image_data_0.parquet','test_image_data_1.parquet','test_image_data_2.parquet','test_image_data_3.parquet']\npredictions = []\nbatch_size=1\nfor fname in test_data:\n    data = pd.read_parquet(f'/kaggle/input/bengaliai-cv19/{fname}')\n    data = Resize(data)\n    test_image = GraphemeDataset(data)\n    test_loader = torch.utils.data.DataLoader(test_image,batch_size=1,shuffle=False)\n    with torch.no_grad():\n        for idx, (inputs) in tqdm(enumerate(test_loader),total=len(test_loader)):\n            inputs.to(device)\n            outputs1,outputs2,outputs3 = model(inputs.unsqueeze(1).float().cuda())\n            predictions.append(outputs3.argmax(1).cpu().detach().numpy())\n            predictions.append(outputs2.argmax(1).cpu().detach().numpy())\n            predictions.append(outputs1.argmax(1).cpu().detach().numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/bengaliai-cv19/sample_submission.csv')\nsubmission.target = np.hstack(predictions)\nsubmission.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"gputest","language":"python","name":"gputest"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"}},"nbformat":4,"nbformat_minor":1}