{"cells":[{"metadata":{},"cell_type":"markdown","source":"**This is a simple resnext50 baseline kernel for bengali.ai competition,i will try to gradually update this kernel**"},{"metadata":{},"cell_type":"markdown","source":"\n# If you find this kernel interesting, please drop an UPVOTE. It motivates me to produce more quality content :)\n"},{"metadata":{},"cell_type":"markdown","source":"# se_resnext50 PyTorch baseline\n\n\n- References (ResNet):\n  - https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py\n  - https://arxiv.org/pdf/1512.03385.pdf\n  \n  \n- Acknowledgements:\n  - Original kernels: https://www.kaggle.com/hanjoonchoe/grapheme-resnet-18-n-l-inference-lb-0-8566 and https://www.kaggle.com/hanjoonchoe/grapheme-resnet-18-naive-learning-3\n  \n  \n- **Kindly upvote the kernel if you found it helpful, including the original author's!**"},{"metadata":{},"cell_type":"markdown","source":"# Part 1"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from __future__ import print_function, division, absolute_import\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch.nn.functional as F\nimport os\n\n# Any results you write to the current directory are saved as output.\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torchvision import transforms,models\nfrom tqdm import tqdm_notebook as tqdm\nimport math\nimport torch.utils.model_zoo as model_zoo\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/bengaliai-cv19/train.csv')\ndata0 = pd.read_feather('/kaggle/usr/lib/resize_and_load_with_feather_format_much_faster/train_data_0.feather')\ndata1 = pd.read_feather('/kaggle/usr/lib/resize_and_load_with_feather_format_much_faster/train_data_1.feather')\ndata2 = pd.read_feather('/kaggle/usr/lib/resize_and_load_with_feather_format_much_faster/train_data_2.feather')\ndata3 = pd.read_feather('/kaggle/usr/lib/resize_and_load_with_feather_format_much_faster/train_data_3.feather')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls /kaggle/usr/lib/resize_and_load_with_feather_format_much_faster/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_full = pd.concat([data0,data1,data2,data3],ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class GraphemeDataset(Dataset):\n    def __init__(self,df,label,_type='train'):\n        self.df = df\n        self.label = label\n    def __len__(self):\n        return len(self.df)\n    def __getitem__(self,idx):\n        label1 = self.label.vowel_diacritic.values[idx]\n        label2 = self.label.grapheme_root.values[idx]\n        label3 = self.label.consonant_diacritic.values[idx]\n        image = self.df.iloc[idx][1:].values.reshape(64,64).astype(np.float)\n        return image,label1,label2,label3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## resnext50_32x4d Model"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"\nclass Selayer(nn.Module):\n\n    def __init__(self, inplanes):\n        super(Selayer, self).__init__()\n        self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n        self.conv1 = nn.Conv2d(inplanes, int(inplanes / 16), kernel_size=1, stride=1)\n        self.conv2 = nn.Conv2d(int(inplanes / 16), inplanes, kernel_size=1, stride=1)\n        self.relu = nn.ReLU(inplace=True)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n\n        out = self.global_avgpool(x)\n\n        out = self.conv1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.sigmoid(out)\n\n        return x * out\n\n\nclass Bottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, cardinality, stride=1, downsample=None):\n        super(Bottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes * 2)\n\n        self.conv2 = nn.Conv2d(planes * 2, planes * 2, kernel_size=3, stride=stride,\n                               padding=1, groups=cardinality, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes * 2)\n\n        self.conv3 = nn.Conv2d(planes * 2, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n\n        self.selayer = Selayer(planes * 4)\n\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        out = self.selayer(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n        out = self.relu(out)\n\n        return out\n\n\nclass SeResNeXt(nn.Module):\n\n    def __init__(self, block, layers, cardinality=32, num_classes=1000):\n        super(SeResNeXt, self).__init__()\n        self.cardinality = cardinality\n        self.inplanes = 64\n\n        self.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        self.bn1 = nn.BatchNorm2d(64)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n\n        self.layer1 = self._make_layer(block, 64, layers[0])\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n\n        self.avgpool = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(512 * block.expansion, num_classes)\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n                m.weight.data.normal_(0, math.sqrt(2. / n))\n                if m.bias is not None:\n                    m.bias.data.zero_()\n            elif isinstance(m, nn.BatchNorm2d):\n                m.weight.data.fill_(1)\n                m.bias.data.zero_()\n\n    def _make_layer(self, block, planes, blocks, stride=1):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes * block.expansion,\n                          kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, self.cardinality, stride, downsample))\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes, self.cardinality))\n                             \n        # vowel_diacritic\n        self.fc1 = nn.Linear(2048,11)\n        # grapheme_root\n        self.fc2 = nn.Linear(2048,168)\n        # consonant_diacritic\n        self.fc3 = nn.Linear(2048,7)\n        return nn.Sequential(*layers)\n        \n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        x = x.view(x.size(0), -1)\n        \n        x1 = self.fc1(x)\n        x2 = self.fc2(x)\n        x3 = self.fc3(x)\n        \n        return x1,x2,x3\n\n\ndef se_resnext50(**kwargs):\n    \"\"\"Constructs a SeResNeXt-50 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = SeResNeXt(Bottleneck, [3, 4, 6, 3], **kwargs)\n    return model\n\n\ndef se_resnext101(**kwargs):\n    \"\"\"Constructs a SeResNeXt-101 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = SeResNeXt(Bottleneck, [3, 4, 23, 3], **kwargs)\n    return model\n\n\ndef se_resnext152(**kwargs):\n    \"\"\"Constructs a SeResNeXt-152 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = SeResNeXt(Bottleneck, [3, 8, 36, 3], **kwargs)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Model\n\n"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"model = se_resnext50().to(device)\noptimizer = optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3)\n#scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=1e-4, max_lr=0.05)\ncriterion = nn.CrossEntropyLoss()\nbatch_size=64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"'''def criterion(input, target, size_average=True):\n    \"\"\"Categorical cross-entropy with logits input and one-hot target\"\"\"\n    l = -(target * torch.log(F.softmax(input, dim=1) + 1e-10)).sum(1)\n    if size_average:\n        l = l.mean()\n    else:\n        l = l.sum()\n    return l'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"%%time\nepochs = 200\nmodel.train()\nlosses = []\naccs = []\nfor epoch in range(epochs):\n    reduced_index =train.groupby(['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']).apply(lambda x: x.sample(5)).image_id.values\n    reduced_train = train.loc[train.image_id.isin(reduced_index)]\n    reduced_data = data_full.loc[data_full.image_id.isin(reduced_index)]\n    train_image = GraphemeDataset(reduced_data,reduced_train)\n    train_loader = torch.utils.data.DataLoader(train_image,batch_size=batch_size,shuffle=True)\n    \n    #print('epochs {}/{} '.format(epoch+1,epochs))\n    running_loss = 0.0\n    running_acc = 0.0\n    for idx, (inputs,labels1,labels2,labels3) in tqdm(enumerate(train_loader),total=len(train_loader)):\n        inputs = inputs.to(device)\n        labels1 = labels1.to(device)\n        labels2 = labels2.to(device)\n        labels3 = labels3.to(device)\n        \n        optimizer.zero_grad()\n        outputs1,outputs2,outputs3 = model(inputs.unsqueeze(1).float())\n        loss1 = criterion(outputs1,labels1)\n        loss2 = criterion(outputs2,labels2)\n        loss3 = criterion(outputs3,labels3)\n        running_loss += loss1+loss2+loss3\n        running_acc += (outputs1.argmax(1)==labels1).float().mean()\n        running_acc += (outputs2.argmax(1)==labels2).float().mean()\n        running_acc += (outputs3.argmax(1)==labels3).float().mean()\n        (loss1+loss2+loss3).backward()\n        optimizer.step()\n    #scheduler.step()\n    losses.append(running_loss/len(train_loader))\n    accs.append(running_acc/(len(train_loader)*3))\n    print('acc : {:.4f}%'.format(running_acc/(len(train_loader)*3)))\n    print('loss : {:.4f}'.format(running_loss/len(train_loader)))\ntorch.save(model.state_dict(), 'se_resnext50.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfig,ax = plt.subplots(1,2,figsize=(15,5))\nax[0].plot(losses)\nax[0].set_title('loss')\nax[1].plot(accs)\nax[1].set_title('acc')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# inference kernel : https://www.kaggle.com/mobassir/se-resnext50-pytorch-inference"},{"metadata":{},"cell_type":"markdown","source":"**References**\n\n* https://www.kaggle.com/khoongweihao/resnet-34-pytorch-starter-kit/data\n* https://www.kaggle.com/hanjoonchoe/grapheme-resnet-18-naive-learning-3"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}