{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from __future__ import absolute_import, division, print_function\n\nimport math\nimport os\nimport cv2\nimport albumentations\nimport joblib\nimport numpy as np  # linear algebra\nimport pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Any results you write to the current directory are saved as output.\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.model_zoo as model_zoo\nfrom PIL import Image\nfrom torch.nn import functional as F\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torchvision import models, transforms\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/bengaliai-cv19/train.csv')\ndata0 = pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_0.parquet')\ndata1 = pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_1.parquet')\ndata2 = pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_2.parquet')\ndata3 = pd.read_parquet('/kaggle/input/bengaliai-cv19/train_image_data_3.parquet')\n\ndata_full = pd.concat([data0, data1, data2, data3], ignore_index=True)\n#data_full = joblib.load(\"data.pkl\")\ndel data0, data1, data2, data3\ndevice = \"cuda\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GraphemeDataset(Dataset):\n    def __init__(self, df, label, _type='train'):\n        self.df = df\n        self.label = label\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        label1 = self.label.vowel_diacritic.values[idx]\n        label2 = self.label.grapheme_root.values[idx]\n        label3 = self.label.consonant_diacritic.values[idx]\n        image = self.df.iloc[idx][1:].values.astype(np.float).reshape(137,236)\n        return image, label2, label1, label3\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):\n    \"\"\"3x3 convolution with padding\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n                     padding=dilation, groups=groups, bias=False, dilation=dilation)\n\n\ndef conv1x1(in_planes, out_planes, stride=1):\n    \"\"\"1x1 convolution\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)\n\n\nclass BasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        super(BasicBlock, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        if groups != 1 or base_width != 64:\n            raise ValueError(\n                'BasicBlock only supports groups=1 and base_width=64')\n        if dilation > 1:\n            raise NotImplementedError(\n                \"Dilation > 1 not supported in BasicBlock\")\n        # Both self.conv1 and self.downsample layers downsample the input when stride != 1\n        self.conv1 = conv3x3(inplanes, planes, stride)\n        self.bn1 = norm_layer(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = conv3x3(planes, planes)\n        self.bn2 = norm_layer(planes)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        identity = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass Bottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        super(Bottleneck, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        width = int(planes * (base_width / 64.)) * groups\n        # Both self.conv2 and self.downsample layers downsample the input when stride != 1\n        self.conv1 = conv1x1(inplanes, width)\n        self.bn1 = norm_layer(width)\n        self.conv2 = conv3x3(width, width, stride, groups, dilation)\n        self.bn2 = norm_layer(width)\n        self.conv3 = conv1x1(width, planes * self.expansion)\n        self.bn3 = norm_layer(planes * self.expansion)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        identity = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass ResNet(nn.Module):\n\n    def __init__(self, block, layers, num_classes=1000, zero_init_residual=False,\n                 groups=1, width_per_group=64, replace_stride_with_dilation=None,\n                 norm_layer=None):\n        super(ResNet, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        self._norm_layer = norm_layer\n\n        self.inplanes = 64\n        self.dilation = 1\n        if replace_stride_with_dilation is None:\n            # each element in the tuple indicates if we should replace\n            # the 2x2 stride with a dilated convolution instead\n            replace_stride_with_dilation = [False, False, False]\n        if len(replace_stride_with_dilation) != 3:\n            raise ValueError(\"replace_stride_with_dilation should be None \"\n                             \"or a 3-element tuple, got {}\".format(replace_stride_with_dilation))\n        self.groups = groups\n        self.base_width = width_per_group\n        self.conv1 = nn.Conv2d(1, self.inplanes, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        self.bn1 = norm_layer(self.inplanes)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0])\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2,\n                                       dilate=replace_stride_with_dilation[0])\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2,\n                                       dilate=replace_stride_with_dilation[1])\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2,\n                                       dilate=replace_stride_with_dilation[2])\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc1 = nn.Linear(512 * block.expansion, 168)\n        self.fc2 = nn.Linear(512 * block.expansion, 11)\n        self.fc3 = nn.Linear(512 * block.expansion, 7)\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(\n                    m.weight, mode='fan_out', nonlinearity='relu')\n            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n        # Zero-initialize the last BN in each residual branch,\n        # so that the residual branch starts with zeros, and each residual block behaves like an identity.\n        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677\n        if zero_init_residual:\n            for m in self.modules():\n                if isinstance(m, Bottleneck):\n                    nn.init.constant_(m.bn3.weight, 0)\n                elif isinstance(m, BasicBlock):\n                    nn.init.constant_(m.bn2.weight, 0)\n\n    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):\n        norm_layer = self._norm_layer\n        downsample = None\n        previous_dilation = self.dilation\n        if dilate:\n            self.dilation *= stride\n            stride = 1\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                conv1x1(self.inplanes, planes * block.expansion, stride),\n                norm_layer(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,\n                            self.base_width, previous_dilation, norm_layer))\n        self.inplanes = planes * block.expansion\n        for _ in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups=self.groups,\n                                base_width=self.base_width, dilation=self.dilation,\n                                norm_layer=norm_layer))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        # See note [TorchScript super()]\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        x = torch.flatten(x, 1)\n        p = self.fc1(x)\n        q = self.fc2(x)\n        r = self.fc3(x)\n        return p, q, r\n\n\ndef _resnet(arch, block, layers, pretrained, progress, **kwargs):\n    model = ResNet(block, layers, **kwargs)\n    if pretrained:\n        state_dict = load_state_dict_from_url(model_urls[arch],\n                                              progress=progress)\n        model.load_state_dict(state_dict)\n    return model\n\n\ndef resnet34(pretrained=False, progress=True, **kwargs):\n    r\"\"\"ResNet-34 model from\n    `\"Deep Residual Learning for Image Recognition\" <https://arxiv.org/pdf/1512.03385.pdf>`_\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n    \"\"\"\n    return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress,\n                   **kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = resnet34().cuda()\noptimizer = optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3)\n#scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=1e-4, max_lr=0.05)\ncriterion = nn.CrossEntropyLoss()\nbatch_size = 32","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 100\nmodel.train()\nlosses = []\naccs = []\nreduced_index = train.groupby(['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']).apply(\n    lambda x: x.sample(5)).image_id.values\nreduced_train = train.loc[train.image_id.isin(reduced_index)]\nreduced_data = data_full.loc[data_full.image_id.isin(reduced_index)]\ntrain_image = GraphemeDataset(reduced_data, reduced_train)\nvalid_size = 0.2\nnum_train = len(train_image)\nindices = list(range(num_train))\nnp.random.shuffle(indices)\nsplit = int(np.floor(valid_size * num_train))\ntrain_idx, valid_idx = indices[split:], indices[:split]\ntrain_sampler = SubsetRandomSampler(train_idx)\nvalid_sampler = SubsetRandomSampler(valid_idx)\n\n\ntrain_loader = torch.utils.data.DataLoader(\n    train_image, batch_size=batch_size, sampler=train_sampler)\nvalidloader = torch.utils.data.DataLoader(train_image, batch_size=64,\n                                          sampler=valid_sampler)\nvalid_loss_min = np.Inf  # track change in validation loss\nmodel.train()\nfor epoch in range(epochs):\n\n    print('epochs {}/{} '.format(epoch+1, epochs))\n    running_loss = 0.0\n    running_acc = 0.0\n    running_loss1 = 0.0\n    running_acc1 = 0.0\n    for idx, (inputs, labels1, labels2, labels3) in tqdm(enumerate(train_loader), total=len(train_loader)):\n        inputs = inputs.to(device)\n        labels1 = labels1.to(device)\n        labels2 = labels2.to(device)\n        labels3 = labels3.to(device)\n\n        optimizer.zero_grad()\n        outputs1, outputs2, outputs3 = model(inputs.unsqueeze(1).float())\n        loss1 = criterion(outputs1, labels1)\n        loss2 = criterion(outputs2, labels2)\n        loss3 = criterion(outputs3, labels3)\n        running_loss += loss1+loss2+loss3\n        running_acc += (outputs1.argmax(1) == labels1).float().mean()\n        running_acc += (outputs2.argmax(1) == labels2).float().mean()\n        running_acc += (outputs3.argmax(1) == labels3).float().mean()\n        (loss1+loss2+loss3).backward()\n        optimizer.step()\n\n    # scheduler.step()\n    model.eval()\n    for idx, (inputs, labels1, labels2, labels3) in tqdm(enumerate(validloader), total=len(validloader)):\n        inputs = inputs.to(device)\n        labels1 = labels1.to(device)\n        labels2 = labels2.to(device)\n        labels3 = labels3.to(device)\n\n        with torch.no_grad():\n\n            outputs1, outputs2, outputs3 = model(inputs.unsqueeze(1).float())\n            loss1 = criterion(outputs1, labels1)\n            loss2 = criterion(outputs2, labels2)\n            loss3 = criterion(outputs3, labels3)\n        running_loss1 += loss1+loss2+loss3\n        running_acc1 += (outputs1.argmax(1) == labels1).float().mean()\n        running_acc1 += (outputs2.argmax(1) == labels2).float().mean()\n        running_acc1 += (outputs3.argmax(1) == labels3).float().mean()\n\n    train_loss = running_loss/len(train_loader)\n    valid_loss = running_loss1/len(train_loader)\n\n    print('train acc : {:.4f}%'.format(running_acc/(len(train_loader)*3)))\n    print('train loss : {:.4f}'.format(running_loss/len(train_loader)))\n    print('valid acc : {:.4f}%'.format(running_acc1/(len(train_loader)*3)))\n    print('valid loss : {:.4f}'.format(running_loss1/len(train_loader)))\n    if valid_loss <= valid_loss_min:\n        print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n            valid_loss_min,\n            valid_loss))\n        torch.save(model.state_dict(), 'validation50epoch.pt')\n        valid_loss_min = valid_loss\n    torch.save(model.state_dict(), 'novalidation50epoch.pt')\n\ntorch.save(model.state_dict(), '50resnet34full.pth')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfig,ax = plt.subplots(1,2,figsize=(15,5))\nax[0].plot(losses)\nax[0].set_title('loss')\nax[1].plot(accs)\nax[1].set_title('acc')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/bengaliai-cv19/test.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GraphemeDataset(Dataset):\n    def __init__(self,df,_type='train'):\n        self.df = df\n    def __len__(self):\n        return len(self.df)\n    def __getitem__(self,idx):\n        image = self.df.iloc[idx][1:].values.reshape(64,64).astype(float)\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel = resnet34(pretrained=False).to(device)\nmodel.load_state_dict(torch.load('./novalidation50epoch.pt'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Resize(df,size=64):\n    resized = {} \n    df = df.set_index('image_id')\n    for i in tqdm(range(df.shape[0])):\n        image = cv2.resize(df.loc[df.index[i]].values.reshape(137,236),(size,size))\n        resized[df.index[i]] = image.reshape(-1)\n    resized = pd.DataFrame(resized).T.reset_index()\n    resized.columns = resized.columns.astype(str)\n    resized.rename(columns={'index':'image_id'},inplace=True)\n    return resized","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodel.eval()\ntest_data = ['test_image_data_0.parquet','test_image_data_1.parquet','test_image_data_2.parquet','test_image_data_3.parquet']\npredictions = []\nbatch_size=1\nfor fname in test_data:\n    data = pd.read_parquet(f'/kaggle/input/bengaliai-cv19/{fname}')\n    data = Resize(data)\n    test_image = GraphemeDataset(data)\n    test_loader = torch.utils.data.DataLoader(test_image,batch_size=1,shuffle=False)\n    with torch.no_grad():\n        for idx, (inputs) in tqdm(enumerate(test_loader),total=len(test_loader)):\n            inputs.to(device)\n            \n            outputs1,outputs2,outputs3 = model(inputs.unsqueeze(1).float().cuda())\n            predictions.append(outputs3.argmax(1).cpu().detach().numpy())\n            predictions.append(outputs2.argmax(1).cpu().detach().numpy())\n            predictions.append(outputs1.argmax(1).cpu().detach().numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/bengaliai-cv19/sample_submission.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.target = np.hstack(predictions)\nsubmission.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}