{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import torch\nimport torch.utils as utils\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport torch.autograd as autograd\nfrom torch.utils.data.dataset import Dataset\nfrom torch.optim.optimizer import Optimizer, required\n\nimport torchvision\nimport torchvision.utils as vutils\nimport torchvision.datasets as dset\nimport torchvision.transforms as transforms\nimport torchvision.models as models\n\n\nimport math\n\n\nimport os\nimport argparse\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm, tqdm_notebook\nimport pyarrow\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"parser = argparse.ArgumentParser()\nparser.add_argument('--device', default = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu'))\n\nparser.add_argument('--size', default = 64)\nparser.add_argument('--criterion', default = nn.CrossEntropyLoss())\nparser.add_argument('--lr', default = 0.0004)\nparser.add_argument('--batch_size', default = 32)\nparser.add_argument('--Epoch', default = 140)\nparser.add_argument('--weight', default = [1, 1/3, 1/5])\n\n\nparser.add_argument('--name', type = tuple, default = ('grapheme_root','vowel_diacritic', 'consonant_diacritic'))\nparser.add_argument('--dim', type = tuple, default = (168, 11, 7))\n\nparser.add_argument('--transform', default = torchvision.transforms.ToTensor())\n\n\n\nargs, _ = parser.parse_known_args()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/bengaliai-cv19/train.csv')\ndata0 = pd.read_feather('/kaggle/usr/lib/resize_and_load_with_feather_format_much_faster/train_data_0.feather')\ndata1 = pd.read_feather('/kaggle/usr/lib/resize_and_load_with_feather_format_much_faster/train_data_1.feather')\ndata2 = pd.read_feather('/kaggle/usr/lib/resize_and_load_with_feather_format_much_faster/train_data_2.feather')\ndata3 = pd.read_feather('/kaggle/usr/lib/resize_and_load_with_feather_format_much_faster/train_data_3.feather')\n\ndata_full = pd.concat([data0,data1,data2,data3],ignore_index=True)\n\nclass GraphemeDataset(Dataset):\n    def __init__(self,df,label,device,transform, _type='train'):\n        self.df = df\n        self.label = label\n        self.device = device\n        self.transform = transform\n    def __len__(self):\n        return len(self.df)\n    def __getitem__(self,idx):\n        label1 = self.label.grapheme_root.values[idx]\n        label2 = self.label.vowel_diacritic.values[idx]\n        label3 = self.label.consonant_diacritic.values[idx]\n        \n        image = self.transform(255 - self.df.iloc[idx][1:].values.reshape(args.size,args.size).astype(np.float))\n\n\n        return image,(label1, label2, label3)\n    \ndef make_test_set(loader, device = args.device):\n    data = next(iter(loader))\n    data[0] = data[0].float()\n    data[0] = data[0].to(device = device)\n    for d in data[1]:\n        d = d.float()\n        d = d.to(device = device)\n        \n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reduced_index =train.groupby(['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']).apply(lambda x: x.sample(5)).image_id.values\nreduced_train = train.loc[train.image_id.isin(reduced_index)]\nreduced_data = data_full.loc[data_full.image_id.isin(reduced_index)]\ntrain_image = GraphemeDataset(reduced_data,reduced_train, device = args.device, transform = args.transform)\ntrain_loader = torch.utils.data.DataLoader(train_image,batch_size=30,shuffle=True, num_workers = 4)\n\nfix_data = make_test_set(train_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def accuracy(y, t):\n    acc = []\n    for y_, t_ in zip(y,t):\n        pred_label = torch.argmax(y_, dim=1)\n        t_ = t_.to(device = args.device)\n        count = pred_label.shape[0]\n        correct = (pred_label == t_).sum().float()\n        acc_ = correct / count\n        acc.append(acc_.item())\n    return acc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResidualBlock(nn.Module):\n    def __init__(self,in_channels,out_channels,stride=1,kernel_size=3,padding=1,bias=False):\n        super(ResidualBlock,self).__init__()\n        self.cnn1 =nn.Sequential(\n            nn.Conv2d(in_channels,out_channels,kernel_size,stride,padding,bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(True)\n            \n        )\n        self.cnn2 = nn.Sequential(\n            nn.Conv2d(out_channels,out_channels,kernel_size,1,padding,bias=False),\n            nn.BatchNorm2d(out_channels)\n        )\n        if stride != 1 or in_channels != out_channels:\n            self.shortcut = nn.Sequential(\n                nn.Conv2d(in_channels,out_channels,kernel_size=1,stride=stride,bias=False),\n                nn.BatchNorm2d(out_channels)\n            )\n        else:\n            self.shortcut = nn.Sequential()\n            \n    def forward(self, x):\n        residual = x\n        x = self.cnn1(x)\n        x = self.cnn2(x)\n        x += self.shortcut(residual)\n        x = nn.ReLU(True)(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResNet34(nn.Module):\n    def __init__(self):\n        super(ResNet34,self).__init__()\n        \n        self.block1 = nn.Sequential(\n            nn.Conv2d(1,64,kernel_size=2,stride=2,padding=3,bias=False),\n            nn.BatchNorm2d(64),\n            nn.ReLU(True)\n        )\n        \n        self.block2 = nn.Sequential(\n            nn.MaxPool2d(1,1),\n            ResidualBlock(64,64),\n            ResidualBlock(64,64,2)\n        )\n        \n        self.block3 = nn.Sequential(\n            ResidualBlock(64,128),\n            ResidualBlock(128,128,2)\n        )\n        \n        self.block4 = nn.Sequential(\n            ResidualBlock(128,256),\n            ResidualBlock(256,256,2)\n        )\n        self.block5 = nn.Sequential(\n            ResidualBlock(256,512),\n            ResidualBlock(512,512,2)\n        )\n        \n        self.avgpool = nn.AvgPool2d(2)\n    def dropout(self, x):\n        return F.dropout2d(x, 0.1)\n        \n    def forward(self,x):\n        x = self.block1(x)\n        x = self.dropout(x)\n        x = self.block2(x)\n        x = self.dropout(x)\n        x = self.block3(x)\n        x = self.dropout(x)\n        x = self.block4(x)\n        x = self.dropout(x)\n        x = self.block5(x)\n        x = self.dropout(x)\n        x = self.avgpool(x)\n        x = x.view(x.size(0),-1)\n\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self,args):\n        super(Model, self).__init__()\n        self.lr = args.lr\n        self.dim = args.dim\n        self.device = args.device\n        self.criterion = args.criterion\n        \n#        resnet34 = models.resnet34(pretrained = True)\n#        cashe = resnet34.conv1.weight[:, 0:1].float()\n#        resnet34.conv1 = nn.Conv2d(1, 64, kernel_size = (7, 7), stride = (2, 2), padding = (3, 3), bias = False)\n#        resnet34.conv1.weight = nn.Parameter(cashe)\n\n\n        self.model = ResNet34().to(device = self.device)\n        self.init_(self.model)\n\n\n        \n\n            \n        \n        self.grapheme = nn.Sequential()\n        self.grapheme.add_module('grapheme_classifier', nn.Linear(in_features = 512, out_features = args.dim[0]))\n        self.init_(self.grapheme)\n#        self.grapheme.add_module('grapheme_sparsemax', Sparsemax())\n\n        \n        self.vowel = nn.Sequential()\n        self.vowel.add_module('vowel_classifier', nn.Linear(in_features = 512, out_features = args.dim[1]))\n        self.init_(self.grapheme)\n#        self.vowel.add_module('vowel_sparsemax', Sparsemax())\n\n        self.consonant = nn.Sequential()\n        self.consonant.add_module('consonant_classifier', nn.Linear(in_features = 512, out_features = args.dim[2]))\n        self.init_(self.consonant)\n#        self.consonant.add_module('consonant_sparsemax', Sparsemax())\n\n        \n\n        \n        self.model.to(self.device)\n        self.grapheme.to(self.device)\n        self.vowel.to(self.device)\n        self.consonant.to(self.device)\n\n\n        \n\n\n\n\n\n    def loss(self, outputs, target, weight = [1, 1, 1]):\n        self.l0ss = [self.criterion(outputs[0], target[0].long().to(device = self.device))* weight[0],\n                    self.criterion(outputs[1], target[1].long().to(device = self.device))* weight[1],\n                    self.criterion(outputs[2], target[2].long().to(device = self.device))* weight[2]]\n\n    \n    def backward(self):\n        \n        (self.l0ss[0] + self.l0ss[1] + self.l0ss[2]).backward()\n    \n    def zero_grad(self, optim):\n        optim.zero_grad()\n            \n    def step(self, optim):\n        optim.step()\n        \n    \n    def forward(self, inputs, training = True):\n        inputs = inputs.to(device = self.device)\n        if training:\n            self.train()\n        else:\n            self.eval()\n        outputs = self.model(inputs)\n        grapheme = self.grapheme(outputs)\n        vowel = self.vowel(outputs)\n        consonant = self.consonant(outputs)\n        \n        return [grapheme, vowel, consonant, outputs]\n    \n    def train_(self, inputs, target, optim, training = True):\n        self.zero_grad(optim)\n#        self.loss(self.forward(inputs), target)\n        self.loss(self.forward(inputs, training), target, weight = args.weight)\n        self.backward()\n        self.step(optim)\n        \n    def test(self, inputs, target, training = False):\n        with torch.no_grad():\n            outputs = self.forward(inputs.to(args.device), training = training)\n            ac1 = (outputs[0].cpu().argmax(1)==target[0]).float().mean()\n            ac2 = (outputs[1].cpu().argmax(1)==target[1]).float().mean()\n            ac3 = (outputs[2].cpu().argmax(1)==target[2]).float().mean()\n#            print('[{:.3f}, {:.3f}, {:.3f}, {:.3f}]'.format(ac1, ac2, ac3, (ac1*2+ac2+ac3)/4))\n        return (ac1*2+ac2+ac3)/4\n        \n        return (ac1*2+ac2+ac3)/4\n    def init_(self, model):\n        def init_func(m):  # define the initialization function\n            classname = m.__class__.__name__\n            if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1):\n                if init_type == 'normal':\n                    init.normal_(m.weight.data, 0.0, init_gain)\n                elif init_type == 'xavier':\n                    init.xavier_normal_(m.weight.data, gain=init_gain)\n                elif init_type == 'kaiming':\n                    init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')\n                elif init_type == 'orthogonal':\n                    init.orthogonal_(m.weight.data, gain=init_gain)\n                else:\n                    raise NotImplementedError('initialization method [%s] is not implemented' % init_type)\n\n                if hasattr(m, 'bias') and m.bias is not None:\n                    init.constant_(m.bias.data, 0.0)\n            elif classname.find('BatchNorm2d') != -1:  # BatchNorm Layer's weight is not a matrix; only normal distribution applies.\n                init.normal_(m.weight.data, 1.0, init_gain)\n                init.constant_(m.bias.data, 0.0)\n            elif classname.find('InstanceNorm2d') != -1:  # InstanceNorm Layer's weight is not a matrix; only normal distribution applies.\n                init.normal_(m.weight.data, 1.0, init_gain)\n                init.constant_(m.bias.data, 0.0)\n        \n        return init_func(model)\n        \n        \n        \nmodel = Model(args)\n#scheduler = torch.optim.lr_scheduler.LambdaLR(optim, lr_lambda=lambda epoch: (1-epoch/args.Epoch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RAdam(Optimizer):\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, degenerated_to_sgd=True):\n        if not 0.0 <= lr:\n            raise ValueError(\"Invalid learning rate: {}\".format(lr))\n        if not 0.0 <= eps:\n            raise ValueError(\"Invalid epsilon value: {}\".format(eps))\n        if not 0.0 <= betas[0] < 1.0:\n            raise ValueError(\"Invalid beta parameter at index 0: {}\".format(betas[0]))\n        if not 0.0 <= betas[1] < 1.0:\n            raise ValueError(\"Invalid beta parameter at index 1: {}\".format(betas[1]))\n        \n        self.degenerated_to_sgd = degenerated_to_sgd\n        if isinstance(params, (list, tuple)) and len(params) > 0 and isinstance(params[0], dict):\n            for param in params:\n                if 'betas' in param and (param['betas'][0] != betas[0] or param['betas'][1] != betas[1]):\n                    param['buffer'] = [[None, None, None] for _ in range(10)]\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, buffer=[[None, None, None] for _ in range(10)])\n        super(RAdam, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(RAdam, self).__setstate__(state)\n\n    def step(self, closure=None):\n\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('RAdam does not support sparse gradients')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n                state['step'] += 1\n                buffered = group['buffer'][int(state['step'] % 10)]\n                if state['step'] == buffered[0]:\n                    N_sma, step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n\n                    if N_sma >= 5:\n                        step_size = math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    elif self.degenerated_to_sgd:\n                        step_size = 1.0 / (1 - beta1 ** state['step'])\n                    else:\n                        step_size = -1\n                    buffered[2] = step_size\n\n                if N_sma >= 5:\n                    if group['weight_decay'] != 0:\n                        p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    p_data_fp32.addcdiv_(-step_size * group['lr'], exp_avg, denom)\n                    p.data.copy_(p_data_fp32)\n                elif step_size > 0:\n                    if group['weight_decay'] != 0:\n                        p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n                    p_data_fp32.add_(-step_size * group['lr'], exp_avg)\n                    p.data.copy_(p_data_fp32)\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nreduced_index =train.groupby(['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']).apply(lambda x: x.sample(1)).image_id.values\nreduced_train = train.loc[train.image_id.isin(reduced_index)]\nreduced_data = data_full.loc[data_full.image_id.isin(reduced_index)]\nvalid_image = GraphemeDataset(reduced_data,reduced_train, device = args.device, transform = args.transform)\nvalid_loader = torch.utils.data.DataLoader(valid_image,batch_size=100,shuffle=True, num_workers = 4)\n\nnot_valid = train.loc[~train.image_id.isin(reduced_index)]\n\nfor epoch in range(args.Epoch):\n    \n    reduced_index =not_valid.groupby(['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']).apply(lambda x: x.sample(5)).image_id.values\n    reduced_train = not_valid.loc[not_valid.image_id.isin(reduced_index)]\n    reduced_data = data_full.loc[data_full.image_id.isin(reduced_index)]\n    train_image = GraphemeDataset(reduced_data,reduced_train, device = args.device, transform = args.transform)\n    train_loader = torch.utils.data.DataLoader(train_image,batch_size = args.batch_size,shuffle=True, num_workers = 4)\n    \n    optim = RAdam(model.parameters(), lr = args.lr * math.cos((epoch/args.Epoch)*math.pi/2) )\n    for idx, [data, label] in tqdm(enumerate(train_loader), total = len(train_loader)):\n        data = data.float()\n        data = data.to(device = args.device)\n        model.train_(data, label, optim, True)\n\n    acc = 0\n    for idx, [data, label] in tqdm(enumerate(valid_loader), total = len(valid_loader)):\n        acc += (model.test(data.float().to(device = args.device), label, False))/len(valid_loader)\n    \n    print('[{}/{}] : [loss : {:.2f}, {:.2f}, {:.2f}] [acc : {:.1f}%]'.format(epoch+1, args.Epoch, model.l0ss[0],model.l0ss[1],model.l0ss[2], acc*100))\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_state_dict(torch.load('/kaggle/input/train-model/ResNet34_1.pth',map_location=args.device))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\nimport cv2\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torchvision import transforms,models\nfrom tqdm import tqdm_notebook as tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GraphemeDataset_(Dataset):\n    def __init__(self,df,device,transform, _type='train'):\n        self.df = df\n        self.device = device\n        self.transform = transform\n    def __len__(self):\n        return len(self.df)\n    def __getitem__(self,idx):\n\n        \n        \n        image = self.transform(255 - self.df.iloc[idx][1:].values.reshape(args.size,args.size).astype(np.float))/255\n\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Resize(df,size=64):\n    resized = {} \n    df = df.set_index('image_id')\n    for i in tqdm(range(df.shape[0])):\n        image = cv2.resize(df.loc[df.index[i]].values.reshape(137,236),(size,size))\n        resized[df.index[i]] = image.reshape(-1)\n    resized = pd.DataFrame(resized).T.reset_index()\n    resized.columns = resized.columns.astype(str)\n    resized.rename(columns={'index':'image_id'},inplace=True)\n    return resized","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = ['test_image_data_0.parquet','test_image_data_1.parquet','test_image_data_2.parquet','test_image_data_3.parquet']\npredictions = []\nbatch_size=1\nfor fname in test_data:\n    data = pd.read_parquet(f'/kaggle/input/bengaliai-cv19/{fname}')\n    data = Resize(data)\n    test_image = GraphemeDataset_(data,device = args.device, transform = args.transform)\n    test_loader = torch.utils.data.DataLoader(test_image,batch_size=1,shuffle=False)\n    with torch.no_grad():\n        for idx, (inputs) in tqdm(enumerate(test_loader),total=len(test_loader)):\n            inputs.to(args.device)\n            inputs = inputs.float()\n            outputs1, outputs2, outputs3, _ = model.forward(inputs, training = False)\n            \n            predictions.append(outputs3.argmax(1).cpu().detach().numpy())\n            predictions.append(outputs1.argmax(1).cpu().detach().numpy())\n            predictions.append(outputs2.argmax(1).cpu().detach().numpy())\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/bengaliai-cv19/sample_submission.csv')\nsubmission.target = np.hstack(predictions) \nsubmission.head(100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}