{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename));\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import fastai\nfrom fastai.vision import *\nfrom fastai.callbacks import SaveModelCallback\nfrom csvlogger import *\nfrom radam import *\n#from mish_activation import *\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfastai.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nimport zipfile\nfrom tqdm import tqdm_notebook as tqdm\nimport random\nimport torchvision\n\nsz = 128\nbs = 128\nSEED = 42\nTRAIN = '../input/bengali-grapheme/'\nLABELS = '../input/bengaliai-cv19/train.csv'\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_label = pd.read_csv(LABELS)\nnunique = list(df_label.nunique())[1:-1]\nprint(nunique)\ndf_label.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_map_df = pd.read_csv('../input/bengaliai-cv19/class_map.csv')\nclass_map_df.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stats, fold, nfolds = ([0.08547], [0.22490]), 0, 4\ndata = (ImageList.from_df(df_label, path='.', folder=TRAIN, suffix='.png', cols='image_id')#, convert_mode='L')\n        .split_by_idx(range(fold*len(df_label)//nfolds,(fold+1)*len(df_label)//nfolds))\n        .label_from_df(cols=['grapheme_root','vowel_diacritic','consonant_diacritic'])\n        .transform(get_transforms(do_flip=False,max_warp=0.1), size=sz, padding_mode='zeros')\n        .databunch(bs=bs)).normalize(imagenet_stats)\n\ndata.show_batch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyModel(nn.Module):\n    def __init__(self):\n        super(MyModel, self).__init__()\n        self.model_resnet = models.resnet34(pretrained=False)\n        num_ftrs = self.model_resnet.fc.in_features\n        self.model_resnet.fc = nn.Identity()\n        self.fc_graph = nn.Linear(num_ftrs, 168)\n        self.fc_vowel = nn.Linear(num_ftrs, 11)\n        self.fc_conso = nn.Linear(num_ftrs, 7)\n\n    def forward(self, x):\n        x = self.model_resnet(x)\n        out1 = self.fc_graph(x)\n        out2 = self.fc_vowel(x)\n        out3 = self.fc_conso(x)\n        return out1, out2, out3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Loss_combine(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n    def forward(self, input, target,reduction='mean'):\n        x1,x2,x3 = input\n        x1,x2,x3 = x1.float(),x2.float(),x3.float()\n        y = target.long()\n        return 0.7*F.cross_entropy(x1,y[:,0],reduction=reduction) + 0.1*F.cross_entropy(x2,y[:,1],reduction=reduction) + \\\n          0.2*F.cross_entropy(x3,y[:,2],reduction=reduction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Metric_idx(Callback):\n    def __init__(self, idx, average='macro'):\n        super().__init__()\n        self.idx = idx\n        self.n_classes = 0\n        self.average = average\n        self.cm = None\n        self.eps = 1e-9\n        \n    def on_epoch_begin(self, **kwargs):\n        self.tp = 0\n        self.fp = 0\n        self.cm = None\n    \n    def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):\n        last_output = last_output[self.idx]\n        last_target = last_target[:,self.idx]\n        preds = last_output.argmax(-1).view(-1).cpu()\n        targs = last_target.long().cpu()\n        \n        if self.n_classes == 0:\n            self.n_classes = last_output.shape[-1]\n            self.x = torch.arange(0, self.n_classes)\n        cm = ((preds==self.x[:, None]) & (targs==self.x[:, None, None])) \\\n          .sum(dim=2, dtype=torch.float32)\n        if self.cm is None: self.cm =  cm\n        else:               self.cm += cm\n\n    def _weights(self, avg:str):\n        if self.n_classes != 2 and avg == \"binary\":\n            avg = self.average = \"macro\"\n            warn(\"average=`binary` was selected for a non binary case. \\\n                 Value for average has now been set to `macro` instead.\")\n        if avg == \"binary\":\n            if self.pos_label not in (0, 1):\n                self.pos_label = 1\n                warn(\"Invalid value for pos_label. It has now been set to 1.\")\n            if self.pos_label == 1: return Tensor([0,1])\n            else: return Tensor([1,0])\n        elif avg == \"micro\": return self.cm.sum(dim=0) / self.cm.sum()\n        elif avg == \"macro\": return torch.ones((self.n_classes,)) / self.n_classes\n        elif avg == \"weighted\": return self.cm.sum(dim=1) / self.cm.sum()\n        \n    def _recall(self):\n        rec = torch.diag(self.cm) / (self.cm.sum(dim=1) + self.eps)\n        if self.average is None: return rec\n        else:\n            if self.average == \"micro\": weights = self._weights(avg=\"weighted\")\n            else: weights = self._weights(avg=self.average)\n            return (rec * weights).sum()\n    \n    def on_epoch_end(self, last_metrics, **kwargs): \n        return add_metrics(last_metrics, self._recall())\n    \nMetric_grapheme = partial(Metric_idx,0)\nMetric_vowel = partial(Metric_idx,1)\nMetric_consonant = partial(Metric_idx,2)\n\nclass Metric_tot(Callback):\n    def __init__(self):\n        super().__init__()\n        self.grapheme = Metric_idx(0)\n        self.vowel = Metric_idx(1)\n        self.consonant = Metric_idx(2)\n        \n    def on_epoch_begin(self, **kwargs):\n        self.grapheme.on_epoch_begin(**kwargs)\n        self.vowel.on_epoch_begin(**kwargs)\n        self.consonant.on_epoch_begin(**kwargs)\n    \n    def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):\n        self.grapheme.on_batch_end(last_output, last_target, **kwargs)\n        self.vowel.on_batch_end(last_output, last_target, **kwargs)\n        self.consonant.on_batch_end(last_output, last_target, **kwargs)\n        \n    def on_epoch_end(self, last_metrics, **kwargs): \n        return add_metrics(last_metrics, 0.5*self.grapheme._recall() +\n                0.25*self.vowel._recall() + 0.25*self.consonant._recall())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MixUpLoss(Module):\n    \"Adapt the loss function `crit` to go with mixup.\"\n    \n    def __init__(self, crit, reduction='mean'):\n        super().__init__()\n        if hasattr(crit, 'reduction'): \n            self.crit = crit\n            self.old_red = crit.reduction\n            setattr(self.crit, 'reduction', 'none')\n        else: \n            self.crit = partial(crit, reduction='none')\n            self.old_crit = crit\n        self.reduction = reduction\n        \n    def forward(self, output, target):\n        if len(target.shape) == 2 and target.shape[1] == 7:\n            loss1, loss2 = self.crit(output,target[:,0:3].long()), self.crit(output,target[:,3:6].long())\n            d = loss1 * target[:,-1] + loss2 * (1-target[:,-1])\n        else:  d = self.crit(output, target)\n        if self.reduction == 'mean':    return d.mean()\n        elif self.reduction == 'sum':   return d.sum()\n        return d\n    \n    def get_old(self):\n        if hasattr(self, 'old_crit'):  return self.old_crit\n        elif hasattr(self, 'old_red'): \n            setattr(self.crit, 'reduction', self.old_red)\n            return self.crit\n\nclass MixUpCallback(LearnerCallback):\n    \"Callback that creates the mixed-up input and target.\"\n    def __init__(self, learn:Learner, alpha:float=0.4, stack_x:bool=False, stack_y:bool=True):\n        super().__init__(learn)\n        self.alpha,self.stack_x,self.stack_y = alpha,stack_x,stack_y\n    \n    def on_train_begin(self, **kwargs):\n        if self.stack_y: self.learn.loss_func = MixUpLoss(self.learn.loss_func)\n        \n    def on_batch_begin(self, last_input, last_target, train, **kwargs):\n        \"Applies mixup to `last_input` and `last_target` if `train`.\"\n        if not train: return\n        lambd = np.random.beta(self.alpha, self.alpha, last_target.size(0))\n        lambd = np.concatenate([lambd[:,None], 1-lambd[:,None]], 1).max(1)\n        lambd = last_input.new(lambd)\n        shuffle = torch.randperm(last_target.size(0)).to(last_input.device)\n        x1, y1 = last_input[shuffle], last_target[shuffle]\n        if self.stack_x:\n            new_input = [last_input, last_input[shuffle], lambd]\n        else: \n            out_shape = [lambd.size(0)] + [1 for _ in range(len(x1.shape) - 1)]\n            new_input = (last_input * lambd.view(out_shape) + x1 * (1-lambd).view(out_shape))\n        if self.stack_y:\n            new_target = torch.cat([last_target.float(), y1.float(), lambd[:,None].float()], 1)\n        else:\n            if len(last_target.shape) == 2:\n                lambd = lambd.unsqueeze(1).float()\n            new_target = last_target.float() * lambd + y1.float() * (1-lambd)\n        return {'last_input': new_input, 'last_target': new_target}  \n    \n    def on_train_end(self, **kwargs):\n        if self.stack_y: self.learn.loss_func = self.learn.loss_func.get_old()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = MyModel()\n#model.conv1 = torch.nn.Conv2d(1,64, kernel_size=(7,7),stride=(2,2),padding=(3,3),bias=False\nmodel;","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = Learner(data, model, loss_func=Loss_combine(), opt_func=Over9000,\n        metrics=[Metric_grapheme(),Metric_vowel(),Metric_consonant(),Metric_tot()])\nlogger = CSVLogger(learn,f'log0')\nlearn.clip_grad = 1.0\nlearn.split([model.fc_graph]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(5, slice(3e-2), \ncallbacks = [logger, SaveModelCallback(learn,monitor='metric_tot',mode='max',name=f'model_0')])\n#metrics: Metric_grapheme, Metric_vowel, Metric_consonant, Metric_tot (competition metric)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = 3e-2\nlearn.fit_one_cycle(5, max_lr=slice(1e-5,lr/5), \ncallbacks = [logger, SaveModelCallback(learn,monitor='metric_tot',mode='max',name='model_1')])\n#metrics: Metric_grapheme, Metric_vowel, Metric_consonant, Metric_tot (competition metric)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torchvision\n\n\nclass MyRn34(nn.Module):\n    def __init__(self):\n        super(MyRn34, self).__init__()\n        self.model_resnet = torchvision.models.resnet34()\n        num_ftrs = self.model_resnet.fc.in_features\n        self.model_resnet.fc = nn.Identity()\n        self.fc_graph = nn.Linear(num_ftrs, 168)\n        self.fc_vowel = nn.Linear(num_ftrs, 11)\n        self.fc_conso = nn.Linear(num_ftrs, 7)\n\n    def forward(self, x):\n        x = self.model_resnet(x)\n        out1 = self.fc_graph(x)\n        out2 = self.fc_vowel(x)\n        out3 = self.fc_conso(x)\n        return out1, out2, out3\n\nmy_rn34=MyRn34()\nmy_rn34;","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save({'state_dict': learn.model.state_dict()}, '/kaggle/working/model_1.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=my_rn34\nweighties = torch.load('/kaggle/working/model_1.pth')\nmodel.load_state_dict(weighties['state_dict'], strict=False)\nmodel.cuda();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nworkers = 2\nPATH = '/kaggle/input/bengaliai-cv19/'\nTEST = [PATH+'test_image_data_0.parquet',\n        PATH+'test_image_data_1.parquet',\n        PATH+'test_image_data_2.parquet',\n        PATH+'test_image_data_3.parquet']\n\ndf_test = pd.read_csv(PATH+'test.csv')\ndf_test.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"HEIGHT = 137\nWIDTH = 236\nSIZE = 128\n\ndef bbox(img):\n    rows = np.any(img, axis=1)\n    cols = np.any(img, axis=0)\n    rmin, rmax = np.where(rows)[0][[0, -1]]\n    cmin, cmax = np.where(cols)[0][[0, -1]]\n    return rmin, rmax, cmin, cmax\n\ndef crop_resize(img0, size=SIZE, pad=16):\n    #crop a box around pixels large than the threshold \n    #some images contain line at the sides\n    ymin,ymax,xmin,xmax = bbox(img0[5:-5,5:-5] > 80)\n    #cropping may cut too much, so we need to add it back\n    xmin = xmin - 13 if (xmin > 13) else 0\n    ymin = ymin - 10 if (ymin > 10) else 0\n    xmax = xmax + 13 if (xmax < WIDTH - 13) else WIDTH\n    ymax = ymax + 10 if (ymax < HEIGHT - 10) else HEIGHT\n    img = img0[ymin:ymax,xmin:xmax]\n    #remove lo intensity pixels as noise\n    img[img < 28] = 0\n    lx, ly = xmax-xmin,ymax-ymin\n    l = max(lx,ly) + pad\n    #make sure that the aspect ratio is kept in rescaling\n    img = np.pad(img, [((l-ly)//2,), ((l-lx)//2,)], mode='constant')\n    return cv2.resize(img,(size,size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GraphemeDataset(Dataset):\n    def __init__(self, fname):\n        self.df = pd.read_parquet(fname)\n        self.data = 255 - self.df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH).astype(np.uint8)\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        name = self.df.iloc[idx,0]\n        #normalize each image by its max val\n        img = (self.data[idx]*(255.0/self.data[idx].max())).astype(np.uint8)\n        img = crop_resize(img)\n        img = (img.astype(np.float32)/255.0 - imagenet_stats[0][0])/imagenet_stats[1][0]\n        return img, name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"row_id,target = [],[]\nfor fname in TEST:\n    ds = GraphemeDataset(fname)\n    dl = DataLoader(ds, batch_size=bs, num_workers=nworkers, shuffle=False)\n    with torch.no_grad():\n        for x,y in tqdm(dl):\n            x = x.unsqueeze(1).cuda()\n            x = x.repeat(1,3,1,1)\n            p1,p2,p3 = model(x)\n            p1 = p1.argmax(-1).view(-1).cpu()\n            p2 = p2.argmax(-1).view(-1).cpu()\n            p3 = p3.argmax(-1).view(-1).cpu()\n            for idx,name in enumerate(y):\n                row_id += [f'{name}_grapheme_root',f'{name}_vowel_diacritic',\n                           f'{name}_consonant_diacritic']\n                target += [p1[idx].item(),p2[idx].item(),p3[idx].item()]\n                \nsub_df = pd.DataFrame({'row_id': row_id, 'target': target})\nsub_df.to_csv('submission.csv', index=False)\nsub_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}