{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Bengali.AI: Transfer Learning with Context Encoders\n\nIn this kernel, I build upon my [previous kernel](https://www.kaggle.com/lextoumbourou/self-supervised-pretraining-with-context-encoders) which implements the paper [Context Encoders: Feature Learning by Inpainting](https://arxiv.org/abs/1604.07379) by Deepak Pathak, Philipp Krahenbuhl, Jeff Donahue, Trevor Darrell and Alexei A. Efros.\n\nHere I am attempting to take the encoder used on the pretext task and repurpose it as a multi-label classification task. I compare 3 results on the same validation set:\n\n1. Results after 10 epochs with random pretrained weights.\n2. Results after 10 epochs with ImageNet pretrained weights.\n3. Results after 10 epochs with Context Encoder pretrained weights.\n\nThanks to the [fastai2 starter](https://www.kaggle.com/mnpinto/bengali-ai-fastai2-starter-lb0-9598) kernel by [mnpinto](https://www.kaggle.com/mnpinto) and the "},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"## Imports and params"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install git+https://github.com/fastai/fastai2 > /dev/null\n!pip install efficientnet-pytorch > /dev/null\n!pip install scikit-learn --upgrade > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path\n\nimport pandas as pd\n\nimport torch\nfrom efficientnet_pytorch import EfficientNet\nfrom efficientnet_pytorch.utils import get_same_padding_conv2d, round_filters, load_pretrained_weights\nfrom sklearn.metrics import recall_score\nfrom torch.utils import model_zoo\n\nfrom fastai2.basics import *\nfrom fastai2.data.all import *\nfrom fastai2.callback.all import *\nfrom fastai2.vision.all import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VALID_PCT = 0.2\nSEED = 420\nBATCH_SIZE = 64\nIMG_SIZE = 128\n\nDATA_PATH = Path('/kaggle/input/bengaliai-cv19')\nIMAGE_DATA_PATH = Path('/kaggle/input/grapheme-imgs-128x128')\nOUTPUT_PATH = Path('/kaggle/working')\nLABELS_PATH  = Path('/kaggle/input/iterative-stratification')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create Datasets and Dataloaders"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(LABELS_PATH/'train_with_fold.csv')#.sample(n=50000).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imagenet_stats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datablock = DataBlock(\n    blocks=(ImageBlock(cls=PILImageBW), CategoryBlock, CategoryBlock, CategoryBlock),\n    getters=[\n        ColReader('image_id', pref=IMAGE_DATA_PATH, suff='.png'),\n        ColReader('grapheme_root'),\n        ColReader('vowel_diacritic'),\n        ColReader('consonant_diacritic')\n    ],\n    splitter=IndexSplitter(train_df.loc[train_df.fold==0].index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfms = aug_transforms(do_flip=False, size=IMG_SIZE) + [Normalize(mean=0.485, std=0.229)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = datablock.dataloaders(train_df, bs=BATCH_SIZE, batch_tfms=tfms)\ndata.n_inp = 1 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.show_batch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class loss_func(Module):\n    def __init__(self, func=F.cross_entropy, weights=[2, 1, 1]):\n        self.func, self.w = func, weights\n\n    def forward(self, xs, *ys):\n        for i, w, x, y in zip(range(len(xs)), self.w, xs, ys):\n            if i == 0:\n                loss = w*self.func(x, y) \n            else:\n                loss += w*self.func(x, y) \n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"recall_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RecallPartial(Metric):\n    # based on AccumMetric\n    \"\"\"Stores predictions and targets on CPU in accumulate to perform final calculations with `func`.\"\"\"\n    def __init__(self, a=0, **kwargs):\n        self.func = partial(recall_score, average='macro', zero_division=0)\n        self.a = a\n\n    def reset(self): self.targs,self.preds = [],[]\n\n    def accumulate(self, learn):\n        pred = learn.pred[self.a].argmax(dim=-1)\n        targ = learn.y[self.a]\n        pred,targ = to_detach(pred),to_detach(targ)\n        pred,targ = flatten_check(pred,targ)\n        self.preds.append(pred)\n        self.targs.append(targ)\n\n    @property\n    def value(self):\n        if len(self.preds) == 0: return\n        preds,targs = torch.cat(self.preds),torch.cat(self.targs)\n        return self.func(targs, preds)\n\n    @property\n    def name(self): return train_df.columns[self.a+1]\n    \n\nclass RecallCombine(Metric):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.combine = 0\n\n    def accumulate(self, learn):\n        scores = [learn.metrics[i].value for i in range(3)]\n        self.combine = np.average(scores, weights=[2,1,1])\n\n    @property\n    def value(self):\n        return self.combine","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BengaliEfficientNet(EfficientNet):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        \n        # the initial layer to convolve into 3 channels\n        # idea from https://www.kaggle.com/aleksandradeis/bengali-ai-efficientnet-pytorch-starter\n        self.input_conv = nn.Conv2d(in_channels=1, out_channels=3, kernel_size=1)\n\n        self.fc1 = nn.Linear(in_features=1280, out_features=168) # grapheme_root\n        self.fc2 = nn.Linear(in_features=1280, out_features=11) # vowel_diacritic\n        self.fc3 = nn.Linear(in_features=1280, out_features=7) # consonant_diacritic\n    \n    def forward(self, inputs):\n        \"\"\" Calls extract_features to extract features, applies final linear layer, and returns logits. \"\"\"\n        \n        bs = inputs.size(0)\n        \n        # Convolve to 3 channels\n        x = self.input_conv(inputs)\n\n        # Convolution layers\n        x = self.extract_features(x)\n        \n        # Pooling\n        x = self._avg_pooling(x)\n        \n        # Final layers\n        x = x.view(bs, -1)\n\n        return [self.fc1(x), self.fc2(x), self.fc3(x)]\n    \n    @classmethod\n    def load(cls, path=None):\n        model = cls.from_name('efficientnet-b0')\n\n        if path is not None:\n            pretrained = torch.load(path, map_location=torch.device('cpu'))\n            encoder_only = {k[len('encoder.'):]: v for (k, v) in pretrained['model'].items() if k.startswith('encoder.')}\n            encoder_only_no_fc = {k: v for k, v in encoder_only.items() if not k.startswith('_fc')}\n    \n            model_dict = model.state_dict()\n            model_dict.update(encoder_only_no_fc) \n            model.load_state_dict(model_dict)\n\n        return model\n    \n    @classmethod\n    def load_imagenet(cls, advprop=False):\n        model_name = 'efficientnet-b0'\n        model = cls.from_name(model_name, override_params={'num_classes': 1})\n        model_dict = model.state_dict()\n\n        state_dict = model_zoo.load_url('https://publicmodels.blob.core.windows.net/container/aa/efficientnet-b0-355c32eb.pth')\n        state_dict_no_fc = {k: v for k, v in state_dict.items() if not k.startswith('_fc')}\n        model_dict.update(state_dict_no_fc)\n        \n        model.load_state_dict(model_dict)\n\n        return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = BengaliEfficientNet.load_imagenet()\n\nif torch.cuda.is_available():\n    model = model.cuda()\n    data = data.cuda()\n\nlearner = Learner(\n    data,\n    model,\n    loss_func=loss_func(),\n    cbs=CSVLogger('history_imagenet.csv'),\n    metrics=[RecallPartial(a=i) for i in range(len(data.c))] + [RecallCombine()]\n)\n\nlearner.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.fit_one_cycle(4, lr_max=slice(1e-3, 1e-2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.recorder.plot_loss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = BengaliEfficientNet.load('/kaggle/input/self-supervised-pretraining-with-context-encoders/models/model_cycle_1.pth')\n\nif torch.cuda.is_available():\n    model = model.cuda()\n    data = data.cuda()\n\nlearner = Learner(\n    data,\n    model,\n    loss_func=loss_func(),\n    cbs=CSVLogger('history_context_encoders.csv'),\n    metrics=[RecallPartial(a=i) for i in range(len(data.c))] + [RecallCombine()]\n)\n\nlearner.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.fit_one_cycle(4, lr_max=slice(1e-3, 1e-2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.recorder.plot_loss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = BengaliEfficientNet.load()\n\nif torch.cuda.is_available():\n    model = model.cuda()\n    data = data.cuda()\n\nlearner = Learner(\n    data,\n    model,\n    loss_func=loss_func(),\n    cbs=CSVLogger('history_no_pretrained.csv'),\n    metrics=[RecallPartial(a=i) for i in range(len(data.c))] + [RecallCombine()]\n).to_fp16()\n\nlearner.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.fit_one_cycle(4, lr_max=slice(1e-3, 1e-2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learner.recorder.plot_loss()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}