{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/bengaliutils2/timm-0.1.18-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch import nn\nimport timm\nfrom torch.utils.data import DataLoader, Dataset\nimport pandas as pd\nimport cv2\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport torch.nn.functional as F\nimport torch\nfrom timm.models.layers.activations import Swish\nfrom torch.nn import Conv2d, BatchNorm2d, Sequential, Linear\nfrom torch.nn.modules.flatten import Flatten\nfrom albumentations import Compose, Normalize\nfrom albumentations.pytorch import ToTensorV2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MishFunction(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, x):\n        ctx.save_for_backward(x)\n        return x * torch.tanh(F.softplus(x))   # x * tanh(ln(1 + exp(x)))\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        x = ctx.saved_variables[0]\n        sigmoid = torch.sigmoid(x)\n        tanh_sp = torch.tanh(F.softplus(x))\n        return grad_output * (tanh_sp + x * sigmoid * (1 - tanh_sp * tanh_sp))\n\nclass Mish(nn.Module):\n    def forward(self, x):\n        return MishFunction.apply(x)\n\ndef to_Mish(model):\n    for child_name, child in model.named_children():\n        if isinstance(child, Swish):\n            setattr(model, child_name, Mish())\n        else:\n            to_Mish(child)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EfficientNew(nn.Module):\n    def __init__(self, num_classes, encoder):\n        super().__init__()\n        n_channels_dict = {'efficientnet_b0': 1280, 'efficientnet_b1': 1280, 'efficientnet_b2': 1408,\n                           'efficientnet_b3': 1536, 'efficientnet_b4': 1792, 'efficientnet_b5': 2048,\n                           'efficientnet_b6': 2304, 'efficientnet_b7': 2560, 'seresnext50_32x4d': 2048,\n                           'tf_efficientnet_b0_ns': 1280, 'tf_efficientnet_b3_ns': 1536,\n                           'tf_efficientnet_b4_ns': 1792}\n        self.net = timm.create_model(encoder, pretrained=False)\n        to_Mish(self.net)\n\n        out_features = n_channels_dict[encoder]\n        self.head_grapheme_root = AverageHead(num_classes[0], out_features)\n        self.head_vowel_diacritic = AverageHead(num_classes[1], out_features)\n        self.head_consonant_diacritic = AverageHead(num_classes[2], out_features)\n        \n    def forward(self, x):\n        x = self.net.forward_features(x)\n        logit_grapheme_root = self.head_grapheme_root(x)\n        logit_vowel_diacritic = self.head_vowel_diacritic(x)\n        logit_consonant_diacritic = self.head_consonant_diacritic(x)\n\n        return logit_grapheme_root, logit_vowel_diacritic, logit_consonant_diacritic\n\nclass AverageHead(nn.Module):\n    def __init__(self, num_classes, out_features):\n        super().__init__()\n        self.post_layers = Sequential(Flatten(), Linear(out_features, num_classes))\n        self._init_weight()\n\n    def _init_weight(self):\n        for m in self.modules():\n            if isinstance(m, Conv2d):\n                torch.nn.init.kaiming_normal_(m.weight)\n            elif isinstance(m, BatchNorm2d):\n                m.weight.data.fill_(1.0)\n                m.bias.data.zero_()\n\n    def forward(self, x):\n        x = 0.5 * (F.adaptive_avg_pool2d(x, 1) + F.adaptive_max_pool2d(x, 1))\n        return self.post_layers(x)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"HEIGHT = 137\nWIDTH = 236\n\ndef valid_aug(image_size=None):\n    augs_list = [Normalize(), ToTensorV2()]\n    return Compose(augs_list, p=1)\n\nclass GraphemeDatasetTest(Dataset):\n    def __init__(self, fname):\n        self.transform = valid_aug()\n        self.df = pd.read_parquet(fname)\n        self.data = self.df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH).astype(np.uint8)\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        name = self.df.iloc[idx, 0]\n        image = self.data[idx]\n        image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)\n        if self.transform:\n            image = self.transform(image=image)['image']\n        return image, name\n    \nclass Predictor:\n    def __init__(self, model):\n        self.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n        self.model = model.to(self.device, dtype=torch.float32);\n        self.model.eval()\n        print(f'Model prepared. Device is {self.device}')\n    \n    def predict(self, inputs, softmax_after=True):\n        inputs = inputs.to(self.device, dtype=torch.float32)\n        with torch.no_grad():\n            out_gr, out_vd, out_cd = self.model(inputs)\n        if not softmax_after:\n            out_gr = torch.nn.functional.softmax(out_gr, dim=1)\n            out_vd = torch.nn.functional.softmax(out_vd, dim=1)\n            out_cd = torch.nn.functional.softmax(out_cd, dim=1)\n        return out_gr, out_vd, out_cd\n\n    def load(self, path):\n        checkpoint = torch.load(path, map_location=self.device)\n        self.model.load_state_dict(checkpoint['model_state_dict'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictors = []\n\nfor i, model_path in enumerate([\n    '/kaggle/input/bengaliutils2/b3_9863_f0_ft1.pth',\n    '/kaggle/input/bengaliutils2/b3_9855_f1_ft1.pth',\n    '/kaggle/input/bengaliutils2/b3_9865_f2_ft1.pth',\n    '/kaggle/input/bengaliutils2/b3_9854_f3_ft1.pth',\n    '/kaggle/input/bengaliutils2/b3_9857_f4_ft1.pth'\n]):\n    predictor = Predictor(EfficientNew([168, 11, 7], 'tf_efficientnet_b3_ns'))\n    predictor.load(model_path)\n    predictors.append(predictor)\n\npredictors_count = len(predictors)\n\ndef predict_to_numpy(predict):\n    return predict.data.cpu().numpy().argmax(axis=1)\n\ndef predict_to_numpy_softmax(predict):\n    return torch.nn.functional.softmax(predict, dim=1).data.cpu().numpy().argmax(axis=1)\n\ndef make_prediction(images, softmax_after=True):\n    global was\n    outputs_gr = 0\n    outputs_vd = 0\n    outputs_cd = 0\n    for predictor in predictors:\n        gr, vd, cd = predictor.predict(images, softmax_after)\n        outputs_gr += gr\n        outputs_vd += vd\n        outputs_cd += cd\n\n    outputs_gr /= predictors_count\n    outputs_vd /= predictors_count\n    outputs_cd /= predictors_count\n        \n    if not was:\n        print(outputs_vd)\n        was = True    \n        \n    if softmax_after:\n        print(\"Averaging raw then softmax!\")\n        roots = predict_to_numpy_softmax(outputs_gr)\n        vowels = predict_to_numpy_softmax(outputs_vd)\n        consonants = predict_to_numpy_softmax(outputs_cd)\n    else:\n        print(\"Averaging softmax!\")\n        roots = predict_to_numpy(outputs_gr)\n        vowels = predict_to_numpy(outputs_vd)\n        consonants = predict_to_numpy(outputs_cd)    \n    \n    return roots, vowels, consonants","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nimport tqdm\ntarget = []\nrow_id = []\nwas = False\nfor i in range(4):\n    dataset = GraphemeDatasetTest(f'../input/bengaliai-cv19/test_image_data_{i}.parquet')\n    data_loader = DataLoader(dataset, batch_size=256, num_workers=4, shuffle=False)\n    for images, images_id in data_loader:\n        p1, p2, p3 = make_prediction(images, True)\n        for idx, name in enumerate(images_id):\n            row_id += [f'{name}_grapheme_root', f'{name}_vowel_diacritic',\n                               f'{name}_consonant_diacritic']\n            target += [p1[idx].item(), p2[idx].item(), p3[idx].item()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission = pd.DataFrame(\n    {\n        'row_id': row_id,\n        'target': target\n    },\n    columns=['row_id','target']\n)\n\ndf_submission.to_csv('submission.csv', index=False)\n\ndf_submission.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}