{"cells":[{"metadata":{},"cell_type":"markdown","source":"Hi everyone, I'm making a minimal version of my pipeline on this competition here, and making it public.\n\nMy pipeline contains 2 models:\n\n* Seen Model, which is trained here (https://www.kaggle.com/haqishen/bengali-train-seen-model)\n* Unseen Model, which is trained here (https://www.kaggle.com/haqishen/bengali-train-unseen-model)\n\nFor the details of Seen Model and Unseen Model, please refer to those kernels. \n\nI've already trained my models on the kernels showed above, and loaded the output files into this inference kernel. Now I'm going to show you how to combine Seen Model and Unseen Model to make the prediction."},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path.append('../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport math\nimport glob\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm_notebook as tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom efficientnet_pytorch import model as enet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"data_dir = '../input/bengaliai-cv19'\ndevice = torch.device('cuda')\n\nHEIGHT = 137\nWIDTH = 236\n\nimg_size = 128\n\nseen_th = 0.825  # seen / unseen threshold\n\nc0_dim = 1295\nc1_dim = 168\nc2_dim = 11\nc3_dim = 7\nout_dim = c0_dim + c1_dim + c2_dim + c3_dim\n\nnum_workers = 4\nbatch_size = 32\n\nfiles_test = [f'test_image_data_{fid}.parquet' for fid in range(4)]\ndf_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))\ndf_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))\ndf_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))\n\nid2grapheme = {i: grapheme for i, grapheme in enumerate(df_train.grapheme.unique())}\ngrapheme2id = {grapheme: i for i, grapheme in enumerate(df_train.grapheme.unique())}\ndf_train['grapheme_id'] = df_train['grapheme'].map(grapheme2id)\n\ndf_label_map = []\nfor i, df in tqdm(df_train.groupby('grapheme_id')):\n    df_label_map.append(df.iloc[:, 1:6].drop_duplicates())\ndf_label_map = pd.concat(df_label_map).reset_index(drop=True)\n\nif DEBUG:\n    files_test = [f'train_image_data_{fid}.parquet' for fid in range(4)]  # train files\n    df_test = pd.read_csv(os.path.join(data_dir, 'train.csv'))  # train files\n    seen_th = 0.94\n#     device = torch.device('cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_data(f):\n    f = os.path.join(data_dir, f)\n    data = pd.read_parquet(f)\n    data = data.iloc[:, 1:].values\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BengaliDataset(Dataset):\n    def __init__(self, data, image_size=128):\n\n        self.data = data\n        self.image_size = image_size\n\n    def __len__(self):\n        return self.data.shape[0]\n\n    def __getitem__(self, index):\n\n        image = 255 - self.data[index].reshape(HEIGHT, WIDTH)\n\n        image = cv2.resize(image, (self.image_size, self.image_size))\n        image = image.astype(np.float32) / 255\n        image = image[np.newaxis, :, :]\n        image = np.repeat(image, 3, 0)  # 1ch to 3ch\n\n        return torch.tensor(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sigmoid = torch.nn.Sigmoid()\nclass Swish(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, i):\n        result = i * sigmoid(i)\n        ctx.save_for_backward(i)\n        return result\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        i = ctx.saved_variables[0]\n        sigmoid_i = sigmoid(i)\n        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))\n\nswish = Swish.apply\n\nclass Swish_module(nn.Module):\n    def forward(self, x):\n        return swish(x)\n\nswish_layer = Swish_module()\n\ndef relu_fn(x):\n    \"\"\" Swish activation function \"\"\"\n    return swish_layer(x)\n\n\nclass DenseCrossEntropy(nn.Module):\n    def forward(self, x, target, reduction='mean'):\n        x = x.float()\n        target = target.float()\n        logprobs = torch.nn.functional.log_softmax(x, dim=-1)\n\n        loss = -logprobs * target\n        loss = loss.sum(-1)\n        if reduction == 'mean':\n            return loss.mean()\n        elif reduction == 'sum':\n            return loss.sum()\n        elif reduction == 'none':\n            return loss\n\n\nclass ArcFaceLoss(nn.modules.Module):\n    def __init__(self, s=30.0, m=0.5, reduction='mean'):\n        super().__init__()\n        self.reduction = reduction\n        self.s = s\n        self.cos_m = math.cos(m)             #  0.87758\n        self.sin_m = math.sin(m)             #  0.47943\n        self.th = math.cos(math.pi - m)      # -0.87758\n        self.mm = math.sin(math.pi - m) * m  #  0.23971\n\n    def forward(self, logits, labels):\n        logits = logits.float()  # float16 to float32 (if used float16)\n        cosine = logits\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))  # equals to **2\n        phi = cosine * self.cos_m - sine * self.sin_m\n        phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n\n        output = (labels * phi) + ((1.0 - labels) * cosine)\n        output *= self.s\n        loss = DenseCrossEntropy()(output, labels, self.reduction)\n        return loss / 2\n\n\nclass ArcMarginProduct(nn.Module):\n    def __init__(self, in_features, out_features):\n        super().__init__()\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        stdv = 1. / math.sqrt(self.weight.size(1))\n        self.weight.data.uniform_(-stdv, stdv)\n\n    def forward(self, features):\n        cosine = F.linear(F.normalize(features), F.normalize(self.weight))\n        return cosine\n    \n    \nclass enet_3cg(nn.Module):\n\n    def __init__(self, backbone, out_dim_1, out_dim_2):\n        super(enet_3cg, self).__init__()\n        self.enet = enet.EfficientNet.from_name(backbone)\n        self.dropouts = nn.ModuleList([\n            nn.Dropout(0.5) for _ in range(5)\n        ])\n        self.myfc_1 = nn.Linear(self.enet._fc.in_features, out_dim_2)\n        self.activate = Swish_module()\n        self.myfc_2 = nn.Linear(out_dim_2, out_dim_1)\n        self.enet._fc = nn.Identity()\n\n    def extract(self, x):\n        return self.enet(x)\n\n    def forward(self, x):\n        x = self.extract(x)\n        for i, dropout in enumerate(self.dropouts):\n            if i == 0:\n                out_2 = self.myfc_1(dropout(x))\n            else:\n                out_2 += self.myfc_1(dropout(x))\n        out_2 /= len(self.dropouts)\n        out_1 = self.myfc_2(self.activate(out_2))\n        return out_1, out_2\n\n\nclass enet_arcface_v2(nn.Module):\n\n    def __init__(self, backbone, out_dim_1, out_dim_2):\n        super(enet_arcface_v2, self).__init__()\n        self.enet = enet.EfficientNet.from_name(backbone)\n        self.dropouts = nn.ModuleList([\n            nn.Dropout(0.5) for _ in range(5)\n        ])\n\n        self.gfc = nn.Linear(self.enet._fc.in_features, 4096)\n        self.metric_classify = ArcMarginProduct(4096, out_dim_1)\n        self.myfc_1 = nn.Linear(4096, out_dim_1)\n        self.myfc_2_1 = nn.Linear(4096, 512)\n        self.myfc_2_2 = nn.Linear(512, out_dim_2)\n        self.enet._fc = nn.Identity()\n\n    def extract(self, x):\n        return self.enet(x)\n\n    def forward(self, x):\n        x = self.extract(x)\n        x = Swish_module()(self.gfc(x))\n        for i, dropout in enumerate(self.dropouts):\n            if i == 0:\n                out_1 = self.myfc_1(dropout(x))\n                out_2 = self.myfc_2_1(dropout(x))\n            else:\n                out_1 += self.myfc_1(dropout(x))\n                out_2 += self.myfc_2_1(dropout(x))\n        out_1 /= len(self.dropouts)\n        out_2 /= len(self.dropouts)\n        out_2 = self.myfc_2_2(Swish_module()(out_2))\n        metric_output = self.metric_classify(x)\n        return out_1, out_2, metric_output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_models(model_files, model_class):\n    enet_type = 'efficientnet-b1'\n    models = []\n    for model_f in model_files:\n\n        model = model_class(enet_type, out_dim_1=c0_dim, out_dim_2=c1_dim+c2_dim+c3_dim)\n        model = model.to(device)\n        model.load_state_dict(torch.load(model_f, map_location=lambda storage, loc: storage), strict=True)\n        model.eval()\n        models.append(model)\n        print(model_f, 'loaded!')\n\n    return models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_files_unseen = [\n    '../input/bengali-train-unseen-model/effnet-b1-unseen_model_fold0.pth',\n]\nmodel_files_arcface = [\n    '../input/bengali-train-seen-model/effnet-b1-seen_model_fold0.pth',\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('loading unseen models...')\nmodels_unseen = get_models(model_files_unseen, enet_3cg)\nprint('loading arcface models...')\nmodels_arcface = get_models(model_files_arcface, enet_arcface_v2)\nprint(len(models_unseen), len(models_arcface))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FINAL_P = []\nwith torch.no_grad():\n    for file in tqdm(files_test):\n\n        data = read_data(file)\n        dataset_test = BengaliDataset(data, img_size)\n        test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n\n        for (image) in tqdm(test_loader):\n            image = image.to(device)\n\n            logits_1 = torch.zeros(image.shape[0], c0_dim).to(device)\n            logits_metric = torch.zeros(image.shape[0], c0_dim).to(device)\n\n            # predict by arcface models\n            for mid, model in enumerate(models_arcface):\n                l1, l2, l3 = model(image)\n                logits_1 += l1.softmax(1)\n                logits_metric += l3\n            logits_metric /= len(models_arcface)\n\n            # fill predictions with Seen Model prediction as first\n            # I decode 3 components from predicted grapheme here\n            pred = df_label_map.iloc[logits_metric.detach().cpu().numpy().argmax(1), :3].values\n\n\n            # use Arcface prediction threshold to find out unseen samples\n            max_p = logits_metric.cpu().numpy().max(1)\n            unseen_idx = np.where(max_p <= seen_th)[0]\n            # if unseen_idx id not empty, use Unseen Models to predict them\n            if unseen_idx.shape[0] > 0:\n                logits_2_unseen = torch.zeros(unseen_idx.shape[0], c1_dim+c2_dim+c3_dim).to(device)\n                for mid, model in enumerate(models_unseen):\n                    _, l2 = model(image[unseen_idx])\n                    logits_2_unseen[:, :c1_dim] += l2[:, :c1_dim].softmax(1)\n                    logits_2_unseen[:, c1_dim:c1_dim+c2_dim] += l2[:, c1_dim:c1_dim+c2_dim].softmax(1)\n                    logits_2_unseen[:, c1_dim+c2_dim:] += l2[:, c1_dim+c2_dim:].softmax(1)\n                # overwrite prediction for unseen samples\n                pred[unseen_idx, 0] = logits_2_unseen[:, :c1_dim].detach().cpu().numpy().argmax(1)\n                pred[unseen_idx, 1] = logits_2_unseen[:, c1_dim:c1_dim+c2_dim].detach().cpu().numpy().argmax(1)\n                pred[unseen_idx, 2] = logits_2_unseen[:, c1_dim+c2_dim:].detach().cpu().numpy().argmax(1)\n\n            FINAL_P += pred.reshape(-1).tolist()\n        del data\n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub = pd.DataFrame({\n    'row_id': [f'Test_{i}_{p}' for i in range(len(FINAL_P) // 3) for p in ['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']],\n    'target': FINAL_P\n})\n\ndf_sub.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}