{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from PIL import Image\nimport math\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, Subset, TensorDataset\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import notebook\nimport torchvision\nfrom torchvision import transforms\nfrom torchvision import models\nfrom collections import Counter\nfrom pathlib import Path\nfrom sklearn.metrics import confusion_matrix, recall_score\nimport seaborn as sn\nimport pyarrow.parquet as pq\nfrom skimage.filters import threshold_otsu\nfrom skimage.transform import AffineTransform, SimilarityTransform, warp, resize","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv('/kaggle/input/bengaliai-cv19/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_tensordataset_from_dfs(parquet_locs, label_loc=None):\n    ids = []\n    X = []\n    for parquet_loc in parquet_locs:\n        df = pd.read_parquet(parquet_loc)\n        ids.extend(df.image_id.tolist())\n        x = df.iloc[:, 1:].to_numpy(dtype=np.uint8)\n        del df\n        X.append(x)\n    X = np.vstack(X)\n    X = X.reshape(-1, 1, 137, 236)\n    X = torch.from_numpy(X)\n    ids = dict((s,i) for (i,s) in enumerate(ids))\n    if label_loc is None:\n        return TensorDataset(X)\n    else:\n        graphemes = torch.zeros(X.shape[0], dtype=torch.long)\n        vowel_diacs = torch.zeros(X.shape[0], dtype=torch.long)\n        consonant_diacs = torch.zeros(X.shape[0], dtype=torch.long)\n        lbl_df = pd.read_csv(label_loc)\n        for row in lbl_df.itertuples():\n            if row.image_id not in ids:\n                continue\n            idx = ids[row.image_id]\n            graphemes[idx] = row.grapheme_root\n            vowel_diacs[idx] = row.vowel_diacritic\n            consonant_diacs[idx] = row.consonant_diacritic\n        return TensorDataset(X, graphemes, vowel_diacs, consonant_diacs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DatasetWithImageTransforms(Dataset):\n    def __init__(self, ds, transforms):\n        super(DatasetWithImageTransforms, self).__init__()\n        self.ds = ds\n        self.tr = transforms\n        self.nt = len(self.ds[0])\n        self.ln = len(self.ds)\n        \n    def __getitem__(self, index):\n        img, *rest = self.ds[index]\n        img = self.tr(img)\n        return (img,) + tuple(rest)\n    \n    def __len__(self):\n        return self.ln","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Returns binary image\ndef thresh(img):\n    thresh_val = int(threshold_otsu(img))\n    img = (img > thresh_val)\n    return img\n\n# For binary image\ndef bounding_box(img):\n    img = thresh(img).astype(np.uint8)\n    # find the min value of each column\n    col_min_val = np.min(img, axis=0)\n    # find the min value of each row\n    row_min_val = np.min(img, axis=1)\n    # argwhere finds the non-zero elements we want to find the zero elements (zeros are part of character)\n    col = np.argwhere(1 - col_min_val).flatten()\n    row = np.argwhere(1 - row_min_val).flatten()\n    return row.min(), row.max(), col.min(), col.max()\n\ndef scale_to_bb(img):\n    height = img.shape[0]\n    width = img.shape[1]\n    t, b, l, r = bounding_box(img)\n    box_width = r - l\n    box_height = b - t\n    t, l = max(0, t - 10), max(0, l - 10)\n    b, r = min(height, b + 10), min(width, r + 10)\n#     print(l, r, t, b)\n    img = resize(img[t:b, l:r], output_shape=(256, 256), preserve_range=True, order=3, cval=1.0)\n    return img\n\ndef random_scale(img):\n    height = img.shape[0]\n    width = img.shape[1]\n    t, b, l, r = bounding_box(img)\n    box_width = r - l\n    box_height = b - t\n    max_width_scale = (box_width + min(l, width - r)) / box_width\n    max_height_scale = (box_height + min(t, height - b)) / box_height\n    max_scale = min(max_width_scale, max_height_scale)\n    min_scale = min(1.0, 0.25 * max((height / box_height), (width / box_width)))\n    scale = random.uniform(min_scale, max_scale)\n    tfm = SimilarityTransform(\n        scale=(scale, scale),\n    )\n    img = warp(img, tfm.inverse, cval=1.0, order=3)\n    return img\n\ndef random_translate(img):\n    height = img.shape[0]\n    width = img.shape[1]\n    t, b, l, r = bounding_box(img)\n    box_width = r - l\n    box_height = b - t\n    translate_height = random.uniform(-t, height - b)\n    translate_width = random.uniform(-l, width - r)\n    tfm = SimilarityTransform(\n        translation=(translate_width, translate_height),\n    )\n    img = warp(img, tfm.inverse, cval=1.0, order=3)\n    return img\n\ndef random_rotate_and_shear(img):\n    max_theta = math.pi / 16\n    theta = random.uniform(-max_theta, max_theta)\n    max_shear_theta = math.pi / 8\n    shear_theta = random.uniform(-max_shear_theta, max_shear_theta)\n    tfm = AffineTransform(rotation=theta, shear=shear_theta)\n    img = warp(img, tfm.inverse, cval=1.0, order=3)\n    return img\n\ndef invert_color(t):\n    t.mul_(-1)\n    t.add_(255)\n    return t\n\ndef affine_transforms(img):\n    img = img.reshape(137, 236).numpy()\n    img = thresh(img).astype(np.float32)\n#     img = random_translate(random_scale(random_rotate_and_shear(img)))\n    img = random_translate(random_scale(img))\n    img = img.reshape(1, 137, 236)\n    return torch.from_numpy(img)\n\ndef tfms(img):\n    img = img.reshape(137, 236).numpy()\n    img = thresh(img).astype(np.float32)\n    img = scale_to_bb(img)\n    img = img.reshape(1, 256, 256)\n    return torch.from_numpy(img)\n\ndef va_tfms(img):\n    img = img.reshape(137, 236).numpy()\n    img = thresh(img).astype(np.float32)\n    img = img.reshape(1, 137, 236)\n    return torch.from_numpy(img)\n\naffine_transforms = transforms.Lambda(affine_transforms)\nmult = transforms.Lambda(lambda img: img * 255)\nto_float = transforms.Lambda(lambda img: img.float())\ninvert_color = transforms.Lambda(invert_color)\ntfms = transforms.Compose([\n#     transforms.RandomApply([\n        tfms,\n#         mult,\n#     ], p=0.90),\n    to_float,\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ds = make_tensordataset_from_dfs(['/kaggle/input/bengaliai-cv19/test_image_data_{}.parquet'.format(i) for i in range(4)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_graphemes, n_vowel_diacs, n_consonant_diacs = 168, 11, 7","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_linear_block_old(in_size, out_size):\n    block = nn.Sequential(\n        nn.Linear(in_size, out_size), \n        nn.ReLU(), \n        nn.BatchNorm1d(num_features=out_size),\n    )\n    nn.init.xavier_normal_(block[0].weight.data)\n    nn.init.zeros_(block[0].bias.data)\n    return block\n\ndef make_ff_predictor_old(in_size, intermediate_size, out_size, layer_count):\n    layers = [make_linear_block_old(in_size, intermediate_size)]\n    for i in range(layer_count):\n        layers.append(make_linear_block_old(intermediate_size, intermediate_size))\n    layers.append(make_linear_block_old(intermediate_size, out_size))\n    layers = nn.Sequential(*layers)\n    return layers\n\ndef make_squeeze_predictor(in_size, out_size):\n    return nn.Sequential(\n            nn.Dropout(p=0.5),\n            nn.Conv2d(in_size, out_size, kernel_size=(1, 1)),\n            nn.ReLU(),\n            nn.AdaptiveAvgPool2d(output_size=(1,1)),\n            nn.Flatten()\n        )\n\nclass BanglaHandwrittenGraphemeNN(nn.Module):\n    def __init__(self):\n        super(BanglaHandwrittenGraphemeNN, self).__init__()\n#         base = models.squeezenet1_0(pretrained=True).features\n        base = models.resnet18(pretrained=False)\n#         base.load_state_dict(torch.load('/kaggle/input/pretrained-pytorch-models/resnet18-5c106cde.pth'))\n        base.fc = nn.Identity()\n        self.base = base\n        feature_size = 512\n        self.grapheme_predictor = make_ff_predictor_old(feature_size, 512, n_graphemes, 2)\n        self.vowel_diac_predictor = make_ff_predictor_old(feature_size, 512, n_vowel_diacs, 1)\n        self.consonant_diacs = make_ff_predictor_old(feature_size, 512, n_consonant_diacs, 1)\n\n    def convert_to_grayscale(self):\n        with torch.no_grad():\n            conv1 = nn.Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n            conv1.weight.data = torch.sum(model.base.conv1.weight.data, dim=1, keepdim=True)\n            self.base.conv1 = conv1\n            \n    def freeze(self):\n        for p in self.base.parameters():\n            p.requires_grad = False\n        \n    def unfreeze(self):\n        for p in self.base.parameters():\n            p.requires_grad = True\n        \n    def forward(self, x):\n        with torch.no_grad():\n            features = self.base(x)\n        g_pred = self.grapheme_predictor(features)\n        v_pred = self.vowel_diac_predictor(features)\n        c_pred = self.consonant_diacs(features)\n        return g_pred, v_pred, c_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_linear_block(in_size, out_size, dropout, bn):\n    if dropout:\n        if bn:\n            block = nn.Sequential(\n                nn.Linear(in_size, out_size), \n                nn.Dropout(0.5),\n                nn.ReLU(), \n                nn.BatchNorm1d(num_features=out_size),\n            )\n        else:\n            block = nn.Sequential(\n                nn.Linear(in_size, out_size), \n                nn.Dropout(0.5),\n                nn.ReLU(), \n            )\n    else:\n        if bn:\n            block = nn.Sequential(\n                nn.Linear(in_size, out_size),\n                nn.ReLU(), \n                nn.BatchNorm1d(num_features=out_size),\n            )\n        else:\n            block = nn.Sequential(\n                nn.Linear(in_size, out_size),\n                nn.ReLU(),\n            )\n    nn.init.xavier_normal_(block[0].weight.data)\n    nn.init.zeros_(block[0].bias.data)\n    return block\n\ndef make_ff_predictor(in_size, intermediate_size, out_size, layer_count, dropout, bn):\n    layers = [make_linear_block(in_size, intermediate_size, dropout, bn)]\n    for i in range(layer_count):\n        layers.append(make_linear_block(intermediate_size, intermediate_size, dropout, bn))\n    layers.append(nn.Linear(intermediate_size, out_size))\n    layers = nn.Sequential(*layers)\n    return layers\n\ndef make_squeeze_predictor(in_size, out_size):\n    return nn.Sequential(\n            nn.Dropout(p=0.5),\n            nn.Conv2d(in_size, out_size, kernel_size=(1, 1)),\n            nn.ReLU(),\n            nn.AdaptiveAvgPool2d(output_size=(1,1)),\n            nn.Flatten(), \n        )\n\nclass MultiTaskNNR18(nn.Module):\n    def __init__(self, n_classes_tasks, depth_tasks, dropout=False, bn=False):\n        super(MultiTaskNNR18, self).__init__()\n#         base = models.squeezenet1_0(pretrained=True).features\n        base, feature_size = models.resnet18(pretrained=False), 512\n#         base = models.wide_resnet101_2(pretrained=True)\n        base.fc = nn.Identity()\n#         base = models.densenet121(pretrained=True)\n#         base.classifier = nn.Identity()\n        self.base = base\n        self.task_predictors = nn.ModuleList([\n            make_ff_predictor(feature_size, 512, n_classes, depth, dropout, bn) \n            for n_classes, depth in zip(n_classes_tasks, depth_tasks)\n        ])\n        self.n_classes_tasks = n_classes_tasks\n        self.depth_tasks = depth_tasks\n\n    def convert_to_grayscale(self):\n        with torch.no_grad():\n            conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n            conv1.weight.data = torch.sum(self.base.conv1.weight.data, dim=1, keepdim=True)\n            self.base.conv1 = conv1\n#             conv1.weight.data = torch.sum(self.base.features.conv0.weight.data, dim=1, keepdim=True)\n#             self.base.features.conv0 = conv1\n            \n    def freeze(self):\n        for p in self.base.parameters():\n            p.requires_grad = False\n        \n    def unfreeze(self):\n        for p in self.base.parameters():\n            p.requires_grad = True\n        \n    def forward(self, x):\n        features = self.base(x)\n        preds = [predictor(features) for predictor in self.task_predictors]\n        return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiTaskNN(nn.Module):\n    def __init__(self, base, task_predictors):\n        super(MultiTaskNN, self).__init__()\n        self.base = base\n        self.task_predictors = nn.ModuleList(task_predictors)\n        \n    def freeze(self):\n        for p in self.base.parameters():\n            p.requires_grad = False\n        \n    def unfreeze(self):\n        for p in self.base.parameters():\n            p.requires_grad = True\n        \n    def forward(self, x):\n        features = self.base(x)\n        preds = [predictor(features) for predictor in self.task_predictors]\n        return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base = torch.jit.load('/kaggle/input/bhgd-r152-trace/resnet152-corrected-more-transforms-base.trace')\nnum_tasks = 3\ntask_predictors = [torch.jit.load('/kaggle/input/bhgd-r152-trace/resnet152-corrected-more-transforms-task-predictor-{}.trace'.format(i)) for i in range(num_tasks)]\nmodel1 = MultiTaskNN(base, task_predictors)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base, feature_size = torchvision.models.resnet152(pretrained=False), 2048\nbase.fc = nn.Identity()\nconv1 = nn.Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), bias=False)\nconv1.weight.data = torch.sum(base.conv1.weight.data, dim=1, keepdim=True)\nbase.conv1 = conv1\n\nn_classes_tasks = [n_graphemes, n_vowel_diacs, n_consonant_diacs]\n\ntask_predictors = [nn.Linear(feature_size, n_classes, bias=False) for n_classes in n_classes_tasks]\n\nmodel2 = MultiTaskNN(base, task_predictors).to(device)\n\nmodel2.load_state_dict(torch.load('/kaggle/input/bhgd-r152-itstrat-mixup-partial/resnet152-itstrat-mixup.pth', map_location=device))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base, feature_size = torchvision.models.resnet152(pretrained=False), 2048\nbase.fc = nn.Identity()\nconv1 = nn.Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), bias=False)\nconv1.weight.data = torch.sum(base.conv1.weight.data, dim=1, keepdim=True)\nbase.conv1 = conv1\n\ntask_predictors = [nn.Linear(feature_size, n_classes, bias=False) for n_classes in n_classes_tasks]\n\nmodel3 = MultiTaskNN(base, task_predictors).to(device)\n\nmodel3.load_state_dict(torch.load('/kaggle/input/bhgd-r152-itstrat-mixup-partial/resnet152-itstrat-mixup-120-1e-3-0.2-16-8.pth', map_location=device))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiTaskEnsemble(nn.Module):\n    def __init__(self, models):\n        super(MultiTaskEnsemble, self).__init__()\n        self.models = nn.ModuleList(models)\n        self.n = len(models)\n\n    def forward(self, x):\n        preds = [m(x) for m in self.models]\n        preds = [[pred[i] for pred in preds] for i in range(len(preds[0]))]\n        preds = [torch.stack(pred).mean(dim=0) for pred in preds]\n        return preds\n#         xes = [sum(x[i] for x in xes) / self.n for i in range(len(xes[0]))]\n#         return xes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = BanglaHandwrittenGraphemeNN().to(device)\nmodel.convert_to_grayscale()\nmodel.load_state_dict(torch.load('/kaggle/input/bhgd-r18-best/model.pth', map_location=device))\nmodel_r18_best = model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = MultiTaskNNR18([n_graphemes, n_vowel_diacs, n_consonant_diacs], [2, 1, 1], bn=True).to(device)\nmodel.convert_to_grayscale()\nmodel.load_state_dict(torch.load('/kaggle/input/bhgd-r18-best-clones/resnet18-original-lossweight-111.pth', map_location=device))\nmodel_r18_111 = model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = MultiTaskNNR18([n_graphemes, n_vowel_diacs, n_consonant_diacs], [2, 1, 1], bn=True).to(device)\nmodel.convert_to_grayscale()\nmodel.load_state_dict(torch.load('/kaggle/input/bhgd-r18-best-clones/resnet18-original-lossweight-211.pth', map_location=device))\nmodel_r18_211 = model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = MultiTaskNNR18([n_graphemes, n_vowel_diacs, n_consonant_diacs], [2, 1, 1], bn=True).to(device)\nmodel.convert_to_grayscale()\nmodel.load_state_dict(torch.load('/kaggle/input/bhgd-r18-best-clones/resnet18-original-80ep.pth', map_location=device))\nmodel_r18_80ep = model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ens1 = MultiTaskEnsemble([model_r18_best, model_r18_111, model_r18_211, model_r18_80ep]).to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ens2 = MultiTaskEnsemble([model1, model2, model3]).to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_model(name, num_tasks):\n    base = torch.jit.load(name + '-base.trace')\n    task_predictors = [torch.jit.load(name + '-task-predictor-{}.trace'.format(i)) for i in range(num_tasks)]\n    return MultiTaskNN(base, task_predictors)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model1 = load_model('/kaggle/input/bhgd-final-attempt/seresnext101-mixup-convtail-small', 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2 = load_model('/kaggle/input/bhgd-final-attempt/seresnext101-mixup-convtail', 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model3 = load_model('/kaggle/input/bhgd-final-attempt/seresnext101-mixup-40ep-rotate-shear', 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model4 = load_model('/kaggle/input/bhgd-final-attempt/seresnext101-mixup-convtail-small-40ep', 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ens3 = MultiTaskEnsemble([model1, model2, model3, model4]).to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = [ens1, ens2, ens3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = MultiTaskEnsemble(models).to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.eval();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with torch.no_grad():\n    for i in range(4):\n        te_ds = make_tensordataset_from_dfs(\n            ['/kaggle/input/bengaliai-cv19' + '/test_image_data_{}.parquet'.format(i)]\n        )\n        # TTA\n        te_ds = DatasetWithImageTransforms(te_ds, tfms)\n        num_tta = 1\n        te_dl = DataLoader(te_ds, batch_size=256, num_workers=2, pin_memory=True)\n        tta_results = [[] for i in range(num_tta)]\n        for i in range(num_tta):\n            for imgs, in te_dl:\n                imgs = imgs.to(device)\n#                 imgs = imgs / 255.0\n                g_pred, v_pred, c_pred, *rest = model(imgs)\n                tta_results[i].append([g_pred, v_pred, c_pred])\n        \n        for i in range(len(te_dl)):\n            g_preds = [results[i][0] for results in tta_results]\n            v_preds = [results[i][1] for results in tta_results]\n            c_preds = [results[i][2] for results in tta_results]\n            g_pred = torch.stack(g_preds).mean(dim=0)\n            v_pred = torch.stack(v_preds).mean(dim=0)\n            c_pred = torch.stack(c_preds).mean(dim=0)\n            g_pred = g_pred.argmax(1).tolist()\n            v_pred = v_pred.argmax(1).tolist()\n            c_pred = c_pred.argmax(1).tolist()\n            \n            for g, v, c in zip(g_pred, v_pred, c_pred):\n                results.extend([c, g, v])\n        del imgs\n        del te_ds\n        del te_dl\n        del tta_results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv('/kaggle/input/bengaliai-cv19' + '/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['target'] = results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('./submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}