{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install /kaggle/input/timm-model/timm-0.3.2-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport cv2\nimport numpy as np\nfrom tqdm.auto import tqdm\nfrom matplotlib import pyplot as plt\nimport time\nimport pyarrow.parquet as pq","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_SIZE=128\nN_CHANNELS=1\ndef resize(df, size=128, need_progress_bar=True):\n    resized = {}\n    resize_size=128\n    if need_progress_bar:\n        for i in range(df.shape[0]):\n            image=df.loc[df.index[i]].values.reshape(137,236)\n            _, thresh = cv2.threshold(image, 30, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) #  \n            contours, _ = cv2.findContours(thresh,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)[-2:]\n\n            idx = 0 \n            ls_xmin = []\n            ls_ymin = []\n            ls_xmax = []\n            ls_ymax = []\n            for cnt in contours:\n                idx += 1\n                x,y,w,h = cv2.boundingRect(cnt)\n                ls_xmin.append(x)\n                ls_ymin.append(y)\n                ls_xmax.append(x + w)\n                ls_ymax.append(y + h)\n            xmin = min(ls_xmin)\n            ymin = min(ls_ymin)\n            xmax = max(ls_xmax)\n            ymax = max(ls_ymax)\n\n            roi = image[ymin:ymax,xmin:xmax]\n            #roi = image\n            resized_roi = cv2.resize(roi, (resize_size, resize_size),interpolation=cv2.INTER_AREA)\n            resized[df.index[i]] = resized_roi.reshape(-1)\n    resized = pd.DataFrame(resized).T\n    return resized","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport numpy as np\nimport tqdm\nimport csv\nimport os\nimport cv2\nimport pandas as pd\nfrom timm.models import create_model\nimport torch.autograd.profiler as profiler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.datasets import ImageFolder\nfrom torchvision.transforms import transforms\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport math\nimport time\nfrom torch.nn import Parameter\nfrom torch.autograd import Variable\nfrom timm.models import create_model\n\nclass MultiHeadSimpleModel(nn.Module):\n  \n  def __init__(self, input_size, backbone, pretrained=True, dropout=0.2):\n    super(MultiHeadSimpleModel, self).__init__()\n    self.input_size = input_size\n    self.backbone = backbone\n    self.pretrained = pretrained\n    self.dropout=dropout\n\n    self.backbone = create_model(self.backbone, self.pretrained, num_classes=0)\n\n\n    feature_size = self.backbone.state_dict()['bn2.weight'].shape[0]\n    self.root_head = nn.Linear(feature_size, 168, bias=True)\n    self.consonant_head = nn.Linear(feature_size, 168, bias=True)\n    self.vowel_head = nn.Linear(feature_size, 168, bias=True)\n\n\n    intermid = []\n    intermid.append(nn.BatchNorm1d(feature_size))\n    intermid.append(nn.Dropout(self.dropout))\n    intermid.append(nn.Linear(feature_size, 512))\n    intermid.append(nn.BatchNorm1d(512))\n\n\n    self.intermid_unique = nn.ModuleList(intermid)\n    torch.nn.init.kaiming_normal_(self.intermid_unique[2].weight)\n    self.arc_face_unique = Arcface(512, 1295)\n\n  def multi_head(self, input, unique):\n    input = self.backbone(input)\n\n    root = self.root_head(input)\n    consonant = self.consonant_head(input)\n    vowel = self.vowel_head(input)\n\n    x = self.intermid_unique[0](input)\n    for inter in self.intermid_unique[1:]:\n      x = inter(x)\n    x = F.normalize(x, p=2, dim=1)\n    unique_p = self.arc_face_unique(x, unique)\n\n    return root, consonant, vowel, unique_p\n\n\n  def forward(self, input, unique):\n    multi_head_outputs = self.multi_head(input, unique)\n\n    return multi_head_outputs\n\ndef l2_norm(input,axis=1):\n    norm = torch.norm(input,2,axis,True)\n    output = torch.div(input, norm)\n    return output\n\nclass Arcface(nn.Module):\n    # implementation of additive margin softmax loss in https://arxiv.org/abs/1801.05599    \n    def __init__(self, embedding_size=512, classnum=51332,  s=30., m=0.5):\n        super(Arcface, self).__init__()\n        self.classnum = classnum\n        self.weight = Parameter(torch.Tensor(embedding_size,classnum))\n        # initial kernel\n        self.weight.data.uniform_(-1, 1).renorm_(2,1,1e-5).mul_(1e5)\n        self.m = m # the margin value, default is 0.5\n        self.s = s # scalar value default is 64, see normface https://arxiv.org/abs/1704.06369\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.mm = self.sin_m * m  # issue 1\n        self.threshold = math.cos(math.pi - m)\n    def forward(self, embbedings, label):\n        # weights norm\n        nB = len(embbedings)\n        kernel_norm = l2_norm(self.weight,axis=0)\n        # cos(theta+m)\n        cos_theta = torch.mm(embbedings,kernel_norm)\n#         output = torch.mm(embbedings,kernel_norm)\n        cos_theta = cos_theta.clamp(-1,1) # for numerical stability\n        cos_theta_2 = torch.pow(cos_theta, 2)\n        sin_theta_2 = 1 - cos_theta_2\n        sin_theta = torch.sqrt(sin_theta_2)\n        cos_theta_m = (cos_theta * self.cos_m - sin_theta * self.sin_m)\n        # this condition controls the theta+m should in range [0, pi]\n        #      0<=theta+m<=pi\n        #     -m<=theta<=pi-m\n        cond_v = cos_theta - self.threshold\n        cond_mask = cond_v <= 0\n        keep_val = (cos_theta - self.mm) # when theta not in [0,pi], use cosface instead\n        cos_theta_m[cond_mask] = keep_val[cond_mask]\n        output = cos_theta * 1.0 # a little bit hacky way to prevent in_place operation on cos_theta\n        idx_ = torch.arange(0, nB, dtype=torch.long)\n        output[idx_, label] = cos_theta_m[idx_, label]\n        output *= self.s # scale up in order to make softmax work, first introduced in normface\n        return output\n\n\nclass BengaliDataset(Dataset):\n  def __init__(self, label_csv, unique_csv, train_folder, parquet_path, transforms, cache=True, test=True):\n    self.label_csv = label_csv\n    self.unique_csv = unique_csv\n    self.train_folder = train_folder\n    self.parquet_path = parquet_path\n    self.label = pd.read_csv(self.label_csv)\n    self.label = self.label[self.label['component']=='grapheme_root']\n    self.label = self.label.reset_index(drop=True)\n    #self.label = pd.read_csv(self.label_csv)\n    unique_df = pd.read_csv(self.unique_csv)\n\n    self.names = self.label['image_id'].values\n    self.uniques = unique_df.grapheme.unique()\n    self.transforms = transforms\n    self.img = [None] * self.label.shape[0]\n    self.test = test\n\n    if cache:\n      self.cache_images()\n\n  def cache_images(self):\n    count = 0\n    root = './test_graphemes/'\n#     for ii in range(4):\n#         p = pd.read_parquet(f'/kaggle/input/bengaliai-cv19/test_image_data_{ii}.parquet').drop(['image_id'], axis=1)#pd.read_parquet('/kaggle/input/bengaliai-cv19/test_image_data_0.parquet')\n#     #     p2 = pq.read_pandas('/kaggle/input/bengaliai-cv19/train_image_data_1.parquet').to_pandas()#pd.read_parquet('/kaggle/input/bengaliai-cv19/test_image_data_1.parquet')\n#     #     p3 = pq.read_pandas('/kaggle/input/bengaliai-cv19/train_image_data_2.parquet').to_pandas()#pd.read_parquet('/kaggle/input/bengaliai-cv19/test_image_data_2.parquet')\n#     #     p4 = pq.read_pandas('/kaggle/input/bengaliai-cv19/train_image_data_3.parquet').to_pandas()#pd.read_parquet('/kaggle/input/bengaliai-cv19/test_image_data_3.parquet')\n#     #     self.test_df = pd.concat([p1, p2, p3, p4]).reset_index(drop=True).drop(['image_id'], axis=1)\n\n\n#         pbar = tqdm.tqdm(range(p.shape[0]), position=0, leave=True)\n#         pbar.set_description('writing images...')\n#         for i, _ in enumerate(pbar):\n#           name = self.names[count]\n#           #self.img[count] = self.transforms(resize(p.iloc[[i]]).values.astype(np.uint8).reshape(-1).reshape(IMG_SIZE, IMG_SIZE))\n#           cv2.imwrite(root+name+'.jpg', resize(p.iloc[[i]]).values.reshape(-1).reshape(IMG_SIZE, IMG_SIZE).astype(np.uint8))\n#           count += 1\n        \n#         del p\n\n    self.p = p = pd.read_parquet(self.parquet_path).drop(['image_id'], axis=1)\n    \n#     count = 0\n#     pbar = tqdm.tqdm(range(self.label.shape[0]))\n#     pbar.set_description('caching images...')\n#     for i, _ in enumerate(pbar):\n#         name = self.names[i]\n#         img = Image.open(os.path.join('./test_graphemes/', name+'.jpg'))\n#         self.img[i] = self.transforms(img)\n\n  def load_image(self, idx):\n    img = resize(self.p.iloc[[idx]]).values.reshape(-1).reshape(IMG_SIZE, IMG_SIZE).astype(np.uint8)\n    return self.transforms(img)\n        \n    img = self.img[idx]\n    if img is None:\n      #name = self.label.loc[idx]['image_id']\n      img = resize(p.iloc[[i]]).values.reshape(-1).reshape(IMG_SIZE, IMG_SIZE).astype(np.uint8)\n      #img = Image.open(os.path.join(self.train_folder, name+'.jpg'))\n      return self.transforms(img)\n    else:\n      return self.transforms(img)\n\n  def __getitem__(self, idx):\n    if not self.test:\n      img = self.load_image(idx)\n      #img = self.img[idx]\n      root = self.label.loc[idx]['grapheme_root']\n      consonant = self.label.loc[idx]['consonant_diacritic']\n      vowel = self.label.loc[idx]['vowel_diacritic']\n      unique = np.where(self.uniques == self.label.grapheme[idx])[0][0]\n      return transforms.ToTensor()(img), root, consonant, vowel, unique\n    else:\n      img = self.load_image(idx)\n      #img = self.img[idx]\n      root = 0\n      consonant = 0\n      vowel = 0\n      unique = 0\n      return transforms.ToTensor()(img), root, consonant, vowel, unique\n\n  def __len__(self):\n    return self.p.shape[0]\n\ndef postprocess(preds, num_classes, EXP = -1.2):\n    p0 = np.argmax(preds,axis=1)\n\n    s = pd.Series(p0)\n    vc = s.value_counts().sort_index()\n    df = pd.DataFrame({'a':np.arange(num_classes),'b':np.ones(num_classes)})\n    df.b = df.a.map(vc)\n    df.fillna(df.b.min(),inplace=True)\n    mat1 = np.diag(df.b.astype('float32')**EXP)\n\n    p0 = np.argmax(preds.dot(mat1), axis=1)\n    \n    return p0\n\nclass Tester:\n    def __init__(self,\n               dataset_path='./drive/MyDrive/datasets/car classification/train_data', \n               batch_size=1, \n               model_name='tf_efficientnet_b3_ns', \n               test_csv='./train_labels.csv', \n               unique_csv='./train_labels.csv',\n               output_dir='../drive/MyDrive/ckpt/grapheme/submission.csv',\n               ckpt='../drive/MyDrive/ckpt/grapheme/20.pth',\n               multihead_ckpt='../'):\n\n        # initialize attributes\n        self.dataset_path = dataset_path\n        self.batch_size = batch_size\n        self.model_name = model_name\n        self.test_csv = test_csv\n        self.unique_csv = unique_csv\n        self.output_dir = output_dir\n        self.ckpt = ckpt\n        self.multihead_ckpt = multihead_ckpt\n        \n        if model_name == 'tf_efficientnet_b0_ns':\n            self.input_size = (224, 224)\n        elif model_name == 'tf_efficientnet_b2_ns':\n            self.input_size = (260, 260)\n        elif model_name == 'tf_efficientnet_b3_ns':\n            self.input_size = (300, 300)\n        elif model_name == 'tf_efficientnet_b4_ns':\n            self.input_size = (380, 380)\n        elif model_name == 'tf_efficientnet_b6_ns':\n            self.input_size = (528, 528)\n        else:\n            raise Exception('non-valid model name')\n        \n        # Compose transforms\n        transform = []\n        transform += [transforms.ToPILImage()]\n        transform += [transforms.Resize(self.input_size)]\n        self.transform = transforms.Compose(transform)\n\n       \n        self.device = torch.device('cuda')\n        self.model_multihead = MultiHeadSimpleModel(self.input_size, self.model_name, pretrained=False, dropout=0).to(self.device)\n\n\n        multihead_ckpt = torch.load(self.multihead_ckpt)\n        self.model_multihead.load_state_dict(multihead_ckpt['model_multihead_state_dict'])\n\n    def test(self):\n        output_roots = []\n        output_consonants = []\n        output_vowels = []\n        count = 0\n        for ii in range(4):\n            self.test_dataset = BengaliDataset(self.test_csv, self.unique_csv, self.dataset_path, f'/kaggle/input/bengaliai-cv19/test_image_data_{ii}.parquet', self.transform, cache=True)\n            self.names = self.test_dataset.names\n            self.test_dataloader = DataLoader(self.test_dataset, batch_size=self.batch_size, num_workers=0, shuffle=False)\n            \n            pbar = tqdm.tqdm(self.test_dataloader)\n            pbar.set_description('testing process')\n            self.model_multihead.eval()\n\n            with torch.no_grad():\n                for it, data in enumerate(pbar):\n                    inputs = data[0].to(self.device)\n                    inputs = inputs.repeat(1, 3, 1, 1)\n                    roots = data[1].to(self.device).long()\n                    consonants = data[2].to(self.device).long()\n                    vowels = data[3].to(self.device).long()\n                    uniques = data[4].to(self.device).long()\n\n                    \n                    root, consonant, vowel, unique = self.model_multihead(inputs, uniques)\n                    \n                \n                    root = postprocess(torch.nn.Softmax(1)(root).cpu().numpy(), 168, -1.2)\n                    consonant = postprocess(torch.nn.Softmax(1)(consonant[:,:8]).cpu().numpy(), 8, -0.5)\n                    vowel = postprocess(torch.nn.Softmax(1)(vowel[:,:11]).cpu().numpy(), 11, -1.2)\n\n                    \n                    for index in range(inputs.shape[0]):\n                \n                        output_roots.append(root[index].item())\n                        output_consonants.append(consonant[index].item())\n                        output_vowels.append(vowel[index].item())                 \n            \n            del self.test_dataset.p\n            del self.test_dataset\n            del self.test_dataloader\n\n        row_id, target = [], []\n        for iid, r, c, v in zip(self.names, output_roots, output_consonants, output_vowels):\n            row_id.append(iid + '_grapheme_root')\n            target.append(int(r))\n            row_id.append(iid + '_vowel_diacritic')\n            target.append(int(v))\n            row_id.append(iid + '_consonant_diacritic')\n            target.append(int(c))\n            count += 1\n\n        sub_fn = self.output_dir\n        sub = pd.DataFrame({'row_id': row_id, 'target': target})\n        sub.to_csv(sub_fn, index=False)\n        print(f'Done wrote to {sub_fn}')\n\ntester = Tester(dataset_path='./test_graphemes', \n               batch_size=128, \n               model_name='tf_efficientnet_b2_ns', \n               test_csv='/kaggle/input/bengaliai-cv19/test.csv', \n               unique_csv='/kaggle/input/bengaliai-cv19/train.csv',\n               output_dir='./submission.csv',\n               ckpt='/kaggle/input/graphemesingle/27.pth',\n               multihead_ckpt='/kaggle/input/graphemesingle/10.pth'\n               )\ntester.test()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}