{"cells":[{"metadata":{},"cell_type":"markdown","source":"Yeah, top 15% sounds much better than 295'th place, so let's leave it for a header :). Actually, there is nothing interesting in this kernel. However, I would like to share it as I have written it myself mainly and, probably, it could be interesting (or even useful) for some beginners like me. \n\nSo, I had just trained resnet101 :). That's all. (pytorch, albumentations)\n\nSome details:\n\n1) Firstly, I had tried resnet18, and my start public score was 0.9060 (of course, after passing a submission quest, which costs me 27 attempts);\n\n2) Then I had trained resnet101 to increase my network complexity;\n\n3) I trained images 224 x 224, RGB (as pytorch docs states at least this size);\n\n4) resnet101 head code is taken from fastai code, as well as initialization;\n\n5) training process:\n\n   a) first 5 epochs (frozen body):lr=2e-3, Adam, no augmentation;\n   \n   b) next 11 epochs (unfrozen body):lr=2e-3, Adam, no augmentation;\n   \n   c) next 8 epochs (unfrozen body), lr=2e-3, Adam, augmentation added;\n   \n   d) next 2 epochs (unfrozed body), lr=2e-4, Adam, augmentation;\n   \n6)didn't use TTA (got timeout at submission time and didn't dig into it)\n   \nSo, my best public score was 0.9596, the private one - 0.9279\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport time\nfrom tqdm.notebook import tqdm_notebook\nimport cv2\nimport torch.utils.data\nfrom torch.utils.data import Dataset,DataLoader\nfrom torchvision import transforms, datasets\nimport torchvision.models as models\nimport re\nimport torch\nimport torch.nn as nn\nfrom typing import Optional\nimport albumentations as A\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntorch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"To switch to train process, and kaggle environment I used  INFERENCE and KAGGLE variables."},{"metadata":{"trusted":true},"cell_type":"code","source":"INFERENCE = True\nKAGGLE = True\nPATH = \"../input/bengaliai-cv19\" if KAGGLE else \"../../data/bengali\" \nMODELS_PATH = \"../input/bengali\" if KAGGLE else \"../../data/bengali\"\nCSV_PATH = \"\" if KAGGLE else \"../../data/bengali/\"\n!dir $PATH\nPREFIX = \"test\" if INFERENCE else \"train\"\nfile = PATH + \"/{0}_image_data_{1}.parquet\"\nWIDTH=236\nHEIGHT=137\nFILES_NUM = 4\nFILE_RECORDS = 50210\nTRAIN_RECORDS_TOTAL = FILES_NUM * FILE_RECORDS\nBS = 8 if INFERENCE else 8\nTRAIN_SIZE = int(FILE_RECORDS * 0.9)\nVALID_SIZE = int(FILE_RECORDS * 0.1)\nnp.random.seed(42)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"everyting in memory to speedup the training process....\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"dfs = None\nif (not INFERENCE):\n    t1 = time.time()\n    dfs = [pd.read_parquet(file.format(PREFIX,i)) for i in range(4)]\n    print(time.time() - t1)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Get data of ind'th file..."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data(ind):\n    df = pd.read_parquet(file.format(PREFIX,ind)) if INFERENCE else dfs[ind] \n    ids = df.iloc[:,0] if INFERENCE else None\n    data = df.iloc[:,1:].values.reshape(-1,HEIGHT, WIDTH).astype(np.uint8)          \n    return data, ids\n\ntrain_csv=None\nif (not INFERENCE):    \n    train_data, ids = get_data(0)\n    train_csv = pd.read_csv(PATH + \"/train.csv\")\n    print(train_csv.head())\n    print(train_csv['grapheme_root'])\n    print(train_csv.max())\n    print(train_data.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"there is some mesh below about vertical flip. Actually, in 295th place solution I didn't use vertial flip at all..."},{"metadata":{"trusted":true},"cell_type":"code","source":"bc=[255,255,255]\n\ndef get_aug(vert_flip = False):\n    p_vert_flip = 1 if vert_flip else 0\n    train_aug =  A.Compose([\n        A.OneOf([A.Blur(blur_limit=10,p=1.0),\n                 A.GaussianBlur(blur_limit=15,p=1.0),\n                 A.GaussNoise(var_limit=200,p=1.0),#???\n                ],p=0.4),\n        A.OneOf([A.Cutout(num_holes=9,  max_h_size=20, max_w_size=20, p=1.0, fill_value=255),\n                 A.CoarseDropout(max_holes=8, max_height=20, max_width=20, p=1.0, fill_value=255),\n                 A.GridDistortion(border_mode=cv2.BORDER_CONSTANT,value=bc, p=0.1)\n                ],p=0.4),\n        A.OneOf([A.RandomBrightness(p=1.0),\n                A.RandomContrast(p=1.0),\n                A.RandomBrightnessContrast(p=1.0)\n               ],p=0.4),\n        A.OneOf([A.IAAPiecewiseAffine(p=1.0),\n                A.ElasticTransform(sigma=30, alpha=1, alpha_affine=30, \n                                 border_mode=cv2.BORDER_CONSTANT,value=bc, p=1.0)\n               ],p=0.4),\n        #A.VerticalFlip(p=p_vert_flip),\n        A.RandomGamma(p=0.8),\n        A.ShiftScaleRotate(shift_limit=0.0625,scale_limit=0.1,\n                            rotate_limit=30,p=1.0,value=bc,border_mode=cv2.BORDER_CONSTANT), \n    ])\n    \n    inf_aug = A.VerticalFlip(p=p_vert_flip)\n    return inf_aug if INFERENCE else train_aug\n\n\n    \n#imgs = [get_image(train_data,i) for i in range(1000)]\n\n#aug(image=imgs[0])  \n\nif (not INFERENCE and False):\n    n_imgs = 8\n    fig, axs = plt.subplots(n_imgs, 2, figsize=(10, 5*n_imgs))\n    for idx in range(n_imgs):        \n        #aug  = A.Blur(blur_limit=10,p=1.0)\n        #aug = A.GaussianBlur(blur_limit=15,p=1.0)\n        #aug = A.GaussNoise(var_limit=200,p=1.0)#???\n        #aug=A.Cutout(num_holes=9,  max_h_size=20, max_w_size=20, p=1.0, fill_value=255)\n        #aug = A.CoarseDropout(max_holes=8, max_height=20, max_width=20, p=1.0, fill_value=255)\n        #aug = A.GridDistortion(border_mode=cv2.BORDER_CONSTANT,value =1, p=0.1)\n        #aug =  A.RandomBrightness(p=1.0)\n        #aug =  A.RandomContrast(p=1.0)\n        #aug =  A.RandomBrightnessContrast(p=1.0)\n        #aug = A.ShiftScaleRotate(shift_limit=0.0625,scale_limit=0.1,rotate_limit=30,p=1.0,border_mode=cv2.BORDER_CONSTANT)\n        #aug =  A.IAAPiecewiseAffine(p=1.0)\n        #aug = A.ElasticTransform(sigma=30, alpha=1, alpha_affine=30, border_mode=cv2.BORDER_CONSTANT,value=[255,255,255], p=1.0)\n        #aug = A.VerticalFlip(p=1.0)\n        #aug = A.RandomGamma(p=1.0)\n        img = get_image(train_data, idx)\n        img0 = train_data[idx]#.reshape(HEIGHT, WIDTH).astype(np.uint8)\n        axs[idx,0].imshow(img)\n        #axs[idx,0].set_title('Original image')\n        axs[idx,0].axis('off')\n        axs[idx,1].imshow(get_aug(True)(image=img)['image'])\n        #axs[idx,1].set_title('Crop & resize')\n        axs[idx,1].axis('off')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#some code of this cell was taken from https://www.kaggle.com/iafoss/image-preprocessing-128x128\n#but I changed a lot I saw more logical....\n\nSIZE=224\n\n\ndef bbox(img):   \n    rows = np.any(img, axis=1)    \n    cols = np.any(img, axis=0)\n    rmin, rmax = np.where(rows)[0][[0, -1]]\n    cmin, cmax = np.where(cols)[0][[0, -1]]\n    return rmin, rmax, cmin, cmax\n\ndef crop_resize(img0, size=SIZE, pad=16):\n    #crop a box around pixels large than the threshold \n    #some images contain line at the sides\n    ymin,ymax,xmin,xmax = bbox(img0[5:-5,5:-5] < (255-60))    \n    #cropping may cut too much, so we need to add it back\n    xmin = xmin - 13 if (xmin > 13) else 0\n    ymin = ymin - 10 if (ymin > 10) else 0\n    xmax = xmax + 13 if (xmax < WIDTH - 13) else WIDTH\n    ymax = ymax + 10 if (ymax < HEIGHT - 10) else HEIGHT\n    img = img0[ymin:ymax,xmin:xmax]\n    #remove lo intensity pixels as noise\n    lx, ly = xmax-xmin,ymax-ymin\n    l = max(lx,ly) + pad\n    #make sure that the aspect ratio is kept in rescaling\n    #img = np.pad(img, [((l-ly)//2,), ((l-lx)//2,)], mode='constant',)\n    \n    return cv2.resize(img,(size,size))\n\ndef get_image(data, idx) :\n    img = data[idx]\n    img = crop_resize(img)\n    img = np.reshape(img, (SIZE,SIZE, -1))\n    img  = np.repeat(img, 3, 2)    \n    return img    \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if (not INFERENCE):\n    n_imgs = 8\n    fig, axs = plt.subplots(n_imgs, 2, figsize=(10, 5*n_imgs))\n    for idx in range(n_imgs):        \n        img = get_image(train_data, idx)\n        img0 = train_data[idx]#.reshape(HEIGHT, WIDTH).astype(np.uint8)\n        axs[idx,0].imshow(img0)\n        axs[idx,0].set_title('Original image')\n        axs[idx,0].axis('off')\n        axs[idx,1].imshow(img)\n        axs[idx,1].set_title('Crop & resize')\n        axs[idx,1].axis('off')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"in 295th place inference only img0 used for inference see below, and no augmentation for img0"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef my():\n    def __init__(self):\n        self.mean = m\n        self.std = s\n        self.n = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                             std=[0.229, 0.224, 0.225])\n    def __call__(self, sample):\n        print(\"!!!called\")\n        return n(sample)\n        \n\ndata_transform = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                             std=[0.229, 0.224, 0.225])\n    ])\n\nclass GraphemeDataset(Dataset):\n    def __init__(self, labels_csv, data, file_num = 0, transform=data_transform, rev = True):\n        self.labels_csv = labels_csv\n        self.data = data\n        self.transform = data_transform\n        self.offset = file_num * FILE_RECORDS\n        self.aug = get_aug(rev)\n        \n        \n    def __len__(self):        \n        return self.data.shape[0]\n    \n    def prepare_image(self,img, do_aug=True):\n        result = img\n        #print(do_aug)\n        if do_aug:\n            #print(\"aug\",self.aug)\n            result = self.aug(image=img)['image']\n        result = (result/255).astype(np.float32)\n        result = self.transform(result)\n        return result\n    \n    def __getitem__(self,idx):\n        image_pure = get_image(self.data, idx)\n        #print(\"img0\")\n        img0 = self.prepare_image(img = image_pure, do_aug=not INFERENCE)  \n        if (INFERENCE):\n            #print(\"img1\")\n            img1 = self.prepare_image(img = image_pure, do_aug=True)\n        #    img2 = self.prepare_image(img = image_pure)\n            return img0,img1\n                   \n        else :    \n            labels = self.labels_csv.iloc[idx + self.offset][1:4]\n            root = labels[0]\n            vowel = labels[1]\n            consonant = labels[2]\n            return img0, root, vowel, consonant\n    \n    \ndef get_dls(ind):\n    data, ids = get_data(ind)\n    rev_ds = GraphemeDataset(train_csv, data, ind, rev = True)\n    if (INFERENCE):\n        rev_dl =  DataLoader(rev_ds, batch_size = BS)\n        \n        return rev_dl, None, ids\n    else:\n        train_ds, valid_ds = torch.utils.data.random_split(rev_ds, [TRAIN_SIZE, VALID_SIZE])\n        train_dl = DataLoader(train_ds, batch_size=BS)\n        valid_dl = DataLoader(valid_ds, batch_size=BS)\n        return train_dl, valid_dl, ids \n    \nif (INFERENCE):\n    check_dl, some_dl, ids = get_dls(0)\n    print(ids[0])\n    x1, x2 = next(iter(check_dl))\n    print(x1.shape, x2.shape)\nelse:    \n    check_dl, valid_dl, ids = get_dls(0)\n    x, l1, l2, l3 = next(iter(check_dl))\n    print(x.shape, l1.shape, l2.shape, l3.shape, type(x[0][0][0][0]))\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Had no time to train densenet carefully, the idea was to try ensemble of densenet162 and resnet101. Had time only for resnet101."},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#This code was taken from fastai (at least at some extent)\n\ndef requires_grad(m:nn.Module, b:Optional[bool]=None)->Optional[bool]:\n    \"If `b` is not set return `requires_grad` of first param, else set `requires_grad` on all params as `b`\"\n    ps = list(m.parameters())\n    if not ps: return None\n    if b is None: return ps[0].requires_grad\n    for p in ps: \n        p.requires_grad=b\n        #print(p.requires_grad)\n\ndef is_pool_type(l): return re.search(r'Pool[123]d$', l.__class__.__name__)\ndef has_pool_type(m):\n    if is_pool_type(m): return True\n    for l in m.children():\n        if has_pool_type(l): return True\n    return False\n\ndef create_body(model):\n    ll = list(enumerate(model.children()))\n    cut = next(i for i,o in reversed(ll) if has_pool_type(o))\n    \n    return nn.Sequential(*list(model.children())[:cut])\n\nclass AdaptiveConcatPool2d(nn.Module):\n    def __init__(self, sz=None):\n        super().__init__()\n        sz = sz or (1,1)\n        self.ap = nn.AdaptiveAvgPool2d(sz)\n        self.mp = nn.AdaptiveMaxPool2d(sz)\n    def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1)\n    \nclass Flatten(nn.Module):\n    def forward(self, x): return x.view(x.size(0), -1)\n    \ndef bn_drop_lin(n_in:int, n_out:int, bn:bool=True, p:float=0., actn:Optional[nn.Module]=None):\n    \"Sequence of batchnorm (if `bn`), dropout (with `p`) and linear (`n_in`,`n_out`) layers followed by `actn`.\"\n    layers = [nn.BatchNorm1d(n_in)] if bn else []\n    if p != 0: layers.append(nn.Dropout(p))\n    layers.append(nn.Linear(n_in, n_out))\n    if actn is not None: layers.append(actn)\n    return layers    \n\ndef create_head(is_densenet):    \n    #lin_ftrs = [1024, 256, 168 + 11 + 7]\n    #lin_ftrs = if [4096, 256, 168 + 11 + 7]\n    lin_ftrs = [4416, 256, 168 + 11 + 7] if is_densenet else [4096, 256, 168 + 11 + 7]\n    ps = [0.5]#listify(ps)\n    bn_final=False\n    concat_pool=True\n    if len(ps) == 1: ps = [ps[0]/2] * (len(lin_ftrs)-2) + ps\n    actns = [nn.ReLU(inplace=True)] * (len(lin_ftrs)-2) + [None]\n    pool = AdaptiveConcatPool2d() if concat_pool else nn.AdaptiveAvgPool2d(1)\n    layers = [pool, Flatten()]\n    for ni,no,p,actn in zip(lin_ftrs[:-1], lin_ftrs[1:], ps, actns):\n        layers += bn_drop_lin(ni, no, True, p, actn)\n        #layers += bn_drop_lin(ni, no, False, 0, actn)\n    if bn_final: layers.append(nn.BatchNorm1d(lin_ftrs[-1], momentum=0.01))\n    head = nn.Sequential(*layers)\n    return head\n\ndef cond_init(m:nn.Module):\n    \"Initialize the non-batchnorm layers of `m` with `init_func`.\"\n    if (not isinstance(m, nn.BatchNorm1d)) and requires_grad(m): \n        if hasattr(m, 'weight'): nn.init.kaiming_normal_(m.weight)\n        if hasattr(m, 'bias') and hasattr(m.bias, 'data'): m.bias.data.fill_(0.)\n\ndef apply_leaf(m:nn.Module, f):\n    \"Apply `f` to children of `m`.\"\n    c = list(m.children())\n    if isinstance(m, nn.Module): f(m)\n    for l in c: apply_leaf(l,f)\n        \ndef apply_init(m):\n    \"Initialize all non-batchnorm layers of `m` with `init_func`.\"\n    apply_leaf(m, cond_init)\n\ndef get_resnet():\n    resnet101 = models.resnet101(pretrained=not INFERENCE)\n    body = create_body(resnet101)\n    head = create_head(False)\n    apply_init(head)\n    return nn.Sequential(body, head),body\n\ndef get_densenet() :\n    densenet= models.densenet161(pretrained= not INFERENCE)\n    \n    head = nn.Linear(2208, 168 + 11 + 7)#create_head()\n    apply_init(head)\n    children = densenet.children()\n    \n    body = next(children) \n    head = create_head(True)\n    return nn.Sequential(body, head),body\n    \nm_resnet,body = get_resnet()\nm_densenet,body = get_densenet()\nprint(m_densenet\n     )\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I had not run training in the kaggle environment, only inference. It should work,but \"In theory there is no difference between theory and practice. In practice there is.” (c)"},{"metadata":{"trusted":true},"cell_type":"code","source":"m = m_resnet#m_densenet\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nepochs = 1\n\noptimizer = torch.optim.Adam(m.parameters(), lr=2e-3)\nm.to(device)\ncriterion = nn.CrossEntropyLoss()\n\ndef split(a):\n    return a[:, 0:168], a[:, 168:168+11], a[:, 168+11:]\n\ndef loss_func(roots168, vowels11, consonants7, outputs):    \n    r,v,c = split(outputs)\n    root_loss = criterion(r, roots168)\n    vowel_loss = criterion(v, vowels11)\n    consonant_loss = criterion(c, consonants7)\n    \n    total_loss = root_loss*2 + vowel_loss + consonant_loss\n    #total_loss = root_loss + vowel_loss + consonant_loss\n    return total_loss\n\ndef acc_func(roots168, vowels11, consonants7, outputs, acc):\n    r,v,c = split(outputs)\n    acc_c = [0,0,0,0]\n    acc_c[0] = (r.argmax(1)==roots168).float().mean()\n    acc_c[1] = (v.argmax(1)==vowels11).float().mean()\n    acc_c[2] = (c.argmax(1)==consonants7).float().mean()\n    acc_c[3] = (2*acc_c[0] + acc_c[1] + acc_c[2])/4\n       \n    return np.add(acc, acc_c)\n\ndef getv(v, length):   \n    \n    return round((v/length).item(),4)\n\ndef getv_list(v, length):\n    return [getv(e,length) for e in v]\n\ndef to_d(i, r, v,c):\n    return i.to(device), r.to(device), v.to(device), c.to(device)\n\ndef train_int(train, dl, m, pbar):\n    acc = [0,0,0,0]\n    loss = 0\n    m.train(train)\n    for (idx, (inp, r, v, c)) in (enumerate(dl)):\n        inp, r, v, c = inp.to(device), r.to(device), v.to(device), c.to(device)           \n        outputs = m(inp)\n        #print(outputs.shape)\n        acc = acc_func(r, v, c, outputs, acc) \n        if (train):    \n            total_loss = loss_func(r, v, c, outputs)\n            loss += total_loss\n            total_loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n            \n        pbar.update(BS)\n        \n    return acc, loss    \n\n\ndef train_epoch(num, start3):\n    #from tqdm.notebook import tqdm_notebook\n\n    pbar = tqdm_notebook(total = TRAIN_RECORDS_TOTAL)\n    print(\"acc train\\tloss train\\tacc valid\")\n    if (start3 != 0):\n        m.load_state_dict(torch.load(MODELS_PATH + \"/densenet162.2e-3.4.tmp\")) \n    for file_num in range(start3, FILES_NUM):\n        train_dl, valid_dl, ids = get_dls(file_num)\n        \n        acc_train, loss_train = train_int(True, train_dl, m, pbar)\n        acc_valid, l = train_int(False, valid_dl, m, pbar)            \n        \n        train_length = len(train_dl)\n                \n        acc_p = getv(acc_train[3], train_length)\n        loss_p = getv(loss_train, train_length)        \n        acc_v_p = getv_list(acc_valid, len(valid_dl))\n        \n        print(acc_p,\"\\t\\t\", loss_p, \"\\t\\t\",acc_v_p[3], \n              \"(\",acc_v_p[0], acc_v_p[1], acc_v_p[2],\")\")\n        torch.save(m.state_dict(), MODELS_PATH + \"/densenet162.2e-3.4.tmp\")\n    pbar.close()\n    \ndef train():\n    print(\"Will train a frozen-body model...\")\n    requires_grad(body, False)\n    start1 = 5\n    start3 = 2\n    m.load_state_dict(torch.load(MODELS_PATH + \"/densenet162.2e-3.5.0\"))\n    for num in range(start1, 5):\n        print(\"Frozen, number \" + str(num))\n        train_epoch(num, start3)\n        start3 = 0\n        torch.save(m.state_dict(), MODELS_PATH + \"/densenet162.2e-3.\" + str(num))\n        #torch.save(m.state_dict(), MODELS_PATH + \"/resnet.2e-3.1.1\")\n        \n    print(\"Will train an unfrozen-body model...\")\n    start2 = 5\n    start3 = 2\n    requires_grad(body, True)  \n    for num in range(start2,30):\n        print(\"Unfrozen, number \" + str(num))\n        train_epoch(num, start3)\n        start3 = 0\n        torch.save(m.state_dict(), MODELS_PATH + \"/densenet162.2e-3.5.\" + str(num))   \n    #torch.save(m.state_dict(), MODELS_PATH + \"/model.2e-3.1.1\")\n    \nif (not INFERENCE):    \n    train()   \n#!pip install line_profiler        \n#%load_ext line_profiler        \n#%lprun -f GraphemeDataset.__getitem__ train()\n#%lprun -f get_image train()\n#%lprun -f train train()\n\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##INFERENCE!!!!\ndef handle_single_file(file_num, subm_csv):\n    print(\"file\",file_num)    \n    test_dl, something, ids = get_dls(file_num)\n    m_densenet.train(False)\n    m_resnet.train(False)\n    \n    index = 0\n    for (idx, (img, img_flipped)) in (enumerate(test_dl)):\n        img = img.to(device)\n        #img_flipped = img_flipped.to(device)\n        \n        outputs1 = m_resnet(img)\n        #outputs2 = m_densenet(img_flipped)\n        outputs = outputs1#0.5 * (outputs1 + outputs2) \n        r,v,c = split(outputs)\n        roots = r.argmax(1).tolist()\n        vowels = v.argmax(1).tolist()\n        consonants = c.argmax(1).tolist()\n        current_bs = img.shape[0]        \n        for i in range(current_bs):\n            test_name = ids[index] \n            \n            subm_csv.write(test_name + \"_consonant_diacritic,\" + str(consonants[i]) + \"\\n\")\n            subm_csv.write(test_name + \"_grapheme_root,\" + str(roots[i]) + \"\\n\")\n            subm_csv.write(test_name + \"_vowel_diacritic,\" + str(vowels[i]) + \"\\n\")\n            index += 1\n                \n\ndef inference():\n    m_resnet.load_state_dict(torch.load(MODELS_PATH + \"/resnet101.2e-4.5.21\"))\n    m_densenet.load_state_dict(torch.load(MODELS_PATH + \"/densenet162.2e-3.5.9\"))\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    m_resnet.to(device)\n    m_densenet.to(device)\n    print(\"Inference....\")\n    total_index = 0\n    subm_csv = open(CSV_PATH + \"submission.csv\", 'w')\n    subm_csv.write('row_id,target\\n')\n    for file_num in range(FILES_NUM):\n        handle_single_file(file_num,subm_csv)\n                            \n    subm_csv.flush()        \n    subm_csv.close()   \n    \nif (INFERENCE):\n    inference()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#if (INFERENCE):\n #   !cat {CSV_PATH}submission.csv","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.9"}},"nbformat":4,"nbformat_minor":4}