{"cells":[{"metadata":{},"cell_type":"markdown","source":"<h2 style='background:#FFFFFF; border:0; color:black'><center>If you fork it, please give an upvote!<center><h2>"},{"metadata":{},"cell_type":"markdown","source":"# About\n* In this notebook, I use ArcFace instead of Linear. .\n* I  made datasets public in the inference section .\n\n# Source Kernels\n* This notebook was written by refering these great kernels below, so please don't forget to **check and upvote** them.\n* [Pytorch Efficientnet Baseline [Train] AMP+Aug](https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-train-amp-aug)\n* [Pytorch Efficientnet Baseline [Inference] TTA](https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-inference-tta)\n\n# Train Link\n\n* Inference part is here: https://www.kaggle.com/wenlia/pytorch-baseline-train-arcface-amp-aug"},{"metadata":{"trusted":true},"cell_type":"code","source":"package_path = '../input/pytorch-image-models/pytorch-image-models-master' #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\nimport sys; sys.path.append(package_path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport math\nfrom torch.nn import Parameter\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom  torch.cuda.amp import autocast, GradScaler\nimport torch.nn.functional as F\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\nimport timm #from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom\nfrom sklearn.metrics import log_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 719,\n    'model_arch': 'tf_efficientnet_b4_ns',\n    'img_size': 512,\n    'use_benchmark': False,\n    'use_deterministic': False,\n    'epochs': 10,\n    'train_bs': 32,\n    'valid_bs': 32,\n    'lr': 1e-4,\n    'num_workers': 4,\n    'accum_iter': 1, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'tta': 3,\n    'used_epochs': [6,8],\n    'weights': [1,1],\n    'classifier_type': 'linear', # arcface,linear\n    'cosine_scale': 15,\n    'cosine_margin': 0.4\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> We could do stratified validation split in each fold to make each fold's train and validation set looks like the whole train set in target distributions."},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if CFG['use_benchmark']:\n    torch.backends.cudnn.benchmark = True\nif CFG['use_deterministic']:\n    torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(\n        self, df, data_root, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        \n        img  = get_img(path)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define Train\\Validation Image Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\ndef get_inference_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            #Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            #HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            #RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Classifier module"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# stronger classifier module\n# default is linear \n# other include in {CircleLoss,Arcface,Cosface,AMSoftmax}\n# ====================================================\nclass ContrastiveLoss(nn.Module):\n    def __init__(self, margin=0.3, **kwargs):\n        super(ContrastiveLoss, self).__init__()\n        self.margin = margin\n\n    def forward(self, inputs, targets):\n        n = inputs.size(0)\n        # Compute similarity matrix\n        sim_mat = torch.matmul(inputs, inputs.t())\n        targets = targets\n        loss = list()\n        c = 0\n\n        for i in range(n):\n            pos_pair_ = torch.masked_select(sim_mat[i], targets == targets[i])\n\n            #  move itself\n            pos_pair_ = torch.masked_select(pos_pair_, pos_pair_ < 1)\n            neg_pair_ = torch.masked_select(sim_mat[i], targets != targets[i])\n\n            pos_pair_ = torch.sort(pos_pair_)[0]\n            neg_pair_ = torch.sort(neg_pair_)[0]\n\n            neg_pair = torch.masked_select(neg_pair_, neg_pair_ > self.margin)\n\n            neg_loss = 0\n\n            pos_loss = torch.sum(-pos_pair_ + 1)\n            if len(neg_pair) > 0:\n                neg_loss = torch.sum(neg_pair)\n            loss.append(pos_loss + neg_loss)\n\n        loss = sum(loss) / n\n        return loss\n\n\nclass CircleLoss(nn.Module):\n    def __init__(self, in_features, out_features, s=256, m=0.25):\n        super(CircleLoss, self).__init__()\n        self.weight = Parameter(torch.Tensor(out_features, in_features))\n        self._s = s\n        self._m = m\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5))\n\n    def __call__(self, bn_feat, targets):\n\n        sim_mat = F.linear(F.normalize(bn_feat), F.normalize(self.weight))\n        alpha_p = F.relu(-sim_mat.detach() + 1 + self._m)\n        alpha_n = F.relu(sim_mat.detach() + self._m)\n        delta_p = 1 - self._m\n        delta_n = self._m\n\n        s_p = self._s * alpha_p * (sim_mat - delta_p)\n        s_n = self._s * alpha_n * (sim_mat - delta_n)\n\n        one_hot = torch.zeros(sim_mat.size(), device=targets.device)\n        one_hot.scatter_(1, targets.view(-1, 1).long(), 1)\n\n        pred_class_logits = one_hot * s_p + (1.0 - one_hot) * s_n\n\n        return pred_class_logits\n\n\nclass Arcface(nn.Module):\n    r\"\"\"Implement of large margin arc distance: :\n        Args:\n            in_features: size of each input sample\n            out_features: size of each output sample\n            s: norm of input feature\n            m: margin\n            cos(theta + m)\n        \"\"\"\n    def __init__(self, in_features, out_features, s=30.0, m=0.30, easy_margin=False, ls_eps=0.0):\n        super(Arcface, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps  # label smoothing\n        self.weight = Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, input, label):\n        #print(\"input.shape\",input.shape,label.shape,self.in_features,self.weight.shape)\n        #print(\"label arcface time device\",label.device)\n        # --------------------------- cos(theta) & phi(theta) ---------------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        if not self.training:\n            return cosine\n        \n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        phi = phi.type_as(cosine)\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        # --------------------------- convert label to one-hot ---------------------------\n        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.size(), device='cuda')\n        \n        #print(\"label device is \",label.device)\n        #print(\"one_hot device is \",one_hot.device)\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.out_features\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) -------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n\n        return output\n\n\nclass Cosface(nn.Module):\n    r\"\"\"Implement of large margin cosine distance: :\n    Args:\n        in_features: size of each input sample\n        out_features: size of each output sample\n        s: norm of input feature\n        m: margin\n        cos(theta) - m\n    \"\"\"\n\n    def __init__(self, in_features, out_features, s=30.0, m=0.30):\n        super(Cosface, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.weight = Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n    def forward(self, input, label):\n        # --------------------------- cos(theta) & phi(theta) ---------------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        phi = cosine - self.m\n        # --------------------------- convert label to one-hot ---------------------------\n        one_hot = torch.zeros(cosine.size(), device='cuda')\n        # one_hot = one_hot.cuda() if cosine.is_cuda else one_hot\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) -------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)  # you can use torch.where if your torch.__version__ is 0.4\n        output *= self.s\n        # print(output)\n\n        return output\n\n    def __repr__(self):\n        return self.__class__.__name__ + '(' \\\n               + 'in_features=' + str(self.in_features) \\\n               + ', out_features=' + str(self.out_features) \\\n               + ', s=' + str(self.s) \\\n               + ', m=' + str(self.m) + ')'\n\n\nclass AMSoftmax(nn.Module):\n    def __init__(self, in_features, out_features, s=30.0, m=0.30):\n        super(AMSoftmax, self).__init__()\n        self.m = m\n        self.s = s\n        self.in_feats = in_features\n        self.W = torch.nn.Parameter(torch.randn(in_features, out_features), requires_grad=True)\n        self.ce = nn.CrossEntropyLoss()\n        nn.init.xavier_normal_(self.W, gain=1)\n\n    def forward(self, x, lb):\n        assert x.size()[0] == lb.size()[0]\n        assert x.size()[1] == self.in_feats\n        x_norm = torch.norm(x, p=2, dim=1, keepdim=True).clamp(min=1e-12)\n        x_norm = torch.div(x, x_norm)\n        w_norm = torch.norm(self.W, p=2, dim=0, keepdim=True).clamp(min=1e-12)\n        w_norm = torch.div(self.W, w_norm)\n        costh = torch.mm(x_norm, w_norm)\n        # print(x_norm.shape, w_norm.shape, costh.shape)\n        lb_view = lb.view(-1, 1)\n        delt_costh = torch.zeros(costh.size(), device='cuda').scatter_(1, lb_view, self.m)\n        costh_m = costh - delt_costh\n        costh_m_s = self.s * costh_m\n        return costh_m_s","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# class CassvaImgClassifier(nn.Module):\n#     def __init__(self, model_arch, n_class, pretrained=False):\n#         super().__init__()\n#         self.model = timm.create_model(model_arch, pretrained=pretrained)\n#         n_features = self.model.classifier.in_features\n#         self.model.classifier = nn.Linear(n_features, n_class)\n        \n#     def forward(self, x):\n#         x = self.model(x)\n#         return x\n\nclass CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        \n        backbone = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = backbone.classifier.in_features\n        self.backbone = nn.Sequential(*backbone.children())[:-1]\n#         for key, value in self.backbone.named_parameters():\n#             print(key,value.shape)\n        \n        \n#         for i in self.backbone.children():\n#             print(i)\n        \n        \"\"\"\n        self.backbone = nn.Sequential(*backbone.children())[:-2]\n        if CFG.POOLING_METHOD == \"GeM\":\n            print(\"using GeM pooling\")\n            self.gap = GeM()\n        else:\n            print(\"using AdaptiveAvgPool2D\")\n            self.gap = nn.AdaptiveAvgPool2d((1, 1))\n        \"\"\"\n        #self.model = timm.create_model(model_arch, pretrained=pretrained)\n        #n_features = self.model.classifier.in_features\n        #self.model.classifier = nn.Linear(n_features, n_class)\n        if CFG['classifier_type'] == \"arcface\":\n            print('using {} with s:{}, m: {}'.format(CFG['classifier_type'],CFG['cosine_scale'],CFG['cosine_margin']))\n            self.classifier = Arcface(n_features, n_class,s=CFG['cosine_scale'], m=CFG['cosine_margin'])\n        elif CFG['classifier_type'] == \"cosface\":\n            print('using {} with s:{}, m: {}'.format(CFG['classifier_type'],CFG['cosine_scale'],CFG['cosine_margin']))\n            self.classifier = Cosface(n_features, n_class,s=CFG['cosine_scale'], m=CFG['cosine_margin'])\n        elif CFG['classifier_type'] == \"amsoftmax\":\n            print('using {} with s:{}, m: {}'.format(CFG['classifier_type'],CFG['cosine_scale'],CFG['cosine_margin']))\n            self.classifier = AMSoftmax(n_features, n_class,s=CFG['cosine_scale'], m=CFG['cosine_margin'])\n        elif CFG['classifier_type'] == \"circle\":\n            print('using {} with s:{}, m: {}'.format(CFG['classifier_type'],CFG['cosine_scale'],CFG['cosine_margin']))\n            self.classifier = CircleLoss(n_features, n_class,s=CFG['cosine_scale'], m=CFG['cosine_margin'])\n        else:\n            self.classifier = nn.Linear(n_features, n_class)  # origin\n        '''\n        self.model.classifier = nn.Sequential(\n            nn.Dropout(0.3),\n            #nn.Linear(n_features, hidden_size,bias=True), nn.ELU(),\n            nn.Linear(n_features, n_class, bias=True)\n        )\n        '''\n    def forward_features(self, x):\n        x = self.backbone(x)\n        return x\n    def forward(self, x,label=None):\n        \n        x = self.forward_features(x)\n        # x = self.pool(feats).view(x.size(0), -1) # orgin\n        #x = self.gap(feats)\n        #x = x.view(x.size(0), -1)\n        \n        if CFG['classifier_type'] in ('arcface','cosface','amsoftmax','circle'):\n            #print(\"label classifer input time device\",label.device)\n            x = self.classifier(x,label)\n        else:\n            x = self.classifier(x)\n        return x  #, feats\n#         x = self.model(x)\n#         return x\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Main Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)   #output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        # we'll train fold 0 first\n#         if fold > 0:\n#             break \n\n        print('Inference fold {} started'.format(fold))\n\n        valid_ = train.loc[val_idx,:].reset_index(drop=True)\n        valid_ds = CassavaDataset(valid_, '../input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms(), output_label=False)\n        \n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms(), output_label=False)\n        \n        val_loader = torch.utils.data.DataLoader(\n            valid_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n        \n        tst_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n\n        device = torch.device(CFG['device'])\n        model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique()).to(device)\n        \n        val_preds = []\n        tst_preds = []\n        \n        #for epoch in range(CFG['epochs']-3):\n        for i, epoch in enumerate(CFG['used_epochs']):    \n            #model.load_state_dict(torch.load('../input/arcface-s15-m04/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))\n            model.load_state_dict(torch.load('../input/linear-weight/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))\n            with torch.no_grad():\n                for _ in range(CFG['tta']):\n                    #val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]\n                    tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]\n                    \n\n        #val_preds = np.mean(val_preds, axis=0) \n        tst_preds = np.mean(tst_preds, axis=0) \n        \n        #print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))\n        #print('fold {} validation accuracy = {:.5f}'.format(fold, (valid_.label.values==np.argmax(val_preds, axis=1)).mean()))\n        \n#         del model\n#         torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['label'] = np.argmax(tst_preds, axis=1)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Please upvote the original notebook as well"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}