{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Intro\n\n* In this notebook I have combined Taylor Cross Entropy loss and label Smoothing\n* In General Label Smoothing Loss, we calculate softmax probabilities, take logarithm and then calculate smooth Cross entropy.\n* Here, we replace the Softmax part with Taylor softmax to give the loss a robust nature.\n\nHere are the per fold results with this Loss, Hope it helps \n\n* FOLD 0 - 0.89182\n* FOLD 1 - 0.91191\n* FOLD 2 - 0.93947\n* FOLD 3 - 0.89343\n* FOLD 4 - 0.89319\n\nIn this notebook I have run Fold 2\n\nIf you are looking for the implementation of Taylor Cross Entropy check out Version 4 of this Notebook."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"!pip install -q timm\n!pip install -q pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport copy\nimport time\nimport random\nimport pickle\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport torchvision\nfrom torchvision import models\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.cuda import amp\nfrom tqdm.notebook import tqdm\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.utils import class_weight\n\nfrom tqdm.notebook import tqdm\nfrom collections import defaultdict\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\nimport pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT_DIR = \"../input/cassava-leaf-disease-classification\"\nTRAIN_DIR = \"../input/cassava-leaf-disease-classification/train_images\"\nTEST_DIR = \"../input/cassava-leaf-disease-classification/test_images\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    model_name = 'tf_efficientnet_b4_ns'\n    img_size = 512\n    scheduler = 'CosineAnnealingWarmRestarts'\n    T_max = 10\n    T_0 = 10\n    lr = 1e-4\n    min_lr = 1e-6\n    batch_size = 16\n    weight_decay = 1e-6\n    seed = 42\n    num_classes = 5\n    num_epochs = 10\n    n_fold = 5\n    NUM_FOLDS_TO_RUN = [2,]\n    smoothing = 0.2\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed = 42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(f\"{ROOT_DIR}/train.csv\")\n\nskf = StratifiedKFold(n_splits=CFG.n_fold)\nfor fold, ( _, val_) in enumerate(skf.split(X=df, y=df.label)):\n    df.loc[val_ , \"kfold\"] = int(fold)\n    \ndf['kfold'] = df['kfold'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaLeafDataset(nn.Module):\n    def __init__(self, root_dir, df, transforms=None):\n        self.root_dir = root_dir\n        self.df = df\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_path = os.path.join(self.root_dir, self.df.iloc[index, 0])\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        label = self.df.iloc[index, 1]\n        \n        if self.transforms:\n            img = self.transforms(image=img)[\"image\"]\n            \n        return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_transforms = {\n    \"train\": A.Compose([\n        A.RandomResizedCrop(CFG.img_size, CFG.img_size),\n        A.Transpose(p=0.5),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.ShiftScaleRotate(p=0.5),\n        A.HueSaturationValue(\n                hue_shift_limit=0.2, \n                sat_shift_limit=0.2, \n                val_shift_limit=0.2, \n                p=0.5\n            ),\n        A.RandomBrightnessContrast(\n                brightness_limit=(-0.1,0.1), \n                contrast_limit=(-0.1, 0.1), \n                p=0.5\n            ),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        A.CoarseDropout(p=0.5),\n        A.Cutout(p=0.5),\n        ToTensorV2()], p=1.),\n    \n    \"valid\": A.Compose([\n        A.CenterCrop(CFG.img_size, CFG.img_size, p=1.),\n        A.Resize(CFG.img_size, CFG.img_size),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        ToTensorV2()], p=1.)\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Taylor Cross Entropy Loss Implementation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# implementations reference - https://github.com/CoinCheung/pytorch-loss/blob/master/pytorch_loss/taylor_softmax.py\n# paper - https://www.ijcai.org/Proceedings/2020/0305.pdf\n\nclass TaylorSoftmax(nn.Module):\n\n    def __init__(self, dim=1, n=2):\n        super(TaylorSoftmax, self).__init__()\n        assert n % 2 == 0\n        self.dim = dim\n        self.n = n\n\n    def forward(self, x):\n        \n        fn = torch.ones_like(x)\n        denor = 1.\n        for i in range(1, self.n+1):\n            denor *= i\n            fn = fn + x.pow(i) / denor\n        out = fn / fn.sum(dim=self.dim, keepdims=True)\n        return out\n\nclass LabelSmoothingLoss(nn.Module):\n\n    def __init__(self, classes, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        \"\"\"Taylor Softmax and log are already applied on the logits\"\"\"\n        #pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad(): \n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))\n    \n\nclass TaylorCrossEntropyLoss(nn.Module):\n\n    def __init__(self, n=2, ignore_index=-1, reduction='mean', smoothing=0.2):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n        self.lab_smooth = LabelSmoothingLoss(CFG.num_classes, smoothing=smoothing)\n\n    def forward(self, logits, labels):\n\n        log_probs = self.taylor_softmax(logits).log()\n        #loss = F.nll_loss(log_probs, labels, reduction=self.reduction,\n        #        ignore_index=self.ignore_index)\n        loss = self.lab_smooth(log_probs, labels)\n        return loss\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, criterion, optimizer, scheduler, num_epochs, dataloaders, dataset_sizes, device, fold):\n    start = time.time()\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n    history = defaultdict(list)\n    scaler = amp.GradScaler()\n\n    for epoch in range(1,num_epochs+1):\n        print('Epoch {}/{}'.format(epoch, num_epochs))\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train','valid']:\n            if(phase == 'train'):\n                model.train() # Set model to training mode\n            else:\n                model.eval() # Set model to evaluation mode\n            \n            running_loss = 0.0\n            running_corrects = 0.0\n            \n            # Iterate over data\n            for inputs,labels in tqdm(dataloaders[phase]):\n                inputs = inputs.to(CFG.device)\n                labels = labels.to(CFG.device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    with amp.autocast():\n                        outputs = model(inputs)\n                        _, preds = torch.max(outputs,1)\n                        loss = criterion(outputs, labels)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        scaler.scale(loss).backward()\n                        scaler.step(optimizer)\n                        scaler.update()\n\n\n                running_loss += loss.item()*inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data).double().item()\n\n            \n            epoch_loss = running_loss/dataset_sizes[phase]\n            epoch_acc = running_corrects/dataset_sizes[phase]\n\n            history[phase + ' loss'].append(epoch_loss)\n            history[phase + ' acc'].append(epoch_acc)\n\n            if phase == 'train' and scheduler != None:\n                scheduler.step()\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n                phase, epoch_loss, epoch_acc))\n            \n            # deep copy the model\n            if phase=='valid' and epoch_acc >= best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n                PATH = f\"Fold{fold}_{best_acc}_epoch{epoch}.bin\"\n                torch.save(model.state_dict(), PATH)\n\n        print()\n\n    end = time.time()\n    time_elapsed = end - start\n    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    print(\"Best Accuracy \",best_acc)\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, history, best_acc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_fold(model, criterion, optimizer, scheduler, device, fold, num_epochs=10):\n    valid_df = df[df.kfold == fold]\n    train_df = df[df.kfold != fold]\n    \n    train_data = CassavaLeafDataset(TRAIN_DIR, train_df, transforms=data_transforms[\"train\"])\n    valid_data = CassavaLeafDataset(TRAIN_DIR, valid_df, transforms=data_transforms[\"valid\"])\n    \n    dataset_sizes = {\n        'train' : len(train_data),\n        'valid' : len(valid_data)\n    }\n    \n    train_loader = DataLoader(dataset=train_data, batch_size=CFG.batch_size, num_workers=4, pin_memory=True, shuffle=True)\n    valid_loader = DataLoader(dataset=valid_data, batch_size=CFG.batch_size, num_workers=4, pin_memory=True, shuffle=False)\n    \n    dataloaders = {\n        'train' : train_loader,\n        'valid' : valid_loader\n    }\n\n    model, history, best_acc = train_model(model, criterion, optimizer, scheduler, num_epochs, dataloaders, dataset_sizes, device, fold)\n    \n    return model, history, best_acc\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = timm.create_model(CFG.model_name, pretrained=True)\nnum_features = model.classifier.in_features\nmodel.classifier = nn.Linear(num_features, CFG.num_classes)\nmodel.to(CFG.device);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = optim.Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)\ncriterion = TaylorCrossEntropyLoss(n=2, smoothing=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fetch_scheduler(optimizer):\n    if CFG.scheduler == 'CosineAnnealingLR':\n        scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr)\n    elif CFG.scheduler == 'CosineAnnealingWarmRestarts':\n        scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr)\n    elif CFG.scheduler == None:\n        return None\n        \n    return scheduler\n\nscheduler = fetch_scheduler(optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"accs = []\nfor fold in CFG.NUM_FOLDS_TO_RUN:\n    print(f\"\\n\\nFOLD: {fold}\\n\\n\")\n    model, history, ba = run_fold(model, criterion, optimizer, scheduler, device=CFG.device, fold=fold, num_epochs=CFG.num_epochs)\n    accs.append(ba)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"MEAN_ACC - {sum(accs)/len(accs)}\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}