{"cells":[{"metadata":{},"cell_type":"markdown","source":"Inference at : https://www.kaggle.com/krisho007/simple-pytorch-lightning-inference"},{"metadata":{"trusted":true,"scrolled":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install timm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"scrolled":false,"_kg_hide-output":true},"cell_type":"code","source":"import os\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport random\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\nfrom pytorch_lightning.metrics.functional import accuracy\nimport torch\n# from torchvision import models\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as albu\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom sklearn.model_selection import StratifiedKFold\nimport timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_kg_hide-output":true},"cell_type":"code","source":"import random\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_kg_hide-output":true},"cell_type":"code","source":"TRAIN_CSV = \"../input/cassava-leaf-disease-classification/train.csv\"\nTRAIN_IMAGE_FOLDER = '../input/cassava-leaf-disease-classification/train_images'\nCLASSES = 5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Symmetric Cross Entropy Loss\nhttps://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/208239"},{"metadata":{"trusted":true},"cell_type":"code","source":"class SymmetricCrossEntropy(nn.Module):\n\n    def __init__(self, alpha=0.1, beta=1.0, num_classes= 5):\n        super(SymmetricCrossEntropy, self).__init__()\n        self.alpha = alpha\n        self.beta = beta\n        self.num_classes = num_classes\n\n    def forward(self, logits, targets, reduction='mean'):\n        onehot_targets = torch.eye(self.num_classes)[targets].cuda()\n        ce_loss = F.cross_entropy(logits, targets, reduction=reduction)\n        rce_loss = (-onehot_targets*logits.softmax(1).clamp(1e-7, 1.0).log()).sum(1)\n        if reduction == 'mean':\n            rce_loss = rce_loss.mean()\n        elif reduction == 'sum':\n            rce_loss = rce_loss.sum()\n        return self.alpha * ce_loss + self.beta * rce_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Taylor Smooth cross entropy"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/173733#965200\n# implementations reference - https://github.com/CoinCheung/pytorch-loss/blob/master/pytorch_loss/taylor_softmax.py\n# paper - https://www.ijcai.org/Proceedings/2020/0305.pdf\n\nclass TaylorSoftmax(nn.Module):\n\n    def __init__(self, dim=1, n=2):\n        super(TaylorSoftmax, self).__init__()\n        assert n % 2 == 0\n        self.dim = dim\n        self.n = n\n\n    def forward(self, x):\n        \n        fn = torch.ones_like(x)\n        denor = 1.\n        for i in range(1, self.n+1):\n            denor *= i\n            fn = fn + x.pow(i) / denor\n        out = fn / fn.sum(dim=self.dim, keepdims=True)\n        return out\n\nclass LabelSmoothingLoss(nn.Module):\n\n    def __init__(self, classes, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        \"\"\"Taylor Softmax and log are already applied on the logits\"\"\"\n        with torch.no_grad(): \n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))\n    \n\nclass TaylorCrossEntropyLoss(nn.Module):\n\n    def __init__(self, n=2, ignore_index=-1, reduction='mean', smoothing=0.2):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n        self.lab_smooth = LabelSmoothingLoss(5, smoothing=smoothing)\n\n    def forward(self, logits, labels):\n\n        log_probs = self.taylor_softmax(logits).log()\n        loss = self.lab_smooth(log_probs, labels)\n        return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Hyper parameters"},{"metadata":{"trusted":true,"scrolled":false,"_kg_hide-output":true},"cell_type":"code","source":"FOLDS = 10\nBATCH_SIZE =24\nLR = 0.0001\nEPOCHS=4\nSMOOTHING = 0.1\n\nLOSS_FUNCTION = nn.CrossEntropyLoss()\nLOSS_FUNCTION = TaylorCrossEntropyLoss(n=2, smoothing=SMOOTHING)\nLOSS_FUNCTION = SymmetricCrossEntropy()\n\nIMG_SIZE = 128\nIMG_SIZE = 240\nIMG_SIZE = 512\n# IMG_SIZE = 380\n\nEARLY_STOPPING = True\n\nMODEL_ARCH = 'resnet50'\nMODEL_ARCH = 'tf_efficientnet_b1_ns'\nMODEL_ARCH = 'efficientnet_b3'\nMODEL_ARCH = 'tf_efficientnet_b4_ns'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# # These are the available model architectures in timm\n# from pprint import pprint\n# model_names = timm.list_models(pretrained=True)\n# pprint(model_names)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset"},{"metadata":{"trusted":true,"scrolled":false,"_kg_hide-output":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, train, train_mode=True, transforms=None):\n        self.train = train\n        self.transforms = transforms\n        self.train_mode = train_mode\n    \n    def __len__(self):\n        return self.train.shape[0]\n    \n    def __getitem__(self, index):\n        image_path = os.path.join(TRAIN_IMAGE_FOLDER, self.train.iloc[index].image_id)\n        image = cv2.imread(image_path, cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if (self.transforms):\n            image = self.transforms(image=image)[\"image\"]\n        \n        if not(self.train_mode):\n            return {\"x\":image}\n        \n        return {\n            \"x\": image,\n            \"y\": torch.tensor(self.train.iloc[index, self.train.columns.str.startswith('label')], dtype=torch.float64)\n        }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Transforms"},{"metadata":{"trusted":true,"scrolled":false,"_kg_hide-output":true},"cell_type":"code","source":"def get_augmentations():\n    \n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225) \n    \n    train_augmentations = albu.Compose([\n        albu.RandomResizedCrop(IMG_SIZE, IMG_SIZE, p=1.0),\n        albu.Transpose(p=0.5),\n        albu.HorizontalFlip(p=0.5),\n#         albu.VerticalFlip(0.5),\n        albu.CoarseDropout (p=0.5),\n        albu.Normalize(always_apply=True),        \n        ToTensorV2(p=1.0)\n    ], p=1.0)\n    \n    valid_augmentations = albu.Compose([\n        albu.Resize(IMG_SIZE, IMG_SIZE),\n        albu.Normalize(always_apply=True),        \n        ToTensorV2(p=1.0)\n    ], p=1.0)   \n    \n    return train_augmentations, valid_augmentations\n\ntrain_augs, val_augs = get_augmentations()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### NN Model"},{"metadata":{"trusted":true,"scrolled":false,"_kg_hide-output":true},"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.model = timm.create_model(MODEL_ARCH, pretrained=True)\n#         self.model = base_model\n\n#         Efficientnets\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, CLASSES)\n        \n#         Resnets\n#         n_features = self.model.fc.in_features\n#         self.model.fc = nn.Linear(n_features, CLASSES)\n        \n        self._freeze_batchnorm()  # NEW NEW NEW NEW NEW NEW NEW NEW NEW\n        \n    def _freeze_batchnorm(self):\n        for module in self.model.modules():\n            if isinstance(module, nn.BatchNorm2d):\n                if hasattr(module, 'weight'):\n                    module.weight.requires_grad_(False)\n                if hasattr(module, 'bias'):\n                    module.bias.requires_grad_(False)\n                module.eval()\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### K-Fold CV"},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"traincsv = pd.read_csv(TRAIN_CSV)\ntraincsv['kfold'] = -1\ntraincsv = traincsv.sample(frac=1).reset_index(drop=True)\nstratifier = StratifiedKFold(n_splits=FOLDS)\n\nfor fold, (train_index, val_index) in enumerate(stratifier.split(X=traincsv.image_id.values, y=traincsv.label.values)):\n    traincsv.loc[val_index, \"kfold\"] = fold\n\ntraincsv.to_csv(\"train_folds.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### PL Data module"},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"class CassavaDataModule(pl.LightningDataModule):\n    def __init__(self, fold):\n        super().__init__()\n        self.train_aug, self.valid_aug = get_augmentations()\n        self.fold = fold\n        self.batch_size = BATCH_SIZE\n    \n    def setup(self, stage=None):\n        folds = pd.read_csv('./train_folds.csv')\n#         folds = pd.get_dummies(folds, columns=['label'])\n        train_fold = folds.loc[folds[\"kfold\"] != self.fold]\n        val_fold = folds.loc[folds[\"kfold\"] == self.fold]\n        \n        self.train_ds = CassavaDataset(train_fold, transforms=train_augs)\n        self.val_ds = CassavaDataset(val_fold, transforms=val_augs)\n        \n    def train_dataloader(self):\n        return DataLoader(self.train_ds, self.batch_size, num_workers=4, shuffle=True)\n        \n    def val_dataloader(self):\n        return DataLoader(self.val_ds, self.batch_size, num_workers=4, shuffle=False)        \n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### PL Module"},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"class CassavaPLModule(pl.LightningModule):\n    def __init__(self, hparams, model):\n        super(CassavaPLModule, self).__init__()\n        self.hparams = hparams\n        self.model = model\n        self.criterion = LOSS_FUNCTION\n        self.accuracy = pl.metrics.Accuracy()\n        \n    def forward(self, x):\n        return self.model(x)\n    \n#     def configure_optimizers(self):\n#         optimizer = torch.optim.Adam(self.model.parameters(), lr=self.hparams.lr)\n# #         optimizer = torch.optim.AdamW(self.model.parameters(), lr=self.hparams.lr, weight_decay=0.001)\n#         scheduler = {\n#             'scheduler': \n#                 torch.optim.lr_scheduler.ReduceLROnPlateau(\n#                     optimizer, \n#                     patience=2,\n#                     factor=0.25,\n#                     threshold=0.01,\n#                     mode='min', verbose=True\n#                 ),\n#             'interval': 'epoch',\n#             'monitor' : 'val_loss'\n#         }\n#         return [optimizer], [scheduler]\n\n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.model.parameters(), lr=self.hparams.lr)\n        scheduler = {\n            'scheduler': \n                torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(\n                    optimizer, \n                    15,\n                    verbose=False\n                ),\n            'interval': 'step',\n            'monitor' : 'train_loss'\n        }\n        return [optimizer], [scheduler]    \n    \n    def training_step(self, batch, batch_index):\n        # One batch at a time\n        features = batch['x']\n        targets = batch['y']\n        out = self(features)\n        loss = self.criterion(out, targets.squeeze().long())\n        self.log(\"train_loss\", loss, on_step=False, on_epoch=True, prog_bar=True, logger=True)   \n#         import pdb; pdb.set_trace()        \n        metric_acc = accuracy(out, targets.squeeze().long())\n#         metric_acc = accuracy(torch.argmax(out, dim=1), torch.argmax(targets.squeeze(), dim=1))\n        self.log(\"train_accuracy\", metric_acc, on_step=False, on_epoch=True, prog_bar=True,logger=True)\n        return loss\n        \n    def validation_step(self, batch, batch_index):\n        # One batch at a time\n        features = batch['x']\n        targets = batch['y']\n        out = self(features)\n        loss = self.criterion(out, targets.squeeze().long())\n        self.log(\"val_loss\", loss, on_step=False, on_epoch=True, prog_bar=True, logger=True) \n        metric_acc = accuracy(out, targets.squeeze().long())\n#         metric_acc = accuracy(torch.argmax(out, dim=1), torch.argmax(targets.squeeze(), dim=1))\n        self.log(\"val_accuracy\", metric_acc, on_step=False, on_epoch=True, prog_bar=True, logger=True)        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def train(fold): \n    callbacks=[]\n    checkpoint_callback = ModelCheckpoint(\n        dirpath='checkpoints/',\n        filename='model_{val_loss:.2f}',\n        monitor='val_loss', verbose=True,\n        save_last=False, save_top_k=1, save_weights_only=False,\n        mode='min', period=1, prefix=''\n    )        \n    callbacks.append(checkpoint_callback)\n    \n    early_stopping = EarlyStopping('val_accuracy', patience=3, verbose=True, mode='max')\n\n    if EARLY_STOPPING == True:\n        callbacks.append(early_stopping)    \n    \n    tpu_core = fold + 1\n    \n    trainer = pl.Trainer(\n                        gpus=-1 if torch.cuda.is_available() else None, \n                        precision=16,\n                        max_epochs=EPOCHS,\n#                         accumulate_grad_batches=4, # NEW NEW NEW NEW NEW NEW NEW NEW NEW\n                        callbacks=callbacks)\n    model = Model()\n    pl_dm = CassavaDataModule(fold=fold)\n    pl_module = CassavaPLModule(hparams={'lr':LR, 'batch_size':BATCH_SIZE}, model=model)\n    \n    trainer.use_native_amp = False\n    trainer.fit(pl_module, pl_dm)\n    \n    print(checkpoint_callback.best_model_path, checkpoint_callback.best_model_score)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"train(0)\ntrain(1)\ntrain(2)\ntrain(3)\ntrain(4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"RandAugment "},{"metadata":{},"cell_type":"markdown","source":"1. Remove error images  \n2. https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-inference-tta - Augments"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}