{"cells":[{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# !pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PyTorch Lightning\n\nIn this notebook I have tried using pytorch-lightning, I was amazed to see the features it provides ,i spent some time going through the docs and implement the basics of it, i am yet to go through and implement the advanced features it provides which can help in finding the optimal hyperparameters.\n\nI have used EfficientNet-b5 trained model, saved the efficientnet_pytorch package wheel (.whl) file as this competition involved no internet. \n\n### Accuracy: 88% on public test images "},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"efficientnet_path='../input/efficientnetlib/efficientnet_pytorch-0.7.0-py3-none-any.whl'\n\nsys.path.append(efficientnet_path)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Importing Libraries\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport pandas\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\n# from sklearn.metrics import accuracy_score\n\nimport cv2\nimport albumentations as albu\nfrom albumentations.pytorch.transforms import ToTensorV2\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision.models import resnext50_32x4d\nfrom torch.utils.data import Dataset,DataLoader\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint,EarlyStopping\nfrom pytorch_lightning.metrics.functional import accuracy\nfrom efficientnet_pytorch import EfficientNet\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Constants"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"TRAIN_IMAGES_DIR='../input/cassava-leaf-disease-classification/train_images'\nTEST_IMAGES_DIR='../input/cassava-leaf-disease-classification/test_images'\nTRAIN_CSV='../input/cassava-leaf-disease-classification/train.csv'\nPRETRAINED_PATH='../input/efficientnet-pytorch/efficientnet-b5-586e6cc6.pth'\nBATCH_SIZE=8\nIMG_SIZE=512\nCLASSES=5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LightningModule\n\n* Use LightningModule as base class and create your own training class\n* Output model in forward method \n* Use configure_optimizers for adding any optimization and scheduler\n* Use training_step for computing and returning loss (everything else like calculating gradient, updating params and device setting is handled by Lightning )\n* Similarly validation_step for inference\n* Add logging using .log\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaLite(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n        self.efficient_net = EfficientNet.from_name('efficientnet-b5')\n        self.efficient_net.load_state_dict(torch.load(PRETRAINED_PATH))\n#         self.efficient_net=EfficientNet.from_pretrained('efficientnet-b3',num_classes=CLASSES)\n        in_features=self.efficient_net._fc.in_features\n        self.efficient_net._fc=nn.Linear(in_features,CLASSES)\n    \n    def forward(self,x):\n        out=self.efficient_net(x)\n        return out\n    \n    def configure_optimizers(self):\n        optimizer=torch.optim.AdamW(self.parameters(),lr=1e-4,weight_decay=0.0001)\n        scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1)\n        return [optimizer],[scheduler]\n    \n    def training_step(self,batch,batch_idx):\n        x,y=batch[\"x\"],batch[\"y\"]\n        y_hat=self(x)\n        loss=F.cross_entropy(y_hat,y)\n        acc=accuracy(y_hat,y)\n        self.log(\"train_acc\",acc,on_step=False,on_epoch=True,prog_bar=True,logger=True)\n        self.log(\"train_loss\",loss,on_step=False,on_epoch=True,prog_bar=True,logger=True)\n        return loss\n    \n    def validation_step(self,batch,batch_idx):\n        x,y=batch[\"x\"],batch[\"y\"]\n        y_hat=self(x)\n        loss=F.cross_entropy(y_hat,y)\n        acc=accuracy(y_hat,y)\n        self.log(\"val_acc\",acc,prog_bar=True,logger=True),\n        self.log(\"val_loss\",loss,prog_bar=True,logger=True)\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self,df:pd.DataFrame,imfolder:str,train:bool = True, transforms=None):\n        self.df=df\n        self.imfolder=imfolder\n        self.train=train\n        self.transforms=transforms\n        \n    def __getitem__(self,index):\n        im_path=os.path.join(self.imfolder,self.df.iloc[index]['image_id'])\n        x=cv2.imread(im_path,cv2.IMREAD_COLOR)\n        x=cv2.cvtColor(x,cv2.COLOR_BGR2RGB)\n        \n        if(self.transforms):\n            x=self.transforms(image=x)['image']\n        \n        if(self.train):\n            y=self.df.iloc[index]['label']\n            return {\n                    \"x\":x,\n                    \"y\":y,\n                }    \n        else:\n            return {\"x\":x}\n        \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Lightning Data Module\n\nLightningDataModule can be used to prepare and dispatch data, which can then be passed in fit method of Trainer."},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataModule(pl.LightningDataModule):\n    def __init__(self):\n        super().__init__()\n        self.train_transform = albu.Compose([\n                        albu.RandomResizedCrop(IMG_SIZE,IMG_SIZE, p=1.0),\n                        albu.Transpose(p=0.5),\n                        albu.HorizontalFlip(p=0.5),\n                        albu.VerticalFlip(p=0.5),\n                        albu.ShiftScaleRotate(p=0.5),\n                        albu.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n                        albu.RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n                        albu.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n                        albu.CoarseDropout(p=0.5),\n                        albu.Cutout(p=0.5),\n                        ToTensorV2(p=1.0),\n                        ], p=1.)\n\n        self.valid_transform = albu.Compose([\n                         albu.CenterCrop(IMG_SIZE,IMG_SIZE, p=1.),\n                         albu.Resize(IMG_SIZE, IMG_SIZE),\n                         albu.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n                         ToTensorV2(p=1.0),\n                         ], p=1.)\n        \n        \n    def prepare_data(self):\n        # prepare_data is called only once on 1- GPU in a distributed computing\n        df=pd.read_csv(TRAIN_CSV)\n        df[\"kfold\"]=-1\n        df=df.sample(frac=1).reset_index(drop=True)\n        stratify=StratifiedKFold(n_splits=5)\n        for i,(t_idx,v_idx) in enumerate(stratify.split(X=df.image_id.values,y=df.label.values)):\n            df.loc[v_idx,\"kfold\"]=i\n            df.to_csv(\"train_folds.csv\",index=False)\n    \n    def setup(self,stage=None):\n        dfx=pd.read_csv(\"train_folds.csv\")\n        train=dfx.loc[dfx[\"kfold\"]!=1]\n        val=dfx.loc[dfx[\"kfold\"]==1]\n        \n        self.train_dataset=CassavaDataset(\n                            train,\n                            TRAIN_IMAGES_DIR,\n                            train=True,\n                            transforms=self.train_transform)\n        \n        self.valid_dataset=CassavaDataset(\n                            val,\n                            TRAIN_IMAGES_DIR,\n                            train=True,\n                            transforms=self.valid_transform)\n    \n    def train_dataloader(self):\n        return DataLoader(self.train_dataset,\n                          batch_size=BATCH_SIZE,\n                          num_workers=4,\n                          shuffle=True)\n    \n    def val_dataloader(self):\n        return DataLoader(self.valid_dataset,\n                          batch_size=BATCH_SIZE,\n                          num_workers=4)\n                         ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_checkpoint=ModelCheckpoint(monitor=\"val_loss\",\n                                 verbose=True,\n                                 filename=\"{epoch}_{val_loss:.4f}\")\nearly_stopping = EarlyStopping('val_loss',patience=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# early_stop_callback=True\n# model=resnext50_32x4d(pretrained=True)#Add Pretrained=True to use pretrained with internet enabled\ndm=CassavaDataModule()\ncassava_model=CassavaLite()\ntrainer=pl.Trainer(gpus=-1,max_epochs=12,callbacks=[model_checkpoint,early_stopping])\ntrainer.fit(cassava_model,dm)\n\n#manually you can save best checkpoints - \ntrainer.save_checkpoint(\"cassava_efficient_net.ckpt\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")\n\ntest_transform = albu.Compose([\n                    albu.RandomResizedCrop(IMG_SIZE,IMG_SIZE),\n                    albu.Transpose(p=0.5),\n                    albu.HorizontalFlip(p=0.5),\n                    albu.VerticalFlip(p=0.5),\n                    albu.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n                    albu.RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n                    albu.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n                    ToTensorV2(p=1.0),\n                    ], p=1.)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = CassavaDataset(test_df,\n                            TEST_IMAGES_DIR,\n                            train=False,\n                            transforms=test_transform)\n\ntest_loader = DataLoader(test_dataset,\n                        batch_size=16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# model=resnext50_32x4d()#Add Pretrained=True to use pretrained with internet enabled\nbest_checkpoints = trainer.checkpoint_callback.best_model_path\npretrained_model = CassavaLite().load_from_checkpoint(checkpoint_path = best_checkpoints)\npretrained_model = pretrained_model.to(\"cuda\")\npretrained_model.eval()\npretrained_model.freeze()\n\nfin_out = []\nfor data in test_loader:\n    y_hat = pretrained_model(data[\"x\"].to(\"cuda\"))\n    y_hat = torch.argmax(y_hat,dim=1)\n    fin_out.extend(y_hat.cpu().detach().numpy().tolist())\ntest_df[\"label\"] = fin_out\ntest_df[[\"image_id\",\"label\"]].to_csv(\"submission.csv\",index=False)\ntest_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}