{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# NF net training\n","metadata":{}},{"cell_type":"code","source":"import sys; \nsys.path.insert(0,'../input/timm-all-models/pytorch-image-models-master/pytorch-image-models-master')\n# sys.path.insert(0, '../input/timm-all-models')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\n\nimport cv2\nfrom tqdm.notebook import tqdm\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\nfrom timm.utils.agc import adaptive_clip_grad\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.cuda.amp import autocast, GradScaler\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nwarnings.simplefilter(\"ignore\")\nfrom pytorch_lightning import Trainer, seed_everything\nimport torch.nn.functional as F","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    seed=42\n    n_epoch=1\n    img_size= 224\n    train_path='../input/plant-pathology-2021-fgvc8/train_images'\n    test_path='../input/plant-pathology-2021-fgvc8/test_images'\n    lr=1e-4\n    weight_decay=0.001\n    debug=False\n    debug_sample=100\n    train_batch=16\n    test_batch=32\n    path='../input/plant-pathology-2021-fgvc8/'\n    \n        \ndevice = torch.device(\"cuda\")\n\nseed_everything(Config.seed)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = pd.read_csv(Config.path + \"train.csv\")\ndf_all = df_all.sample(frac=1).reset_index(drop=True)\nlabels = list(df_all['labels'].value_counts().keys())\nlabels_dict = dict(zip(labels, range(12)))\nif Config.debug:\n    train_split = df_all[0:500]\n    valid_split = df_all[500:550]\nelse:\n    valid_split=df_all[:1000]\n    train_split=df_all[1000:]\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Augments:\n    \"\"\"\n    Contains Train, Validation Augments\n    \"\"\"\n    train_augments = Compose([\n            Resize(Config.img_size, Config.img_size),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ],p=1.)\n    \n    valid_augments = Compose([\n            Resize(Config.img_size, Config.img_size),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class NFNetModel(nn.Module):\n    \"\"\"\n    Model Class for the newly introduced Normalization Free Network (NFNet) Model Architecture\n    \"\"\"\n    def __init__(self, num_classes=12, model_name='nfnet_f1', pretrained=True):\n        super(NFNetModel, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        self.model.head.fc = nn.Linear(self.model.head.fc.in_features, num_classes)\n#         self.optimizer=torch.optim.AdamW(self.model.parameters(), lr=1e-4, weight_decay=0.001)\n        \n\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x\n    def save(self,optim):\n        self.eval()\n        torch.save({\n            'epoch': epoch,\n            'model_state_dict': self.state_dict(),\n            'optimizer_state_dict': optim.state_dict(),\n            'loss': 0,\n            }, './nfnet.pth')\n    def load(self,optim,path):\n        checkpoint = torch.load(path)\n        self.load_state_dict(checkpoint['model_state_dict'])\n        optim.load_state_dict(checkpoint['optimizer_state_dict'])\n        self.epoch = checkpoint['epoch']\n        self.loss = checkpoint['loss']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Plant_data(Dataset):\n    def __init__(self, df, num_classes=12, is_train=True, augments=None, img_size=Config.img_size, img_path=\"../input/plant-pathology-2021-fgvc8/train_images\"):\n        super().__init__()\n        self.df = df.sample(frac=1).reset_index(drop=True)\n        self.num_classes = num_classes\n        self.is_train = is_train\n        self.augments = augments\n        self.img_size = img_size\n        self.img_path = img_path\n        self.image_id = df['image'].values\n        self.labels = df['labels'].values\n        \n        \n        \n    def __getitem__(self, idx):\n        image_id = self.image_id[idx]\n        image = cv2.imread(os.path.join(self.img_path, image_id ))\n        image = image[:, :, ::-1]\n        \n        # Augments must be albumentations\n        if self.augments:\n            img = self.augments(image=image)['image']\n        \n        \n#         label = self.labels[idx]\n        label=labels_dict[self.labels[idx]]\n        return img, torch.tensor(label)\n        \n        \n    \n    def __len__(self):\n        return len(self.df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer:\n    def __init__(self, train_dataloader, valid_dataloader, model, optimizer, loss_fn, val_loss_fn, agc=False, device=\"cuda:0\"):\n        \"\"\"\n        Constructor for Trainer class\n        \"\"\"\n        self.train = train_dataloader\n        self.valid = valid_dataloader\n        self.optim = optim\n        self.loss_fn = loss_fn\n        self.val_loss_fn = val_loss_fn\n        self.device = device\n        self.agc = agc\n    \n    def train_one_cycle(self):\n        \"\"\"\n        Runs one epoch of training, backpropagation and optimization\n        \"\"\"\n        model.train()\n        train_prog_bar = tqdm(self.train, total=len(self.train))\n\n        all_train_labels = []\n        all_train_preds = []\n        all_acc=[]\n        \n        running_loss = 0\n        \n        for iteration,xytrain in enumerate(train_prog_bar):\n            xtrain = xytrain[0].to(device).float()\n            hy= F.one_hot(xytrain[1] ,num_classes=12)\n            ytrain = hy.to(device).float()\n            \n            with autocast():\n                # Get predictions\n                z = model(xtrain)\n\n                # Training\n                train_loss = self.loss_fn(z, ytrain)\n                scaler.scale(train_loss).backward()\n                \n                if self.agc:\n                    adaptive_clip_grad(model.parameters(), clip_factor=0.01, eps=1e-3, norm_type=2.0)\n                \n                scaler.step(self.optim)\n                scaler.update()\n                self.optim.zero_grad()\n\n                # For averaging and reporting later\n                running_loss += train_loss\n\n                # Convert the predictions and corresponding labels to right form\n                train_predictions = torch.argmax(z, 1).detach().cpu().numpy()\n                train_labels = torch.argmax(ytrain, 1).detach().cpu().numpy()\n                acc_per_iteration=np.sum(train_predictions==train_labels)/16                          ##batch size\n\n                # Append current predictions and current labels to a list\n                all_train_labels += [train_predictions]\n                all_train_preds += [train_labels]\n                all_acc.append(acc_per_iteration)\n                if iteration%5==0:\n                    \n                    print('acc per iter ={}'.format(acc_per_iteration))\n\n            # Show the current loss to the progress bar\n            train_pbar_desc = f'loss: {train_loss.item():.4f}'\n            train_prog_bar.set_description(desc=train_pbar_desc)\n        \n        # Now average the running loss over all batches and return\n        train_running_loss = running_loss / len(self.train)\n        print(f\"Final Training Loss: {train_running_loss:.4f}\")\n        print(f\"Final Training acc: {np.mean(all_acc):.4f}\")\n        train_running_acc=np.mean(all_acc)\n        \n        # Free up memory\n        del all_train_labels, all_train_preds, train_predictions, train_labels, xtrain, ytrain, z,all_acc\n        \n        return train_running_loss, train_running_acc\n\n    def valid_one_cycle(self):\n        \"\"\"\n        Runs one epoch of prediction\n        \"\"\"        \n        model.eval()\n        \n        valid_prog_bar = tqdm(self.valid, total=len(self.valid))\n        \n        with torch.no_grad():\n            all_valid_labels = []\n            all_valid_preds = []\n            all_valid_acc=[]\n            \n            running_loss = 0\n            \n            for xval, y in valid_prog_bar:\n                xval = xval.to(device).float()\n                \n                yval=F.one_hot(y ,num_classes=12)\n                yval = yval.to(device).float()\n                \n                val_z = model(xval)\n                \n                val_loss = self.val_loss_fn(val_z, yval)\n                \n                running_loss += val_loss.item()\n                \n                val_pred = torch.argmax(val_z, 1).detach().cpu().numpy()\n                val_label =  torch.argmax(yval, 1).detach().cpu().numpy()\n                \n                \n                acc_per_iteration=np.sum(val_pred==val_label)/32\n                \n                \n                \n                all_valid_labels += [val_label]\n                all_valid_preds += [val_pred]\n                all_valid_acc.append(acc_per_iteration)\n                \n            \n                # Show the current loss\n                valid_pbar_desc = f\"loss: {val_loss.item():.4f}\"\n                valid_prog_bar.set_description(desc=valid_pbar_desc)\n            \n            # Get the final loss\n            final_loss_val = running_loss / len(self.valid)\n            \n            # Get Validation Accuracy\n            all_valid_labels = np.concatenate(all_valid_labels)\n            all_valid_preds = np.concatenate(all_valid_preds)\n            \n            print(f\"Final Validation Loss: {final_loss_val:.4f}\")\n            print(f\"acc: {np.mean(all_valid_acc):.4f}\")\n            final_val_acc=np.mean(all_valid_acc)\n            \n            # Free up memory\n            del all_valid_labels, all_valid_preds, val_label, val_pred, xval, yval, val_z,all_valid_acc\n            \n        return (final_loss_val,final_val_acc, model)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodel = NFNetModel().to(device)\noptim = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.001)\nloss_fn_train = nn.BCEWithLogitsLoss()\nloss_fn_val = nn.BCEWithLogitsLoss()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodel = NFNetModel().to(device)\noptim = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.001)\nmodel.load(optim,'./nfnet.pth')\n\nloss_fn_train = nn.BCEWithLogitsLoss()\nloss_fn_val = nn.BCEWithLogitsLoss()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_epoch_loss_training=[]\nall_epoch_loss_val=[]\nall_epoch_acc_training=[]\nall_epoch_acc_val=[]\n\n\ntrain_set = Plant_data(df=train_split, augments=Augments.train_augments)\nvalid_set = Plant_data(df=valid_split, augments=Augments.valid_augments)\n\ntrain = DataLoader(\n    train_set,\n    batch_size=16,\n    shuffle=True,\n    pin_memory=False,\n    drop_last=False,\n    num_workers=8\n)\n\nvalid = DataLoader(\n    valid_set,\n    batch_size=32,\n    shuffle=False,\n    pin_memory=False,\n    num_workers=8\n)\n\nmodel = NFNetModel().to(device)\noptim = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.001)\nloss_fn_train = nn.BCEWithLogitsLoss()\nloss_fn_val = nn.BCEWithLogitsLoss()\n\ntrainer = Trainer(\n    train_dataloader=train,\n    valid_dataloader=valid,\n    model=model,\n    optimizer=optim,\n    loss_fn=loss_fn_train,\n    val_loss_fn=loss_fn_val,\n    agc=True,\n    device=device,\n)\n\ntrain_losses_nfn = []\nvalid_losses_nfn = []\ntrain_acc_nfn = []\nvalid_acc_nfn = []\n\nscaler = GradScaler()\n\nfor epoch in range(Config.n_epoch):\n    print(f\"{'-'*20} EPOCH: {epoch+1}/{Config.n_epoch} {'-'*20}\")\n\n    # Run one training epoch\n    current_train_loss,current_train_acc = trainer.train_one_cycle()\n    train_losses_nfn.append(current_train_loss)\n    train_acc_nfn.append(current_train_acc)\n\n    # Run one validation epoch\n    current_val_loss,current_val_acc, op_model = trainer.valid_one_cycle()\n    valid_losses_nfn.append(current_val_loss)\n    valid_acc_nfn.append(current_val_acc)\n    \n\n    # Empty CUDA cache\n    torch.cuda.empty_cache()\n    \n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(optim)","metadata":{},"execution_count":null,"outputs":[]}]}