{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Comparing Efficient Nets and Normalization Free Nets + Adaptive Gradient Clipping in PyTorch\n\nIn this notebook, I am comparing the performance of Normalization Free Network (f-1) + Adaptive Gradient Clipping with Efficient Network (f-1) with no Adaptive Gradient Clipping.\n\nNote: I am using just a subset of data for training and validation because doing that on all samples leads to notebook crashing mid-run."},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys; \nsys.path.insert(0,'../input/timm-nfnet')\nsys.path.insert(0, '../input/nfnet-pretrained')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\n\nimport cv2\nfrom tqdm.notebook import tqdm\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\nfrom timm.utils.agc import adaptive_clip_grad\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.cuda.amp import autocast, GradScaler\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nwarnings.simplefilter(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    CFG = {\n        'img_size': 224,\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Augments:\n    \"\"\"\n    Contains Train, Validation Augments\n    \"\"\"\n    train_augments = Compose([\n            Resize(Config.CFG['img_size'], Config.CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ],p=1.)\n    \n    valid_augments = Compose([\n            Resize(Config.CFG['img_size'], Config.CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class NFNetModel(nn.Module):\n    \"\"\"\n    Model Class for the newly introduced Normalization Free Network (NFNet) Model Architecture\n    \"\"\"\n    def __init__(self, num_classes=11, model_name='nfnet_f1', pretrained=True):\n        super(NFNetModel, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        self.model.load_state_dict(torch.load(\"../input/nfnet-pretrained/NFNet-f1.pt\"))\n        self.model.head.fc = nn.Linear(self.model.head.fc.in_features, num_classes)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \nclass EfficientNetModel(nn.Module):\n    \"\"\"\n    Model Class for EfficientNet Model\n    \"\"\"\n    def __init__(self, num_classes=11, model_name='efficientnet_b1', pretrained=True):\n        super(EfficientNetModel, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        self.model.classifier = nn.Linear(self.model.classifier.in_features, num_classes)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANCZRData(Dataset):\n    def __init__(self, df, num_classes=5, is_train=True, augments=None, img_size=Config.CFG['img_size'], img_path=\"../input/ranzcr-clip-catheter-line-classification/train\"):\n        super().__init__()\n        self.df = df.sample(frac=1).reset_index(drop=True)\n        self.num_classes = num_classes\n        self.is_train = is_train\n        self.augments = augments\n        self.img_size = img_size\n        self.img_path = img_path\n    def __getitem__(self, idx):\n        image_id = self.df['StudyInstanceUID'].values[idx]\n        image = cv2.imread(os.path.join(self.img_path, image_id + \".jpg\"))\n        image = image[:, :, ::-1]\n        \n        # Augments must be albumentations\n        if self.augments:\n            img = self.augments(image=image)['image']\n        \n        if self.is_train:\n            label = self.df[self.df['StudyInstanceUID'] == image_id].values.tolist()[0][1:-1]\n            return img, torch.tensor(label)\n        \n        return img\n    \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Trainer:\n    def __init__(self, train_dataloader, valid_dataloader, model, optimizer, loss_fn, val_loss_fn, agc=False, device=\"cuda:0\"):\n        \"\"\"\n        Constructor for Trainer class\n        \"\"\"\n        self.train = train_dataloader\n        self.valid = valid_dataloader\n        self.optim = optim\n        self.loss_fn = loss_fn\n        self.val_loss_fn = val_loss_fn\n        self.device = device\n        self.agc = agc\n    \n    def train_one_cycle(self):\n        \"\"\"\n        Runs one epoch of training, backpropagation and optimization\n        \"\"\"\n        model.train()\n        train_prog_bar = tqdm(self.train, total=len(self.train))\n\n        all_train_labels = []\n        all_train_preds = []\n        \n        running_loss = 0\n        \n        for xtrain, ytrain in train_prog_bar:\n            xtrain = xtrain.to(device).float()\n            ytrain = ytrain.to(device).float()\n            \n            with autocast():\n                # Get predictions\n                z = model(xtrain)\n\n                # Training\n                train_loss = self.loss_fn(z, ytrain)\n                scaler.scale(train_loss).backward()\n                \n                if self.agc:\n                    adaptive_clip_grad(model.parameters(), clip_factor=0.01, eps=1e-3, norm_type=2.0)\n                \n                scaler.step(self.optim)\n                scaler.update()\n                self.optim.zero_grad()\n\n                # For averaging and reporting later\n                running_loss += train_loss\n\n                # Convert the predictions and corresponding labels to right form\n                train_predictions = torch.argmax(z, 1).detach().cpu().numpy()\n                train_labels = ytrain.detach().cpu().numpy()\n\n                # Append current predictions and current labels to a list\n                all_train_labels += [train_predictions]\n                all_train_preds += [train_labels]\n\n            # Show the current loss to the progress bar\n            train_pbar_desc = f'loss: {train_loss.item():.4f}'\n            train_prog_bar.set_description(desc=train_pbar_desc)\n        \n        # Now average the running loss over all batches and return\n        train_running_loss = running_loss / len(self.train)\n        print(f\"Final Training Loss: {train_running_loss:.4f}\")\n        \n        # Free up memory\n        del all_train_labels, all_train_preds, train_predictions, train_labels, xtrain, ytrain, z\n        \n        return train_running_loss\n\n    def valid_one_cycle(self):\n        \"\"\"\n        Runs one epoch of prediction\n        \"\"\"        \n        model.eval()\n        \n        valid_prog_bar = tqdm(self.valid, total=len(self.valid))\n        \n        with torch.no_grad():\n            all_valid_labels = []\n            all_valid_preds = []\n            \n            running_loss = 0\n            \n            for xval, yval in valid_prog_bar:\n                xval = xval.to(device).float()\n                yval = yval.to(device).float()\n                \n                val_z = model(xval)\n                \n                val_loss = self.val_loss_fn(val_z, yval)\n                \n                running_loss += val_loss.item()\n                \n                val_pred = torch.argmax(val_z, 1).detach().cpu().numpy()\n                val_label = yval.detach().cpu().numpy()\n                \n                all_valid_labels += [val_label]\n                all_valid_preds += [val_pred]\n            \n                # Show the current loss\n                valid_pbar_desc = f\"loss: {val_loss.item():.4f}\"\n                valid_prog_bar.set_description(desc=valid_pbar_desc)\n            \n            # Get the final loss\n            final_loss_val = running_loss / len(self.valid)\n            \n            # Get Validation Accuracy\n            all_valid_labels = np.concatenate(all_valid_labels)\n            all_valid_preds = np.concatenate(all_valid_preds)\n            \n            print(f\"Final Validation Loss: {final_loss_val:.4f}\")\n            \n            # Free up memory\n            del all_valid_labels, all_valid_preds, val_label, val_pred, xval, yval, val_z\n            \n        return (final_loss_val, model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nb_epochs = 5\ndevice = torch.device(\"cuda\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv(\"../input/ranzcr-clip-catheter-line-classification/train.csv\")\ndata = data.sample(frac=1).reset_index(drop=True)\n\n# 27,583 in Train, 2500 in Valid\ntrain_split = data[2500:14500]\nvalid_split = data[:2500]\n\nprint(train_split.shape, valid_split.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EfficientNet Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_set = RANCZRData(df=train_split, augments=Augments.train_augments)\nvalid_set = RANCZRData(df=valid_split, augments=Augments.valid_augments)\n\ntrain = DataLoader(\n    train_set,\n    batch_size=16,\n    shuffle=True,\n    pin_memory=False,\n    drop_last=False,\n    num_workers=8\n)\n\nvalid = DataLoader(\n    valid_set,\n    batch_size=32,\n    shuffle=False,\n    pin_memory=False,\n    num_workers=8\n)\n\nmodel = EfficientNetModel().to(device)\noptim = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.001)\nloss_fn_train = nn.BCEWithLogitsLoss()\nloss_fn_val = nn.BCEWithLogitsLoss()\n\ntrainer = Trainer(\n    train_dataloader=train,\n    valid_dataloader=valid,\n    model=model,\n    optimizer=optim,\n    loss_fn=loss_fn_train,\n    val_loss_fn=loss_fn_val,\n    agc=False,\n    device=device,\n)\n\ntrain_losses_eff = []\nvalid_losses_eff = []\n\nscaler = GradScaler()\n\nfor epoch in range(nb_epochs):\n    print(f\"{'-'*20} EPOCH: {epoch+1}/{nb_epochs} {'-'*20}\")\n\n    # Run one training epoch\n    current_train_loss = trainer.train_one_cycle()\n    train_losses_eff.append(current_train_loss)\n\n    # Run one validation epoch\n    current_val_loss, op_model = trainer.valid_one_cycle()\n    valid_losses_eff.append(current_val_loss)\n\n    # Empty CUDA cache\n    torch.cuda.empty_cache()\n    \n    # Save the model every epoch\n#     print(f\"Saving Model for this epoch...\")\n#     torch.save(op_model.state_dict(), f\"effnet_f1_model.pth\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Normalization Free Network Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_set = RANCZRData(df=train_split, augments=Augments.train_augments)\nvalid_set = RANCZRData(df=valid_split, augments=Augments.valid_augments)\n\ntrain = DataLoader(\n    train_set,\n    batch_size=16,\n    shuffle=True,\n    pin_memory=False,\n    drop_last=False,\n    num_workers=8\n)\n\nvalid = DataLoader(\n    valid_set,\n    batch_size=32,\n    shuffle=False,\n    pin_memory=False,\n    num_workers=8\n)\n\nmodel = NFNetModel().to(device)\noptim = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.001)\nloss_fn_train = nn.BCEWithLogitsLoss()\nloss_fn_val = nn.BCEWithLogitsLoss()\n\ntrainer = Trainer(\n    train_dataloader=train,\n    valid_dataloader=valid,\n    model=model,\n    optimizer=optim,\n    loss_fn=loss_fn_train,\n    val_loss_fn=loss_fn_val,\n    agc=True,\n    device=device,\n)\n\ntrain_losses_nfn = []\nvalid_losses_nfn = []\n\nscaler = GradScaler()\n\nfor epoch in range(nb_epochs):\n    print(f\"{'-'*20} EPOCH: {epoch+1}/{nb_epochs} {'-'*20}\")\n\n    # Run one training epoch\n    current_train_loss = trainer.train_one_cycle()\n    train_losses_nfn.append(current_train_loss)\n\n    # Run one validation epoch\n    current_val_loss, op_model = trainer.valid_one_cycle()\n    valid_losses_nfn.append(current_val_loss)\n\n    # Empty CUDA cache\n    torch.cuda.empty_cache()\n    \n    # Save the model every epoch\n#     print(f\"Saving Model for this epoch...\")\n#     torch.save(op_model.state_dict(), f\"nfnet_f1_model.pth\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = [i for i in range(nb_epochs)]\n\nfig, ax = plt.subplots(1, 2)\nfig.set_size_inches(20, 10)\n\nax[0].plot(epochs, train_losses_eff, 'go-', label='Efficient Net')\nax[0].plot(epochs, train_losses_nfn, 'ro-', label='NormFree Net')\nax[0].set_title('Training Losses')\nax[0].legend()\nax[0].set_xlabel('Epochs')\nax[0].set_ylabel('Loss')\n\nax[1].plot(epochs, valid_losses_eff, 'go-', label='Efficient Net')\nax[1].plot(epochs, valid_losses_nfn, 'ro-', label='NormFree Net')\nax[1].set_title('Validation Losses')\nax[1].legend()\nax[1].set_xlabel('Epochs')\nax[1].set_ylabel('Loss')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}