{"cells":[{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"from IPython.display import HTML\nf = open(\"../input/notebookassets/orange.css\").read()\nHTML(f\"<style>{f}</style>\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Torch Trainer + Augementations + Normalization Free Networks (NFNets)\n\nSo Google's Deepmind recently announced a new faster and apparently better family of Convolutional Neural Network architectures called \"Normalization Free Networks\" that can compete with the existing Efficient Network family of networks, so I have decided to try them in this competition and see the results."},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys; \nsys.path.insert(0,'../input/timm-nfnet')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\n\nimport cv2\nfrom tqdm.notebook import tqdm\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, CosineAnnealingWarmRestarts\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nwarnings.simplefilter(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    CFG = {\n        'img_size': 224,\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_results(train_acc, valid_acc, train_loss, valid_loss, nb_epochs):\n    epochs = [i for i in range(nb_epochs)]\n    \n    fig, ax = plt.subplots(1, 2)\n    fig.set_size_inches(20, 10)\n    \n    ax[0].plot(epochs, train_acc, 'go-', label='Training Accuracy')\n    ax[0].plot(epochs, valid_acc, 'ro-', label='Validation Accuracy')\n    ax[0].set_title('Training & Validation Accuracy')\n    ax[0].legend()\n    ax[0].set_xlabel('Epochs')\n    ax[0].set_ylabel('Accuracy')\n    \n    ax[1].plot(epochs, train_loss, 'go-', label='Training Loss')\n    ax[1].plot(epochs, valid_loss, 'ro-', label='Validation Loss')\n    ax[1].set_title('Training & Validation Loss')\n    ax[1].legend()\n    ax[1].set_xlabel('Epochs')\n    ax[1].set_ylabel('Loss')\n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Augments:\n    \"\"\"\n    Contains Train, Validation Augments\n    \"\"\"\n    train_augments = Compose([\n            RandomResizedCrop(Config.CFG['img_size'], Config.CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ],p=1.)\n    \n    valid_augments = Compose([\n            CenterCrop(Config.CFG['img_size'], Config.CFG['img_size'], p=1.),\n            Resize(Config.CFG['img_size'], Config.CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class NFNetModel(nn.Module):\n    \"\"\"\n    Model Class for the newly introduced Normalization Free Network (NFNet) Model Architecture\n    \"\"\"\n    def __init__(self, num_classes=11, model_name='nfnet_f1', pretrained=True):\n        super(NFNetModel, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        self.model.head.fc = nn.Linear(self.model.head.fc.in_features, num_classes)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANCZRData(Dataset):\n    def __init__(self, df, num_classes=5, is_train=True, augments=None, img_size=Config.CFG['img_size'], img_path=\"../input/ranzcr-clip-catheter-line-classification/train\"):\n        super().__init__()\n        self.df = df.sample(frac=1).reset_index(drop=True)\n        self.num_classes = num_classes\n        self.is_train = is_train\n        self.augments = augments\n        self.img_size = img_size\n        self.img_path = img_path\n    def __getitem__(self, idx):\n        image_id = self.df['StudyInstanceUID'].values[idx]\n        image = cv2.imread(os.path.join(self.img_path, image_id + \".jpg\"))\n        image = image[:, :, ::-1]\n        \n        # Augments must be albumentations\n        if self.augments:\n            img = self.augments(image=image)['image']\n        \n        if self.is_train:\n            label = self.df[self.df['StudyInstanceUID'] == image_id].values.tolist()[0][1:-1]\n            return img, torch.tensor(label)\n        \n        return img\n    \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Trainer:\n    def __init__(self, train_dataloader, valid_dataloader, model, optimizer, loss_fn, val_loss_fn, scheduler, device=\"cuda:0\", plot_results=True):\n        \"\"\"\n        Constructor for Trainer class\n        \"\"\"\n        self.train = train_dataloader\n        self.valid = valid_dataloader\n        self.optim = optim\n        self.loss_fn = loss_fn\n        self.val_loss_fn = val_loss_fn\n        self.scheduler = scheduler\n        self.device = device\n        self.plot_results = plot_results\n    \n    def train_one_cycle(self):\n        \"\"\"\n        Runs one epoch of training, backpropagation and optimization\n        \"\"\"\n        model.train()\n        train_prog_bar = tqdm(self.train, total=len(self.train))\n\n        all_train_labels = []\n        all_train_preds = []\n        \n        running_loss = 0\n        \n        for xtrain, ytrain in train_prog_bar:\n            xtrain = xtrain.to(device).float()\n            ytrain = ytrain.to(device).float()\n            \n            with autocast():\n                # Get predictions\n                z = model(xtrain)\n\n                # Training\n                train_loss = self.loss_fn(z, ytrain)\n                scaler.scale(train_loss).backward()\n                \n                scaler.step(self.optim)\n                scaler.update()\n                self.optim.zero_grad()\n\n                # For averaging and reporting later\n                running_loss += train_loss\n\n                # Convert the predictions and corresponding labels to right form\n                train_predictions = torch.argmax(z, 1).detach().cpu().numpy()\n                train_labels = ytrain.detach().cpu().numpy()\n\n                # Append current predictions and current labels to a list\n                all_train_labels += [train_predictions]\n                all_train_preds += [train_labels]\n\n            # Show the current loss to the progress bar\n            train_pbar_desc = f'loss: {train_loss.item():.4f}'\n            train_prog_bar.set_description(desc=train_pbar_desc)\n        \n        # Now average the running loss over all batches and return\n        train_running_loss = running_loss / len(self.train)\n        print(f\"Final Training Loss: {train_running_loss:.4f}\")\n        \n        # Free up memory\n        del all_train_labels, all_train_preds, train_predictions, train_labels, xtrain, ytrain, z\n        \n        return train_running_loss\n\n    def valid_one_cycle(self):\n        \"\"\"\n        Runs one epoch of prediction\n        \"\"\"        \n        model.eval()\n        \n        valid_prog_bar = tqdm(self.valid, total=len(self.valid))\n        \n        with torch.no_grad():\n            all_valid_labels = []\n            all_valid_preds = []\n            \n            running_loss = 0\n            \n            for xval, yval in valid_prog_bar:\n                xval = xval.to(device).float()\n                yval = yval.to(device).float()\n                \n                val_z = model(xval)\n                \n                val_loss = self.val_loss_fn(val_z, yval)\n                \n                running_loss += val_loss.item()\n                \n                val_pred = torch.argmax(val_z, 1).detach().cpu().numpy()\n                val_label = yval.detach().cpu().numpy()\n                \n                all_valid_labels += [val_label]\n                all_valid_preds += [val_pred]\n            \n                # Show the current loss\n                valid_pbar_desc = f\"loss: {val_loss.item():.4f}\"\n                valid_prog_bar.set_description(desc=valid_pbar_desc)\n            \n            # Get the final loss\n            final_loss_val = running_loss / len(self.valid)\n            \n            # Get Validation Accuracy\n            all_valid_labels = np.concatenate(all_valid_labels)\n            all_valid_preds = np.concatenate(all_valid_preds)\n            \n            print(f\"Final Validation Loss: {final_loss_val:.4f}\")\n            \n            # Free up memory\n            del all_valid_labels, all_valid_preds, val_label, val_pred, xval, yval, val_z\n            \n        return (final_loss_val, model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nb_epochs = 5\ndevice = torch.device(\"cuda\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv(\"../input/ranzcr-clip-catheter-line-classification/train.csv\")\ndata = data.sample(frac=1).reset_index(drop=True)\n\n# 27,583 in Train, 2500 in Valid\ntrain_split = data[2500:]\nvalid_split = data[:2500]\n\nprint(train_split.shape, valid_split.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_set = RANCZRData(df=train_split, augments=Augments.train_augments)\nvalid_set = RANCZRData(df=valid_split, augments=Augments.valid_augments)\n\ntrain = DataLoader(\n    train_set,\n    batch_size=16,\n    shuffle=True,\n    pin_memory=False,\n    drop_last=False,\n    num_workers=8\n)\n\nvalid = DataLoader(\n    valid_set,\n    batch_size=32,\n    shuffle=False,\n    pin_memory=False,\n    num_workers=8\n)\n\nmodel = NFNetModel().to(device)\noptim = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.001)\nloss_fn_train = nn.BCEWithLogitsLoss()\nloss_fn_val = nn.BCEWithLogitsLoss()\n\ntrainer = Trainer(\n    train_dataloader=train,\n    valid_dataloader=valid,\n    model=model,\n    optimizer=optim,\n    loss_fn=loss_fn_train,\n    val_loss_fn=loss_fn_val,\n    scheduler=None,\n    device=device,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_losses = []\nvalid_losses = []\n\nscaler = GradScaler()\n\nfor epoch in range(nb_epochs):\n    print(f\"{'-'*20} EPOCH: {epoch+1}/{nb_epochs} {'-'*20}\")\n\n    # Run one training epoch\n    current_train_loss = trainer.train_one_cycle()\n    train_losses.append(current_train_loss)\n\n    # Run one validation epoch\n    current_val_loss, op_model = trainer.valid_one_cycle()\n    valid_losses.append(current_val_loss)\n\n    # Empty CUDA cache\n    torch.cuda.empty_cache()\n    \n    # Save the model every epoch\n    print(f\"Saving Model for this epoch...\")\n    torch.save(op_model.state_dict(), f\"nfnet_f1_model.pth\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}