{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Installing necessary libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"! /opt/conda/bin/python3.7 -m pip install -q --upgrade pip\n! pip install -q timm catalyst iterative-stratification\n! pip install -q --upgrade wandb\n! pip install -q pytorch-gradcam","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We are going to use `wandb` for tracking our model's performance. If you don't have a `wandb` account, go to [this](wandb.ai) link, create an account using either google or github account. Then go to `wandb.ai/[your_username]` -> `Create New Project`. Give a cute little name to your project. Open your project page. You'll find some line like this:\n\n`wandb login e1da498db2dd649a76a04c6e4743e5a4f95a2ae0`\n\nCopy and paste this line to the next cell."},{"metadata":{"trusted":true},"cell_type":"code","source":"! wandb login e1da498db2dd649a76a04c6e4743e5a4f95a2ae0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Config\nThis section contains configuration parameters for my classification pipeline."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport cv2\nimport pandas as pd\nimport albumentations as A\nfrom albumentations.augmentations.transforms import Equalize, Posterize, Downscale\nfrom albumentations import (\n    PadIfNeeded, HorizontalFlip, VerticalFlip, CenterCrop,    \n    RandomCrop, Resize, Crop, Compose, HueSaturationValue,\n    Transpose, RandomRotate90, ElasticTransform, GridDistortion, \n    OpticalDistortion, RandomSizedCrop, Resize, CenterCrop,\n    VerticalFlip, HorizontalFlip, OneOf, CLAHE, Normalize,\n    RandomBrightnessContrast, Cutout, RandomGamma, ShiftScaleRotate ,\n    GaussNoise, Blur, MotionBlur, GaussianBlur, \n)\n\nSEED = 24\nn_epochs = 25\ndevice = 'cuda:0'\ndata_dir = '../input/ranzcr-clip-catheter-line-classification'\nloss_thr = 1e6\nimg_path = '../input/ranzcr-clip-1024-resized/resized_1024'\ndf = pd.read_csv(f'{data_dir}/train.csv')\ndf['path'] = df['StudyInstanceUID'].map(lambda x: f\"{img_path}/{x}.jpg\")\nencoder_model = 'resnest50_fast_1s1x64d'\nfold = 0\nmodel_name= f'Resnest50_fold{fold}' # Will come up with a better name later\nmodel_dir = 'model_dir'\nhistory_dir = 'history_dir'\nload_model = False\nimg_dim = 320\nbatch_size = 40\naccum_step = 1\nlearning_rate = 7.50e-3\nnum_workers = 4\nmixed_precision = True\npatience = 3\nbalanced_sampler = False\ntrain_aug = A.Compose([A.CenterCrop(p=0.3, height=int(0.8*img_dim), width=int(0.8*img_dim)),\nA.augmentations.transforms.RandomCrop(int(0.8*img_dim), int(0.8*img_dim), p=0.3),\nA.augmentations.transforms.Rotate(limit=30, interpolation=1, border_mode=4, value=None, mask_value=None, always_apply=False, p=0.5),\nA.augmentations.transforms.Resize(img_dim, img_dim, interpolation=1, always_apply=True, p=0.6),\nCutout(num_holes=8, max_h_size=20, max_w_size=20, fill_value=0, always_apply=False, p=0.2),\nA.RandomBrightnessContrast(brightness_limit=0.3, contrast_limit=0.3, brightness_by_max=True, always_apply=False, p=0.3),\nA.augmentations.transforms.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=20, always_apply=False, p=0.4),\n# A.HorizontalFlip(p=0.5),\nA.VerticalFlip(p=0.5),                    \nOneOf([\n        GaussNoise(var_limit=0.1),\n        Blur(),\n        GaussianBlur(blur_limit=3),\n        # RandomGamma(p=0.7),\n        ], p=0.3),\nA.HorizontalFlip(p=0.3), Normalize(always_apply=True)],)\nval_aug = Compose([Normalize(always_apply=True)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nfrom matplotlib import pyplot as plt\nfig = plt.figure(figsize = (15,20))\nax = fig.gca()\ndf.hist(ax = ax)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fixing Seed"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport torch\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Stratification"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split, KFold, StratifiedKFold\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold \nmskf = MultilabelStratifiedKFold(n_splits=5, random_state=SEED)\nX = df['path']\ny = df.iloc[:, 1:12]\ntrain_idx = []\nval_idx = []\n\ndf['fold'] = np.nan\n\n#split data\nfor i, (_, test_index) in enumerate(mskf.split(X, y)):\n    df.loc[test_index, 'fold'] = i\n    \ndf['fold'] = df['fold'].astype('int')\n\nvalid_df = df[df['fold']==fold]\ntrain_df = df[df['fold']!=fold]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset,DataLoader\n\nclass CatheterDataset(Dataset):\n    def __init__(self, df, dim=256, transforms=None):\n        super().__init__()\n        self.image_ids = df.path.tolist()\n        try:\n            self.labels = np.array(df.iloc[:, 1:12])\n        except:\n            self.labels = None\n        self.transforms = transforms\n        self.dim = dim\n        \n    def __getitem__(self, idx):\n        image_id = self.image_ids[idx]\n        image = cv2.imread(image_id, cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (self.dim, self.dim))\n        \n        if self.transforms is not None:\n            aug = self.transforms(image=image)\n            image = aug['image'].reshape(self.dim, self.dim, 3).transpose(2, 0, 1)\n        else:\n            image = image.reshape(self.dim, self.dim, 3).transpose(2, 0, 1)\n        if self.labels is not None:\n            target = self.labels[idx]\n            return image_id, image, target\n        else:\n            return image_id, image\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def get_labels(self):\n        return list(self.labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{},"cell_type":"markdown","source":"### Generalized Mean Pooling\n\nI'm using `Resnest` model for training and replacing the last `global_pool` layer with Generalized Mean Pooling(GeM) layer from [here](https://www.kaggle.com/c/aptos2019-blindness-detection/discussion/108065).  "},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch import nn\n\ndef gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM,self).__init__()\n        self.p = Parameter(torch.ones(1)*p)\n        self.eps = eps\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)       \n    def __repr__(self):\n        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch import nn\nfrom torch.nn import *\nfrom torch.nn import functional as F\nfrom torchvision import models\nimport timm\n\nclass Resne_t(nn.Module):\n\n    def __init__(self, model_name):\n        super().__init__()\n        try:\n            self.backbone = timm.create_model(model_name, pretrained=True)\n        except:\n            self.backbone = torch.hub.load('zhanghang1989/ResNeSt', model_name, pretrained=True)\n        self.in_features = self.backbone.fc.in_features\n        self.output = nn.Sequential(nn.Linear(self.in_features, 128), nn.Linear(128, 11))\n        self.backbone.global_pool = GeM()\n\n    def forward(self, x):\n        x = self.backbone.conv1(x)\n        x = self.backbone.bn1(x)\n        try:\n            x = self.backbone.act1(x)\n        except:\n            x = self.backbone.relu(x)\n        x = self.backbone.maxpool(x)\n\n        x = self.backbone.layer1(x)\n        x = self.backbone.layer2(x)\n        \n        x = self.backbone.layer3(x)\n        x = self.backbone.layer4(x)\n        x = self.backbone.global_pool(x)\n        x = x.view(x.size(0), -1)\n        x = self.output(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss function\n\nI'm going to use `Focal Cosine Loss` in this notebook. The implementation is borrowed from [here](https://github.com/byeongjokim/VIPriors-Image-Classification-Challenge/blob/332e04fd3e82b20d312128bad302a9081f5c37ce/timm/loss/cosine.py) and modified for multi-label classification."},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalCosineLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, xent=.1, reduction=\"mean\"):\n        super(FocalCosineLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n        self.xent = xent\n        self.reduction = reduction\n        \n        self.y = torch.Tensor([1]).cuda()\n        \n    def forward(self, input, target):\n        cosine_loss = F.cosine_embedding_loss(input, target, self.y, reduction=self.reduction)\n        cent_loss = nn.BCEWithLogitsLoss()(input, target)\n        pt = torch.exp(-cent_loss)\n        focal_loss = self.alpha * (1-pt)**self.gamma * cent_loss\n\n        if self.reduction == \"mean\":\n            focal_loss = torch.mean(focal_loss)\n        \n        return cosine_loss + self.xent * focal_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{},"cell_type":"markdown","source":"After running this cell, you will get a link titled `Run Page` that looks like this: `https://wandb.ai/[user_name]/[project_name]/runs/********`. If you commit this notebook, then you won't be able to see this link. In that case, go to your project page i.e., `https://wandb.ai/[user_name]/[project_name]/` and find the running project (the one with a green knob). Your Training logs should be available there."},{"metadata":{"trusted":true},"cell_type":"code","source":"import logging\nlogging.basicConfig(level=logging.ERROR)\nimport wandb\nfrom functools import partial\nfrom collections import Counter\nimport gc\nimport time\nimport pandas as pd\nfrom torch import optim\nfrom catalyst.data.sampler import BalanceClassSampler\n\nwandb.init(project=\"catheter\")\nwandb.run.name= model_name\n\nm_p = mixed_precision\nif m_p:\n  scaler = torch.cuda.amp.GradScaler() \n\nnp.random.seed(SEED)\n\ntrain_ds = CatheterDataset(train_df, img_dim, train_aug)\nif balanced_sampler:\n  print('Using Balanced Sampler....')\n  train_loader = torch.utils.data.DataLoader(train_ds,batch_size=batch_size, sampler=BalanceClassSampler(labels=train_ds.get_labels(), mode=\"upsampling\"), shuffle=False, num_workers=4)\nelse:\n  train_loader = torch.utils.data.DataLoader(train_ds, batch_size=batch_size, shuffle=True, num_workers=num_workers, pin_memory=True)\n\nval_ds = CatheterDataset(valid_df, img_dim, val_aug)\nvalid_loader = torch.utils.data.DataLoader(\nval_ds, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True)\n\nos.makedirs(model_dir, exist_ok=True)\nos.makedirs(history_dir, exist_ok=True)\n\nresult = pd.DataFrame(columns=['name', 'prediction', 'label', 'difference'])\nif os.path.exists(f'{history_dir}/history_{model_name}_{img_dim}.csv'):\n    history = pd.read_csv(f'{history_dir}/history_{model_name}_{img_dim}.csv')\nelse:\n    history = pd.DataFrame(columns=['train_loss','train_time','val_loss','val_roc_auc', 'val_time'])\n\nmodel = Resne_t(encoder_model).to(device)\nwandb.watch(model)\n# criterion = nn.BCEWithLogitsLoss()\ncriterion = FocalCosineLoss(reduction='sum')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_model(valid_loss, valid_roc, best_valid_loss, best_valid_roc, best_state, savepath):\n    if valid_loss<best_valid_loss:\n        print(f'Validation loss has decreased from:  {best_valid_loss:.4f} to: {valid_loss:.4f}. Saving checkpoint')\n        torch.save(best_state, savepath+'_loss.pth')\n        best_valid_loss = valid_loss\n    if valid_roc>best_valid_roc:\n        print(f'Validation ROC_AUC score has increased from:  {best_valid_roc:.4f} to: {valid_roc:.4f}. Saving checkpoint')\n        torch.save(best_state, savepath + '_roc_auc.pth')\n        best_valid_roc = valid_roc\n    else:\n        torch.save(best_state, savepath + '_last.pth')\n    return best_valid_loss, best_valid_roc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ROC_AUC"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\ndef ROC(predictions, labels):\n    tmp_roc = []\n    for i in range(11):\n        tmp_roc.append(roc_auc_score(np.array(labels)[:, i], np.array(predictions)[:, i], ))\n    return np.mean(tmp_roc)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_val(epoch, dataloader, optimizer, pretrained=None, train=True, mode='train', record=True):\n    global m_p\n    global result\n    global batch_size\n    global accum_step\n    t1 = time.time()\n    running_loss = 0\n    epoch_samples = 0\n    pred = []\n    lab = []\n    if pretrained:\n        model.load_state_dict(pretrained)\n    if train:\n        model.train()\n        print(\"Initiating train phase ...\")\n    else:\n        model.eval()\n        print(\"Initiating val phase ...\")\n    for idx, (_, img, labels) in enumerate(dataloader):\n        with torch.set_grad_enabled(train):\n            img = img.to(device, dtype=torch.float32)\n            labels = labels.to(device, dtype=torch.float32)\n            epoch_samples += len(img)\n            optimizer.zero_grad()\n            with torch.cuda.amp.autocast(m_p):\n                if m_p:\n                    img = img.half()\n                else:\n                    img = img.float()\n                outputs = model(img)\n\n                loss = criterion(outputs, labels).sum()\n                running_loss += loss.item()*len(img)\n                loss = loss/accum_step\n      \n                if train:\n                     if m_p:\n                         scaler.scale(loss).backward()\n                         if (idx+1) % accum_step == 0:\n                             scaler.step(optimizer)\n                             scaler.update() \n                             optimizer.zero_grad()\n                     else:\n                         loss.backward()\n                         if (idx+1) % accum_step == 0:\n                             optimizer.step()\n                             optimizer.zero_grad()\n\n        elapsed = int(time.time() - t1)\n        eta = int(elapsed / (idx+1) * (len(dataloader)-(idx+1)))\n        pred.extend(torch.sigmoid(outputs).detach().cpu().numpy())\n        lab.extend(labels.cpu().numpy())\n\n        if train:\n            msg = f\"Epoch: {epoch} Progress: [{idx}/{len(dataloader)}] loss: {(running_loss/epoch_samples):.4f} Time: {elapsed}s ETA: {eta} s\"\n        else:\n            msg = f'Epoch {epoch} Progress: [{idx}/{len(dataloader)}] loss: {(running_loss/epoch_samples):.4f} Time: {elapsed}s ETA: {eta} s'\n        wandb.log({\"Train Loss\": running_loss/epoch_samples, \"Epoch\":epoch})\n        print(msg, end= '\\r')\n    roc = ROC(np.array(pred), np.array(lab))\n    history.loc[epoch, f'{mode}_loss'] = running_loss/epoch_samples\n    history.loc[epoch, f'{mode}_time'] = elapsed\n    if mode=='val' or mode=='test':\n        lr_reduce_scheduler.step(roc)\n        msg = f'{mode} Loss: {running_loss/epoch_samples:.4f} \\n {mode} ROC_AUC: {roc:.4f}'\n        print(msg)\n        wandb.log({f\"{mode} Loss\": running_loss/epoch_samples, f\"{mode} ROC_AUC\":roc, \"Epoch\":epoch})\n        history.loc[epoch, f'{mode}_loss'] = running_loss/epoch_samples\n        history.loc[epoch, f'{mode}_roc_auc'] = roc\n        # NaN check\n        if running_loss/epoch_samples > loss_thr or running_loss!=running_loss:\n            print('\\033[91mMixed Precision\\033[0m rendering nan value. Forcing \\033[91mMixed Precision\\033[0m to be False ...')\n            m_p = False\n            batch_size = batch_size//2\n            accum_step = accum_step*2\n            print('Loading last best model ...')\n            tmp = torch.load(os.path.join(model_dir, model_name+'_loss.pth'))\n            model.load_state_dict(tmp['model'])\n            optimizer.load_state_dict(tmp['optim'])\n            lr_reduce_scheduler.load_state_dict(tmp['scheduler'])\n            del tmp\n            \n        if record:\n            history.to_csv(f'{history_dir}/history_{model_name}_{img_dim}.csv', index=False)\n        return running_loss/epoch_samples, roc\n\n\nplist = [ \n        {'params': model.backbone.parameters(),  'lr': learning_rate/50},\n        {'params': model.output.parameters(),  'lr': learning_rate}\n    ]\noptimizer = optim.Adam(plist, lr=learning_rate)\nlr_reduce_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=patience, verbose=True, threshold=1e-4, threshold_mode='rel', cooldown=0, min_lr=1e-7, eps=1e-08)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def main():\n  prev_epoch_num = 0\n  best_valid_loss = np.inf\n  best_valid_roc = 0.0\n\n  if load_model:\n    tmp = torch.load(os.path.join(model_dir, model_name+'_roc_auc.pth'))\n    model.load_state_dict(tmp['model'])\n    optimizer.load_state_dict(tmp['optim'])\n    lr_reduce_scheduler.load_state_dict(tmp['scheduler'])\n    scaler.load_state_dict(tmp['scaler'])\n    prev_epoch_num = tmp['epoch']\n    best_valid_loss = tmp['best_loss']\n    best_valid_loss, best_valid_roc = train_val(prev_epoch_num+1, valid_loader, optimizer=optimizer, train=False, mode='val')\n    del tmp\n    print('Model Loaded!')\n  \n  for epoch in range(prev_epoch_num, n_epochs):\n    torch.cuda.empty_cache()\n    print(gc.collect())\n\n    train_val(epoch, train_loader, optimizer=optimizer, train=True, mode='train')\n    valid_loss, valid_roc = train_val(epoch, valid_loader, optimizer=optimizer, train=False, mode='val')\n    print(\"#\"*20)\n    print(f\"Epoch {epoch} Report:\")\n    print(f\"Validation Loss: {valid_loss :.4f} Validation ROC_AUC: {valid_roc :.4f}\")\n    best_state = {'model': model.state_dict(), 'optim': optimizer.state_dict(), 'scheduler':lr_reduce_scheduler.state_dict(), \n          'scaler': scaler.state_dict(),\n    'best_loss':valid_loss, 'best_acc':valid_roc, 'epoch':epoch}\n    best_valid_loss, best_valid_roc = save_model(valid_loss, valid_roc, best_valid_loss, best_valid_roc, best_state, os.path.join(model_dir, model_name))\n    print(\"#\"*20)\n   \nif __name__== '__main__':\n  main()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}