{"cells":[{"metadata":{"trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"!pip install timm\n!pip install adabelief-pytorch==0.2.0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This is a simple notebook to train the model. The idea is to understand and get upto speed with latest ideas. All experiment results will be logged on to this google doc. We are following this [notebook](https://www.kaggle.com/yasufuminakama/ranzcr-resnext50-32x4d-starter-training)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\nimport cv2\nimport matplotlib.pyplot as plt\nimport ast\nimport tqdm\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"path = '../input/ranzcr-clip-catheter-line-classification/'\ntrain_annot = pd.read_csv(path+'train_annotations.csv')\ntrain_annot.head()\ntrain_images = os.listdir(path+'train')\ntrain_labels = pd.read_csv(path+'train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import ast\ndef plot_row(path,df_annotations,row=5):\n    colors = [(228, 26, 28), (55, 126, 184), (152, 78, 163), (255, 255, 51), (166, 86, 40), (247, 129, 191), (153, 153, 153)]\n    studyid = df_annotations.loc[row, 'StudyInstanceUID']\n#     img = cv2.imread(df_annotations.loc[row, 'image_path'], 1)\n    img_name = path+'train/'+studyid+'.jpg'\n    img = cv2.imread(img_name,1)\n    h,w,_ = img.shape\n#     img = cv2.resize(img,(256,256))\n    scale_x,scale_y = 256/h,256/w\n    data = df_annotations[df_annotations['StudyInstanceUID'] == studyid]\n    for index, row in data.reset_index().iterrows():\n        print(row['label'])\n        pts = np.array(ast.literal_eval(row['data']), np.int32)\n        if row['label'][:3] == 'NGT':\n            color_index = 0\n        elif row['label'][:3] == 'CVC':\n            color_index = 1\n        elif row['label'][:3] == 'ETT':\n            color_index = 3\n        else:\n            color_index = 4\n        for i, pt in enumerate(pts):\n            if i < len(pts)-1:\n                a,b = pts[i]\n#                 a = int(a*scale_x)\n#                 b = int(b*scale_y)\n                c,d = pts[i+1]\n#                 c = int(c*scale_x)\n#                 d = int(d*scale_y)\n                cv2.line(img,tuple(pts[i]), tuple(pts[i+1]), colors[color_index], 30)\n    plt.imshow(img)\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_row(path,train_annot,5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Most of the images are in the ball park of 2500x2500. They will be rescaled to smaller dimension to make training possible. The annotations would also be scaled down to make it possible. Also we dont have to work with the annotation. The annotation is an extra piece of information but its not necessary and is mostly used for our understanding. The entire game is based on image and multi label.  "},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    debug=False\n    print_freq=100\n    num_workers=4\n    model_name='efficientnet_b2'\n    size=512\n    scheduler='CosineAnnealingLR'#['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts']\n    epochs=5\n    #factor=0.2 # ReduceLROnPlateau\n    #patience=4 # ReduceLROnPlateau\n    #eps=1e-6 # ReduceLROnPlateau\n    T_max=6 # CosineAnnealingLR\n    #T_0=6 # CosineAnnealingWarmRestarts\n    lr=1e-4\n    min_lr=1e-6\n    batch_size=32\n    weight_decay=1e-6\n    gradient_accumulation_steps=1\n    max_grad_norm=1000\n    seed=42\n    target_size=11\n    target_cols=['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',\n                 'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal', \n                 'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',\n                 'Swan Ganz Catheter Present']\n    n_fold=4\n    trn_fold=[0, 1, 2, 3]\n    train=True\n    \nif CFG.debug:\n    CFG.epochs = 1\n    train = train.sample(n=100, random_state=CFG.seed).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose\n    )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport timm\nfrom adabelief_pytorch import AdaBelief\n\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OUTPUT_DIR='./'\ndef get_score(y_true, y_pred):\n    scores = []\n    for i in range(y_true.shape[1]):\n        score = roc_auc_score(y_true[:,i], y_pred[:,i])\n        scores.append(score)\n    avg_score = np.mean(scores)\n    return avg_score, scores\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}]start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\ndef init_logger(log_file=OUTPUT_DIR+'train.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = init_logger()\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### CV SPLIT"},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = train_labels.copy()\nFold = GroupKFold(n_splits=CFG.n_fold)\ngroups = folds['PatientID'].values\nfor n, (train_index, val_index) in enumerate(Fold.split(folds,folds[CFG.target_cols],groups)):\n    folds.loc[val_index,'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype('int32')\nfolds.groupby('fold').size()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### DATASET"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainDataset(Dataset):\n    def __init__(self,df,transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.labels = df[CFG.target_cols].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self,idx):\n        file_name = self.file_names[idx]\n        train_path = path+'/train'\n#         print(file_name)\n        file_path = f'{train_path}/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).float()\n        return image,label\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"# These are albumentation based and not the pytorch ones\ndef get_transforms(*,data):\n    if data == 'train':\n        return Compose([\n            Resize(CFG.size,CFG.size),\n            RandomResizedCrop(CFG.size,CFG.size,scale=(0.85,1.0)),\n            HorizontalFlip(p=0.5),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n    if data == 'valid':\n        return Compose([\n            Resize(CFG.size, CFG.size),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = TrainDataset(train_labels,get_transforms(data='train'))\nfor i in range(3):\n    image, label = train_dataset[3*i]\n    plt.title(f'label: {label}')\n    plt.imshow(image[0])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomModel(nn.Module):\n    def __init__(self,model,pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model,pretrained=pretrained,num_classes=CFG.target_size)\n#         n_features = self.model.fc.in_features\n#         self.model.fc = nn.Linear(n_features, CFG.target_size)\n#         n_features = self.model._fc.in_features\n#         self.model.fc = nn.Linear(n_features, CFG.target_size)\n        \n    def forward(self,x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter(object):\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n        \n    def update(self,val,n=1):\n        self.val += val\n        self.sum += val*n\n        self.count += n\n        self.avg = self.sum/self.count\n        \ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_epoch(train_loader,model,criterion,optimizer,epoch,scheduler,device):\n#     GradScaler is a pytorch multiprecision construct. \n    scaler = GradScaler()\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    \n    model.train()\n    start = end = time.time()\n    global_step = 0\n    for step, (images,labels) in enumerate(tqdm(train_loader)):\n        data_time.update(time.time()-end)\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        with autocast():\n            y_preds = model(images)\n            loss = criterion(y_preds,labels)\n        losses.update(loss.item(),batch_size)\n        if CFG.gradient_accumulation_steps >1 :\n            loss /= CFG.gradient_accumulation_steps\n        scaler.scale(loss).backward()\n        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(),CFG.max_grad_norm)\n        if (step+1)%CFG.gradient_accumulation_steps == 0:\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n            global_step += 1\n        batch_time.update(time.time()-end)\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(train_loader)-1):\n            print('Epoch: [{0}][{1}/{2}] '\n                  'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  'Grad: {grad_norm:.4f}  '\n                  #'LR: {lr:.6f}  '\n                  .format(\n                   epoch+1, step, len(train_loader), batch_time=batch_time,\n                   data_time=data_time, loss=losses,\n                   remain=timeSince(start, float(step+1)/len(train_loader)),\n                   grad_norm=grad_norm,\n                   #lr=scheduler.get_lr()[0],\n                   ))\n    return losses.avg\n\ndef valid_epoch(valid_loader,model,criterion,device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    \n    model.eval()\n    preds = []\n    start = end = time.time()\n    for step,(images,labels) in enumerate(tqdm(valid_loader)):\n        data_time.update(time.time()-end)\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        \n        with torch.no_grad():\n            y_preds = model(images)\n        loss = criterion(y_preds,labels)\n        losses.update(loss.item(),batch_size)\n        preds.append(y_preds.sigmoid().to('cpu').numpy())\n        if CFG.gradient_accumulation_steps > 1:\n            loss /= CFG.gradient_accumulation_steps\n        batch_time.update(time.time()-end)\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(valid_loader)-1):\n            print('EVAL: [{0}/{1}] '\n                  'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  .format(\n                   step, len(valid_loader), batch_time=batch_time,\n                   data_time=data_time, loss=losses,\n                   remain=timeSince(start, float(step+1)/len(valid_loader)),\n                   ))\n    predictions = np.concatenate(preds)\n    return losses.avg,predictions","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_loop(folds,fold):\n    LOGGER.info(f\"========== fold: {fold} training ==========\")\n    train_idx = folds[folds['fold']!=fold].index\n    val_idx = folds[folds['fold']==fold].index\n#     print(\"val index\",val_idx.shape)\n    train_folds = folds.loc[train_idx].reset_index(drop=True)\n    val_folds = folds.loc[val_idx].reset_index(drop=True)\n#     print(val_folds.shape)\n    val_labels = val_folds[CFG.target_cols].values\n    \n    train_dataset = TrainDataset(train_folds,transform=get_transforms(data='train'))\n    val_dataset = TrainDataset(val_folds,transform=get_transforms(data='valid'))\n    \n    train_loader = DataLoader(train_dataset, batch_size=CFG.batch_size,shuffle=True,num_workers = CFG.num_workers,pin_memory=True, drop_last=True)\n    val_loader = DataLoader(val_dataset,batch_size=CFG.batch_size * 2,shuffle=False,num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n    \n    def get_scheduler(optimizer):\n        if CFG.scheduler=='ReduceLROnPlateau':\n            scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=CFG.factor, patience=CFG.patience, verbose=True, eps=CFG.eps)\n        elif CFG.scheduler=='CosineAnnealingLR':\n            scheduler = CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr, last_epoch=-1)\n        elif CFG.scheduler=='CosineAnnealingWarmRestarts':\n            scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr, last_epoch=-1)\n        return scheduler\n    \n    model = CustomModel(model=CFG.model_name,pretrained=True)\n    model = model.to(device)\n    \n    optimizer = Adam(model.parameters(),lr=CFG.lr,weight_decay=CFG.weight_decay,amsgrad=False)\n    scheduler = get_scheduler(optimizer)\n    \n    criterion = nn.BCEWithLogitsLoss()\n    \n    best_score=0\n    best_loss = np.inf\n    \n    for epoch in tqdm(range(CFG.epochs)):\n        \n        start_time = time.time()\n        \n        # train\n        avg_loss = train_epoch(train_loader, model, criterion, optimizer, epoch, scheduler, device)\n\n        # eval\n        avg_val_loss, preds = valid_epoch(val_loader, model, criterion, device)\n        \n        if isinstance(scheduler, ReduceLROnPlateau):\n            scheduler.step(avg_val_loss)\n        elif isinstance(scheduler, CosineAnnealingLR):\n            scheduler.step()\n        elif isinstance(scheduler, CosineAnnealingWarmRestarts):\n            scheduler.step()\n        \n        # scoring\n#         print(val_labels.shape,preds.shape,gt_labels.shape)\n#         exit()\n        score, scores = get_score(val_labels, preds)\n        \n\n        elapsed = time.time() - start_time\n\n        LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n        LOGGER.info(f'Epoch {epoch+1} - Score: {score:.4f}  Scores: {np.round(scores, decimals=4)}')\n\n        \"\"\"\n        if score > best_score:\n            best_score = score\n            LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n            torch.save({'model': model.state_dict(), \n                        'preds': preds},\n                        OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.pth')\n        \"\"\"\n        \n        if avg_val_loss < best_loss:\n            best_loss = avg_val_loss\n            LOGGER.info(f'Epoch {epoch+1} - Save Best Loss: {best_loss:.4f} Model')\n            torch.save({'model': model.state_dict(), \n                        'preds': preds},\n                        OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.pth')\n    \n    check_point = torch.load(OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.pth')\n    for c in [f'pred_{c}' for c in CFG.target_cols]:\n        val_folds[c] = np.nan\n    val_folds[[f'pred_{c}' for c in CFG.target_cols]] = check_point['preds']\n\n    return val_folds\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def main():\n\n    \"\"\"\n    Prepare: 1.train  2.folds\n    \"\"\"\n\n    def get_result(result_df):\n        preds = result_df[[f'pred_{c}' for c in CFG.target_cols]].values\n        labels = result_df[CFG.target_cols].values\n        score, scores = get_score(labels, preds)\n        LOGGER.info(f'Score: {score:<.4f}  Scores: {np.round(scores, decimals=4)}')\n    \n    if CFG.train:\n        # train \n        oof_df = pd.DataFrame()\n        for fold in range(CFG.n_fold):\n            if fold in CFG.trn_fold:\n                _oof_df = train_loop(folds, fold)\n                oof_df = pd.concat([oof_df, _oof_df])\n                LOGGER.info(f\"========== fold: {fold} result ==========\")\n                get_result(_oof_df)\n        # CV result\n        LOGGER.info(f\"========== CV ==========\")\n        get_result(oof_df)\n        # save result\n        oof_df.to_csv(OUTPUT_DIR+'oof_df.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n    main()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}