{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Train a EfficientNet baseline model with timm"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Competition: https://www.kaggle.com/c/ranzcr-clip-catheter-line-classification\n\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = 'all'\n\nimport os\nimport sys\nimport gc\nimport math\nimport pickle\nimport random\nimport time\nimport psutil\nimport pytz\nfrom datetime import datetime\nfrom collections import defaultdict\nfrom contextlib import contextmanager\n\nimport warnings\nwarnings.filterwarnings('ignore')  # warnings.filterwarnings(action='once')\n\nfrom tqdm import tqdm, tqdm_notebook\n\nimport numpy as np\nimport pandas as pd\n_ = np.seterr(divide='ignore', invalid='ignore')\n\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.style as style\nstyle.use('fivethirtyeight')\nimport seaborn as sns\nfrom IPython.display import Image  \n\nimport lightgbm as lgb\nfrom sklearn.metrics import roc_auc_score\n\nimport tensorflow as tf\nfrom tensorflow import keras\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n    \ndef seed_all(random_seed=42):\n    os.environ['PYTHONHASHSEED'] = str(random_seed)\n    random.seed(random_seed)\n    np.random.seed(random_seed)\n    tf.random.set_seed(random_seed)\n    torch.manual_seed(random_seed)\n    torch.cuda.manual_seed(random_seed)\n    torch.backends.cudnn.deterministic = True\n\nsys.path.append('../input/gputil/GPUtil')\nfrom GPUtil import showUtilization as gpu_usage\n\nglobal_start_t = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ==============================================\n# CFG\n# ==============================================\nclass CFG:\n    debug = True  # False\n    device = 'GPU'  # ['TPU', 'GPU']\n    nprocs = 1  # [1, 8]\n    print_freq = 100\n    model_name = 'EffnetModel'\n    num_workers = 4\n    size = 640\n    scheduler = 'CosineAnnealingLR'  # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmupRestarts']\n    epochs = 4\n    T_max = 4  # CosineAnnealingLR\n    lr = 5e-4 # 1e-4\n    min_lr = 1e-6\n    batch_size = 16  # 64\n    weight_decay = 1e-6\n    gradient_accumulation_steps = 1 # 1 \n    max_grad_norm = 1000\n    seed = 2021\n    target_size = 11\n    target_cols = ['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',\n                   'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal', \n                   'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',\n                   'Swan Ganz Catheter Present']\n    n_fold = 5\n    trn_fold = [0] # [0, 1, 2, 3, 4]\n    train = True\n    \nif CFG.debug:\n    CFG.epochs = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# =======================================================\n# Library\n# =======================================================\nimport sys\nsys.path.append('../input/pytorch-image-models/')\n\nimport ast\nimport copy\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n\nfrom tqdm.auto import tqdm\nimport cv2\nfrom PIL import Image\n\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose, HueSaturationValue, CoarseDropout\n    )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport timm\nfrom torch.cuda.amp import autocast, GradScaler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n    \nRANDOM_SEED = 53113\nseed_all(RANDOM_SEED)   # to make the fold split the same every run\n    \nTRAIN_PATH = '../input/ranzcr-clip-catheter-line-classification/train'\ntrain = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/train.csv')\n\nif CFG.debug:  # for quick test\n    train = train.sample(n=300, random_state=2021).reset_index(drop=True)\nelse:\n    train = train.sample(n=len(train), random_state=2021).reset_index(drop=True)\n\nfolds = train.copy()\nFold = GroupKFold(n_splits=CFG.n_fold)\ngroups = folds['PatientID'].values\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds[CFG.target_cols], groups)):\n    folds.loc[val_index, 'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype(int)\ndisplay(folds.groupby('fold').size())\n\ndef get_time_random_seed():\n    t = int(time.time() * 1000.0)\n    return  (((t & 0xff000000) >> 24) +\n             ((t & 0x00ff0000) >>  8) +\n             ((t & 0x0000ff00) <<  8) +\n             ((t & 0x000000ff) << 24))\n\ntime_random_seed = get_time_random_seed()\nprint(f'time_random_seed: {time_random_seed}')\nseed_all(time_random_seed)  # for model results reproducible\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'device: {device}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZCR_EfficientNet(nn.Module):\n    def __init__(self, model_name='efficientnet_b3', out_dim=11, pretrained=False):  \n        '''\n        model_name: ['efficientnet_b1', 'efficientnet_b2', 'efficientnet_b3', \n                     'efficientnet_b1_pruned', 'efficientnet_b2_pruned', 'efficientnet_b3_pruned', ...]\n        For more, checkout https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/efficientnet.py\n        '''\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.classifier = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, out_dim)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output\n\nmodel_name = 'efficientnet_b3'\nglobal_model = RANZCR_EfficientNet(model_name, pretrained=True).to(device)\n\n# show total number of parameters in the model\nmodel_param_num_sum = sum(p.numel() for p in global_model.parameters())\nmodel_trainable_param_num_sum = sum(p.numel() for p in global_model.parameters() if p.requires_grad)\n\nmodel_name, model_param_num_sum, model_trainable_param_num_sum ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_auc_score(y_true, y_pred):\n    scores = []\n    for i in range(y_true.shape[1]):\n        if len(np.unique(y_true[:, i]))==1:  # if there is just one value, set the auc to default 0.5\n            score = 0.5\n        else:\n            score = roc_auc_score(y_true[:, i], y_pred[:, i])\n        scores.append(score)\n    avg_score = np.mean(scores)\n    return avg_score, scores\n\ndef get_accuracy_score(y_true, y_pred):\n    y_pred = (y_pred >= 0.5).astype(int)\n    scores = []\n    for i in range(y_true.shape[1]):\n        score = (y_true[:, i]==y_pred[:, i]).sum()/y_true.shape[0]\n        scores.append(score)\n    avg_score = np.mean(scores)\n    return avg_score, scores\n\nclass TrainDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.labels = df[CFG.target_cols].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TRAIN_PATH}/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).float()\n\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================\n# Transforms\n# ====================================\ndef get_transforms(*, data):\n    if data == 'train':\n        return Compose([\n            #Resize(CFG.size, CFG.size),\n            RandomResizedCrop(CFG.size, CFG.size, scale=(0.85, 1.0)),\n            HorizontalFlip(p=0.5),\n            RandomBrightnessContrast(p=0.2, brightness_limit=(-0.2, 0.2), contrast_limit=(-0.2, 0.2)),\n            HueSaturationValue(p=0.2, hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2),\n            ShiftScaleRotate(p=0.2, shift_limit=0.0625, scale_limit=0.2, rotate_limit=20),\n            CoarseDropout(p=0.2),\n            Cutout(p=0.2, max_h_size=16, max_w_size=16, fill_value=(0., 0., 0.), num_holes=16),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n    elif data == 'valid':\n        return Compose([\n            Resize(CFG.size, CFG.size),\n            Normalize(\n                mean = [0.485, 0.456, 0.406],\n                std = [0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# =============================================\n# Helper functions\n# =============================================\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n        \n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n        \n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n        \ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return f'{m}m {s}s'\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return f'{asMinutes(s)} (remain {asMinutes(rs)})'\n\ndef train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device):\n    if CFG.device == 'GPU':\n        scaler = GradScaler()\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to train mode\n    model.train()\n    start = end = time.time()\n    global_step = 0\n    len_train_loader = len(train_loader)\n    print(f'in train_fn() len_train_loader: {len_train_loader}')\n    for step, (images, labels) in enumerate(train_loader):\n        # measure data loading time\n        if step%50==0:\n            print(f'in train_fn() step: {step} len_train_loader: {len_train_loader}')\n            gpu_usage()\n        data_time.update(time.time() - end)\n        images, labels = images.to(device), labels.to(device)\n        batch_size = labels.size(0)\n        if CFG.device == 'GPU':\n            with autocast():\n                y_preds = model(images)\n                loss = criterion(y_preds, labels)\n                # record loss\n                losses.update(loss.item(), batch_size)\n                if CFG.gradient_accumulation_steps > 1:\n                    loss = loss / CFG.gradient_accumulation_steps\n                scaler.scale(loss).backward()\n                grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), CFG.max_grad_norm)\n                if (step + 1) % CFG.gradient_accumulation_steps == 0:\n                    scaler.step(optimizer)\n                    scaler.update()\n                    optimizer.zero_grad()\n                    global_step += 1\n\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if CFG.device == 'GPU':\n            if step % CFG.print_freq == 0 or step == (len(train_loader)-1):\n                print('Epoch: [{0}][{1}/{2}] '\n                      'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                      'Elapsed {remain:s} '\n                      'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                      'Grad: {grad_norm:.4f}  '\n                      #'LR: {lr:.6f}  '\n                      .format(\n                       epoch+1, step, len(train_loader), batch_time=batch_time,\n                       data_time=data_time, loss=losses,\n                       remain=timeSince(start, float(step+1)/len(train_loader)),\n                       grad_norm=grad_norm,\n                       #lr=scheduler.get_lr()[0],\n                       ))\n                \n    return losses.avg\n\n\ndef valid_fn(valid_loader, model, criterion, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to evaluation mode\n    model.eval()\n    trues, preds = [], []\n    start = end = time.time()\n    for step, (images, labels) in enumerate(valid_loader):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        images, labels = images.to(device), labels.to(device)\n        batch_size = labels.size(0)\n        # compute loss\n        with torch.no_grad():\n            y_preds = model(images)\n        loss = criterion(y_preds, labels)\n        losses.update(loss.item(), batch_size)\n        # record accuracy\n        trues.append(labels.to('cpu').numpy())\n        preds.append(y_preds.sigmoid().to('cpu').numpy())\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if CFG.device == 'GPU':\n            if step % CFG.print_freq == 0 or step == (len(valid_loader)-1):\n                print('EVAL: [{0}/{1}] '\n                      'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                      'Elapsed {remain:s} '\n                      'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                      .format(\n                       step, len(valid_loader), batch_time=batch_time,\n                       data_time=data_time, loss=losses,\n                       remain=timeSince(start, float(step+1)/len(valid_loader)),\n                       ))\n                \n    trues = np.concatenate(trues)\n    predictions = np.concatenate(preds)\n    return losses.avg, predictions, trues","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_optimizer_lr(optimizer):\n    for param_group in optimizer.param_groups:\n        return param_group['lr']\n    \n# ==================================================\n# Train loop\n# ==================================================\ndef train_loop(folds, fold):\n    print('in train_loop()')\n    gpu_usage()\n    if CFG.device == 'GPU':\n        print(f'========== fold: {fold} training ============')\n            \n    # =============================================\n    # loader\n    # =============================================\n    trn_idx = folds[folds['fold'] != fold].index\n    val_idx = folds[folds['fold'] == fold].index\n    \n    train_folds = folds.loc[trn_idx].reset_index(drop=True)\n    valid_folds = folds.loc[val_idx].reset_index(drop=True)\n    \n    valid_labels = valid_folds[CFG.target_cols].values\n    \n    train_dataset = TrainDataset(train_folds, transform=get_transforms(data='train'))\n    valid_dataset = TrainDataset(valid_folds, transform=get_transforms(data='valid'))\n    \n    if CFG.device == 'GPU':\n        train_loader = DataLoader(train_dataset,\n                                  batch_size=CFG.batch_size,\n                                  shuffle=True, num_workers=CFG.num_workers, \n                                  pin_memory=True, drop_last=True)\n        valid_loader = DataLoader(valid_dataset,\n                                  batch_size=CFG.batch_size * 2,\n                                  shuffle=False, num_workers=CFG.num_workers, \n                                  pin_memory=True, drop_last=False)\n        \n    # ==============================================\n    # scheduler\n    # ==============================================\n    def get_scheduler(optimizer):\n        if CFG.scheduler=='ReduceLROnPlateau':\n            scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=CFG.factor, patience=CFG.patience, verbose=True, eps=CFG.eps)\n        elif CFG.scheduler=='CosineAnnealingLR':\n            scheduler = CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr, last_epoch=-1)\n        elif CFG.scheduler=='CosineAnnealingWarmpRestarts':\n            scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr, last_epoch=-1)\n        return scheduler\n    \n    # ==============================================\n    # model & optimizer\n    # ==============================================\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    print('device: ', device)\n    \n    model = global_model\n    \n    optimizer = Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)\n    scheduler = get_scheduler(optimizer)\n    \n    # =============================================\n    # loop\n    # =============================================\n    criterion = nn.BCEWithLogitsLoss()\n    best_score, best_loss = 0., np.inf\n    \n    for epoch in range(CFG.epochs):\n        start_time = time.time()\n        \n        # train\n        avg_loss = train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device)\n            \n        # eval\n        avg_val_loss, preds, _ = valid_fn(valid_loader, model, criterion, device)\n        \n        if isinstance(scheduler, ReduceLROnPlateau):\n            scheduler.step(avg_val_loss)\n        elif isinstance(scheduler, CosineAnnealingLR):\n            scheduler.step()\n        elif isinstance(scheduler, CosineAnnealingWarmRestarts):\n            scheduler.step()\n            \n        # scoring\n        score, scores = get_auc_score(valid_labels, preds)\n        accuracy_score, accuracy_scores = get_accuracy_score(valid_labels, preds)\n        \n        print(f'score:{score}, scores: {scores}')\n        print(f'accuracy_score:{accuracy_score}, accuracy_scores: {accuracy_scores}')\n    \n        elapsed = time.time() - start_time\n\n        if CFG.device == 'GPU':\n            print(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.3f}s')\n            print(f'Epoch {epoch+1} - Score: {score:.4f}  Scores: {np.round(scores, decimals=4)}')\n                \n        if score > best_score:\n            best_score = score\n            if CFG.device == 'GPU':\n                print(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n                torch.save({'model': model.state_dict(), \n                            'preds': preds, \n                            'scheduler': scheduler.state_dict(), \n                            'optimizer': optimizer.state_dict()},\n                           OUTPUT_DIR + f'{CFG.model_name}_fold{fold}_best_score.pth')\n                print('save best_score model, epoch: ', epoch, \n                      'lr_optimizer',  get_optimizer_lr(optimizer), \n                      'scheduler.last_lr is ', scheduler.get_last_lr())\n                \n        if avg_val_loss < best_loss:\n            best_loss = avg_val_loss\n            if CFG.device == 'GPU':\n                print(f'Epoch {epoch+1} - Save Best Loss: {best_loss:.4f} Model')\n                torch.save({'model': model.state_dict(), \n                            'preds': preds, \n                            'scheduler': scheduler.state_dict(), \n                            'optimizer': optimizer.state_dict()},\n                           OUTPUT_DIR + f'{CFG.model_name}_fold{fold}_best_loss.pth')\n                print('save best_loss model, epoch: ', epoch, \n                      'lr_optimizer',  get_optimizer_lr(optimizer), \n                      'scheduler.last_lr is ', scheduler.get_last_lr())\n    if CFG.nprocs != 8:\n        check_point = torch.load(OUTPUT_DIR + f'{CFG.model_name}_fold{fold}_best_score.pth')\n        for c in [f'pred_{c}' for c in CFG.target_cols]:\n            valid_folds[c] = np.nan\n        valid_folds[[f'pred_{c}' for c in CFG.target_cols]] = check_point['preds']\n    \n    return valid_folds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def main():\n    for fold in range(CFG.n_fold):\n        if fold in CFG.trn_fold:\n            train_loop(folds, fold)\n\nif __name__=='__main__':\n    print('before main()')\n    main_start_t = time.time()\n    gpu_usage()\n    main()\n    print(f'main total cost time: {time.time()-main_start_t: .3f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dummy submission\ntest = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\ntarget_cols = test.iloc[:, 1:12].columns.tolist()\ntest[target_cols] = 0\ntest[['StudyInstanceUID'] + target_cols].to_csv('submission.csv', index=False)\n\nprint(f'total cost time: {time.time()-global_start_t:.5f} sec')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}