{"cells":[{"metadata":{},"cell_type":"markdown","source":"## This notebook trains:\n- PyTorch Resnet200d\n- The folds used is based on @abhishek training kernel [here](https://www.kaggle.com/abhishek/ranzcr-tez-training-efficientnet-5/data)\n- This notebook is based on kernel made by @yasufuminakama ([check it out here](https://www.kaggle.com/yasufuminakama/ranzcr-resnext50-32x4d-starter-training))\n\n### Hope this is useful!"},{"metadata":{"trusted":true},"cell_type":"code","source":"package_paths = ['../input/pytorch-image-models/pytorch-image-models-master']\nOUTPUT_DIR = './'\n\nimport sys;\nfor pth in package_paths:\n    sys.path.append(pth)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom contextlib import contextmanager\nfrom datetime import datetime\nimport time\nimport math\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom torch.optim import Adam, SGD\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\nfrom torch.cuda.amp import autocast, GradScaler\n\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose, ToGray\n    )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport timm\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\n#from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configuration"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"CFG = {\n    'output_path': './',\n    'train_path': '../input/ranzcr-clip-catheter-line-classification/train',\n    'print_freq': 100,\n    'fold_num': 4,\n    'seed': 2003,\n    'model_arch': 'resnet200d',\n    'img_size': 640,\n    'epochs': 1,\n    'scheduler': 'ReduceLROnPlateau', #['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts']\n    'factor': 0.2, # ReduceLROnPlateau\n    'patience': 4, # ReduceLROnPlateau\n    'eps': 1e-6, # ReduceLROnPlateau\n    'T_max': 6, # CosineAnnealingLR\n    'T_0': 6, # CosineAnnealingWarmRestarts\n    'gradient_accumulation_steps': 1,\n    'max_grad_norm': 1000,\n    'batch_size': 8,\n    'lr': 1e-4,\n    'min_lr': 1e-4,\n    'weight_decay': 1e-6,\n    'num_workers': 4,\n    'accum_iter': 2,\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'target_size': 11,\n    'target_cols': ['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',\n                    'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal', \n                    'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',\n                    'Swan Ganz Catheter Present'],\n    'train': True,\n    'trn_fold': [0] # Train only fold-0\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CV"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/train.csv')\n# folds = pd.read_csv('../input/ranzcr-folds/train_folds.csv')\ntest = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = train.copy()\nFold = GroupKFold(n_splits=CFG['fold_num'])\ngroups = folds['PatientID'].values\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds[CFG['target_cols']], groups)):\n    folds.loc[val_index, 'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nfolds.columns = ['StudyInstanceUID', 'ETT - Abnormal', 'ETT - Borderline',\n       'ETT - Normal', 'NGT - Abnormal', 'NGT - Borderline',\n       'NGT - Incompletely Imaged', 'NGT - Normal', 'CVC - Abnormal',\n       'CVC - Borderline', 'CVC - Normal', 'Swan Ganz Catheter Present',\n       'PatientID', 'fold']\n'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_score(y_true, y_pred):\n    scores = []\n    for i in range(y_true.shape[1]):\n        score = roc_auc_score(y_true[:,i], y_pred[:,i])\n        scores.append(score)\n    avg_score = np.mean(scores)\n    return avg_score, scores\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n\ndef init_logger(log_file=CFG['output_path']+'train.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = init_logger()\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG['seed'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.labels = df[CFG['target_cols']].values\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{CFG[\"train_path\"]}/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = image[:,:,::-1]\n        #image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).float()\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_transforms(mode):\n    if mode == 'train':\n        return Compose([\n            Resize(CFG['img_size'], CFG['img_size']),\n            #RandomResizedCrop(CFG['img_size'], CFG['img_size'], scale=(0.85, 1.0)),\n            HorizontalFlip(p=0.5),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n                p=1.0,\n            ),\n            ToTensorV2()\n        ])\n    elif mode == 'valid':\n        return Compose([\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n                p=1.0,\n            ),\n            ToTensorV2(),\n        ])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RanzcrModel(nn.Module):\n    def __init__(self, model_name, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        # self.model .conv1[0].in_channels = 1\n        # weight = self.model.conv1[0].weight.mean(1, keepdim=True)\n        # self.model.conv1[0].weight = torch.nn.Parameter(weight)\n        # self.dropout = nn.Dropout(0.1)\n        # n_features = self.model.classifier.in_features\n        n_features = self.model.fc.in_features\n        # self.model.classifier = nn.Linear(n_features, CFG['target_size'])\n        self.model.fc = nn.Linear(n_features, CFG['target_size'])\n    \n    def forward(self, x):\n        # batch_size, _, _, _ = x.shape\n        x = self.model(x)\n        #x = nn.functional.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)\n        #x = self.dropout(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter(object):\n    def __init__(self):\n        self.reset()\n        \n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n        \n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n        \ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))\n\n\ndef train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device):\n    scaler = GradScaler()\n    \n    # Measurements\n    batch_time = AverageMeter() # to measure elapsed time per batch\n    data_time = AverageMeter() # to measure data loading time\n    losses = AverageMeter()\n    scores = AverageMeter()\n    \n    model.train()\n    start = end = time.time()\n    global_step = 0\n    \n    for step, (images, labels) in enumerate(train_loader):\n        \n        data_time.update(time.time() - end)\n        \n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        \n        with autocast():\n            y_preds = model(images)\n            loss = criterion(y_preds, labels)\n        \n        losses.update(loss.item(), batch_size)\n        \n        if CFG['gradient_accumulation_steps'] > 1:\n            loss = loss / CFG['gradient_accumulation_steps']\n            \n        scaler.scale(loss).backward()\n        # grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), CFG['max_grad_norm'])\n        grad_norm = 0\n        \n        if (step + 1) % CFG['gradient_accumulation_steps'] == 0:\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n            global_step += 1\n            \n        batch_time.update(time.time() - end)\n        end = time.time()\n        \n        if step % CFG['print_freq'] == 0 or step == (len(train_loader)-1):\n            print('Epoch: [{0}][{1}/{2}] '\n                  'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  'Grad: {grad_norm:.4f}  '\n                  #'LR: {lr:.6f}  '\n                  .format(\n                   epoch+1, step, len(train_loader), batch_time=batch_time,\n                   data_time=data_time, loss=losses,\n                   remain=timeSince(start, float(step+1)/len(train_loader)),\n                   grad_norm=grad_norm,\n                   #lr=scheduler.get_lr()[0],\n                   ))\n    return losses.avg\n\ndef valid_fn(valid_loader, model, criterion, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    \n    model.eval()\n    preds = []\n    start = end = time.time()\n    for step, (images, labels) in enumerate(valid_loader):\n        data_time.update(time.time() - end)\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        \n        with torch.no_grad():\n            y_preds = model(images)\n        loss = criterion(y_preds, labels)\n        losses.update(loss.item(), batch_size)\n        \n        preds.append(y_preds.sigmoid().to('cpu').numpy())\n        if CFG['gradient_accumulation_steps'] > 1:\n            loss = loss / CFG['gradient_accumulation_steps']\n\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG['print_freq'] == 0 or step == (len(valid_loader)-1):\n            print('EVAL: [{0}/{1}] '\n                  'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  .format(\n                   step, len(valid_loader), batch_time=batch_time,\n                   data_time=data_time, loss=losses,\n                   remain=timeSince(start, float(step+1)/len(valid_loader)),\n                   ))\n    predictions = np.concatenate(preds)\n    return losses.avg, predictions","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_loop(folds, fold):\n    \n    LOGGER.info(f\"========== fold: {fold} training ==========\")\n\n    # ====================================================\n    # loader\n    # ====================================================\n    train_idx = folds[folds['fold'] != fold].index\n    valid_idx = folds[folds['fold'] == fold].index\n    \n    train_folds = folds.loc[train_idx].reset_index(drop=True)\n    valid_folds = folds.loc[valid_idx].reset_index(drop=True)\n    valid_labels = valid_folds[CFG['target_cols']].values\n    \n    train_dataset = TrainDataset(train_folds, transform=get_transforms(mode='train'))\n    valid_dataset = TrainDataset(valid_folds, transform=get_transforms(mode='valid'))\n    \n    train_loader = DataLoader(train_dataset, \n                              batch_size=CFG['batch_size'], \n                              shuffle=True, \n                              num_workers=CFG['num_workers'], pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, \n                              batch_size=CFG['batch_size'], \n                              shuffle=False, \n                              num_workers=CFG['num_workers'], pin_memory=True, drop_last=False)\n    \n    # ====================================================\n    # scheduler\n    # ====================================================\n    def get_scheduler(optimizer):\n        if CFG['scheduler'] == 'ReduceLROnPlateau':\n            scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=CFG['factor'], patience=CFG['patience'], verbose=True, eps=CFG['eps'])\n        elif CFG['scheduler'] == 'CosineAnnealingLR':\n            scheduler = CosineAnnealingLR(optimizer, T_max=CFG['T_max'], eta_min=CFG['min_lr'], last_epoch=-1)\n        elif CFG['scheduler'] == 'CosineAnnealingWarmRestarts':\n            scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)\n        return scheduler\n    \n    # ====================================================\n    # model & optimizer\n    # ====================================================\n    model = RanzcrModel(model_name=CFG['model_arch'], pretrained=True)\n    model.to(device)\n    \n    optimizer = Adam(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'], amsgrad=False)\n    \n    scheduler = get_scheduler(optimizer)\n    \n    # ====================================================\n    # loop\n    # ====================================================\n    criterion = nn.BCEWithLogitsLoss()\n\n    best_score = 0.\n    best_loss = np.inf\n    \n    for epoch in range(CFG['epochs']):\n        \n        start_time = time.time()\n        \n        # train\n        # train\n        avg_loss = train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device)\n\n        # eval\n        avg_val_loss, preds = valid_fn(valid_loader, model, criterion, device)\n        \n        if isinstance(scheduler, ReduceLROnPlateau):\n            scheduler.step(avg_val_loss)\n        elif isinstance(scheduler, CosineAnnealingLR):\n            scheduler.step()\n        elif isinstance(scheduler, CosineAnnealingWarmRestarts):\n            scheduler.step()\n\n        # scoring\n        score, scores = get_score(valid_labels, preds)\n\n        elapsed = time.time() - start_time\n\n        LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n        LOGGER.info(f'Epoch {epoch+1} - Score: {score:.4f}  Scores: {np.round(scores, decimals=4)}')\n\n        \"\"\"\n        if score > best_score:\n            best_score = score\n            LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n            torch.save({'model': model.state_dict(), \n                        'preds': preds},\n                        OUTPUT_DIR+f'{CFG.model_arch}_fold{fold}_best.pth')\n        \"\"\"\n        \n        if avg_val_loss < best_loss:\n            best_loss = avg_val_loss\n            LOGGER.info(f'Epoch {epoch+1} - Save Best Loss: {best_loss:.4f} Model')\n            torch.save({'model': model.state_dict(), \n                        'preds': preds}, \n                       OUTPUT_DIR+CFG['model_arch']+f'_fold{fold}_best.pth')\n    \n    check_point = torch.load(OUTPUT_DIR+CFG['model_arch']+f'_fold{fold}_best.pth')\n    for c in [f'pred_{c}' for c in CFG['target_cols']]:\n        valid_folds[c] = np.nan\n    valid_folds[[f'pred_{c}' for c in CFG['target_cols']]] = check_point['preds']\n\n    return valid_folds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def main():\n\n    def get_result(result_df):\n        preds = result_df[[f'pred_{c}' for c in CFG['target_cols']]].values\n        labels = result_df[CFG['target_cols']].values\n        score, scores = get_score(labels, preds)\n        LOGGER.info(f'Score: {score:<.4f}  Scores: {np.round(scores, decimals=4)}')\n    \n    if CFG['train']:\n        # train \n        oof_df = pd.DataFrame()\n        for fold in range(CFG['fold_num']):\n            if fold in CFG['trn_fold']:\n                _oof_df = train_loop(folds, fold)\n                oof_df = pd.concat([oof_df, _oof_df])\n                LOGGER.info(f\"========== fold: {fold} result ==========\")\n                get_result(_oof_df)\n        # CV result\n        LOGGER.info(f\"========== CV ==========\")\n        get_result(oof_df)\n        # save result\n        oof_df.to_csv(OUTPUT_DIR+'oof_df.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n    main()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}