{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Summary\nHi, Kagglers. This is a public benchmark with resnet200d using `image_size = 512`. Since it's only the beginning of the competition, I feel like giving an idea of what can be achieved with solely images. Weights are available [here](https://www.kaggle.com/underwearfitting/resnet200d-baseline-benchmark-public). Inference is available [here](https://www.kaggle.com/underwearfitting/resnet200d-public-benchmark-2xtta).\n\nHere is the single fold training notebook. However, It cannot be run on kaggle because of the limiting computing resources. It can reach CV around 0.955 every fold and LB 0.965 in 5 folds. Notebook modified from https://www.kaggle.com/haqishen/1st-place-soluiton-code-small-ver. It's slightly different from my local version but I believe you are smart enough to figure it out."},{"metadata":{},"cell_type":"markdown","source":"## Configuration"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"fold_id = 0\n\nimage_size = 512\nseed = 42\nwarmup_epo = 1\ninit_lr = 1e-4/3\nbatch_size = 8 # 64\nvalid_batch_size = 32\nn_epochs = 30\nwarmup_factor = 10\nnum_workers = 4\n\nuse_amp = True\ndebug = True # change this to run on full data\nearly_stop = 5\n\nkernel_type = 'resnet200d'\ndata_dir = '../input/ranzcr-clip-catheter-line-classification/train'\nmodel_dir = f'weights/'\n! mkdir $model_dir","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install -q pip install git+https://github.com/ildoonet/pytorch-gradual-warmup-lr.git","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport os\nimport time\nimport cv2\nimport PIL.Image\nimport random\nfrom sklearn.metrics import accuracy_score\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import CosineAnnealingLR \nfrom warmup_scheduler import GradualWarmupScheduler\nimport albumentations\nfrom albumentations import *\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport gc\nfrom sklearn.metrics import roc_auc_score\nimport seaborn as sns\nfrom pylab import rcParams\nimport timm\nfrom warnings import filterwarnings\nfilterwarnings(\"ignore\")\n\ndevice = torch.device('cuda')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZCRResNet200D(nn.Module):\n    def __init__(self, model_name='resnet200d', out_dim=11, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        if pretrained:\n            pretrained_path = '../input/resnet200d-pretrained-weight/resnet200d_ra2-bdba9bf9.pth'\n            self.model.load_state_dict(torch.load(pretrained_path))\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, out_dim)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"class GradualWarmupSchedulerV2(GradualWarmupScheduler):\n    def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None):\n        super(GradualWarmupSchedulerV2, self).__init__(optimizer, multiplier, total_epoch, after_scheduler)\n    def get_lr(self):\n        if self.last_epoch > self.total_epoch:\n            if self.after_scheduler:\n                if not self.finished:\n                    self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs]\n                    self.finished = True\n                return self.after_scheduler.get_lr()\n            return [base_lr * self.multiplier for base_lr in self.base_lrs]\n        if self.multiplier == 1.0:\n            return [base_lr * (float(self.last_epoch) / self.total_epoch) for base_lr in self.base_lrs]\n        else:\n            return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in self.base_lrs]\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True # for faster training, but not deterministic\n    \nseed_everything(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_train = albumentations.Compose([\n   albumentations.RandomResizedCrop(image_size, image_size, scale=(0.9, 1), p=1), \n   albumentations.HorizontalFlip(p=0.5),\n   albumentations.ShiftScaleRotate(p=0.5),\n   albumentations.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=10, val_shift_limit=10, p=0.7),\n   albumentations.RandomBrightnessContrast(brightness_limit=(-0.2,0.2), contrast_limit=(-0.2, 0.2), p=0.7),\n   albumentations.CLAHE(clip_limit=(1,4), p=0.5),\n   albumentations.OneOf([\n       albumentations.OpticalDistortion(distort_limit=1.0),\n       albumentations.GridDistortion(num_steps=5, distort_limit=1.),\n       albumentations.ElasticTransform(alpha=3),\n   ], p=0.2),\n   albumentations.OneOf([\n       albumentations.GaussNoise(var_limit=[10, 50]),\n       albumentations.GaussianBlur(),\n       albumentations.MotionBlur(),\n       albumentations.MedianBlur(),\n   ], p=0.2),\n  albumentations.Resize(image_size, image_size),\n  albumentations.OneOf([\n      JpegCompression(),\n      Downscale(scale_min=0.1, scale_max=0.15),\n  ], p=0.2),\n  IAAPiecewiseAffine(p=0.2),\n  IAASharpen(p=0.2),\n  albumentations.Cutout(max_h_size=int(image_size * 0.1), max_w_size=int(image_size * 0.1), num_holes=5, p=0.5),\n  albumentations.Normalize(),\n])\n\ntransforms_valid = albumentations.Compose([\n    albumentations.Resize(image_size, image_size),\n    albumentations.Normalize()\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZERDataset(Dataset):\n    def __init__(self, df, mode, transform=None):\n        \n        self.df = df.reset_index(drop=True)\n        self.mode = mode\n        self.transform = transform\n        self.labels = df[target_cols].values\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        row = self.df.loc[index]\n        img = cv2.imread(row.file_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        if self.transform is not None:\n            res = self.transform(image=img)\n            img = res['image']\n                \n        img = img.astype(np.float32)\n        img = img.transpose(2,0,1)\n        label = torch.tensor(self.labels[index]).float()\n        if self.mode == 'test':\n            return torch.tensor(img).float()\n        else:\n            return torch.tensor(img).float(), label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/how-to-properly-split-folds/train_folds.csv')\ndf_train['file_path'] = df_train.StudyInstanceUID.apply(lambda x: os.path.join(data_dir, f'{x}.jpg'))\nif debug:\n    df_train = df_train.sample(frac=0.1)\ntarget_cols = df_train.iloc[:, 1:12].columns.tolist()\ndataset = RANZERDataset(df_train, 'train', transform=transforms_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def macro_multilabel_auc(label, pred):\n    aucs = []\n    for i in range(len(target_cols)):\n        aucs.append(roc_auc_score(label[:, i], pred[:, i]))\n    print(np.round(aucs, 4))\n    return np.mean(aucs)\n\n\ndef train_func(train_loader):\n    model.train()\n    bar = tqdm(train_loader)\n    if use_amp:\n        scaler = torch.cuda.amp.GradScaler()\n    losses = []\n    for batch_idx, (images, targets) in enumerate(bar):\n\n        images, targets = images.to(device), targets.to(device)\n        \n        if use_amp:\n            with torch.cuda.amp.autocast():\n                logits = model(images)\n                loss = criterion(logits, targets)\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n        else:\n            logits = model(images)\n            loss = criterion(logits, targets)\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n\n        losses.append(loss.item())\n        smooth_loss = np.mean(losses[-30:])\n\n        bar.set_description(f'loss: {loss.item():.5f}, smth: {smooth_loss:.5f}')\n\n    loss_train = np.mean(losses)\n    return loss_train\n\n\ndef valid_func(valid_loader):\n    model.eval()\n    bar = tqdm(valid_loader)\n\n    PROB = []\n    TARGETS = []\n    losses = []\n    PREDS = []\n    \n    with torch.no_grad():\n        for batch_idx, (images, targets) in enumerate(bar):\n\n            images, targets = images.to(device), targets.to(device)\n            logits = model(images)\n            PREDS += [logits.sigmoid()]\n            TARGETS += [targets.detach().cpu()]\n            loss = criterion(logits, targets)\n            losses.append(loss.item())\n            smooth_loss = np.mean(losses[-30:])\n            bar.set_description(f'loss: {loss.item():.5f}, smth: {smooth_loss:.5f}')\n            \n    PREDS = torch.cat(PREDS).cpu().numpy()\n    TARGETS = torch.cat(TARGETS).cpu().numpy()\n    #roc_auc = roc_auc_score(TARGETS.reshape(-1), PREDS.reshape(-1))\n    roc_auc = macro_multilabel_auc(TARGETS, PREDS)\n    loss_valid = np.mean(losses)\n    return loss_valid, roc_auc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = RANZCRResNet200D(out_dim=len(target_cols), pretrained=True)\nmodel = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(model.parameters(), lr=init_lr/warmup_factor)\nscheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, n_epochs, eta_min=1e-7)\nscheduler_warmup = GradualWarmupSchedulerV2(optimizer, multiplier=10, total_epoch=warmup_epo, after_scheduler=scheduler_cosine)\n\ndf_train_this = df_train[df_train['fold'] != fold_id]\ndf_valid_this = df_train[df_train['fold'] == fold_id]\n\ndataset_train = RANZERDataset(df_train_this, 'train', transform=transforms_train)\ndataset_valid = RANZERDataset(df_valid_this, 'valid', transform=transforms_valid)\n\ntrain_loader = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size, shuffle=True,  num_workers=num_workers, pin_memory=True)\nvalid_loader = torch.utils.data.DataLoader(dataset_valid, batch_size=valid_batch_size, shuffle=False, num_workers=num_workers, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"log = {}\nroc_auc_max = 0.\nloss_min = 99999\nnot_improving = 0\n\nfor epoch in range(1, n_epochs+1):\n    scheduler_warmup.step(epoch-1)\n    loss_train = train_func(train_loader)\n    loss_valid, roc_auc = valid_func(valid_loader)\n\n    log['loss_train'] = log.get('loss_train', []) + [loss_train]\n    log['loss_valid'] = log.get('loss_valid', []) + [loss_valid]\n    log['lr'] = log.get('lr', []) + [optimizer.param_groups[0][\"lr\"]]\n    log['roc_auc'] = log.get('roc_auc', []) + [roc_auc]\n\n    content = time.ctime() + ' ' + f'Fold {fold_id}, Epoch {epoch}, lr: {optimizer.param_groups[0][\"lr\"]:.7f}, loss_train: {loss_train:.5f}, loss_valid: {loss_valid:.5f}, roc_auc: {roc_auc:.6f}.'\n    print(content)\n    not_improving += 1\n    \n    if roc_auc > roc_auc_max:\n        print(f'roc_auc_max ({roc_auc_max:.6f} --> {roc_auc:.6f}). Saving model ...')\n        torch.save(model.state_dict(), f'{model_dir}{kernel_type}_fold{fold_id}_best_AUC.pth')\n        roc_auc_max = roc_auc\n        not_improving = 0\n\n    if loss_valid < loss_min:\n        loss_min = loss_valid\n        torch.save(model.state_dict(), f'{model_dir}{kernel_type}_fold{fold_id}_best_loss.pth')\n        \n    if not_improving == early_stop:\n        print('Early Stopping...')\n        break\n        \n    ## only run 1 epoch here\n    break\n\ntorch.save(model.state_dict(), f'{model_dir}{kernel_type}_fold{fold_id}_final.pth')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}