{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Configuration"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"fold_id = 2\nimage_size = 512\nseed = 42\nwarmup_epo = 1\ninit_lr = 5e-4\nbatch_size = 20 # 64\nvalid_batch_size = 32\nn_epochs = 7\nwarmup_factor = 10\nnum_workers = 4\n\nuse_amp = True\ndebug = False # change this to run on full data\nearly_stop = 5\n\nkernel_type = 'resnet200d'\ndata_dir = '../input/ranzcr-clip-catheter-line-classification/train'\nmodel_dir = f'weights/'\n! mkdir $model_dir\n\neach_epoch = f'epoch/'\n! mkdir $each_epoch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Imports"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport os\nimport time\nimport cv2\nimport PIL.Image\nimport random\nfrom sklearn.metrics import accuracy_score\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import CosineAnnealingLR \n# from warmup_scheduler import GradualWarmupScheduler\nimport albumentations\nfrom albumentations import *\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport gc\nfrom sklearn.metrics import roc_auc_score\nimport seaborn as sns\nfrom pylab import rcParams\nimport timm\nfrom warnings import filterwarnings\nfilterwarnings(\"ignore\")\n\ndevice = torch.device('cuda')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZCRResNet200D(nn.Module):\n    def __init__(self, model_name='resnet200d', out_dim=11, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)         \n        if pretrained:\n            pretrained_path = '../input/resnet200d-pretrained-weight/resnet200d_ra2-bdba9bf9.pth'\n            checkpoint = torch.load(pretrained_path, map_location='cuda:0')\n            for key in list(checkpoint.keys()):\n                 if 'model.' in key:\n                    checkpoint[key.replace('model.', '')] = checkpoint[key]\n                    del checkpoint[key]\n            self.model.load_state_dict(checkpoint)\n          #  self.model.load_state_dict(torch.load(pretrained_path))\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, out_dim)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True # for faster training, but not deterministic\n    \nseed_everything(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_train = albumentations.Compose([\n   albumentations.RandomResizedCrop(image_size, image_size, scale=(0.9, 1), p=1), \n   albumentations.HorizontalFlip(p=0.5),\n   albumentations.ShiftScaleRotate(p=0.5),\n   albumentations.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=10, val_shift_limit=10, p=0.7),\n   albumentations.RandomBrightnessContrast(brightness_limit=(-0.2,0.2), contrast_limit=(-0.2, 0.2), p=0.7),\n   albumentations.CLAHE(clip_limit=(1,4), p=0.5),\n  albumentations.Resize(image_size, image_size),\n  IAAPiecewiseAffine(p=0.2),\n  IAASharpen(p=0.2),\n  albumentations.Cutout(max_h_size=int(image_size * 0.1), max_w_size=int(image_size * 0.1), num_holes=5, p=0.5),\n  albumentations.Normalize(),\n])\n\ntransforms_valid = albumentations.Compose([\n    albumentations.Resize(image_size, image_size),\n    albumentations.Normalize()\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZERDataset(Dataset):\n    def __init__(self, df, mode, transform=None):\n        \n        self.df = df.reset_index(drop=True)\n        self.mode = mode\n        self.transform = transform\n        self.labels = df[target_cols].values\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        row = self.df.loc[index]\n        img = cv2.imread(row.file_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        if self.transform is not None:\n            res = self.transform(image=img)\n            img = res['image']\n                \n        img = img.astype(np.float32)\n        img = img.transpose(2,0,1)\n        label = torch.tensor(self.labels[index]).float()\n        if self.mode == 'test':\n            return torch.tensor(img).float()\n        else:\n            return torch.tensor(img).float(), label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/data-after-2-splits/full training set ranczr and nih positive 2 splits.csv')\n# df_train['file_path'] = df_train.StudyInstanceUID.apply(lambda x: os.path.join(data_dir, f'{x}.jpg'))\nif debug:\n    df_train = df_train.sample(frac=0.1)\ntarget_cols = df_train.iloc[:, 2:13].columns.tolist()\ndataset = RANZERDataset(df_train, 'train', transform=transforms_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def macro_multilabel_auc(label, pred):\n    aucs = []\n    for i in range(len(target_cols)):\n        aucs.append(roc_auc_score(label[:, i], pred[:, i]))\n    print(np.round(aucs, 4))\n    return np.mean(aucs)\n\n\ndef train_func(train_loader):\n    model.train()\n    bar = tqdm(train_loader)\n    if use_amp:\n        scaler = torch.cuda.amp.GradScaler()\n    losses = []\n    for batch_idx, (images, targets) in enumerate(bar):\n\n        images, targets = images.to(device), targets.to(device)\n        \n        if use_amp:\n            with torch.cuda.amp.autocast():\n                logits = model(images)\n                loss = criterion(logits, targets)\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n        else:\n            logits = model(images)\n            loss = criterion(logits, targets)\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n\n        losses.append(loss.item())\n        smooth_loss = np.mean(losses[-30:])\n\n        bar.set_description(f'loss: {loss.item():.5f}, smth: {smooth_loss:.5f}')\n\n    loss_train = np.mean(losses)\n    return loss_train\n\n\ndef valid_func(valid_loader):\n    model.eval()\n    bar = tqdm(valid_loader)\n\n    PROB = []\n    TARGETS = []\n    losses = []\n    PREDS = []\n    \n    with torch.no_grad():\n        for batch_idx, (images, targets) in enumerate(bar):\n\n            images, targets = images.to(device), targets.to(device)\n            logits = model(images)\n            PREDS += [logits.sigmoid()]\n            TARGETS += [targets.detach().cpu()]\n            loss = criterion(logits, targets)\n            losses.append(loss.item())\n            smooth_loss = np.mean(losses[-30:])\n            bar.set_description(f'loss: {loss.item():.5f}, smth: {smooth_loss:.5f}')\n            \n    PREDS = torch.cat(PREDS).cpu().numpy()\n    TARGETS = torch.cat(TARGETS).cpu().numpy()\n    roc_auc = macro_multilabel_auc(TARGETS, PREDS)\n    loss_valid = np.mean(losses)\n    return loss_valid, roc_auc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = RANZCRResNet200D(out_dim=len(target_cols), pretrained=True)\ncriterion = nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(model.parameters(), lr=init_lr)\nscheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, 15, eta_min=1e-6)\ncheckpoint = torch.load('../input/all-data-fold-2/epoch/resnet200d_fold2parameters.pth')\nmodel.load_state_dict(checkpoint['model'])\noptimizer.load_state_dict(checkpoint['optimizer'])\nepoch = checkpoint['epoch']\nscheduler_cosine.load_state_dict(checkpoint['scheduler'])\n\n\nfor state in optimizer.state.values():\n    for k, v in state.items():\n        if isinstance(v, torch.Tensor):\n            state[k] = v.cuda()\nmodel = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_this = df_train[df_train['fold'] != fold_id]\ndf_valid_this = df_train[df_train['fold'] == fold_id]\n\ndataset_train = RANZERDataset(df_train_this, 'train', transform=transforms_train)\ndataset_valid = RANZERDataset(df_valid_this, 'valid', transform=transforms_valid)\n\ntrain_loader = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size, shuffle=True,  num_workers=num_workers, pin_memory=True)\nvalid_loader = torch.utils.data.DataLoader(dataset_valid, batch_size=valid_batch_size, shuffle=False, num_workers=num_workers, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"log = {}\nroc_auc_max = 0.95787\nloss_min = 0.11630\nnot_improving = 0\n\nfor epoch in range(1,n_epochs+1):\n    loss_train = train_func(train_loader)\n    loss_valid, roc_auc = valid_func(valid_loader)\n\n    log['loss_train'] = log.get('loss_train', []) + [loss_train]\n    log['loss_valid'] = log.get('loss_valid', []) + [loss_valid]\n    log['lr'] = log.get('lr', []) + [optimizer.param_groups[0][\"lr\"]]\n    log['roc_auc'] = log.get('roc_auc', []) + [roc_auc]\n\n\n    content = time.ctime() + ' ' + f'Fold {fold_id}, Epoch {epoch}, lr: {optimizer.param_groups[0][\"lr\"]:.7f}, loss_train: {loss_train:.5f}, loss_valid: {loss_valid:.5f}, roc_auc: {roc_auc:.6f}.'\n    print(content)\n    not_improving += 1\n    \n    if roc_auc > roc_auc_max:\n        print(f'roc_auc_max ({roc_auc_max:.6f} --> {roc_auc:.6f}). Saving model ...')\n        torch.save(model.state_dict(), f'{model_dir}{kernel_type}_fold{fold_id}_best_AUC CV{np.round(roc_auc,6)}.pth')\n\n        roc_auc_max = roc_auc\n        not_improving = 0\n\n    if loss_valid < loss_min:\n        loss_min = loss_valid\n        torch.save(model.state_dict(), f'{model_dir}{kernel_type}_fold{fold_id}_best_loss CV {np.round(roc_auc,6)}.pth')\n\n        \n    if not_improving == early_stop:\n        print('Early Stopping...')\n        break\n\n    \n    scheduler_cosine.step()\n    \n    torch.save({\n    'epoch': epoch,\n    'model': model.state_dict(),\n    'optimizer': optimizer.state_dict(),\n    'scheduler': scheduler_cosine.state_dict(),\n    'scheduler2':scheduler_cosine}, f'{each_epoch}{kernel_type}_fold{fold_id}parameters.pth')\n\ntorch.save(model.state_dict(), f'{model_dir}{kernel_type}_fold{fold_id}_final.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}