{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Configuration"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"fold_id = 2\n\nimage_size = 720\nseed = 42\nwarmup_epo = 1\ninit_lr = 0.00001\nbatch_size = 8 # 64\nvalid_batch_size = 32\nn_epochs = 5\nwarmup_factor = 10\nnum_workers = 4\n\nuse_amp = True\ndebug = False # change this to run on full data\nearly_stop = 1\n\nkernel_type = 'resnet200d'\ndata_dir = '../input/ranzcr-clip-catheter-line-classification/train'\nmodel_dir = f'weights/'\n! mkdir $model_dir","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport os\nimport time\nimport cv2\nimport PIL.Image\nimport random\nfrom sklearn.metrics import accuracy_score\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n# from warmup_scheduler import GradualWarmupScheduler\nimport albumentations\nfrom albumentations import *\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport gc\nfrom sklearn.metrics import roc_auc_score\nimport seaborn as sns\nfrom pylab import rcParams\nimport timm\nfrom warnings import filterwarnings\nfilterwarnings(\"ignore\")\n\ndevice = torch.device('cuda')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MODEL"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZCRResNet200D(nn.Module):\n    def __init__(self, model_name='resnet200d', out_dim=11, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        if pretrained:\n            pretrained_path = '../input/resnet200d-pretrained-weight/resnet200d_ra2-bdba9bf9.pth'\n            self.model.load_state_dict(torch.load(pretrained_path))\n        n_features = self.model.fc.in_features\n        self.model.global_pool = nn.Identity()\n        self.model.fc = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(n_features, out_dim)\n\n    def forward(self, x):\n        bs = x.size(0)\n        features = self.model(x)\n        pooled_features = self.pooling(features).view(bs, -1)\n        output = self.fc(pooled_features)\n        return output\n    \n    \n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True # for faster training, but not deterministic\n    \nseed_everything(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_train = albumentations.Compose([\n   albumentations.Resize(image_size, image_size,p=1), \n#    albumentations.HorizontalFlip(p=0.5),\n#    albumentations.ShiftScaleRotate(p=0.5),\n#    albumentations.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=10, val_shift_limit=10, p=0.7),\n#    albumentations.RandomBrightnessContrast(brightness_limit=(-0.2,0.2), contrast_limit=(-0.2, 0.2), p=0.7),\n#    albumentations.CLAHE(clip_limit=(1,4), p=0.5),\n#    albumentations.OneOf([\n#        albumentations.OpticalDistortion(distort_limit=1.0),\n#        albumentations.GridDistortion(num_steps=5, distort_limit=1.),\n#        albumentations.ElasticTransform(alpha=3),\n#    ], p=0.2),\n#    albumentations.OneOf([\n#        albumentations.GaussNoise(var_limit=[10, 50]),\n#        albumentations.GaussianBlur(),\n#        albumentations.MotionBlur(),\n#        albumentations.MedianBlur(),\n#    ], p=0.2),\n#   albumentations.Resize(image_size, image_size),\n#   albumentations.OneOf([\n#       JpegCompression(),\n#       Downscale(scale_min=0.1, scale_max=0.15),\n#   ], p=0.2),\n#   IAAPiecewiseAffine(p=0.2),\n#   IAASharpen(p=0.2),\n#   albumentations.Cutout(max_h_size=int(image_size * 0.1), max_w_size=int(image_size * 0.1), num_holes=5, p=0.5),\n  albumentations.Normalize(),\n])\n\ntransforms_valid = albumentations.Compose([\n    albumentations.Resize(image_size, image_size),\n    albumentations.Normalize()\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZERDataset(Dataset):\n    def __init__(self, df, mode, transform=None):\n        \n        self.df = df.reset_index(drop=True)\n        self.mode = mode\n        self.transform = transform\n        self.labels = df[target_cols].values\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        row = self.df.loc[index]\n        img = cv2.imread(row.file_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        if self.transform is not None:\n            res = self.transform(image=img)\n            img = res['image']\n                \n        img = img.astype(np.float32)\n        img = img.transpose(2,0,1)\n        label = torch.tensor(self.labels[index]).float()\n        if self.mode == 'test':\n            return torch.tensor(img).float()\n        else:\n            return torch.tensor(img).float(), label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/how-to-properly-split-folds/train_folds.csv')\ndf_train['file_path'] = df_train.StudyInstanceUID.apply(lambda x: os.path.join(data_dir, f'{x}.jpg'))\nif debug:\n    df_train = df_train.sample(frac=0.25)\ntarget_cols = df_train.iloc[:, 1:12].columns.tolist()\ndataset = RANZERDataset(df_train, 'train', transform=transforms_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def macro_multilabel_auc(label, pred):\n    aucs = []\n    for i in range(len(target_cols)):\n        aucs.append(roc_auc_score(label[:, i], pred[:, i]))\n    print(np.round(aucs, 4))\n    return np.mean(aucs)\n\n\ndef train_func(train_loader):\n    model.train()\n    bar = tqdm(train_loader)\n    if use_amp:\n        scaler = torch.cuda.amp.GradScaler()\n    losses = []\n    for batch_idx, (images, targets) in enumerate(bar):\n\n        images, targets = images.to(device), targets.to(device)\n        \n        if use_amp:\n            with torch.cuda.amp.autocast():\n                logits = model(images)\n                loss = criterion(logits, targets)\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n        else:\n            logits = model(images)\n            loss = criterion(logits, targets)\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n\n        losses.append(loss.item())\n        smooth_loss = np.mean(losses[-30:])\n\n        bar.set_description(f'loss: {loss.item():.5f}, smth: {smooth_loss:.5f}')\n\n    loss_train = np.mean(losses)\n    return loss_train\n\n\ndef valid_func(valid_loader):\n    model.eval()\n    bar = tqdm(valid_loader)\n\n    PROB = []\n    TARGETS = []\n    losses = []\n    PREDS = []\n    \n    with torch.no_grad():\n        for batch_idx, (images, targets) in enumerate(bar):\n\n            images, targets = images.to(device), targets.to(device)\n            logits = model(images)\n            PREDS += [logits.sigmoid()]\n            TARGETS += [targets.detach().cpu()]\n            loss = criterion(logits, targets)\n            losses.append(loss.item())\n            smooth_loss = np.mean(losses[-30:])\n            bar.set_description(f'loss: {loss.item():.5f}, smth: {smooth_loss:.5f}')\n            \n    PREDS = torch.cat(PREDS).cpu().numpy()\n    TARGETS = torch.cat(TARGETS).cpu().numpy()\n    #roc_auc = roc_auc_score(TARGETS.reshape(-1), PREDS.reshape(-1))\n    roc_auc = macro_multilabel_auc(TARGETS, PREDS)\n    loss_valid = np.mean(losses)\n    return loss_valid, roc_auc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '../input/resnet200d-baseline-benchmark-public/resnet200d_fold2_cv955.pth'\nmodel = RANZCRResNet200D(out_dim=len(target_cols), pretrained=True)\nmodel.load_state_dict(torch.load(path,map_location = 'cuda:0'))\nmodel = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(model.parameters(), lr=init_lr,amsgrad=True)\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.5, last_epoch=-1, verbose=True)\n\ndf_train_this = df_train[df_train['fold'] != fold_id]\ndf_valid_this = df_train[df_train['fold'] == fold_id]\n\ndataset_train = RANZERDataset(df_train_this, 'train', transform=transforms_train)\ndataset_valid = RANZERDataset(df_valid_this, 'valid', transform=transforms_valid)\n\ntrain_loader = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size, shuffle=True,  num_workers=num_workers, pin_memory=True)\nvalid_loader = torch.utils.data.DataLoader(dataset_valid, batch_size=valid_batch_size, shuffle=False, num_workers=num_workers, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"log = {}\nroc_auc_max = 0.955\nloss_min = 99999\nnot_improving = 0\n\nfor epoch in range(1, n_epochs+1):\n#     scheduler_warmup.step(epoch-1)\n    loss_train = train_func(train_loader)\n    loss_valid, roc_auc = valid_func(valid_loader)\n\n    log['loss_train'] = log.get('loss_train', []) + [loss_train]\n    log['loss_valid'] = log.get('loss_valid', []) + [loss_valid]\n    log['lr'] = log.get('lr', []) + [optimizer.param_groups[0][\"lr\"]]\n    log['roc_auc'] = log.get('roc_auc', []) + [roc_auc]\n\n    content = time.ctime() + ' ' + f'Fold {fold_id}, Epoch {epoch}, lr: {optimizer.param_groups[0][\"lr\"]:.7f}, loss_train: {loss_train:.5f}, loss_valid: {loss_valid:.5f}, roc_auc: {roc_auc:.6f}.'\n    print(content)\n    not_improving += 1\n    \n    if roc_auc > roc_auc_max:\n        print(f'roc_auc_max ({roc_auc_max:.6f} --> {roc_auc:.6f}). Saving model ...')\n        torch.save(model.state_dict(), f'{model_dir}{kernel_type}_fold{fold_id}_best_AUC.pth')\n        roc_auc_max = roc_auc\n        not_improving = 0\n\n    if loss_valid < loss_min:\n        loss_min = loss_valid\n        torch.save(model.state_dict(), f'{model_dir}{kernel_type}_fold{fold_id}_best_loss.pth')\n        \n    if not_improving == early_stop:\n        print('Early Stopping...')\n        break\n    scheduler.step()\n    ## only run 1 epoch here\n#     break\n\ntorch.save(model.state_dict(), f'{model_dir}{kernel_type}_fold{fold_id}_final.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}