{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Melanoma classification with PyTorch 1.6 Native AMP\n\n### [STABLE] AUTOMATIC MIXED PRECISION (AMP) TRAINING\n\nAMP allows users to easily enable automatic mixed precision training enabling higher performance and memory savings of up to 50% on Tensor Core GPUs. Using the natively supported `torch.cuda.amp` API, AMP provides convenience methods for mixed precision, where some operations use the `torch.float32 (float)` datatype and other operations use `torch.float16 (half)`. Some ops, like linear layers and convolutions, are much faster in `float16`. Other ops, like reductions, often require the dynamic range of `float32`. Mixed precision tries to match each op to its appropriate datatype.\n\n- Design doc ([Link](https://github.com/pytorch/pytorch/issues/25081))\n- Documentation ([Link](https://pytorch.org/docs/stable/amp.html))\n- Usage examples ([Link](https://pytorch.org/docs/stable/notes/amp_examples.html))","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html > /dev/null\n!pip install -U git+https://github.com/albumentations-team/albumentations > /dev/null\n!pip install timm > /dev/null\n!pip install catalyst==20.4.2 > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nimport cv2\nimport sklearn\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport albumentations as A\n\n\nfrom timm.models.efficientnet import tf_efficientnet_b5_ns\nfrom albumentations.pytorch.functional import img_to_tensor\nfrom sklearn.model_selection import KFold\nfrom torch.utils.data import Dataset\nfrom torch.nn.modules.dropout import Dropout\nfrom torch.nn.modules.linear import Linear\nfrom torch.nn.modules.pooling import AdaptiveAvgPool2d\nfrom catalyst.data.sampler import DistributedSampler, BalanceClassSampler\nfrom functools import partial\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class args:\n    \n    exp_name = \"E5_512_300\"\n    output_dir = \"outputs\"\n    \n    folds = 2\n    \n    train_image_path = \"../input/jpeg-melanoma-512x512/train\"\n    test_image_path = \"../input/jpeg-melanoma-512x512/test\"\n    \n    network = \"MelanomaClassifier\"\n    encoder = \"tf_efficientnet_b5_ns\"\n    \n    train_csv = \"../input/jpeg-melanoma-512x512/train.csv\"\n    test_csv = \"../input/jpeg-melanoma-512x512/test.csv\"\n    \n    label_smoothing = 0.01\n    epochs = 50\n    size = 300\n    batch_size = 30\n    learning_rate = 0.00002\n    \n    \n    normalize = {\n        \"mean\": [0.485, 0.456, 0.406],\n        \"std\": [0.229, 0.224, 0.225]\n    }\n    \n    # CUSTOM LEARNING SCHEUDLE\n    LR_START = 0.00001\n    LR_MAX = 0.00005\n    LR_MIN = 0.00001\n    LR_RAMPUP_EPOCHS = 5\n    LR_SUSTAIN_EPOCHS = 0\n    LR_EXP_DECAY = .8\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder_params = {\n    \"tf_efficientnet_b5_ns\": {\n        \"features\": 2048,\n        \"init_op\": partial(tf_efficientnet_b5_ns, pretrained=True, drop_path_rate=0.2)\n    }\n}\n\n\nclass MelanomaClassifier(nn.Module):\n    def __init__(self, encoder, dropout_rate=0.0) -> None:\n        super().__init__()\n        self.encoder = encoder_params[encoder][\"init_op\"]()\n        self.avg_pool = AdaptiveAvgPool2d((1, 1))\n        self.dropout = Dropout(dropout_rate)\n        self.fc = Linear(encoder_params[encoder][\"features\"], 1)\n\n    def forward(self, x):\n        x = self.encoder.forward_features(x)\n        x = self.avg_pool(x).flatten(1)\n        x = self.dropout(x)\n        x = self.fc(x)\n        return x\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaClassifierDataset(Dataset):\n    def __init__(\n        self,\n        df,\n        label_smoothing,\n        normalize,\n        mode=\"train\",\n        transforms=None,\n        data_root=None\n    ):\n        \n        super().__init__()\n        self.df = df\n        self.mode = mode\n        self.label_smoothing = label_smoothing\n        self.normalize = normalize\n        self.transforms = transforms\n        self.data_root = data_root\n        \n        self.image_name = self.df[\"image_name\"].values\n        self.label = self.df[\"target\"].values\n        \n    def __getitem__(self, index: int):\n\n        image_name, label = self.image_name[index], self.label[index]\n        if self.mode == \"train\":\n            label = np.clip(label, self.label_smoothing, 1 - self.label_smoothing)\n        image = cv2.imread(f\"{self.data_root}/{image_name}.jpg\", cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        if self.transforms:\n            image = self.transforms(image=image)[\"image\"]\n\n        image = img_to_tensor(image, self.normalize)\n\n        return {\n            \"image_name\": image_name,\n            \"image\": image,\n            \"label\": label\n        }\n    \n    def __len__(self):\n        return len(self.image_name)\n    \n    def __get_labels__(self):\n        return self.label.tolist()\n\n\nclass MelanomaClassifierDatasetTest(Dataset):\n    def __init__(\n        self,\n        df,\n        normalize={\"mean\": [0.485, 0.456, 0.406],\n                    \"std\": [0.229, 0.224, 0.225]},\n        transforms=None,\n        data_root=None\n    ):\n        super().__init__()\n        self.df = df\n        self.normalize = normalize\n        self.transforms = transforms\n        self.data_root = data_root\n\n        self.image_name = self.df[\"image_name\"]\n    \n    def __getitem__(self, index: int):\n        \n        image = cv2.imread(f\"{self.data_root}/{self.image_name[index]}.jpg\", cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        if self.transforms:\n            image = self.transforms(image=image)[\"image\"]\n\n        image = img_to_tensor(image, self.normalize)\n\n        return {\n            \"image_name\": self.image_name[index],\n            \"image\": image\n        }\n    \n    def __len__(self):\n        return len(self.image_name)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Utils","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\nclass RocAucMeter(object):\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.y_true = [0,1]\n        self.y_pred = [0.5, 0.5]\n        self.score = 0\n    \n    def update(self, y_true, y_pred):\n        self.y_true.extend(y_true.cpu().detach().numpy().round().tolist())\n        self.y_pred.extend(y_pred.cpu().detach().numpy().reshape(-1).tolist())\n        y_pred1 = np.array(self.y_pred)\n        y_true1 = np.array(self.y_true)\n        y_pred1[np.isnan(y_pred1)]=0.5\n        self.score = sklearn.metrics.roc_auc_score(y_true1, y_pred1)\n    @property\n    def avg(self):\n        return self.score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Augmentations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_transforms(size=300):\n    return A.Compose([\n        A.ImageCompression(quality_lower=60, quality_upper=100, p=0.5),\n        A.GaussianBlur(blur_limit=3 , p=0.05),\n        A.GaussNoise(p=0.1),\n        A.Resize(size, size),\n        A.PadIfNeeded(min_height=size, min_width=size, border_mode=cv2.BORDER_CONSTANT),\n        A.OneOf([\n            A.VerticalFlip(),\n            A.HorizontalFlip(),\n            A.Flip()\n        ], p=0.5),\n        A.Transpose(p=0.33),\n        A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),\n        A.RandomRotate90(p=0.11),\n        A.ElasticTransform(alpha_affine=60, p=0.33),\n        A.Cutout(num_holes=8, max_h_size=size//8, max_w_size=size//8, fill_value=0, p=0.3),\n        A.Rotate(limit=80)\n    ])\n\ndef valid_transforms(size=300):\n    return A.Compose([\n        A.Resize(size, size),\n        A.PadIfNeeded(min_height=size, min_width=size, border_mode=cv2.BORDER_CONSTANT),\n    ])\n\ndef test_transforms(size=300):\n    return A.Compose([\n        A.ImageCompression(quality_lower=60, quality_upper=100, p=0.5),\n        A.GaussianBlur(blur_limit=3 , p=0.05),\n        A.GaussNoise(p=0.1),\n        A.Resize(size, size),\n        A.PadIfNeeded(min_height=size, min_width=size, border_mode=cv2.BORDER_CONSTANT),\n        A.OneOf([\n            A.VerticalFlip(),\n            A.HorizontalFlip(),\n            A.Flip()\n        ], p=0.5),\n        A.Transpose(p=0.33),\n        A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),\n        A.RandomRotate90(p=0.11),\n        A.ElasticTransform(alpha_affine=60, p=0.33),\n        A.Rotate(limit=80)\n    ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def loss_fn(output, target):\n    return nn.BCEWithLogitsLoss()(output, target.view(-1, 1))\n\ndef lrfn(epoch):\n    if epoch < args.LR_RAMPUP_EPOCHS:\n        lr = (args.LR_MAX - args.LR_START) / args.LR_RAMPUP_EPOCHS * epoch + args.LR_START\n    elif epoch < args.LR_RAMPUP_EPOCHS + args.LR_SUSTAIN_EPOCHS:\n        lr = args.LR_MAX\n    else:\n        lr = (args.LR_MAX - args.LR_MIN) * args.LR_EXP_DECAY**(epoch - args.LR_RAMPUP_EPOCHS - args.LR_SUSTAIN_EPOCHS) + args.LR_MIN\n    return lr\n\ndef adjust_learning_rate(optimizer, epoch):\n    \"\"\"Sets the learning rate to the initial LR decayed by 10 every 30 epochs\"\"\"\n    lr = lrfn(epoch)\n    for param_group in optimizer.param_groups:\n        param_group['lr'] = lr ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_epoch(args, model, train_loader, optimizer, scheduler, device, epoch):\n    losses = AverageMeter()\n    scores = RocAucMeter()\n\n    model.train()\n    scaler = torch.cuda.amp.GradScaler()\n\n    t = tqdm(train_loader)\n    for i, sample in enumerate(t):\n        imgs = sample[\"image\"].to(device)\n        labels = sample[\"label\"].to(device)\n\n        optimizer.zero_grad()\n\n        # Casts operations to mixed precision\n        with torch.cuda.amp.autocast():\n            outputs = model(imgs)\n            loss = loss_fn(outputs, labels)\n\n        bs = imgs.size(0)\n        scores.update(labels, torch.sigmoid(outputs))\n        losses.update(loss.item(), bs)\n\n        # Scales the loss, and calls backward()\n        # to create scaled gradients\n        scaler.scale(loss).backward()\n\n        # Uncales gradients and calls\n        # or skips optimizer.step()\n        scaler.step(optimizer)\n\n        # Updates the scale for next iteration\n        scaler.update()\n\n        t.set_description(f\"Train E:{epoch} - Loss:{losses.avg:0.4f} - AUC:{scores.avg:0.4f} \")\n\n    return scores.avg, losses.avg\n\ndef valid_epoch(args, model, valid_loader, device, epoch):\n    losses = AverageMeter()\n    scores = RocAucMeter()\n\n    model.eval()\n    with torch.no_grad():\n        t = tqdm(valid_loader)\n        for i, sample in enumerate(t):\n            imgs = sample[\"image\"].to(device)\n            labels = sample[\"label\"].to(device)\n\n            outputs = model(imgs)\n\n            bs = imgs.size(0)\n            scores.update(labels, torch.sigmoid(outputs))\n\n            t.set_description(f\"Valid E:{epoch} - AUC:{scores.avg:0.4f} \")\n\n    return scores.avg\n\ndef test_epoch(args, model, test_loader, device):\n\n    probs = []\n\n    model.eval()\n\n    with torch.no_grad():\n        t = tqdm(test_loader)\n        for i, sample in enumerate(t):\n            imgs = sample[\"image\"].to(device)\n            img_names = sample[\"image_name\"]\n\n            out = model(imgs)\n            preds = torch.sigmoid(out).cpu().numpy().tolist()\n\n            probs.extend(preds)\n    \n    return probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def main(fold, idxT, idxV):\n    \n    device = \"cuda\"\n    model = MelanomaClassifier(args.encoder)\n    model = model.cuda()\n    \n    args.save_path = os.path.join(args.output_dir, args.exp_name)\n    os.makedirs(args.save_path, exist_ok=True)\n    \n    train_df = pd.read_csv(args.train_csv)\n    test_df = pd.read_csv(args.test_csv)\n    \n    train_folds = train_df[train_df.tfrecord.isin(idxT)]\n    valid_folds = train_df[train_df.tfrecord.isin(idxV)]\n    \n    train_dataset = MelanomaClassifierDataset(\n        df=train_folds,\n        mode=\"train\",\n        label_smoothing=args.label_smoothing,\n        normalize=args.normalize,\n        transforms=train_transforms(size=args.size),\n        data_root=args.train_image_path\n    )\n\n    valid_dataset = MelanomaClassifierDataset(\n        df=valid_folds,\n        mode=\"valid\",\n        label_smoothing=args.label_smoothing,\n        normalize=args.normalize,\n        transforms=valid_transforms(size=args.size),\n        data_root=args.train_image_path\n    )\n\n    test_dataset = MelanomaClassifierDatasetTest(\n        df=test_df,\n        normalize=args.normalize,\n        transforms=test_transforms(size=args.size),\n        data_root=args.test_image_path   \n    )\n    \n    optimizer = torch.optim.AdamW(\n        model.parameters(),\n        lr=args.learning_rate\n    )\n    scheduler = None\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=args.batch_size,\n        sampler=BalanceClassSampler(labels=train_dataset.__get_labels__(), mode=\"downsampling\"),\n        drop_last=True,\n        num_workers=4\n    )\n    \n    valid_loader = torch.utils.data.DataLoader(\n        valid_dataset,\n        batch_size=args.batch_size,\n        drop_last=False,\n        num_workers=4\n    )\n    \n    test_loader = torch.utils.data.DataLoader(\n        test_dataset,\n        batch_size=args.batch_size,\n        drop_last=False,\n        num_workers=4\n    )\n    \n    best_auc = 0\n    preds_list = []\n    \n    print(\"Training started ..... \")\n    \n    for epoch in range(args.epochs):\n        \n        adjust_learning_rate(optimizer, epoch)\n        \n        train_auc, train_loss = train_epoch(\n            args,\n            model,\n            train_loader,\n            optimizer,\n            scheduler,\n            device,\n            epoch\n        )\n        \n        if epoch >= 40:\n            \n            valid_auc = valid_epoch(\n                args,\n                model,\n                valid_loader,\n                device,\n                epoch\n            )\n            \n            print(f\"Epoch : {epoch} - AUC : {valid_auc}\")\n            \n            if valid_auc > best_auc:\n                print(f\"###***### Model Improved from {best_auc} to {valid_auc}\")\n                torch.save(model.state_dict(), os.path.join(args.save_path, f\"fold-{fold}.bin\"))\n                best_auc = valid_auc\n            \n            preds = test_epoch(\n                args,\n                model,\n                test_loader,\n                device\n            )\n            \n            preds_list.append(preds)\n    \n    final_preds = np.mean(preds_list, axis=0)       \n    np.save(os.path.join(args.save_path, f\"test-pred-fold-{fold}.npy\"), final_preds)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# clean up gpu in case you are debugging \nimport gc\ntorch.cuda.empty_cache(); gc.collect()\ntorch.cuda.empty_cache(); gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = KFold(n_splits=args.folds, shuffle=True, random_state=42)\nfor fold, (idxT, idxV) in enumerate(skf.split(np.arange(15))):\n    main(fold, idxT, idxV)\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}