{"cells":[{"metadata":{"id":"tkQEDQA44qlC"},"cell_type":"markdown","source":"# Melanoma classification with PyTorch Lightning\n\nUsing EfficientNet on PyTorch Lightning, with its amazing hardware agnostic and mixed precision implementation.\n\nThis is still work in progress, so please bear with me","execution_count":null},{"metadata":{"trusted":true,"id":"ftZsAlKR4qlF"},"cell_type":"code","source":"fold_number = 1\nseed  = 66\ndebug = False\ntta   = 2 if debug else 20\n\nbatch_size = {\n    'tpu': 10, # x8\n    'gpu': 22, # 10 without AMP\n    'cpu': 4,\n}\n\narch = 'efficientnet-b5'\nresolution = 456  # orignal res for B5\ninput_res  = 512\n\nlr = 8e-6   # * batch_size\nweight_decay = 2e-5\npos_weight   = 3.2\nlabel_smoothing = 0.03\n\nmax_epochs = 7","execution_count":null,"outputs":[]},{"metadata":{"id":"UnEoyD1a4qlO"},"cell_type":"markdown","source":"# Why PyTorch Lightning?\nLightning is simply organized PyTorch code. There's NO new framework to learn.\nFor more details about Lightning visit the repo:\n\nhttps://github.com/PyTorchLightning/pytorch-lightning\n\n- Run on CPU, GPU clusters or TPU, without any code changes\n- Transparent use of AMP (automatic mixed precision)\n\n![lightning structure](https://raw.githubusercontent.com/PyTorchLightning/pytorch-lightning/master/docs/source/_images/lightning_module/pt_to_pl.png)","execution_count":null},{"metadata":{"id":"oa7QSur04qlR"},"cell_type":"markdown","source":"# Install modules\n\nUpdate PyTorch to enable its native support to Mixed Precision or XLA for TPU","execution_count":null},{"metadata":{"trusted":true,"id":"_lmyKFye4qlT","outputId":"a008eca4-4ec3-4ac7-a1a8-7f72cfa3c337","_kg_hide-input":false,"_kg_hide-output":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\nimport os\n\nif 'TPU_NAME' in os.environ.keys():\n    try:\n        import torch_xla\n    except:\n        # XLA powers the TPU support for PyTorch\n        !curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n        !python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev\nelse:\n    # Update PyTorch to enable its native support to Mixed Precision\n    !pip install --pre torch==1.7.0.dev20200701+cu101 torchvision==0.8.0.dev20200701+cu101 -f https://download.pytorch.org/whl/nightly/cu101/torch_nightly.html\n\n!pip install -U pip albumentations==0.4.5 PyYAML pytorch-lightning==0.8.5 efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"id":"NRdfuFiR4qlf"},"cell_type":"markdown","source":"# Hardware lookup","execution_count":null},{"metadata":{"trusted":true,"id":"t9FS-xGM4qlh","outputId":"5802bc07-8b6e-4254-ac49-2139d11911f8","_kg_hide-input":true},"cell_type":"code","source":"import os\nimport torch\n\nnum_workers = 2  # os.cpu_count()\ngpus = 1 if torch.cuda.is_available() else None\n\ntry:\n    import torch_xla.core.xla_model as xm\n    tpu_cores = 8 #xm.xrt_world_size()\nexcept:\n    tpu_cores = None\n\nif isinstance(batch_size, dict):\n    if tpu_cores:\n        batch_size = batch_size['tpu']\n        lr *= tpu_cores\n        num_workers = 1\n    elif gpus:\n        batch_size = batch_size['gpu']\n        # support for free Colab GPU's\n        if 'K80' in torch.cuda.get_device_name():\n            batch_size = batch_size//3\n        elif 'T4' in torch.cuda.get_device_name():\n            batch_size = int(batch_size * 0.66)\n    else:\n        batch_size = batch_size['cpu']\n\nlr *= batch_size\n\ndict(\n    num_workers=num_workers,\n    tpu_cores=tpu_cores,\n    gpus=gpus,\n    batch_size=batch_size,\n    lr=lr,\n)","execution_count":null,"outputs":[]},{"metadata":{"id":"fa--zHZ64qln"},"cell_type":"markdown","source":"# Automatic Mixed Precision\n\nNVIDIA Apex is required only prior to PyTorch 1.6","execution_count":null},{"metadata":{"_kg_hide-output":true,"trusted":true,"id":"2jNntNJb4qln","_kg_hide-input":true},"cell_type":"code","source":"# check for torch's native mixed precision support (pt1.6+)\nif gpus and not hasattr(torch.cuda, \"amp\"):\n    try:\n        from apex import amp\n    except:\n        !git clone https://github.com/NVIDIA/apex  nv_apex\n        !pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./nv_apex\n        from apex import amp\n    # with PyTorch Lightning all you need to do now is set precision=16","execution_count":null,"outputs":[]},{"metadata":{"id":"SaoinaAu4qlu"},"cell_type":"markdown","source":"# Imports","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true,"id":"UFXXIDJt4qlv","outputId":"b5e99e4c-016c-440c-e2f4-c3ed45ba50eb","_kg_hide-output":true},"cell_type":"code","source":"import os\nimport time\nimport random\nfrom datetime import datetime\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nimport cv2\nfrom skimage import io\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\n\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import DataLoader\n\nfrom glob import glob\nimport sklearn\n\nimport pytorch_lightning as pl\nimport warnings\n\nwarnings.filterwarnings(\"ignore\") \nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning) \n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(seed*6 + fold_number)\n\ntorch.__version__","execution_count":null,"outputs":[]},{"metadata":{"id":"oeM18dHg4qlz"},"cell_type":"markdown","source":"# Dataset\n\nWe will be using @shonenkov dataset with external data: https://www.kaggle.com/shonenkov/melanoma-merged-external-data-512x512-jpeg \n\nthank you @shonenkov","execution_count":null},{"metadata":{"trusted":true,"id":"MQ0g7uTd4ql0","_kg_hide-input":true},"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\n\n\nclass ImageDataset(Dataset):\n    def __init__(self, path, image_ids, labels=None, transforms=None):\n        super().__init__()\n        self.path = path\n        self.image_ids = image_ids\n        self.labels = labels\n        self.transforms = transforms\n\n    def __getitem__(self, idx: int):\n        image_id = self.image_ids[idx]\n        image = cv2.imread(f'{self.path}/{image_id}.jpg', cv2.IMREAD_COLOR)\n\n        if self.transforms:\n            sample = self.transforms(image=image)\n            image  = sample['image']\n\n        label = self.labels[idx] if self.labels is not None else 0.5\n        return image, label\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\n    def get_labels(self):\n        return list(self.labels)","execution_count":null,"outputs":[]},{"metadata":{"id":"vlBehFHZ4ql4"},"cell_type":"markdown","source":"# Augmentations","execution_count":null},{"metadata":{"trusted":true,"id":"9We_dtmL4ql6","_kg_hide-input":false},"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose([\n            A.JpegCompression(p=0.5),\n            A.Rotate(limit=80, p=1.0),\n            A.OneOf([\n                A.OpticalDistortion(),\n                A.GridDistortion(),\n                A.IAAPiecewiseAffine(),\n            ]),\n            A.RandomSizedCrop(min_max_height=(int(resolution*0.7), input_res),\n                              height=resolution, width=resolution, p=1.0),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.GaussianBlur(p=0.3),\n            A.OneOf([\n                A.RandomBrightnessContrast(),   \n                A.HueSaturationValue(),\n            ]),\n            A.Cutout(num_holes=8, max_h_size=resolution//8, max_w_size=resolution//8, fill_value=0, p=0.3),\n            A.Normalize(),\n            ToTensorV2(),\n        ], p=1.0)\n\ndef get_valid_transforms():\n    return A.Compose([\n            A.CenterCrop(height=resolution, width=resolution, p=1.0),\n            A.Normalize(),\n            ToTensorV2(),\n        ], p=1.0)\n\ndef get_tta_transforms():\n    return A.Compose([\n            A.JpegCompression(p=0.5),\n            A.RandomSizedCrop(min_max_height=(int(resolution*0.9), int(resolution*1.1)),\n                              height=resolution, width=resolution, p=1.0),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.Transpose(p=0.5),\n            A.Normalize(),\n            ToTensorV2(),\n        ], p=1.0)","execution_count":null,"outputs":[]},{"metadata":{"id":"6K-p9nPl4ql-"},"cell_type":"markdown","source":"# Setup dataset","execution_count":null},{"metadata":{"trusted":true,"id":"Dq3S8RAF4ql_","outputId":"00122522-4e2b-4e79-adcb-6501fea3ff78"},"cell_type":"code","source":"DATA_PATH = '../input/melanoma-merged-external-data-512x512-jpeg'\nTRAIN_ROOT_PATH = f'{DATA_PATH}/512x512-dataset-melanoma/512x512-dataset-melanoma'\nTEST_ROOT_PATH = f'{DATA_PATH}/512x512-test/512x512-test'\n\ndf_folds = pd.read_csv(f'{DATA_PATH}/folds.csv', index_col='image_id',\n                       usecols=['image_id', 'fold', 'target'], dtype={'fold': np.byte, 'target': np.byte})\n\n_ = df_folds.groupby('fold').target.hist(alpha=0.4)\ndf_folds.groupby('fold').target.mean().to_frame('ratio').T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"3h2Zcdod4qmE"},"cell_type":"code","source":"df_test = pd.read_csv(f'../input/siim-isic-melanoma-classification/test.csv', index_col='image_name')\n\nif debug:\n    df_folds = df_folds.sample(batch_size * 80)\n\ndf_folds = df_folds.sample(frac=1.0, random_state=seed*6+fold_number)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"5i8UhuvM4qmJ","outputId":"c0ea1e24-01ca-4660-af87-df919cdc884c"},"cell_type":"code","source":"ds_train = ImageDataset(\n    path=TRAIN_ROOT_PATH,\n    image_ids=df_folds[df_folds['fold'] != fold_number].index.values,\n    labels=df_folds[df_folds['fold'] != fold_number].target.values,\n    transforms=get_train_transforms(),\n)\n\nds_val = ImageDataset(\n    path=TRAIN_ROOT_PATH,\n    image_ids=df_folds[df_folds['fold'] == fold_number].index.values,\n    labels=df_folds[df_folds['fold'] == fold_number].target.values,\n    transforms=get_valid_transforms(),\n)\n\nds_test = ImageDataset(\n    path=TEST_ROOT_PATH,\n    image_ids=df_test.index.values,\n    transforms=get_tta_transforms(),\n)\n\ndel df_folds\nlen(ds_train), len(ds_val), len(ds_test)","execution_count":null,"outputs":[]},{"metadata":{"id":"dqfK4rxh4qmO"},"cell_type":"markdown","source":"# Model","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"id":"vMgS0GCj4qmP"},"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\nfrom pytorch_lightning.metrics.classification import AUROC\nfrom sklearn.metrics import roc_auc_score\n\nclass Model(pl.LightningModule):\n    def __init__(self, *args, **kwargs):\n        super().__init__()\n        self.net = EfficientNet.from_pretrained(arch, advprop=True)\n        self.net._fc = nn.Linear(in_features=self.net._fc.in_features, out_features=1, bias=True)\n\n    def forward(self, x):\n        return self.net(x)\n\n    def configure_optimizers(self):\n        optimizer = torch.optim.AdamW(self.parameters(), lr=lr, weight_decay=weight_decay)\n        scheduler = torch.optim.lr_scheduler.OneCycleLR(\n            max_lr=lr,\n            epochs=max_epochs,\n            optimizer=optimizer,\n            steps_per_epoch=int(len(ds_train) / batch_size),\n            pct_start=0.1,\n            div_factor=10,\n            final_div_factor=100,\n            base_momentum=0.90,\n            max_momentum=0.95,\n        )\n        return [optimizer], [scheduler]\n\n    def step(self, batch):\n        # return batch loss\n        x, y  = batch\n        y_hat = self(x).flatten()\n        y_smo = y.float() * (1 - label_smoothing) + 0.5 * label_smoothing\n        loss  = F.binary_cross_entropy_with_logits(y_hat, y_smo.type_as(y_hat),\n                                                   pos_weight=torch.tensor(pos_weight))\n        return loss, y, y_hat.sigmoid()\n\n    def training_step(self, batch, batch_nb):\n        # hardware agnostic training\n        loss, y, y_hat = self.step(batch)\n        acc = (y_hat.round() == y).float().mean().item()\n        tensorboard_logs = {'train_loss': loss, 'acc': acc}\n        return {'loss': loss, 'acc': acc, 'log': tensorboard_logs}\n\n    def validation_step(self, batch, batch_nb):\n        loss, y, y_hat = self.step(batch)\n        return {'val_loss': loss,\n                'y': y.detach(), 'y_hat': y_hat.detach()}\n\n    def validation_epoch_end(self, outputs):\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        y = torch.cat([x['y'] for x in outputs])\n        y_hat = torch.cat([x['y_hat'] for x in outputs])\n        auc = AUROC()(pred=y_hat, target=y) if y.float().mean() > 0 else 0.5 # skip sanity check\n        acc = (y_hat.round() == y).float().mean().item()\n        print(f\"Epoch {self.current_epoch} acc:{acc} auc:{auc}\")\n        tensorboard_logs = {'val_loss': avg_loss, 'val_auc': auc, 'val_acc': acc}\n        return {'avg_val_loss': avg_loss,\n                'val_auc': auc, 'val_acc': acc,\n                'log': tensorboard_logs}\n\n    def test_step(self, batch, batch_nb):\n        x, _ = batch\n        y_hat = self(x).flatten().sigmoid()\n        return {'y_hat': y_hat}\n\n    def test_epoch_end(self, outputs):\n        y_hat = torch.cat([x['y_hat'] for x in outputs])\n        df_test['target'] = y_hat.tolist()\n        N = len(glob('submission*.csv'))\n        df_test.target.to_csv(f'submission{N}.csv')\n        return {'tta': N}\n\n    def train_dataloader(self):\n        return DataLoader(ds_train, batch_size=batch_size, num_workers=num_workers,\n                          drop_last=True, shuffle=True, pin_memory=True)\n\n    def val_dataloader(self):\n        return DataLoader(ds_val, batch_size=batch_size, num_workers=num_workers,\n                          drop_last=False, shuffle=False, pin_memory=True)\n\n    def test_dataloader(self):\n        return DataLoader(ds_test, batch_size=batch_size, num_workers=num_workers,\n                          drop_last=False, shuffle=False, pin_memory=False)\n\nmodel = Model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot some training images\nimport torchvision.utils as vutils\nbatch, targets = next(iter(model.train_dataloader()))\n\nplt.figure(figsize=(16, 8))\nplt.axis(\"off\")\nplt.title(\"Training Images\")\n_ = plt.imshow(vutils.make_grid(\n    batch[:16], nrow=8, padding=2, normalize=True).cpu().numpy().transpose((1, 2, 0)))\n\ntargets[:16].reshape([2, 8]) if len(targets) >= 16 else targets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# # test the same images\n# with torch.no_grad():\n#     print(model(batch[:16]).reshape([len(targets)//8,8]).sigmoid())\ndel batch; del targets","execution_count":null,"outputs":[]},{"metadata":{"id":"_1ib_IBN4qmS"},"cell_type":"markdown","source":"# Train\nThe Trainer automates the rest.\n\nTrains on 8 TPU cores, GPU or CPU - whatever is available.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# # View logs life in tensorboard\n# Unfortunately broken again in the Kaggle notebooks :(\n# however, it still works nicely in Colab or locally :)\n\n# if gpus:\n#     !pip install -qU tensorboard-plugin-profile\n# %reload_ext tensorboard\n# %tensorboard --logdir lightning_logs/","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"id":"4AEUW-iB4qmT","outputId":"49468221-86ef-4e7b-b627-7d51397fad08"},"cell_type":"code","source":"checkpoint_callback = pl.callbacks.ModelCheckpoint(\"{epoch:02d}_{val_auc:.4f}\",\n                                                   save_top_k=1, monitor='val_auc', mode='max')\ntrainer = pl.Trainer(\n    tpu_cores=tpu_cores,\n    gpus=gpus,\n    precision=16 if gpus else 32,\n    max_epochs=max_epochs,\n    num_sanity_val_steps=1 if debug else 0,\n    checkpoint_callback=checkpoint_callback,\n#     val_check_interval=0.25, # check validation 4 times per epoch\n)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# clean up gpu in case you are debugging \nimport gc\ntorch.cuda.empty_cache(); gc.collect()\ntorch.cuda.empty_cache(); gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"GpASvAkY4qmb","outputId":"ed7fc27d-3b35-4ca9-d479-b2318582d38c"},"cell_type":"code","source":"trainer.fit(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"apwpS9S94qmf"},"cell_type":"code","source":"# import pdb; pdb.pm()","execution_count":null,"outputs":[]},{"metadata":{"id":"h4WrS30T4qmj"},"cell_type":"markdown","source":"# Submission\nInfer on test set using a simple random TTA (test-time augmentation)","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"%%time\nfor _ in range(tta):\n    trainer.test(ckpt_path='best')","execution_count":null,"outputs":[]},{"metadata":{"id":"2HEluAlSiRsr","trusted":true},"cell_type":"code","source":"# merge TTA\nsubmission = df_test[['target']]\nsubmission.target = 0.0\nfor sub in glob('submission*.csv'):\n    submission.target += pd.read_csv(sub, index_col='image_name').target\n\n# min-max norm\nsubmission.target -= submission.target.min()\nsubmission.target /= submission.target.max()\n\nsubmission.to_csv(f'submission_fold{fold_number}.csv')\n\nsubmission.hist(bins=100, log=True, alpha=0.6)\nsubmission.target.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# K-Fold blend","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"folds_path = '../input/melanoma-neat-pytorch-lightning'\n!cp {folds_path}/*_fold*.csv .\n!cp {folds_path}/*.ckpt .","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folds_sub = pd.read_csv(f'{folds_path}/submission.csv', index_col='image_name')\n\n# incremental blend with equal weights for all folds\nsubmission.target += folds_sub.target * (fold_number + 4)\nsubmission.target /= (fold_number + 5)\n\nsubmission.to_csv('submission.csv')\n\nsubmission.hist(bins=100, log=True, alpha=0.6)\nsubmission.target.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"329ls2VE4qmu"},"cell_type":"code","source":"if not debug and gpus:\n    !rm nv_apex -rf\n!ls -sh","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}