{"cells":[{"metadata":{},"cell_type":"markdown","source":"# About this notebook\n\nThis notebook is a sample code rewritten using pytorch-lightning, based on [this notebook by nakama](https://www.kaggle.com/yasufuminakama/ranzcr-resnext50-32x4d-starter-training).\n\nUsing pytorch-lightning provides the following benefits\n\n- can train on TPU\n- easy to manage\n- log metrics\n- test ideas\n\nThis notebook is a sample code and has some incomplete parts. (e.g. validation, configuration)\n\nrefered to: https://www.kaggle.com/vishnus/cassava-pytorch-lightning-starter-notebook-0-895"},{"metadata":{},"cell_type":"markdown","source":"# Library"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!pip install timm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np  # linear algebra\nimport pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport time\nimport random\nfrom contextlib import contextmanager\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nfrom collections import defaultdict, Counter\nimport sys\nfrom typing import Tuple\nimport PIL\nfrom torch.utils.data import Dataset\nfrom pathlib import Path\nfrom PIL import Image\nfrom PIL.Image import Image as PILImage\nfrom torch.utils.data.dataloader import DataLoader\nimport cv2\nfrom pytorch_lightning import LightningDataModule\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nimport albumentations as A\nfrom sklearn.metrics import roc_auc_score\nfrom pytorch_lightning.core.lightning import LightningModule\n\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip,\n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout,\n    IAAAdditiveGaussianNoise, Transpose\n)\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\nimport matplotlib.pyplot as plt\nimport timm\nfrom torchvision import models\nimport torch.nn as nn\nimport torch\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\nimport pytorch_lightning as pl\nfrom torch import optim\nfrom pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# config & setting"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_PATH = '../input/ranzcr-clip-catheter-line-classification/train'\ntarget_cols = ['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal', 'NGT - Abnormal',\n               'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal', 'CVC - Abnormal',\n               'CVC - Borderline', 'CVC - Normal', 'Swan Ganz Catheter Present']\ntrain = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/train.csv')\ntest = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n\n# ====================================================\n# CFG\n# ====================================================\nclass CFG:\n    debug = False\n    print_freq = 100\n    num_workers = 4\n    model_name = 'resnext50_32x4d'\n    size = 600\n    scheduler = 'CosineAnnealingLR'  # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts']\n    epochs = 6\n    # factor=0.2 # ReduceLROnPlateau\n    # patience=4 # ReduceLROnPlateau\n    # eps=1e-6 # ReduceLROnPlateau\n    T_max = 6  # CosineAnnealingLR\n    # T_0=6 # CosineAnnealingWarmRestarts\n    lr = 1e-4\n    min_lr = 1e-6\n    batch_size = 32\n    weight_decay = 1e-6\n    gradient_accumulation_steps = 1\n    max_grad_norm = 1000\n    seed = 42\n    target_size = 11\n    target_cols = ['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',\n                   'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal',\n                   'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',\n                   'Swan Ganz Catheter Present']\n    n_fold = 4\n    trn_fold = [0, 1, 2, 3]\n    train = True\n\n\nif CFG.debug:\n    CFG.epochs = 1\n    train = train.sample(n=100, random_state=CFG.seed).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef get_score(y_true, y_pred):\n    scores = []\n    for i in range(y_true.shape[1]):\n        score = roc_auc_score(y_true[:, i].astype(int), y_pred[:, i])\n        scores.append(score)\n    avg_score = np.mean(scores)\n    return avg_score, scores\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n\ndef init_logger(log_file='train.log'):\n    from logging import getLogger, INFO, FileHandler, Formatter, StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\n\nLOGGER = init_logger()\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n\nseed_torch(seed=CFG.seed)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Transforms\n# ====================================================\ndef get_transforms(img_size, data):\n    if data == 'train':\n        return Compose([\n            Resize(img_size, img_size),\n            RandomResizedCrop(img_size, img_size, scale=(0.85, 1.0)),\n            HorizontalFlip(p=0.5),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n\n    elif data == 'valid':\n        return Compose([\n            Resize(img_size, img_size),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n\n\n# ====================================================\n# Dataset\n# ====================================================\nclass TrainDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['StudyInstanceUID'].values\n        self.labels = df[CFG.target_cols].values\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TRAIN_PATH}/{file_name}.jpg'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).float()\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DataModule"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZCRDataModule(LightningDataModule):\n    def __init__(\n            self,\n            train_df,\n            val_df,\n            aug_p: float = 0.5,\n            val_pct: float = 0.2,\n            img_sz: int = 224,\n            batch_size: int = 64,\n            num_workers: int = 4,\n            fold_id: int = 0,\n    ):\n        super().__init__()\n        self.aug_p = aug_p\n        self.val_pct = val_pct\n        self.img_sz = img_sz\n        self.batch_size = batch_size\n        self.num_workers = num_workers\n        self.fold_id = fold_id\n\n        self.train_df = train_df\n        self.val_df = val_df\n\n    def train_dataloader(self):\n        train_dataset = TrainDataset(self.train_df, transform=get_transforms(self.img_sz, data=\"train\"))\n        return DataLoader(\n            train_dataset,\n            batch_size=self.batch_size,\n            num_workers=self.num_workers,\n            shuffle=True,\n            pin_memory=True,\n            drop_last=True\n        )\n\n    def val_dataloader(self):\n        valid_dataset = TrainDataset(self.val_df, transform=get_transforms(self.img_sz, data=\"valid\"))\n        return DataLoader(\n            valid_dataset,\n            batch_size=self.batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n            pin_memory=True,\n            drop_last=False\n        )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RANZCRModel(LightningModule):\n    def __init__(self, cfg, model_name=\"resnext50_32x4d\", pretrained=False):\n        super().__init__()\n\n        self.cfg = cfg\n        self.lr = cfg.lr\n        self.wd = 1e-6\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, cfg.target_size)\n\n        self.optimizer = Adam(self.model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay, amsgrad=False)\n        self.criterion = nn.BCEWithLogitsLoss()\n\n        self.scheduler = self.get_scheduler()\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n\n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        loss = self.criterion(y_hat, y)\n        self.log(\"train_loss\", loss, prog_bar=True)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        loss = self.criterion(y_hat, y)\n        self.log(\"valid_loss\", loss, prog_bar=True)\n\n    def configure_optimizers(self):\n        optimizer = optim.AdamW(\n            self.model.parameters(), lr=self.lr, weight_decay=self.wd\n        )\n        scheduler = optim.lr_scheduler.CosineAnnealingLR(\n            optimizer, self.trainer.max_epochs, 0\n        )\n\n        return [optimizer], [scheduler]\n\n    def get_scheduler(self):\n        if self.cfg.scheduler == 'ReduceLROnPlateau':\n            scheduler = ReduceLROnPlateau(self.optimizer, mode='min', factor=CFG.factor, patience=CFG.patience,\n                                          verbose=True,\n                                          eps=CFG.eps)\n        elif self.cfg.scheduler == 'CosineAnnealingLR':\n            scheduler = CosineAnnealingLR(self.optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr, last_epoch=-1)\n        elif self.cfg.scheduler == 'CosineAnnealingWarmRestarts':\n            scheduler = CosineAnnealingWarmRestarts(self.optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr,\n                                                    last_epoch=-1)\n        return scheduler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nWe have to use KFold instead of this approach.\n\"\"\"\ndata_module = RANZCRDataModule(\n    train.iloc[:-1000, :],  # train\n    train.iloc[-1000:, :],  # validation\n    aug_p=0.5,\n    img_sz=320,\n    batch_size=64,\n    num_workers=4,\n    # fold_id=fold_id,\n)\nmodel = RANZCRModel(\n    CFG,\n    model_name=\"resnext50_32x4d\",\n)\n\ntrainer = pl.Trainer(\n    gpus=1,\n    max_epochs=1,\n    gradient_clip_val=0.1,\n    precision=16,\n)\n\ntrainer.fit(model=model, datamodule=data_module)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}