{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Melanoma starter pipeline by [@shonenkov](https://www.kaggle.com/shonenkov)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Main ideas\n\n\n- Using External Data\n- StratifyGroupKFold\n- Focal Loss / Label Smoothing\n- BalanceClassSampler\n- SimpleAugs\n- 512x512 image size\n- EfficientNet","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Dependencies","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install -q efficientnet_pytorch > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"from glob import glob\nimport pandas as pd\nfrom sklearn.model_selection import GroupKFold\nimport cv2\nfrom skimage import io\nimport albumentations as A\nimport torch\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.nn import functional as F\nfrom glob import glob\nimport sklearn\nfrom torch import nn\nimport warnings\n\nwarnings.filterwarnings(\"ignore\") \nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning) \n\nSEED = 42\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# External data\n\nI have prepared kernel with merging data. Don't forget to read [this kernel](https://www.kaggle.com/shonenkov/merge-external-data) ;)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = '../input/melanoma-merged-external-data-512x512-jpeg'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# StratifyGroupKFold\n\nI think group by patient_id is very important. Also I think that stratify by sex, target, source, anatom_site_general_challenge also useful.\nCode with getting folds you can find [here](https://www.kaggle.com/shonenkov/merge-external-data)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_folds = pd.read_csv(f'{DATA_PATH}/folds.csv', index_col='image_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"set(df_folds[df_folds['fold'] == 0]['patient_id'].values).intersection(df_folds[df_folds['fold'] == 1]['patient_id'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_folds[df_folds['fold'] == 0]['target'].hist();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_folds[df_folds['fold'] == 1]['target'].hist();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose([\n            A.RandomSizedCrop(min_max_height=(400, 400), height=512, width=512, p=0.5),\n            A.RandomRotate90(p=0.5),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.Resize(height=512, width=512, p=1),\n            A.Cutout(num_holes=8, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),\n            ToTensorV2(p=1.0),                  \n        ], p=1.0)\n\ndef get_valid_transforms():\n    return A.Compose([\n            A.Resize(height=512, width=512, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_ROOT_PATH = f'{DATA_PATH}/512x512-dataset-melanoma/512x512-dataset-melanoma'\n\ndef onehot(size, target):\n    vec = torch.zeros(size, dtype=torch.float32)\n    vec[target] = 1.\n    return vec\n\nclass DatasetRetriever(Dataset):\n\n    def __init__(self, image_ids, labels, transforms=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.labels = labels\n        self.transforms = transforms\n\n    def __getitem__(self, idx: int):\n        image_id = self.image_ids[idx]\n        image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = image.astype(np.float32) / 255.0\n\n        label = self.labels[idx]\n\n        if self.transforms:\n            sample = {'image': image}\n            sample = self.transforms(**sample)\n            image = sample['image']\n\n        target = onehot(2, label)\n        return image, target\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\n    def get_labels(self):\n        return list(self.labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Metrics","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\nclass RocAucMeter(object):\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.y_true = np.array([0,1])\n        self.y_pred = np.array([0.5,0.5])\n        self.score = 0\n\n    def update(self, y_true, y_pred):\n        y_true = y_true.cpu().numpy().argmax(axis=1).clip(min=0, max=1).astype(int)\n        # y_pred = 1 - nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:,0]\n        y_pred = nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:,1]\n        self.y_true = np.hstack((self.y_true, y_true))\n        self.y_pred = np.hstack((self.y_pred, y_pred))\n        self.score = sklearn.metrics.roc_auc_score(self.y_true, self.y_pred)\n\n    @property\n    def avg(self):\n        return self.score\n\n    \nclass APScoreMeter(RocAucMeter):\n    def __init__(self):\n        super(APScoreMeter, self).__init__()\n\n    def update(self, y_true, y_pred):\n        y_true = y_true.cpu().numpy().argmax(axis=1).clip(min=0, max=1).astype(int)\n        y_pred = nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:,1]\n        self.y_true = np.hstack((self.y_true, y_true))\n        self.y_pred = np.hstack((self.y_pred, y_pred))\n        self.score = sklearn.metrics.average_precision_score(self.y_true, self.y_pred)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, logits=False, reduce=True):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.logits = logits\n        self.reduce = reduce\n\n    def forward(self, inputs, targets):\n        if self.logits:\n            BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets)\n        else:\n            BCE_loss = F.binary_cross_entropy(inputs, targets)\n\n        pt = torch.exp(-BCE_loss)\n        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n\n        if self.reduce:\n            return torch.mean(F_loss)\n        else:\n            return F_loss\n        \nclass LabelSmoothing(nn.Module):\n    def __init__(self, smoothing = 0.1):\n        super(LabelSmoothing, self).__init__()\n        self.confidence = 1.0 - smoothing\n        self.smoothing = smoothing\n\n    def forward(self, x, target):\n        if self.training:\n            x = x.float()\n            target = target.float()\n            logprobs = torch.nn.functional.log_softmax(x, dim = -1)\n\n            nll_loss = -logprobs * target\n            nll_loss = nll_loss.sum(-1)\n    \n            smooth_loss = -logprobs.mean(dim=-1)\n\n            loss = self.confidence * nll_loss + self.smoothing * smooth_loss\n\n            return loss.mean()\n        else:\n            return torch.nn.functional.cross_entropy(x, target)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Net","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\n\ndef get_net():\n    net = EfficientNet.from_pretrained('efficientnet-b5')\n    net._fc = nn.Linear(in_features=2048, out_features=2, bias=True)\n    return net\n\nnet = get_net().cuda()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fitter","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Fitter:\n    \n    def __init__(self, model, device, config, folder):\n        self.config = config\n        self.epoch = 0\n\n        self.base_dir = f'./{folder}'\n        if not os.path.exists(self.base_dir):\n            os.makedirs(self.base_dir)\n\n        self.log_path = f'{self.base_dir}/log.txt'\n        self.best_score = 0\n        self.best_loss = 10**5\n        self.best_ap = 0\n        \n        self.model = model\n        self.device = device\n\n        param_optimizer = list(self.model.named_parameters())\n        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n        optimizer_grouped_parameters = [\n            {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n            {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n        ] \n\n        self.optimizer = torch.optim.AdamW(self.model.parameters(), lr=config.lr)\n        self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)\n\n#         self.criterion = FocalLoss(logits=True).to(self.device)\n        self.criterion = LabelSmoothing().to(self.device)\n        self.log(f'Fitter prepared. Device is {self.device}')\n\n    def fit(self, train_loader, validation_loader):\n        for e in range(self.config.n_epochs):\n            if self.config.verbose:\n                lr = self.optimizer.param_groups[0]['lr']\n                timestamp = datetime.utcnow().isoformat()\n                self.log(f'\\n{timestamp}\\nLR: {lr}')\n\n            t = time.time()\n            summary_loss, roc_auc_scores, ap_scores = self.train_one_epoch(train_loader)\n            self.log(f'[RESULT]: Train. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, roc_auc: {roc_auc_scores.avg:.5f}, ap: {ap_scores.avg:.5f}, time: {(time.time() - t):.5f}')\n\n            t = time.time()\n            summary_loss, roc_auc_scores, ap_scores = self.validation(validation_loader)\n\n            self.log(f'[RESULT]: Val. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, roc_auc: {roc_auc_scores.avg:.5f}, ap: {ap_scores.avg:.5f}, time: {(time.time() - t):.5f}')\n            if summary_loss.avg < self.best_loss:\n                self.best_loss = summary_loss.avg\n                self.save_model(f'{self.base_dir}/best-loss-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-loss-checkpoint-*epoch.bin'))[:-2]:\n                    os.remove(path)\n                    \n            if roc_auc_scores.avg > self.best_score:\n                self.best_score = roc_auc_scores.avg\n                self.save_model(f'{self.base_dir}/best-score-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-score-checkpoint-*epoch.bin'))[:-2]:\n                    os.remove(path)\n                    \n            if ap_scores.avg > self.best_ap:\n                self.best_ap = ap_scores.avg\n                self.save_model(f'{self.base_dir}/best-ap-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-ap-checkpoint-*epoch.bin'))[:-2]:\n                    os.remove(path)\n\n            if self.config.validation_scheduler:\n                self.scheduler.step(metrics=summary_loss.avg)\n\n            self.epoch += 1\n\n    def validation(self, val_loader):\n        self.model.eval()\n        summary_loss = AverageMeter()\n        roc_auc_scores = RocAucMeter()\n        ap_scores = APScoreMeter()\n        t = time.time()\n        for step, (images, targets) in enumerate(val_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Val Step {step}/{len(val_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, roc_auc: {roc_auc_scores.avg:.5f}, ap: {ap_scores.avg:.5f} ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            with torch.no_grad():\n                targets = targets.to(self.device).float()\n                batch_size = images.shape[0]\n                images = images.to(self.device).float()\n                outputs = self.model(images)\n                loss = self.criterion(outputs, targets)\n                roc_auc_scores.update(targets, outputs)\n                ap_scores.update(targets, outputs)\n                summary_loss.update(loss.detach().item(), batch_size)\n\n        return summary_loss, roc_auc_scores, ap_scores\n\n    def train_one_epoch(self, train_loader):\n        self.model.train()\n        summary_loss = AverageMeter()\n        roc_auc_scores = RocAucMeter()\n        ap_scores = APScoreMeter()\n        t = time.time()\n        for step, (images, targets) in enumerate(train_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Train Step {step}/{len(train_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, roc_auc: {roc_auc_scores.avg:.5f}, ap: {ap_scores.avg:.5f} ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            \n            targets = targets.to(self.device).float()\n            images = images.to(self.device).float()\n            batch_size = images.shape[0]\n\n            self.optimizer.zero_grad()\n            outputs = self.model(images)\n            loss = self.criterion(outputs, targets)\n            loss.backward()\n            \n            roc_auc_scores.update(targets, outputs)\n            ap_scores.update(targets, outputs)\n            summary_loss.update(loss.detach().item(), batch_size)\n\n            self.optimizer.step()\n\n            if self.config.step_scheduler:\n                self.scheduler.step()\n\n        return summary_loss, roc_auc_scores, ap_scores\n    \n    def save_model(self, path):\n        self.model.eval()\n        torch.save(self.model.state_dict(),path)\n\n    def save(self, path):\n        self.model.eval()\n        torch.save({\n            'model_state_dict': self.model.state_dict(),\n            'optimizer_state_dict': self.optimizer.state_dict(),\n            'scheduler_state_dict': self.scheduler.state_dict(),\n            'best_score': self.best_score,\n            'best_ap': self.best_ap,\n            'best_loss': self.best_loss,\n            'epoch': self.epoch,\n        }, path)\n\n    def load(self, path):\n        checkpoint = torch.load(path)\n        self.model.load_state_dict(checkpoint['model_state_dict'])\n        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n        self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n        self.best_score = checkpoint['best_score']\n        self.best_ap = checkpoint['best_ap']\n        self.best_loss = checkpoint['best_loss']\n        self.epoch = checkpoint['epoch']\n        \n    def log(self, message):\n        if self.config.verbose:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainGlobalConfig:\n    num_workers = 2\n    batch_size = 8 \n    n_epochs = 20\n    lr = 0.00003\n\n    # -------------------\n    verbose = True\n    verbose_step = 1\n    # -------------------\n\n    # --------------------\n    step_scheduler = False  # do scheduler.step after optimizer.step\n    validation_scheduler = True  # do scheduler.step after validation stage loss\n\n#     SchedulerClass = torch.optim.lr_scheduler.OneCycleLR\n#     scheduler_params = dict(\n#         max_lr=0.001,\n#         epochs=n_epochs,\n#         steps_per_epoch=int(len(train_dataset) / batch_size),\n#         pct_start=0.1,\n#         anneal_strategy='cos', \n#         final_div_factor=10**5\n#     )\n    \n    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n    scheduler_params = dict(\n        mode='min',\n        factor=0.8,\n        patience=1,\n        verbose=False, \n        threshold=0.0001,\n        threshold_mode='abs',\n        cooldown=0, \n        min_lr=1e-8,\n        eps=1e-08\n    )\n    # --------------------","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Save all states for \"honest\" training of folds","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fitter = Fitter(model=net, device=torch.device('cuda:0'), config=TrainGlobalConfig, folder='base_state')\nBASE_STATE_PATH = f'{fitter.base_dir}/base_state.bin'\nfitter.save(BASE_STATE_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from catalyst.data.sampler import BalanceClassSampler\n\ndef train_fold(fold_number):\n\n    train_dataset = DatasetRetriever(\n        image_ids=df_folds[df_folds['fold'] != fold_number].index.values,\n        labels=df_folds[df_folds['fold'] != fold_number].target.values,\n        transforms=get_train_transforms(),\n    )\n\n    df_val = df_folds[(df_folds['fold'] == fold_number) & (df_folds['source'] == 'ISIC20')]\n\n    validation_dataset = DatasetRetriever(\n        image_ids=df_val.index.values,\n        labels=df_val.target.values,\n        transforms=get_valid_transforms(),\n    )\n\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        sampler=BalanceClassSampler(labels=train_dataset.get_labels(), mode=\"downsampling\"),\n        batch_size=TrainGlobalConfig.batch_size,\n        pin_memory=False,\n        drop_last=True,\n        num_workers=TrainGlobalConfig.num_workers,\n    )\n    val_loader = torch.utils.data.DataLoader(\n        validation_dataset, \n        batch_size=TrainGlobalConfig.batch_size,\n        num_workers=TrainGlobalConfig.num_workers,\n        shuffle=False,\n        sampler=SequentialSampler(validation_dataset),\n        pin_memory=False,\n    )\n\n    fitter = Fitter(model=net, device=torch.device('cuda:0'), config=TrainGlobalConfig, folder=f'fold{fold_number}')\n    fitter.load(BASE_STATE_PATH)\n    fitter.fit(train_loader, val_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning) \n\nfor fold_number in range(1): # range(5)\n    train_fold(fold_number=fold_number)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Thank you for reading my kernel!","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}