{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%%capture\n%%bash\ncurl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\npython pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev\npip install efficientnet_pytorch torchtoolbox pytorch-lightning","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport cv2\nimport random\nimport os\n\nfrom tqdm.notebook import tqdm\nimport multiprocessing as mp\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim.lr_scheduler import LambdaLR\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import _LRScheduler\n\nimport pytorch_lightning as pl \nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.core.lightning import LightningModule\nfrom pytorch_lightning.metrics import AUROC\nfrom pytorch_lightning.metrics.functional import accuracy, auroc\n# from pytorch_lightning.logging import TensorBoardLogger\nfrom pytorch_lightning import loggers as pl_loggers\nfrom pytorch_lightning.callbacks.early_stopping import EarlyStopping\n\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n# import torch_xla.core.xla_model as xm\n# device = xm.xla_device()\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom efficientnet_pytorch import EfficientNet\n\nimport warnings\nwarnings.simplefilter('ignore')\n%matplotlib inline\n\nimport pytorch_lightning as pl\npl.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE = \"efficientnet-b0\"\nEPOCHS = 10\nGRAD_ACCUMULATE = 1\nBS = 16\np = 0.3\nLR_RANGE = [1e-7, 2e-4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"warnings.simplefilter('ignore')\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_tfms = A.Compose([\n    A.Cutout(p=p),\n    A.RandomRotate90(p=p),\n    A.Flip(p=p),\n    A.OneOf([\n        A.RandomBrightnessContrast(brightness_limit=0.2,\n                                   contrast_limit=0.2,\n                                   ),\n        A.HueSaturationValue(\n            hue_shift_limit=20,\n            sat_shift_limit=50,\n            val_shift_limit=50)\n    ], p=p),\n    A.OneOf([\n        A.IAAAdditiveGaussianNoise(),\n        A.GaussNoise(),\n    ], p=p),\n    A.OneOf([\n        A.MedianBlur(blur_limit=3, p=0.1),\n        A.Blur(blur_limit=3, p=0.1),\n    ], p=p),\n    A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=p),\n    A.OneOf([\n        A.OpticalDistortion(p=0.3),\n        A.GridDistortion(p=0.1),\n        A.IAAPiecewiseAffine(p=0.3),\n    ], p=p), \n    A.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],\n    ),\n    ToTensorV2()\n])\n    \ntest_tfms = A.Compose([\n    A.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],\n    ),\n    ToTensorV2()\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None):\n        \"\"\"\n        Class initialization\n        Args:\n            df (pd.DataFrame): DataFrame with data description\n            imfolder (str): folder with images\n            train (bool): flag of whether a training dataset is being initialized or testing one\n            transforms: image transformation method to be applied\n            meta_features (list): list of features with meta information, such as sex and age\n            \n        \"\"\"\n        self.df = df\n        self.imfolder = imfolder\n        self.transforms = transforms\n        self.train = train\n        self.meta_features = meta_features\n        \n    def __getitem__(self, index):\n        im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')\n        x = cv2.cvtColor(cv2.imread(im_path), cv2.COLOR_BGR2RGB)\n#         meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)\n\n        if self.transforms:\n            x = self.transforms(image=x)['image']\n            \n        if self.train:\n            y = self.df.iloc[index]['target']\n#             return (x, meta), y\n            return x, y\n        else:\n#             return (x, meta)\n            return x\n    \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/train.csv')\ntest_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/test.csv')\ntrain_df = train_df[train_df.tfrecord!=-1]\ntrain_df.target = train_df.target.astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = train_df.tfrecord.unique()\nval_idx = np.random.choice(idx, 3, replace=False)\ntrain_idx = np.array([i for i in idx if i not in val_idx])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"a1 = 1 / train_df[\"target\"].mean()\na2 = 1 / (1 - train_df[\"target\"].mean())\n\nclass WeightedFocalLoss(nn.Module):\n    \"Non weighted version of Focal Loss\"\n    def __init__(self, a1=1, a2=1, gamma=1.1):\n        super().__init__()\n        self.a1, self.a2 = a1, a2\n        self.gamma = gamma\n\n    def forward(self, inputs, targets):\n        inputs = inputs.squeeze()\n        targets = targets.squeeze()\n\n        BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction='none')\n        at = self.a1 * targets + self.a2 * (1-targets)\n        pt = torch.exp(-BCE_loss)\n        F_loss = at*(1-pt)**self.gamma * BCE_loss\n\n        return F_loss.mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(LightningModule):\n    def __init__(self, loss_params, base=BASE, freeze=True):\n        super().__init__()\n\n        # EfficientNet\n        self.base = EfficientNet.from_pretrained(base)\n        \n        if freeze:\n            for p in self.base.parameters(): p.requires_grad=False\n        \n        # Replace last layer\n        self.fc = nn.Linear(self.base._fc.in_features, 1)\n        self.loss = WeightedFocalLoss(*loss_params)\n        self.auroc = AUROC()\n    \n    def unfreeze(self):\n        for p in self.base.parameters(): p.requires_grad=True\n        for p in self.base._fc.parameters(): p.requires_grad=False\n    \n    def forward(self, x):\n        pool = F.adaptive_avg_pool2d(self.base.extract_features(x), 1)\n        pool = pool.view(x.shape[0], -1)\n        return self.fc(pool)\n    \n    def configure_optimizers(self):\n        params = [p for p in self.parameters() if p.requires_grad]\n        optimizer = torch.optim.Adam(params, lr=1e-3)\n        return optimizer\n    \n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        y_pred = self.forward(x)\n        loss = self.loss(y_pred, y)\n        \n#         return loss\n        result = pl.EvalResult(checkpoint_on=loss)\n        result.log(\"val_loss\", loss, prog_bar=True)\n#         result.log(\"val_auc\", self.auroc(y_pred.squeeze(), y.squeeze()), prog_bar=True)\n\n        return result\n\n    \n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        y_pred = self.forward(x)\n        loss = self.loss(y_pred, y)\n        \n#         return loss\n    \n        result = pl.TrainResult(loss)\n        result.log(\"train_loss\", loss)\n#         result.log(\"train_auc\", self.auroc(y_pred.squeeze(), y.squeeze()), prog_bar=True)\n        return result\n    \n\nmodel = Model(loss_params=(a1, a2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaData():\n    def __init__(self, train_df, train_idx, val_idx, train_tfms, test_tfms, batch_size=BS):\n        self.train_ds = MelanomaDataset(\n            train_df.loc[train_df.tfrecord.isin(train_idx)].reset_index(drop=True),\n            '/kaggle/input/jpeg-melanoma-256x256/train/', \n            train=True, \n            transforms=train_tfms\n        )\n\n        self.valid_ds = MelanomaDataset(\n            train_df.loc[train_df.tfrecord.isin(val_idx)].reset_index(drop=True),\n            '/kaggle/input/jpeg-melanoma-256x256/train/', \n            train=True, \n            transforms=test_tfms\n        )\n        \n        self.batch_size = batch_size\n    \n    def train_dataloader(self):\n        return DataLoader(self.train_ds, self.batch_size, shuffle=True, drop_last=True)\n    \n    def val_dataloader(self):\n        return DataLoader(self.valid_ds, self.batch_size, drop_last=True)\n    \ndata = MelanomaData(train_df, train_idx, val_idx, train_tfms, test_tfms, batch_size=BS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.hparams.lr = 2e-3\n# early_stopping = EarlyStopping('val_loss')\ntrainer = Trainer(tpu_cores=8, max_epochs=1, val_check_interval=0.5, accumulate_grad_batches=GRAD_ACCUMULATE)\ntrainer.fit(model, data.train_dataloader(), data.val_dataloader())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ys = []\ny_preds = []\nmodel = model.to(device)\nmodel.eval()\nwith torch.no_grad():\n    for x, y in tqdm(data.val_dataloader()):\n        x, y = x.to(device), y.to(device)\n        y_pred = model(x)\n        ys.extend(y)\n        y_preds.extend(y_pred)\n\n# y_preds = torch.stack(y_preds)\n# ys = torch.stack(ys)\n# auc = auroc(y_preds.squeeze(), ys.squeeze())\n# print(f\"Initial AUC is {auc:.4f}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_preds = torch.stack(y_preds)\nys = torch.stack(ys)\n# auc = auroc(y_preds.squeeze(), ys.squeeze())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.stack(ys)[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weights = []\nfor param in model.parameters():\n    weights.append(param.clone())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_optimizer(model, lr_range=LR_RANGE):\n    blocks = []\n\n    for n,p in model.base.named_parameters():\n        if p.requires_grad:\n            if n.startswith(\"_blocks.\"):\n                n = \".\".join(n.split(\".\", maxsplit=2)[:2])\n            else:\n                n = n.split(\".\", maxsplit=1)[0]\n            if n not in blocks:\n                blocks.append(n)\n\n    blocks = [\"base.\"+block for block in blocks]\n    blocks += [\"fc\"]\n    blocks = [block+\".\" for block in blocks]\n\n    mul = (lr_range[1] / lr_range[0]) ** (1/(len(blocks)-1))\n    lrs = [lr_range[0]*mul**i for i in range(len(blocks))]\n\n    param_list = []\n    for lr, block in zip(lrs, blocks):\n        param_list.extend([{'params':p ,'lr':lr} for n,p in model.named_parameters() if n.startswith(block)])\n    optimizer = torch.optim.Adam(param_list)\n    \n    return optimizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class OneCycleScheduler(_LRScheduler): \n    def __init__(self, optimizer, n_rounds, max_beta=0.95, min_beta=0.85, div_factor=10.0): \n        self.optimizer = optimizer\n        self.max_lr = [grp['lr'] for grp in optimizer.param_groups]\n        self.min_lr = [lr/div_factor for lr in self.max_lr]\n        # initialise lrs\n        for grp, lr in zip(self.optimizer.param_groups, self.min_lr):\n            grp['lr'] = lr\n            grp['betas'] = (max_beta, 0.999)\n        \n        self.min_beta = min_beta\n        self.max_beta = max_beta\n        \n        self.cutoff1 = int(n_rounds * 0.3)\n        if n_rounds < 20:\n            self.cutoff2 = n_rounds\n        else:\n            self.cutoff2 = int(n_rounds * 0.95)\n\n        self.k = 0\n\n        gaps = [max - min for max, min in zip(self.max_lr, self.min_lr)]\n        gaps2 = [min - min/100 for min in self.min_lr]\n        # movement of learning rate and momentum\n        self.step_up_lr = [gap / self.cutoff1 for gap in gaps]\n        self.step_down_lr1 = [gap / (self.cutoff2 - self.cutoff1) for gap in gaps]\n        self.step_down_lr2 = [gap / (n_rounds - self.cutoff2 + 1e-8) for gap in gaps2]\n        self.step_down_beta = (max_beta - min_beta) / self.cutoff1\n        self.step_up_beta = (max_beta - min_beta) / (self.cutoff2 - self.cutoff1)\n\n    def step(self):\n        self.k += 1\n        if self.k <= self.cutoff1:\n            for grp, d_lr in zip(self.optimizer.param_groups, self.step_up_lr):\n                grp['lr'] += d_lr\n                grp['betas'] = (grp['betas'][0] - self.step_down_beta, 0.999)\n        elif self.k <= self.cutoff2:\n            for grp, d_lr in zip(self.optimizer.param_groups, self.step_down_lr1):\n                grp['lr'] -= d_lr\n                grp['betas'] = (grp['betas'][0] + self.step_up_beta, 0.999)\n        else:\n            for grp, d_lr in zip(self.optimizer.param_groups, self.step_down_lr2):\n                grp['lr'] -= d_lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"steps_per_epoch = len(model.train_dataloader())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class UnfrozenModel(Model):\n    def __init__(self, loss_params, base=BASE, freeze=False):\n        super().__init__(loss_params, base, freeze)\n        for p in self.base._fc.parameters(): p.requires_grad = False\n        \n    def configure_optimizers(self):\n        optimizer = get_optimizer(self)\n        one_cycle_scheduler = OneCycleScheduler(optimizer, EPOCHS * steps_per_epoch // GRAD_ACCUMULATE)\n        scheduler = {'scheduler': one_cycle_scheduler, \"interval\": \"step\"}\n\n        return [optimizer], [scheduler]\n\nmodel2 = UnfrozenModel(loss_params=(a1, a2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for p, w in zip(model2.parameters(), weights): p.data = w","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# early_stopping = EarlyStopping('val_loss')\ntrainer = Trainer(gpus=1, max_epochs=EPOCHS, accumulate_grad_batches=GRAD_ACCUMULATE)\ntrainer.fit(model2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ys = []\ny_preds = []\nmodel2 = model2.to(device)\nmodel2.eval()\nwith torch.no_grad():\n    for x, y in tqdm(model2.val_dataloader()):\n        x, y = x.to(device), y.to(device)\n        y_pred = model2(x)\n        ys.extend(y)\n        y_preds.extend(y_pred)\n\ny_preds = torch.stack(y_preds)\nys = torch.stack(ys)\nauc = auroc(y_preds.squeeze(), ys.squeeze())\nprint(f\"Initial AUC is {auc:.4f}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = MelanomaDataset(test_df,\n   imfolder='/kaggle/input/jpeg-melanoma-256x256/test/', \n   train=False,\n   transforms=test_tfms\n                      )\ntest_loader = DataLoader(dataset=test, batch_size=BS, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_preds = []\nmodel2 = model2.to(device)\nmodel2.eval()\nwith torch.no_grad():\n    for x in tqdm(test_loader):\n        x = x.to(device)\n        y_pred = model2(x)\n        y_preds.extend(y_pred)\n        \ntest_df['target'] = torch.sigmoid(torch.stack(y_preds)).cpu().numpy()\ntest_df[['image_name', 'target']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}