{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install efficientnet_pytorch\n! pip install pytorch_lightning","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import torch\nimport torchvision\nimport torch.nn.functional as F\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nimport pandas as pd\nimport numpy as np\nimport gc\nimport os\nimport cv2\nimport time\nimport datetime\nimport warnings\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom efficientnet_pytorch import EfficientNet\nfrom pathlib import Path\nimport torchvision.transforms as ttransforms \nimport PIL\nfrom torch.utils.data.sampler import WeightedRandomSampler\nimport albumentations.pytorch\nimport albumentations\nimport math \n\n# At least fixing some random seeds. \n# It is still impossible to make results 100% reproducible when using GPU\nwarnings.simplefilter('ignore')\ntorch.manual_seed(47)\nnp.random.seed(47)\nrandom_state=47\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Directories","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_SIZE = 224\ndata_dir = Path('../input/')\ncsv_dir = data_dir/'siim-isic-melanoma-classification/'\nim_dir_test = data_dir/'siic-isic-224x224-images/test'\nim_dir_train = data_dir/'siic-isic-224x224-images/train'\n\nif 'jpeg' in str(im_dir_test):\n    ext = '.jpg'\nelse:\n    ext='.png'\nim_dir_train.is_dir()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Dataset with Metadata","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(csv_dir/'train.csv')\ntest_df = pd.read_csv(csv_dir/'test.csv')\n\n# One-hot encoding of anatom_site_general_challenge feature\nconcat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)\ndummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')\ntrain_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)\ntest_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)\n# Sex features\ntrain_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})\ntest_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})\ntrain_df['sex'] = train_df['sex'].fillna(-1)\ntest_df['sex'] = test_df['sex'].fillna(-1)\n\n# Age features\ntrain_df['age_approx'] /= train_df['age_approx'].max()\ntest_df['age_approx'] /= test_df['age_approx'].max()\ntrain_df['age_approx'] = train_df['age_approx'].fillna(0)\ntest_df['age_approx'] = test_df['age_approx'].fillna(0)\n\ntrain_df['patient_id'] = train_df['patient_id'].fillna(0)\n\nmeta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]\nmeta_features.remove('anatom_site_general_challenge')\ndel train_df['patient_id'];del train_df['anatom_site_general_challenge']\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, \n                 box_trans=True, meta_features = None, ext='.png'):\n        \"\"\"\n        Class initialization\n        Args:\n            df (pd.DataFrame): DataFrame with data description\n            imfolder (str): folder with images\n            train (bool): flag of whether a training dataset is being initialized or testing one\n            transforms: image transformation method to be applied\n            meta_features (list): list of features with meta information, such as sex and age\n            \n        \"\"\"\n        self.df = df\n        self.imfolder = imfolder\n        self.df['image_path'] = self.df['image_name'].apply(lambda x: os.path.join(self.imfolder, x + ext))\n        self.transforms = transforms\n        self.train = train\n        self.meta_features = meta_features\n                \n    def __getitem__(self, index):\n        im_path = self.df.iloc[index]['image_path']\n        x = cv2.imread(im_path)\n        x = cv2.cvtColor(x, cv2.COLOR_BGR2RGB)\n        meta = np.array(self.df[self.meta_features].iloc[index].values, dtype=np.float32)\n        if self.transforms:\n            x = self.transforms(image=x)['image']\n\n                \n        if self.train:\n            y = self.df.iloc[index]['target']\n            return (x, meta), y\n        else:\n            return (x, meta)\n        \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transforms","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Microscope(albumentations.ImageOnlyTransform):\n    def __init__(self, p: float = 0.3, always_apply=False):\n        super().__init__(always_apply, p)\n\n    def apply(self, img, **params):\n        if random.random() < self.p:\n            circle = cv2.circle((np.ones(img.shape) * 255).astype(np.uint8),\n                        (img.shape[0]//2, img.shape[1]//2),\n                        random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15),\n                        (0, 0, 0),\n                        -1)\n\n            mask = circle - 255\n            img = np.multiply(img, mask)\n\n        return img\n\ncutout = albumentations.Cutout(num_holes=4, max_h_size=int(IMG_SIZE*0.05), \n                      max_w_size=int(IMG_SIZE*0.04), \n                      p=0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MEL_DATASET_STATS = {'mean': [0.8016, 0.6186, 0.5849], 'std': [0.0916, 0.1036, 0.1139]}\nIMAGNET_STATS = {'mean': [0.485, 0.456, 0.406],'std': [0.229, 0.224, 0.225]}\n# normalization stats calculated this competition dataset \ndataset_stats = MEL_DATASET_STATS\nheight = 512\nwidth = 512\n\ntrain_transform = albumentations.Compose(transforms=[\n    albumentations.Flip(p=0.5),\n    albumentations.ShiftScaleRotate(shift_limit=(-0.17, 0.17), scale_limit=0., rotate_limit=0, \n                      border_mode=cv2.BORDER_REFLECT_101, p=0.70),\n    albumentations.RandomBrightnessContrast(brightness_limit=(-0.3, 0.15), contrast_limit=(-0.2, 0.15), p=1),\n    albumentations.RGBShift(p=0.70, r_shift_limit=(-50, 50)),\n    albumentations.GaussNoise(p=0.3),\n    albumentations.core.composition.OneOf([cutout, Microscope()], p=0.3),\n    albumentations.pytorch.transforms.ToTensor(normalize=dataset_stats),\n])\n\nvalid_transform = albumentations.Compose([\n    albumentations.Flip(p=0.5),\n    albumentations.GaussNoise(p=0.3),\n    albumentations.pytorch.transforms.ToTensor(normalize=dataset_stats)\n])\n\ntest_transform = albumentations.Compose(transforms=[\n    albumentations.Flip(p=0.5),\n    albumentations.GaussNoise(p=0.3),\n    albumentations.pytorch.transforms.ToTensor(normalize=dataset_stats),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# test_transform(False, image=np.ones((412,412, 3)).astype(np.uint8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# new_im_dim = 300\n# train_transform = transforms.Compose([\n#     transforms.RandomApply(transforms.RandomResizedCrop(size=new_im_dim, scale=(0.7, 1.0)), p=0.6)\n#     transforms.RandomHorizontalFlip(),\n#     transforms.RandomVerticalFlip(),\n#     transforms.ColorJitter(brightness= (0.3, 1), contrast=(0.3, 1), saturation=0.5),\n# #     Microscope(p=0.6),\n# #     transforms.Cutout(scale=(0.05, 0.007), value=(0, 0)),\n#     transforms.ToTensor(),\n#     transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n# ])\n\n# test_transform = transforms.Compose([\n# #     transforms.RandomResizedCrop(size=256, scale=(0.7, 1.0)),\n# #     transforms.Resize((new_im_dim, new_im_dim)),\n#     transforms.ToTensor(),\n#     transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n# ])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Functions to create weights for PyTorch RandomSampler - NOT USED\n\ndef generate_class_weights(df):\n    B = 0.4\n    C = np.array([(1 - B), B])*2\n    ones = len(df.query('target == 1'))\n    zeros = len(df.query('target == 0'))\n\n    weightage_fn = {0: 1-zeros/len(df), 1: 1-ones/len(df)}\n    return [weightage_fn[target] for target in df.target]\ndef covert_to_prob(r,NewMin=0.0,  NewMax=0.40):\n    OldMax = r.max()\n    OldMin = r.min()\n    OldRange = (OldMax - OldMin)  \n    NewRange = (NewMax - NewMin)  \n    NewValue = (((r - OldMin) * NewRange) / OldRange) + NewMin\n    return NewValue\n\ndef generate_anat_weights(train):\n    train['anatom_site_general_challenge']=train['anatom_site_general_challenge'].fillna('None')\n    site_counts = train['anatom_site_general_challenge'].value_counts()\n    print(site_counts)\n    site_probs = (1-site_counts/len(train)).to_dict()\n    anat_probs = train['anatom_site_general_challenge'].apply(lambda x: site_probs[x])\n    anat_weights=torch.Tensor(anat_probs.to_numpy())\n    \n    return covert_to_prob(anat_weights, NewMax=1, NewMin=0.30)\n\ndef generate_weights(df):\n    class_weights = torch.Tensor(generate_class_weights(df))\n#     anat_weights = generate_anat_weights(df)\n#     weights=class_weights * anat_weights\n    \n    return class_weights\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# To resmple dataframe prior to training. Using this resampled dataset slightly increased performance because all samples\n# are trained on here rather than in random sampling during training, where there is a chance that some examples are \n# are never or trained very less on \n    \ndef resample_df(df, targ_minority_frac=0.30, minority_class=1):\n    minority = df[df['target']==minority_class]\n    majority = df[df['target']!=minority_class]\n    minority_frac = len(minority)/len(df)\n    frac = targ_minority_frac/minority_frac\n    \n    minority = minority.sample(frac=frac, replace=True)\n    df = pd.concat([minority, majority])\n    return df.sample(frac=1).reset_index(drop=True)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pytorch_lightning.core.lightning import LightningModule\nfrom torch.functional import F\nfrom torch import optim\nfrom torch.optim.lr_scheduler import OneCycleLR\n\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n    def forward(self, inputs, targets):\n        BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduce=None)\n        pt = torch.exp(-BCE_loss)\n        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n        return torch.mean(F_loss)\n\n        \nclass LitModel(LightningModule):\n    def __init__(self, backbone, n_meta_features: int, total_steps=None, pos_weight=None):\n        super().__init__()\n        self.total_steps=total_steps\n        self.backbone = backbone\n        if 'ResNet' in str(backbone.__class__):\n            self.backbone.fc = nn.Linear(in_features=512, out_features=500, bias=True)\n        \n        if 'EfficientNet' in str(backbone.__class__):\n            print('Loading for EfficientNet')\n            in_features = self.backbone._fc.in_features\n            self.backbone._fc = nn.Linear(in_features=in_features, out_features=500, bias=True)\n            \n        \n        self.meta = nn.Sequential(nn.Linear(n_meta_features, 500),\n                                  nn.BatchNorm1d(500),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2),\n                                  nn.Linear(500, 250),  # FC layer output will have 250 features\n                                  nn.BatchNorm1d(250),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2))\n        \n        self.ouput = nn.Linear(500 + 250, 1)\n#         self.criterion = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([pos_weight]).cuda())\n        # changing gamma from 2 to 3 increased performance -> it penalizes the more frequent class lesser \n        self.criterion = FocalLoss(gamma=3)\n        \n    def forward(self, inputs):\n        x, meta = inputs\n        image_features = self.backbone(x)\n        meta_features = self.meta(meta)\n        features = torch.cat((image_features, meta_features), dim=-1)\n        output = self.ouput(features)\n        return output.squeeze(1)\n    \n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        y = y.float()\n        y_hat = self(x)\n        loss = self.criterion(y_hat, y)\n        \n        d = {'loss': loss}\n        d.update({'log': {'train_loss': loss}})\n\n        return d\n\n    def configure_optimizers(self):\n        optimizer= torch.optim.Adam(self.parameters(), lr=0.001)\n        default_config = {'interval': 'epoch',  # default every epoch\n                          'frequency': 1,  # default every epoch/batch\n                          'reduce_on_plateau': True,  # most often not ReduceLROnPlateau scheduler\n                          'monitor': 'val_auc'}  # default value to monitor for ReduceLROnPlateau\n#         lr_scheduler = {'scheduler': ReduceLROnPlateau(optimizer=optimizer, mode='max', patience=1, \n#                                                        verbose=True, factor=0.50),\n#                                                     'name': 'reduce_plateau', **default_config}\n        \n        lr_scheduler = {'scheduler': OneCycleLR(optimizer=optimizer, max_lr=1e-3, total_steps=self.total_steps,\n                                                div_factor=15), \n                                                'name': 'OneCycleLR', **default_config}\n        lr_scheduler.update({'interval': 'step', 'reduce_on_plateau': False})\n        \n        return [optimizer], [lr_scheduler]\n\n    \n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        y = y.float()\n        \n        y_hat = self(x)\n        return (y_hat.cpu(), y.cpu())\n    \n    def test_step(self, batch, batch_idx):\n        y_hat = self(batch)\n        \n        return {'preds': torch.sigmoid(y_hat).cpu()}\n    \n    def test_epoch_end(self, outputs):\n        self.preds = torch.cat([out['preds'] for out in outputs])\n        return {'None': [1,2,3]}\n        \n    def validation_epoch_end(self, outputs):\n        y_hat, y = [], []\n        for o in outputs:\n            y_hat.append(o[0])\n            y.append(o[1])\n\n        y_hat, y = torch.cat(y_hat, axis=0), torch.cat(y, axis=0)\n#         y[-1] = 1\n        loss = self.criterion(y_hat, y)\n        d = calculate_metrics(y_hat,y)\n        d.update({'val_loss': loss.item()}) \n        result = {'progress bar': d, 'log': d}\n        return result\n    \ndef calculate_metrics(outputs, true):\n    pred = torch.sigmoid(outputs)\n    pred, true = pred.cpu(), true.cpu().long()\n    val_acc = accuracy_score(true, pred.round())\n    val_roc = roc_auc_score(true,pred)\n    \n    return {f'val_acc': val_acc, 'val_auc': val_roc}\n\nfrom tqdm import tqdm_notebook\ndef predict_model(model, test_loader):\n    model.eval()\n    epoch_preds = []\n    for epochs in tqdm_notebook(range(4)):\n        preds=[]\n        for data in test_loader:\n            data = (data[0].cuda(), data[1].cuda())\n            with torch.no_grad(): \n                pred = torch.sigmoid(model(data))\n                preds.append(pred.cpu())\n\n        preds=torch.cat(preds, axis=0)\n        epoch_preds.append(preds.numpy())\n\n    preds = np.array(epoch_preds).mean(0)\n    return preds ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false},"cell_type":"code","source":"import sklearn \nfrom pytorch_lightning import Trainer, callbacks, loggers\n\nskf = StratifiedKFold(n_splits=6)\n\n# test_df = test_df.iloc[:20]\ntest_dataset = MelanomaDataset(df=test_df,\n                       imfolder=im_dir_test, \n                       train=False,\n                       transforms=test_transform,\n                       meta_features=meta_features,\n                              ext=ext)\nk_fold_preds = []\ni = 0\nBATCH_SIZE = 140//3\nnum_workers=0\ntest_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE*2, shuffle=False, num_workers=num_workers)\nepochs = 7\nnum_folds = 0\n\nfor train_idx, val_idx in skf.split(np.zeros(len(train_df)), y=np.zeros(len(train_df)), groups=train_df['target']):\n    train = train_df.iloc[train_idx]\n    valid = train_df.iloc[val_idx]\n    \n    train = resample_df(train)\n    pos_weight = (train['target']==0).sum()/(train['target']==1).sum()\n#     \"\"\"Remove\"\"\"\n#     valid = valid[:20]\n#     train = train[:20]\n    \n    valid_dataset = MelanomaDataset(df=valid,\n                        imfolder=im_dir_train, \n                        train=True,\n                        transforms=valid_transform,\n                        meta_features=meta_features, ext=ext)\n    \n    train_dataset = MelanomaDataset(df=train,\n                        imfolder=im_dir_train, \n                        train=True,\n                        transforms=train_transform,\n                        meta_features=meta_features, ext=ext)\n    \n#     weights = generate_weights(train)\n#     train_sampler = WeightedRandomSampler(weights, len(train))\n    \n    \n    train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, \n                              num_workers=num_workers, sampler=None)\n    valid_loader = DataLoader(dataset=valid_dataset, batch_size=BATCH_SIZE*2, shuffle=False, num_workers=num_workers)\n    \n    early_stop_callback = callbacks.EarlyStopping(monitor='val_auc',min_delta=0.00,patience=2,\n                                                  verbose=False,mode='max')\n    logger = loggers.TensorBoardLogger(save_dir=os.path.join(os.getcwd(), 'lightning_logs'),\n        version=None,name='224, b1, cuttout and microscope, focal gamma 3, stacked', comment='sampling')\n\n    checkpoint_callback = callbacks.ModelCheckpoint(filepath=logger.log_dir,save_top_k=3,\n        monitor='val_auc',mode='max',prefix='')\n    lr_logger = callbacks.LearningRateLogger()\n    # backbone = EfficientNet.from_name(\"efficientnet-b1\")\n    backbone = EfficientNet.from_pretrained(\"efficientnet-b1\")\n    model = LitModel(backbone=backbone, n_meta_features=len(meta_features), \n                     total_steps=math.ceil(len(train)/BATCH_SIZE)*epochs, pos_weight=pos_weight)\n    trainer = Trainer(gpus=1, num_nodes=1, num_sanity_val_steps=0,  max_epochs=epochs,\n                      callbacks=[early_stop_callback, lr_logger], \n                      checkpoint_callback=checkpoint_callback,\n                      logger=logger, log_save_interval=10, check_val_every_n_epoch=1, precision=32,\n                      weights_summary=None\n                     )\n    # train\n    trainer.fit(model, train_loader, valid_loader)\n    gc.collect()\n    # predict \n    trainer.test(test_dataloaders=test_loader)\n    preds = model.preds'\n    k_fold_preds.append(preds)\n    gc.collect()\n    num_folds+=1\n    if num_folds >= 3:\n        break\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"final_preds = np.array(k_fold_preds).mean(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"sub = pd.read_csv(csv_dir/'sample_submission.csv')\nsub['target'] = final_preds\nprint('Number of 1s:', (sub['target'] > 0.5).sum())\nprint('Number of 0s:', (sub['target'] < 0.5).sum())\nout_fname = os.path.join(logger.log_dir, 'submission_512.csv')\nsub.to_csv(out_fname, index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}