{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install pytorch_lightning torch_optimizer category_encoders efficientnet-pytorch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n## Library Install","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport os\nimport cv2\nimport glob\nimport random\nimport itertools\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nimport category_encoders as ce\n\nimport torch\nfrom torch import nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import RandomSampler, SequentialSampler\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nimport albumentations as albu\nfrom albumentations.pytorch import ToTensorV2\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom pytorch_lightning.metrics.classification import AUROC\nfrom torch_optimizer import RAdam\nfrom efficientnet_pytorch import EfficientNet\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('../input')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n## Utils","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = False\n\nSEED = 42\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n## Load Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_data(data_dir):\n    train = pd.read_csv(os.path.join(data_dir, 'train.csv'))\n    test = pd.read_csv(os.path.join(data_dir, 'test.csv'))\n    \n    # set fold\n    cv = GroupKFold(n_splits=5)\n    train['fold'] = -1\n    for i, (trn_idx, val_idx) in enumerate(cv.split(train, train['target'], groups=train['patient_id'].tolist())):\n        train.loc[val_idx, 'fold'] = i\n\n    img_paths = {\n        'train': glob.glob(os.path.join(data_dir, 'train', '*.jpg')),\n        'test': glob.glob(os.path.join(data_dir, 'test', '*.jpg'))\n    }\n    \n    return train, test, img_paths","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train, test, img_paths = load_data('../input/jpeg-melanoma-384x384')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n## Preprocessing MetaData","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocessing_meta(train, test):\n    train = train[['image_name', 'patient_id', 'sex', 'age_approx', 'anatom_site_general_challenge', 'target', 'fold']]\n    test = test[['image_name', 'patient_id', 'sex', 'age_approx', 'anatom_site_general_challenge']]\n    test.loc[:, 'target'] = 0\n    test.loc[:, 'fold'] = 0\n\n    # Preprocessing\n    train['age_approx'] /= train['age_approx'].max()\n    test['age_approx'] /= test['age_approx'].max()\n    train['age_approx'].fillna(0, inplace=True)\n    test['age_approx'].fillna(0, inplace=True)\n    for c in ['sex', 'anatom_site_general_challenge']:\n        train[c].fillna('Nodata', inplace=True)\n        test[c].fillna('Nodata', inplace=True)\n    encoder = ce.OneHotEncoder(cols=['sex', 'anatom_site_general_challenge'], handle_unknown='impute')\n    train = encoder.fit_transform(train)\n    test = encoder.transform(test)\n\n    test.drop(['target', 'fold'], axis=1, inplace=True)\n\n    return train, test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train, test = preprocessing_meta(train, test)\nfeatures_num = len([f for f in train.columns if f not in ['image_name', 'patient_id', 'target', 'fold']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n## Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    def __init__(self, df, img_paths, transform=None, phase='train'):\n        self.df = df\n        self.features = [f for f in self.df.columns if f not in ['image_name', 'patient_id', 'target', 'fold']]\n        self.img_paths = img_paths\n        self.transform = transform\n        self.phase = phase\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        if 'image_id' in self.df.columns:\n            img_name = row['image_id']\n        else:\n            img_name = row['image_name']\n\n        meta = row[self.features]\n        meta = torch.tensor(meta, dtype=torch.float32)\n\n        img_path = [path for path in self.img_paths if img_name in path][0]\n        img = cv2.imread(img_path, cv2.IMREAD_COLOR)\n\n        if self.transform is not None:\n            img = self.transform(img, self.phase)\n        else:\n            img = torch.from_numpy(img.transpose((2, 0, 1)))\n            img = img / 255.\n\n        if self.phase == 'test':\n            return img, meta, img_name\n        else:\n            label = row['target']\n            label = torch.tensor(label, dtype=torch.float)\n\n        return img, meta, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n## Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class ENet(nn.Module):\n    def __init__(self, output_size=1, model_name='efficientnet-b0', meta_features_num=11):\n        super(ENet, self).__init__()\n        self.enet = EfficientNet.from_name(model_name=model_name)\n        self.fc = nn.Sequential(\n            nn.Linear(in_features=meta_features_num, out_features=500),\n            nn.BatchNorm1d(500),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.2)\n        )\n        self.classification = nn.Linear(1500, out_features=output_size)\n\n    def forward(self, x, d):\n        out1 = self.enet(x)\n        out2 = self.fc(d)\n        out = torch.cat((out1, out2), dim=1)\n\n        out = self.classification(out)\n\n        return out","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n## Data Augmentations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageTransform:\n    def __init__(self, img_size=512, input_res=512, data_dir='./input'):\n        self.data_dir = data_dir\n        self.transform = {\n            'train': albu.Compose([\n                albu.ImageCompression(p=0.5),\n                albu.Rotate(limit=80, p=1.0),\n                albu.OneOf([\n                    albu.OpticalDistortion(),\n                    albu.GridDistortion(),\n                ]),\n                albu.RandomSizedCrop(min_max_height=(int(img_size * 0.7), input_res),\n                                     height=img_size, width=img_size, p=1.0),\n                albu.HorizontalFlip(p=0.5),\n                albu.VerticalFlip(p=0.5),\n                albu.GaussianBlur(p=0.3),\n                albu.OneOf([\n                    albu.RandomBrightnessContrast(),\n                    albu.HueSaturationValue(),\n                ]),\n                albu.CoarseDropout(max_holes=8, max_height=img_size // 8, max_width=img_size // 8, fill_value=0, p=0.3),\n                albu.Normalize(),\n                ToTensorV2(),\n            ], p=1.0),\n\n            'val': albu.Compose([\n                albu.CenterCrop(height=img_size, width=img_size, p=1.0),\n                albu.Normalize(),\n                ToTensorV2(),\n            ], p=1.0),\n\n            'test': albu.Compose([\n                albu.ImageCompression(p=0.5),\n                albu.RandomSizedCrop(min_max_height=(int(img_size * 0.9), input_res),\n                                     height=img_size, width=img_size, p=1.0),\n                albu.HorizontalFlip(p=0.5),\n                albu.VerticalFlip(p=0.5),\n                albu.Transpose(p=0.5),\n                albu.CoarseDropout(max_holes=8, max_height=img_size // 8, max_width=img_size // 8, fill_value=0, p=0.3),\n                albu.Normalize(),\n                ToTensorV2(),\n            ], p=1.0)\n        }\n\n    def __call__(self, img, phase='train'):\n        augmented = self.transform[phase](image=img)\n\n        return augmented['image']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n## Lightning System","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setting  #######################\nlabel_smoothing = 0.2\npos_weight = 3.1\n\n\nclass MelanomaSystem(pl.LightningModule):\n    def __init__(self, net, cfg, img_paths, train_df, test_df, transform):\n        super(MelanomaSystem, self).__init__()\n        self.net = net\n        self.cfg = cfg\n        self.img_paths = img_paths\n        self.train_df = train_df\n        self.test_df = test_df\n        self.transform = transform\n        self.criterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(pos_weight))\n        self.best_loss = 1e+9\n        self.best_auc = None\n        self.best_weight = None\n        self.auc_list = []\n        self.loss_list = []\n\n    def prepare_data(self):\n        # Split Train, Validation\n        fold = self.cfg['fold']\n        train = self.train_df[self.train_df['fold'] != fold].reset_index(drop=True)\n        val = self.train_df[self.train_df['fold'] == fold].reset_index(drop=True)\n\n        self.train_dataset = MelanomaDataset(train, self.img_paths['train'], self.transform, phase='train')\n        self.val_dataset = MelanomaDataset(val, self.img_paths['train'], self.transform, phase='val')\n        self.test_dataset = MelanomaDataset(self.test_df, self.img_paths['test'], self.transform, phase='test')\n\n    def train_dataloader(self):\n        return DataLoader(self.train_dataset,\n                          batch_size=self.cfg['batch_size'],\n                          pin_memory=True,\n                          sampler=RandomSampler(self.train_dataset), drop_last=True)\n\n    def val_dataloader(self):\n        return DataLoader(self.val_dataset,\n                          batch_size=self.cfg['batch_size'],\n                          pin_memory=True,\n                          sampler=SequentialSampler(self.val_dataset), drop_last=False)\n\n    def test_dataloader(self):\n        return DataLoader(self.test_dataset,\n                          batch_size=self.cfg['batch_size'],\n                          pin_memory=False,\n                          shuffle=False, drop_last=False)\n\n    def configure_optimizers(self):\n        self.optimizer = RAdam(self.parameters(), lr=self.cfg['lr'], weight_decay=2e-5)\n        self.scheduler = CosineAnnealingLR(self.optimizer, T_max=self.cfg['epoch_num'], eta_min=0)\n        return [self.optimizer], [self.scheduler]\n\n    def forward(self, x, d):\n        return self.net(x, d)\n\n    def step(self, batch):\n        inp, d, label = batch\n        out = self.forward(inp, d)\n\n        if label is not None:\n            # Label Smoothing\n            label_smo = label.float() * (1 - label_smoothing) + 0.5 * label_smoothing\n            loss = self.criterion(out, label_smo.unsqueeze(1))\n        else:\n            loss = None\n\n        return loss, label, torch.sigmoid(out)\n\n    def training_step(self, batch, batch_idx):\n        loss, label, logits = self.step(batch)\n        logs = {'train/loss': loss.item()}\n\n        return {'loss': loss, 'logits': logits, 'labels': label}\n\n    def validation_step(self, batch, batch_idx):\n        loss, label, logits = self.step(batch)\n        val_logs = {'val/loss': loss.item()}\n\n        return {'val_loss': loss, 'logits': logits.detach(), 'labels': label.detach()}\n\n    def validation_epoch_end(self, outputs):\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        LOGITS = torch.cat([x['logits'] for x in outputs])\n        LABELS = torch.cat([x['labels'] for x in outputs])\n\n        # Skip Sanity Check\n        auc = AUROC()(pred=LOGITS, target=LABELS) if LABELS.float().mean() > 0 else 0.5\n        logs = {'val/epoch_loss': avg_loss.item(), 'val/epoch_auc': auc}\n        \n        self.loss_list.append(avg_loss.item())\n        self.auc_list.append(auc)\n\n        return {'avg_val_loss': avg_loss}\n\n    def test_step(self, batch, batch_idx):\n        inp, d, img_name = batch\n        out = self.forward(inp, d)\n        logits = torch.sigmoid(out)\n\n        return {'preds': logits, 'image_names': img_name}\n\n    def test_epoch_end(self, outputs):\n        PREDS = torch.cat([x['preds'] for x in outputs]).reshape((-1)).detach().cpu().numpy()\n        # [tuple, tuple]\n        IMG_NAMES = [x['image_names'] for x in outputs]\n        # [list, list]\n        IMG_NAMES = [list(x) for x in IMG_NAMES]\n        IMG_NAMES = list(itertools.chain.from_iterable(IMG_NAMES))\n\n        res = pd.DataFrame({\n            'image_name': IMG_NAMES,\n            'target': PREDS\n        })\n\n        try:\n            res['target'] = res['target'].apply(lambda x: x.replace('[', '').replace(']', ''))\n        except:\n            pass\n        \n        N = len(glob.glob(f'submission_*.csv'))\n        filename = f'submission_{N}.csv'\n        res.to_csv(filename, index=False)\n        \n        return {'res': res}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n## Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# config\ncfg = {\n    'img_size': 256,\n    'batch_size': 64,\n    'epoch_num': 20,\n    'lr': 5e-5,\n    'fold': 0\n}\n\nif DEBUG:\n    train = train.sample(100)\n    test = test.sample(100)\n    cfg['epoch_num'] = 2\n\n\nnet = ENet(model_name='efficientnet-b2', meta_features_num=features_num)\ntransform = ImageTransform(img_size=cfg['img_size'], input_res=384)\n\nmodel = MelanomaSystem(net, cfg, img_paths, train, test, transform)\n\ncheckpoint_callback = ModelCheckpoint(\n    filepath='.',\n    save_top_k=1,\n    verbose=True,\n    monitor='avg_val_loss',\n    mode='min'\n)\n\ntrainer = Trainer(\n    max_epochs=cfg['epoch_num'],\n    checkpoint_callback=checkpoint_callback,\n    gpus=[0]\n    )","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# Train\ntrainer.fit(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,axes = plt.subplots(ncols=2, nrows=1, figsize=(16, 6))\naxes[0].plot(model.loss_list, color='b')\naxes[0].set_title('Val Loss')\naxes[0].set_xlabel('Epoch')\n\naxes[1].plot(model.auc_list, color='r')\naxes[1].set_title('Val roc_auc')\naxes[1].set_xlabel('Epoch')\n\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# Predict\nTTA_num = 3 if DEBUG else 20\n\n# Test\nfor i in range(TTA_num):\n    trainer.test(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n## Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def summarize_submit(sub_list, filename='submission.csv'):\n    res = pd.DataFrame()\n    for i, path in enumerate(sub_list):\n        sub = pd.read_csv(path)\n\n        if i == 0:\n            res['image_name'] = sub['image_name']\n            res['target'] = sub['target']\n        else:\n            res['target'] += sub['target']\n        os.remove(path)\n\n    # min-max norm\n    res['target'] -= res['target'].min()\n    res['target'] /= res['target'].max()\n\n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_list = glob.glob(f'submission_*.csv')\n\nres = summarize_submit(sub_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(16, 4))\nplt.hist(res['target'], bins=20)\nplt.title('Histgram of Predict')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}