{"cells":[{"metadata":{},"cell_type":"markdown","source":"### import modules\n---","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport random\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom pathlib import Path\nfrom PIL import Image, ImageEnhance, ImageOps, ImageDraw\nfrom torchvision import transforms, models\nfrom efficientnet_pytorch import EfficientNet\nimport torch\nfrom torch import optim\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.data.sampler import WeightedRandomSampler\nfrom torch import cuda","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### define functions\n---","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#--------------------\n# utils\n#--------------------\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \n\nclass AverageMeter():\n    def __init__(self):\n        self.reset()\n        \n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n        \n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#--------------------\n# augmentations\n#--------------------\nclass UniformAugment():\n    def __init__(self, NumOps=2, fillcolor=(128, 128, 128)):\n        self.NumOps = NumOps\n        self.augs = {\n            'shearX': [-0.3, 0.3],\n            'shearY': [-0.3, 0.3],\n            'translateX': [-0.45, 0.45],\n            'translateY': [-0.45, 0.45],\n            'rotate': [-30, 30],\n            'autocontrast': [0, 0],\n            'invert': [0, 0],\n            'equalize': [0, 0],\n            'solarize': [0, 256],\n            'posterize': [4, 8],\n            'contrast': [0.1, 1.9],\n            'color': [0.1, 1.9],\n            'brightness': [0.1, 1.9],\n            'sharpness': [0.1, 1.9],\n            'cutout': [0, 0.2] \n        }\n\n        def rotate_with_fill(img, magnitude):\n            rot = img.convert('RGBA').rotate(magnitude)\n            return Image.composite(rot, Image.new('RGBA', rot.size, (128,) * 4), rot).convert(img.mode)\n\n        def cutout(img, magnitude, fillcolor):\n            img = img.copy()\n            w, h = img.size\n            v = w * magnitude\n            x0 = np.random.uniform(w)\n            y0 = np.random.uniform(h)\n            x0 = int(max(0, x0 - v / 2.))\n            y0 = int(max(0, y0 - v / 2.))\n            x1 = min(w, x0 + v)\n            y1 = min(h, y0 + v)\n            xy = (x0, y0, x1, y1)\n            ImageDraw.Draw(img).rectangle(xy, fillcolor)\n            return img\n\n        self.func = {\n            'shearX': lambda img, magnitude: img.transform(\n                img.size, Image.AFFINE, (1, magnitude, 0, 0, 1, 0), Image.BICUBIC, fillcolor=fillcolor),\n            'shearY': lambda img, magnitude: img.transform(\n                img.size, Image.AFFINE, (1, 0, 0, magnitude, 1, 0), Image.BICUBIC, fillcolor=fillcolor),\n            'translateX': lambda img, magnitude: img.transform(\n                img.size, Image.AFFINE, (1, 0, magnitude, 0, 1, 0), fillcolor=fillcolor),\n            'translateY': lambda img, magnitude: img.transform(\n                img.size, Image.AFFINE, (1, 0, 0, 0, 1, magnitude), fillcolor=fillcolor),\n            'rotate': lambda img, magnitude: rotate_with_fill(img, magnitude),\n            'color': lambda img, magnitude: ImageEnhance.Color(img).enhance(magnitude),\n            'posterize': lambda img, magnitude: ImageOps.posterize(img, int(magnitude)),\n            'solarize': lambda img, magnitude: ImageOps.solarize(img, int(magnitude)),\n            'contrast': lambda img, magnitude: ImageEnhance.Contrast(img).enhance(magnitude),\n            'sharpness': lambda img, magnitude: ImageEnhance.Sharpness(img).enhance(magnitude),\n            'brightness': lambda img, magnitude: ImageEnhance.Brightness(img).enhance(magnitude),\n            'autocontrast': lambda img, magnitude: ImageOps.autocontrast(img),\n            'equalize': lambda img, magnitude: ImageOps.equalize(img),\n            'invert': lambda img, magnitude: ImageOps.invert(img),\n            'cutout': lambda img, magnitude: cutout(img, magnitude, fillcolor=fillcolor)\n        }\n\n    def __call__(self, img):\n        operations = random.sample(list(self.augs.items()), self.NumOps)\n        for operation in operations:\n            aug, range = operation\n            magnitude = random.uniform(range[0], range[1])\n            probability = random.random()\n            if random.random() < probability:\n                img = self.func[aug](img, magnitude)\n        return img\n  \n  \nclass ImageTransform():\n    def __init__(self, resize, uniform_augment, mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), train=True):\n        self.data_transform = {\n            'train': transforms.Compose([\n                transforms.RandomResizedCrop(size=resize, scale=(0.7, 1.0)),\n                transforms.RandomHorizontalFlip(),\n                transforms.RandomVerticalFlip(),\n                transforms.ToTensor(),\n                transforms.Normalize(mean, std)\n            ]),\n            'valid': transforms.Compose([\n                transforms.ToTensor(),\n                transforms.Normalize(mean, std)\n            ]),\n            'test': transforms.Compose([\n                transforms.RandomResizedCrop(size=resize, scale=(0.7, 1.0)),\n                transforms.RandomHorizontalFlip(),\n                transforms.RandomVerticalFlip(),\n                transforms.ToTensor(),\n                transforms.Normalize(mean, std)\n            ])\n        }\n        if uniform_augment:\n            self.data_transform['train'].transforms.insert(0, UniformAugment())\n            self.data_transform['test'].transforms.insert(0, UniformAugment())\n            \n    def __call__(self, img, phase):\n        return self.data_transform[phase](img=img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#--------------------\n# dataset\n#--------------------\nclass MelanomaDataset(Dataset):\n    def __init__(self, base_dir, info, transform=None, phase='train'):\n        self.base_dir = base_dir\n        self.info = info\n        self.transform = transform\n        self.phase = phase\n\n    def __len__(self):\n        return len(self.info)\n\n    def __getitem__(self, index):\n        p = Path(self.base_dir, self.info.loc[index, 'image_name'] + '.jpg')\n        img = Image.open(p)\n        img_transformed = self.transform(img, self.phase)\n        return {\n            'inputs': img_transformed,\n            'labels': torch.tensor(self.info.loc[index, 'target'], dtype=torch.int64)\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#--------------------\n# models\n#--------------------\ndef load_model(model, out_features):\n    net = None\n    if model.startswith('efficientnet'):\n        net = EfficientNet.from_pretrained(model, num_classes=out_features)\n        for name, param in net.named_parameters():\n            if '_fc' in name:\n                param.requires_grad = True\n            else:\n                param.requires_grad = False                \n                \n    elif model == 'vgg19':\n        net = models.vgg19(pretrained=True)\n        net.classifier[6] = nn.Linear(in_features=net.classifier[6].weight.size(1), out_features=out_features)\n        for name, param in net.named_parameters():\n            if name in ['classifier.6.weight', 'classifier.6.bias']:\n                param.requires_grad = True\n            else:\n                param.requires_grad = False\n\n    elif model == 'resnext':\n        net = models.resnext101_32x8d(pretrained=True)\n        net.fc = nn.Linear(in_features=net.fc.weight.size(1), out_features=out_features)\n        for name, param in net.named_parameters():\n            if 'fc' in name:\n                param.requires_grad = True\n            else:\n                param.requires_grad = False\n                \n    elif model == 'densenet':\n        net = models.densenet161(pretrained=True)\n        net.classifier = nn.Linear(in_features=net.classifier.weight.size(1), out_features=out_features)\n        for name, param in net.named_parameters():\n            if 'classifier' in name:\n                param.requires_grad = True\n            else:\n                param.requires_grad = False\n\n    return net","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#--------------------\n# train and evaluate\n#--------------------\ndef eval_model(model, dataset, batch_size, criterion, num_workers, device=None):\n    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n    losses = AverageMeter()\n    preds = []\n    targets = []\n    with torch.no_grad():\n        for data in dataloader:\n            inputs = data['inputs'].to(device)\n            labels = data['labels'].to(device)\n\n            # forward\n            outputs = model(inputs)\n\n            # loss\n            loss = criterion(outputs, labels)\n            losses.update(loss.item(), inputs.size(0))\n\n            # prediction\n            preds.append(outputs[:, 1])\n            targets.append(labels)\n            \n        # auc\n        preds = torch.cat(preds, dim=-1).cpu().numpy()\n        targets = torch.cat(targets, dim=-1).cpu().numpy()\n        auc = roc_auc_score(targets, preds)\n      \n    return losses.avg, auc\n  \n\ndef train_model(\n    model_id, \n    dataset_train, \n    dataset_valid, \n    batch_size, \n    model, \n    criterion, \n    optimizer, \n    scheduler,\n    num_epochs, \n    freezed_epochs,\n    base_dir, \n    num_workers, \n    sampler, \n    device, \n    early_stopping\n    ):\n\n    model.to(device)\n    \n    # create a dataloader\n    if sampler != None:\n        dataloader_train = DataLoader(dataset_train, batch_size=batch_size, sampler=sampler, num_workers=num_workers)\n    else:\n        dataloader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n\n    # train\n    for epoch in range(1, num_epochs + 1):\n        losses = AverageMeter()\n        s_time = time.time()\n        \n        if epoch == freezed_epochs + 1:\n            # unfreeze upstream layers\n            model.load_state_dict(torch.load(f'./state_dict_{model_id}.pt', map_location=device))\n            for param in model.parameters():\n                param.requires_grad = True\n            for g in optimizer.param_groups:\n                g['lr'] = 1e-4\n\n        # set the train mode\n        model.train()\n        \n        for data in dataloader_train:\n            # zero the parameter gradients\n            optimizer.zero_grad()\n                        \n            # forward + backward + optimize\n            inputs = data['inputs'].to(device)\n            labels = data['labels'].to(device)\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            losses.update(loss.item(), inputs.size(0))\n        \n        # set the eval mode\n        model.eval()\n\n        # calculate validation loss and auc\n        loss_valid, auc_valid = eval_model(model, dataset_valid, batch_size, criterion, num_workers, device)\n\n        # save the checkpoint\n        if epoch == 1 or auc_valid > max_auc:\n            saved = True\n            max_auc = auc_valid\n            torch.save(model.state_dict(), f'./state_dict_{model_id}.pt')\n            counter = 0\n        else:\n            saved = False\n            counter += 1\n\n        # print statistics\n        e_time = time.time()\n        print(f'epoch: {epoch}, loss_train: {losses.avg:.4f}, loss_valid: {loss_valid:.4f}, auc_valid: {auc_valid:.4f}, saved: {saved}, {(e_time - s_time):.4f}sec') \n\n        # no operation in freezed epochs\n        if epoch < freezed_epochs + 1:\n            counter = 0\n        else:\n            # step the scheduler            \n            scheduler.step(auc_valid)\n    \n            # early stopping\n            if early_stopping != None:\n                if counter == early_stopping:\n                    break\n        \n        \ndef predict(dataset, batch_size, model, device=None):\n    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False)\n    model.to(device)\n    model.eval()\n    preds = []\n    with torch.no_grad():\n        for data in dataloader:\n            inputs = data['inputs'].to(device)\n            outputs = model(inputs)\n            preds.append(outputs)\n        preds = torch.cat(preds)\n\n    return preds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### main\n---","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# set parameters\nseed_everything(123)\ntorch.backends.cudnn.benchmark = True\n\nconfig = {\n    'INPUT_DIR'      : '../input/jpeg-melanoma-256x256',\n    'MODEL'          : 'efficientnet-b0',\n    'SIZE'           : 256,\n    'BATCH_SIZE'     : 128,\n    'NUM_FOLDS'      : 5,\n    'NUM_EPOCHS'     : 20,\n    'FREEZED_EPOCHS' : 5,\n    'LEARNING_RATE'  : 1e-3,\n    'EARLY_STOPPING' : 3,\n    'UNIFORM_AUGMENT': True,\n    'TTA'            : 5,\n    'NUM_WORKERS'    : 4,\n    'DEVICE'         : 'cuda'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load data\ntrain = pd.read_csv(Path(config['INPUT_DIR'], 'train.csv'))\ntest = pd.read_csv(Path(config['INPUT_DIR'], 'test.csv'))\nsub = pd.read_csv(Path(config['INPUT_DIR'], 'sample_submission.csv'))\n\n# define transformer\ntransform = ImageTransform(config['SIZE'], config['UNIFORM_AUGMENT'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cross validation\nprediction = pd.DataFrame()\nskf = StratifiedKFold(n_splits=config['NUM_FOLDS'], shuffle=True)\nfor i, (tr_idx, va_idx) in enumerate(skf.split(train, train['target'])):\n    tr = train.loc[tr_idx, :]\n    va = train.loc[va_idx, :]\n    tr.reset_index(drop=True, inplace=True)\n    va.reset_index(drop=True, inplace=True)\n\n    # create datasets\n    dataset_train = MelanomaDataset(Path(config['INPUT_DIR'], 'train'), tr, transform=transform, phase='train')\n    dataset_valid = MelanomaDataset(Path(config['INPUT_DIR'], 'train'), va, transform=transform, phase='valid')\n    \n    # load a pretrained model\n    net = load_model(config['MODEL'], 2)\n\n    # define a loss function\n    criterion = nn.CrossEntropyLoss()\n\n    # define an optimizer\n    optimizer = optim.Adam(net.parameters(), lr=config['LEARNING_RATE'])\n\n    # define a scheduler\n    scheduler = ReduceLROnPlateau(optimizer=optimizer, mode='max', patience=2, factor=0.2)\n\n    # create a sampler\n    class_sample_count = np.array([len(np.where(tr['target'] == t)[0]) for t in np.unique(tr['target'])])\n    weight = 1. / class_sample_count\n    samples_weight = np.array([weight[t] for t in tr['target']])\n    sampler = WeightedRandomSampler(samples_weight, len(samples_weight))\n\n    # train the network\n    print(f\"---- fold: {i + 1} ------------\")\n    train_model(\n        f\"{config['MODEL']}_{i + 1}\",\n        dataset_train,\n        dataset_valid,\n        config['BATCH_SIZE'],\n        net,\n        criterion,\n        optimizer,\n        scheduler,\n        config['NUM_EPOCHS'],\n        config['FREEZED_EPOCHS'],\n        config['INPUT_DIR'],\n        config['NUM_WORKERS'],\n        sampler,\n        config['DEVICE'],\n        config['EARLY_STOPPING']\n    )\n    \n    # predict on test dataset\n    test['target'] = 0\n    dataset_test = MelanomaDataset(Path(config['INPUT_DIR'], 'test'), test, transform=transform, phase='test')\n    for _ in range(config['TTA']):                                        \n        pred_test = predict(dataset_test, config['BATCH_SIZE'], net, device=config['DEVICE'])\n        pred_test = pd.DataFrame(torch.softmax(pred_test, 1)[:, 1].cpu().numpy())\n        prediction = pd.concat([prediction, pred_test], axis=1)\n    \n# output\nsub['target'] = prediction.mean(axis=1)\nsub.to_csv('./submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}