{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install timm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport cv2\nimport torch.nn.init as init\nimport torch\nimport torch.nn as nn\nfrom torch.nn.modules.loss import _WeightedLoss\nfrom PIL import Image, ImageFilter\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom torch.optim import Adam, SGD, RMSprop\nimport time\nfrom torch.autograd import Variable\nimport torch.functional as F\nfrom tqdm import tqdm\nfrom sklearn import metrics\nimport urllib\nimport pickle\nimport cv2\nimport torch.nn.functional as F\nfrom torchvision import models\nimport seaborn as sns\nimport random\nimport timm\nfrom sklearn.metrics import roc_auc_score\nimport sys\nsys.path.append('../input/autoaug')\nfrom auto_augment import AutoAugment, Cutout\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom catalyst.data.sampler import BalanceClassSampler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.benchmark = True\n    torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(0)\nnum_classes = 2\nbs = 32\nlr = 1e-4\nIMG_SIZE = 256","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = '../input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/512x512-dataset-melanoma/'\ntest_path = '../input/melanoma-merged-external-data-512x512-jpeg/512x512-test/512x512-test/'\ntrain_csv = pd.read_csv('../input/melanoma-merged-external-data-512x512-jpeg/folds_13062020.csv')\ntest_csv = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\nsample = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyDataset(Dataset):\n    \n    def __init__(self, dataframe, transform=None, test=False):\n        self.df = dataframe\n        self.transform = transform\n        self.test = test\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        \n        label = self.df.target.values[idx]\n        \n        \n        if self.test == False:\n            p = self.df.image_id.values[idx]\n            p_path = train_path + p + '.jpg'\n        else:\n            p = self.df.image_name.values[idx]\n            p_path = test_path + p + '.jpg'\n            \n        image = cv2.imread(p_path, cv2.IMREAD_COLOR)\n        image = image.astype(np.float32) / 255.0\n#         image = transforms.ToPILImage()(image)\n        \n        if self.transform:\n            sample = {'image': image}\n            sample = self.transform(**sample)\n            image = sample['image']\n        \n        return image, label\n    \n    def get_labels(self):\n        return list(self.df.target.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AdvancedHairAugmentation:\n    def __init__(self, hairs: int = 4, hairs_folder: str = \"\"):\n        self.hairs = hairs\n        self.hairs_folder = hairs_folder\n\n    def __call__(self, img):\n        n_hairs = random.randint(0, self.hairs)\n\n        if not n_hairs:\n            return img\n\n        height, width, _ = img.shape  # target image width and height\n        hair_images = [im for im in os.listdir(self.hairs_folder) if 'png' in im]\n\n        for _ in range(n_hairs):\n            hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))\n            hair = cv2.flip(hair, random.choice([-1, 0, 1]))\n            hair = cv2.rotate(hair, random.choice([0, 1, 2]))\n\n            h_height, h_width, _ = hair.shape  # hair image width and height\n            roi_ho = random.randint(0, img.shape[0] - hair.shape[0])\n            roi_wo = random.randint(0, img.shape[1] - hair.shape[1])\n            roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]\n\n            img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)\n            ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)\n            mask_inv = cv2.bitwise_not(mask)\n            img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n            hair_fg = cv2.bitwise_and(hair, hair, mask=mask)\n\n            dst = cv2.add(img_bg, hair_fg)\n            img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst\n\n        return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transform = A.Compose([\n#             AdvancedHairAugmentation(hairs_folder='/kaggle/input/melanoma-hairs/'),\n            A.RandomSizedCrop(min_max_height=(200, 200), height=256, width=256, p=0.5),\n            A.RandomRotate90(p=0.5),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.Resize(height=256, width=256, p=1),\n            A.Cutout(num_holes=4, max_h_size=16, max_w_size=16, fill_value=0, p=0.5),\n            ToTensorV2(p=1.0),                  \n            ], p=1.0)\n\ntest_transform = A.Compose([\n            A.Resize(height=256, width=256, p=1.0),\n            ToTensorV2(p=1.0),\n            ], p=1.0)\n\n\ntestset      = MyDataset(sample, transform=test_transform, test=True)\ntest_loader  = torch.utils.data.DataLoader(testset, batch_size=bs, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n    return bbx1, bby1, bbx2, bby2\n\ndef cutmix(data, targets, alpha):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_targets = targets[indices]\n    \n    lam = np.random.beta(alpha, alpha)\n    bbx1, bby1, bbx2, bby2 = rand_bbox(data.size(), lam)\n    data[:, :, bbx1:bbx2, bby1:bby2] = data[indices, :, bbx1:bbx2, bby1:bby2]\n    # adjust lambda to exactly match pixel ratio\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (data.size()[-1] * data.size()[-2]))\n\n    targets = [targets, shuffled_targets, lam]\n    return data, targets\n\n# loss 变化\ndef cutmix_criterion(preds, targets):\n    targets1, targets2, lam = targets[0], targets[1], targets[2]\n    #criterion = nn.CrossEntropyLoss(reduction='mean')\n    return lam * criterion(preds, targets1) + (1 - lam) * criterion(preds, targets2)\n\n\ndef mixup(data, targets, alpha):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_targets = targets[indices]\n    \n    lam = np.random.beta(alpha, alpha)\n    data = data * lam + shuffled_data * (1 - lam)\n    targets1 = [targets, shuffled_targets, lam]\n\n    return data, targets1\n\ndef mixup_criterion(preds, targets):\n    targets1, targets2, lam = targets[0], targets[1], targets[2]\n    #criterion = nn.CrossEntropyLoss(reduction='mean')\n    return lam * criterion(preds, targets1) + (1 - lam) * criterion(preds, targets2)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SmoothCrossEntropyLoss(_WeightedLoss):\n    def __init__(self, weight=None, reduction='mean', smoothing=0.1):\n        super().__init__(weight=weight, reduction=reduction)\n        self.smoothing = smoothing\n        self.weight = weight\n        self.reduction = reduction\n\n    @staticmethod\n    def _smooth_one_hot(targets: torch.Tensor, n_classes: int, smoothing=0.1):\n        assert 0 <= smoothing < 1\n        with torch.no_grad():\n            targets = torch.empty(size=(targets.size(0), n_classes),\n                                  device=targets.device) \\\n                .fill_(smoothing / (n_classes - 1)) \\\n                .scatter_(1, targets.data.unsqueeze(1), 1. - smoothing)\n        return targets\n\n    def forward(self, inputs, targets):\n        targets = SmoothCrossEntropyLoss._smooth_one_hot(targets, inputs.size(-1),\n                                                         self.smoothing)\n        lsm = F.log_softmax(inputs, -1)\n\n        if self.weight is not None:\n            lsm = lsm * self.weight.unsqueeze(0)\n\n        loss = -(targets * lsm).sum(-1)\n\n        if self.reduction == 'sum':\n            loss = loss.sum()\n        elif self.reduction == 'mean':\n            loss = loss.mean()\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def margin_focal_binary_cross_entropy(logit, truth):\n    weight_pos=2\n    weight_neg=1\n    gamma=2\n    margin=0.2\n    em = np.exp(margin)\n    \n    l1 = nn.CrossEntropyLoss()(logit, truth)\n    \n    logit = logit[:,1].view(-1)\n    truth = truth.view(-1)\n    log_pos = -F.logsigmoid( logit)\n    log_neg = -F.logsigmoid(-logit)\n\n    log_prob = truth*log_pos + (1-truth)*log_neg\n    prob = torch.exp(-log_prob)\n    margin = torch.log(em +(1-em)*prob)\n\n    weight = truth*weight_pos + (1-truth)*weight_neg\n    loss = margin + weight*(1 - prob) ** gamma * log_prob\n\n    loss = loss.mean()\n    \n    loss - loss*0.5+l1*0.5\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter:\n    \"\"\"\n    Computes and stores the average and current value\n    \"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, epoch):\n    model.train() \n    \n    losses = AverageMeter()\n    avg_loss = 0.\n\n    optimizer.zero_grad()\n    \n    tk = tqdm(train_loader, total=len(train_loader), position=0, leave=True)\n    for idx, (imgs, labels) in enumerate(tk):\n        imgs_train, labels_train = imgs.cuda(), labels.cuda().long()\n        \n        if np.random.rand()<0.5:\n            with torch.no_grad():\n                imgs_train, targets = mixup(imgs_train, labels_train, 0.2)\n\n            output_train = model(imgs_train)\n            loss = mixup_criterion(output_train, targets) \n        else: # elif np.random.rand()<0.75:\n            with torch.no_grad():\n                imgs_train, targets = cutmix(imgs_train, labels_train, 1.0)\n            output_train = model(imgs_train)\n            loss = mixup_criterion(output_train, targets) \n            \n#         output_train = model(imgs_train)\n#         loss = criterion(output_train, labels_train)\n        loss.backward()\n\n        optimizer.step() \n        optimizer.zero_grad() \n        \n        avg_loss += loss.item() / len(train_loader)\n        \n        losses.update(loss.item(), imgs_train.size(0))\n\n        tk.set_postfix(loss=losses.avg)\n        \n    return avg_loss\n\n\ndef test_model(model):    \n    model.eval()\n    \n    losses = AverageMeter()\n    avg_val_loss = 0.\n    \n    valid_preds, valid_targets = [], []\n    \n    with torch.no_grad():\n        tk = tqdm(val_loader, total=len(val_loader), position=0, leave=True)\n        for idx, (imgs, labels) in enumerate(tk):\n            imgs_valid, labels_valid = imgs.cuda(), labels.cuda().long()\n            output_valid = model(imgs_valid)\n            \n            loss = criterion(output_valid, labels_valid)\n            \n            avg_val_loss += loss.item() / len(val_loader)\n\n            losses.update(loss.item(), imgs_valid.size(0))\n            \n            tk.set_postfix(loss=losses.avg)\n            \n            valid_preds.append(torch.softmax(output_valid,1)[:,1].detach().cpu().numpy())\n            valid_targets.append(labels_valid.detach().cpu().numpy())\n            \n        valid_preds = np.concatenate(valid_preds)\n        valid_targets = np.concatenate(valid_targets)\n        auc =  roc_auc_score(valid_targets, valid_preds) \n            \n    return avg_val_loss, auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(1):\n    fold = i+1\n    print('fold:', fold)\n\n    train_df = train_csv[(train_csv['fold'] != i)]\n    val_df   = train_csv[(train_csv['fold'] == i) & (train_csv['source'] == 'ISIC20')]\n    train_df.reset_index(drop=True, inplace=True)\n    val_df.reset_index(drop=True, inplace=True)\n\n    trainset = MyDataset(train_df, transform=train_transform)\n    train_loader = torch.utils.data.DataLoader(trainset, batch_size=bs, shuffle=True,\n#                 sampler=BalanceClassSampler(labels=trainset.get_labels(), mode=\"downsampling\"),\n                num_workers=4)\n   \n    valset = MyDataset(val_df, transform=test_transform)\n    val_loader = torch.utils.data.DataLoader(valset, batch_size=bs, shuffle=False, num_workers=4)\n\n    model = timm.create_model('resnext50_32x4d', pretrained=True, num_classes=num_classes)\n    model.cuda()\n\n    optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4)\n#     criterion = nn.CrossEntropyLoss()\n    criterion = margin_focal_binary_cross_entropy\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n        optimizer,\n        patience=1,\n        threshold=1e-4,\n        mode=\"max\"\n    )\n    \n    best_auc = 0\n    n_epochs = 25\n\n    for epoch in range(n_epochs):\n        avg_loss = train_model(model, epoch)\n        avg_val_loss, auc = test_model(model)\n\n        if auc > best_auc:\n            best_auc = auc\n            torch.save(model.state_dict(), str(fold) + 'weight.pt')\n       \n        print('current_val_auc:', auc, 'best_val_auc:', best_auc)\n        \n        scheduler.step(auc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(best_auc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model1 = timm.create_model('resnext50_32x4d', pretrained=True, num_classes=num_classes)\nmodel1.cuda()\nmodel1.load_state_dict(torch.load(\"./1weight.pt\"))\n\n# model2 = timm.create_model('resnext50_32x4d', pretrained=True, num_classes=num_classes)\n# model2.cuda()\n# model2.load_state_dict(torch.load(\"./2weight.pt\"))\n\n# model3 = timm.create_model('resnext50_32x4d', pretrained=True, num_classes=num_classes)\n# model3.cuda()\n# model3.load_state_dict(torch.load(\"./3weight.pt\"))\n\n# model4 = timm.create_model('resnext50_32x4d', pretrained=True, num_classes=num_classes)\n# model4.cuda()\n# model4.load_state_dict(torch.load(\"./4weight.pt\"))\n\n# model5 = timm.create_model('resnext50_32x4d', pretrained=True, num_classes=num_classes)\n# model5.cuda()\n# model5.load_state_dict(torch.load(\"./5weight.pt\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"model1.eval()\n# model2.eval()\n# model3.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# models = [model1, model2, model3]\n# ensemble = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for j in range(len(models)):\n    test_pred = np.zeros((len(sample),))\n\n    with torch.no_grad():\n        for i, data in enumerate(tqdm(test_loader, position=0, leave=True)):\n            images, _ = data\n            images = images.cuda()\n\n            pred = model1(images)\n            \n            pred = torch.softmax(pred,1).cpu().detach().numpy()[:,1]\n\n            test_pred[i*bs: (i+1)*bs] = pred\n\n    sample.target = test_pred\n    sample.to_csv('submission1.csv',index=False)\n    \n#     ensemble.append(test_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ensemble = np.array(ensemble).mean(0)\n\n# sample.target = ensemble\n# sample.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}