{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install timm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport cv2\nimport torch.nn.init as init\nimport torch\nimport torch.nn as nn\nfrom PIL import Image, ImageFilter\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom torch.optim import Adam, SGD, RMSprop\nimport time\nfrom torch.autograd import Variable\nimport torch.functional as F\nfrom tqdm import tqdm\nfrom sklearn import metrics\nimport urllib\nimport pickle\nimport cv2\nimport torch.nn.functional as F\nfrom torchvision import models\nimport seaborn as sns\nimport random\nimport timm\nfrom sklearn.metrics import roc_auc_score\nimport sys\nsys.path.append('../input/autoaug')\nfrom auto_augment import AutoAugment, Cutout\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.benchmark = True\n    torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(2020)\nnum_classes = 2\nbs = 80\nlr = 1e-3\nIMG_SIZE = 224","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = '../input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/512x512-dataset-melanoma/'\ntest_path = '../input/melanoma-merged-external-data-512x512-jpeg/512x512-test/512x512-test/'\ntrain_csv = pd.read_csv('../input/melanoma-merged-external-data-512x512-jpeg/folds.csv')\ntest_csv = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\nsample = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyDataset(Dataset):\n    \n    def __init__(self, dataframe, transform=None, test=False):\n        self.df = dataframe\n        self.transform = transform\n        self.test = test\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        \n        label = self.df.target.values[idx]\n        p = self.df.image_id.values[idx]\n        \n        if self.test == False:\n            p_path = train_path + p + '.jpg'\n        else:\n            p_path = test_path + p + '.jpg'\n            \n#         image = cv2.imread(p_path)\n#         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.imread(p_path, cv2.IMREAD_COLOR)\n        image = image.astype(np.float32) / 255.0\n#         image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n#         image = transforms.ToPILImage()(image)\n        \n        if self.transform:\n            sample = {'image': image}\n            sample = self.transform(**sample)\n            image = sample['image']\n        \n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transform = A.Compose([\n            A.RandomSizedCrop(min_max_height=(200, 200), height=224, width=224, p=0.5),\n            A.RandomRotate90(p=0.5),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.Resize(height=224, width=224, p=1),\n            A.Cutout(num_holes=4, max_h_size=16, max_w_size=16, fill_value=0, p=0.5),\n            ToTensorV2(p=1.0),                  \n            ], p=1.0)\n\ntest_transform = A.Compose([\n            A.Resize(height=224, width=224, p=1.0),\n            ToTensorV2(p=1.0),\n            ], p=1.0)\n\n\ntestset      = MyDataset(sample, transform=test_transform, test=True)\ntest_loader  = torch.utils.data.DataLoader(testset, batch_size=bs, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter:\n    \"\"\"\n    Computes and stores the average and current value\n    \"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, epoch):\n    model.train() \n    \n    losses = AverageMeter()\n    avg_loss = 0.\n\n    optimizer.zero_grad()\n    \n    tk = tqdm(train_loader, total=len(train_loader), position=0, leave=True)\n    for idx, (imgs, labels) in enumerate(tk):\n        imgs_train, labels_train = imgs.cuda(), labels.cuda().long()\n        output_train = model(imgs_train)\n\n        loss = criterion(output_train, labels_train)\n        loss.backward()\n\n        optimizer.step() \n        optimizer.zero_grad() \n        \n        avg_loss += loss.item() / len(train_loader)\n        \n        losses.update(loss.item(), imgs_train.size(0))\n\n        tk.set_postfix(loss=losses.avg)\n        \n    return avg_loss\n\n\ndef test_model(model):    \n    model.eval()\n    \n    losses = AverageMeter()\n    avg_val_loss = 0.\n    \n    valid_preds, valid_targets = [], []\n    \n    with torch.no_grad():\n        tk = tqdm(val_loader, total=len(val_loader), position=0, leave=True)\n        for idx, (imgs, labels) in enumerate(tk):\n            imgs_valid, labels_valid = imgs.cuda(), labels.cuda().long()\n            output_valid = model(imgs_valid)\n            \n            loss = criterion(output_valid, labels_valid)\n            \n            avg_val_loss += loss.item() / len(val_loader)\n\n            losses.update(loss.item(), imgs_valid.size(0))\n            \n            tk.set_postfix(loss=losses.avg)\n            \n            valid_preds.append(torch.softmax(output_valid,1)[:,1].detach().cpu().numpy())\n            valid_targets.append(labels_valid.detach().cpu().numpy())\n            \n        valid_preds = np.concatenate(valid_preds)\n        valid_targets = np.concatenate(valid_targets)\n        auc =  roc_auc_score(valid_targets, valid_preds) \n            \n    return avg_val_loss, auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kf = StratifiedKFold(5, shuffle=True, random_state=0)\n\ncv = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(5):\n    fold = i+1\n    print('fold:', fold)\n\n    train_df = train_csv[train_csv['fold'] != i]\n    val_df = train_csv[train_csv['fold'] == i]\n    train_df.reset_index(drop=True, inplace=True)\n    val_df.reset_index(drop=True, inplace=True)\n\n    trainset = MyDataset(train_df, transform=train_transform)\n    train_loader = torch.utils.data.DataLoader(trainset, batch_size=bs, shuffle=True, num_workers=4)\n   \n    valset = MyDataset(val_df, transform=test_transform)\n    val_loader = torch.utils.data.DataLoader(valset, batch_size=bs, shuffle=False, num_workers=4)\n\n    model = timm.create_model('tf_efficientnet_b2_ns', pretrained=True, num_classes=num_classes)\n    model.cuda()\n\n    optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=0.001)\n    criterion = nn.CrossEntropyLoss()\n    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.5)\n#     scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n#         optimizer,\n#         patience=1,\n#         threshold=1e-4,\n#         mode=\"max\"\n#     )\n    \n    best_auc = 0\n    n_epochs = 10\n    es = 0\n\n    for epoch in range(n_epochs):\n        avg_loss = train_model(model, epoch)\n        avg_val_loss, auc = test_model(model)\n\n        if auc > best_auc:\n            es = 0\n            best_auc = auc\n            torch.save(model.state_dict(), str(fold) + 'weight.pt')\n        else:\n            es += 1\n            if es > 2:\n                break\n        print('current_val_auc:', auc, 'best_val_auc:', best_auc)\n        \n        scheduler.step(auc)\n\n    cv.append(best_auc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(cv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model1 = timm.create_model('tf_efficientnet_b2_ns', pretrained=True, num_classes=num_classes)\nmodel1.cuda()\nmodel1.load_state_dict(torch.load(\"./1weight.pt\"))\n\nmodel2 = timm.create_model('tf_efficientnet_b2_ns', pretrained=True, num_classes=num_classes)\nmodel2.cuda()\nmodel2.load_state_dict(torch.load(\"./2weight.pt\"))\n\nmodel3 = timm.create_model('tf_efficientnet_b2_ns', pretrained=True, num_classes=num_classes)\nmodel3.cuda()\nmodel3.load_state_dict(torch.load(\"./3weight.pt\"))\n\nmodel4 = timm.create_model('tf_efficientnet_b2_ns', pretrained=True, num_classes=num_classes)\nmodel4.cuda()\nmodel4.load_state_dict(torch.load(\"./4weight.pt\"))\n\nmodel5 = timm.create_model('tf_efficientnet_b2_ns', pretrained=True, num_classes=num_classes)\nmodel5.cuda()\nmodel5.load_state_dict(torch.load(\"./5weight.pt\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"model1.eval()\nmodel2.eval()\nmodel3.eval()\nmodel4.eval()\nmodel5.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pred = np.zeros((len(sample),))\n\nwith torch.no_grad():\n    for i, data in enumerate(tqdm(test_loader, position=0, leave=True)):\n        images, _ = data\n        images = images.cuda()\n        \n        pred = (model1(images) + model2(images) + model3(images) + model4(images) + model5(images)) \n        \n        pred = torch.softmax(pred,1).cpu().detach().numpy()[:,1]\n    \n        test_pred[i*bs: (i+1)*bs] = pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test_pred[:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.target = test_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}