{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 1. Install","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Import Library","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn\nimport gc\nimport cv2\nimport random\nfrom collections import defaultdict\n\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch import optim\nimport torchvision\nfrom torchvision import transforms\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score,roc_auc_score,log_loss\n\nfrom efficientnet_pytorch import EfficientNet\n\nimport sys\nsys.path.append('../input/autoaug')\nfrom auto_augment import AutoAugment\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.benchmark = True\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(2020)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Config","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    train_batch_size = 64\n    test_batch_size = 32\n    epochs = 15\n    lr = 1e-4\n    \nconfig = Config()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Data","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## 4.1 Path and csv data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = '../input/siic-isic-224x224-images/train'\ntest_path = '../input/siic-isic-224x224-images/test'\ntrain_df = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntest_df = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['image_path'] = train_path +'/'+train_df['image_name']+'.png'\ntest_df['image_path'] = test_path +'/'+test_df['image_name']+'.png'\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"anatom = pd.get_dummies(train_df['anatom_site_general_challenge'], prefix='anotom')\nanatom_cols = list(anatom.columns)\nsex = pd.get_dummies(train_df['sex'], prefix='sex')\nsex_cols = list(sex.columns)\nmeta_cols = anatom_cols + sex_cols + ['age_approx']\npd.concat([train_df,sex,anatom],axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4.2 Data Transform","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transform = transforms.Compose([\n    #transforms.RandomResizedCrop(size=224, scale=(0.7, 1.0)),\n    #transforms.RandomAffine(90, translate=(0.1,0.1)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    AutoAugment(),\n    #transforms.ColorJitter(brightness=32. / 255.,saturation=0.5),\n    #transforms.Cutout(scale=(0.05, 0.1), value=(0, 0)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])\n\ntest_transform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4.3 Data Helper","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Data(Dataset):\n    def __init__(self,df=train_df,is_train=True,transform=None):\n        super(Data,self).__init__()\n        self.df = df\n        self.is_train = is_train\n        self.transform = transform\n        pass\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self,i):\n        path = self.df.image_path.iloc[i]\n        img = cv2.imread(path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = transforms.ToPILImage()(img)\n        img = self.transform(img)\n        \n        if self.is_train:\n            y = self.df.target.iloc[i]\n            return img,y\n        else:\n            return img\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kf = StratifiedKFold(5,shuffle=True,random_state=0)\n\ntest_data = Data(test_df,is_train=False,transform=test_transform)\ntest_loader = DataLoader(test_data,batch_size=config.test_batch_size,shuffle=False,num_workers=6)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyModel(nn.Module):\n    def __init__(self,):\n        super(MyModel,self).__init__()\n        self.backbone = EfficientNet.from_pretrained('efficientnet-b0')\n        self.backbone._fc = nn.Linear(in_features=1280,out_features=1,bias=True)\n    \n    def forward(self,x):\n        x = self.backbone(x)\n        x = torch.sigmoid(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6. Function for train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter:\n    def __init__(self):\n        self.reset()\n        \n    def reset(self):\n        self.sum = 0\n        self.n = 0\n        \n    def update(self,val,n=1):\n        self.sum+=val*n\n        self.n+=n\n        self.avg = self.sum/self.n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def eval_model(val_loader,model):\n    model.eval()\n    metrics = {}\n    preds = []\n    targets = []\n    avg_loss = AverageMeter()\n    with torch.no_grad():\n        for it,(imgs,target) in enumerate(val_loader):\n            imgs,target = imgs.cuda(),target.cuda().float().unsqueeze(1)\n            pred = model(imgs)\n            avg_loss.update(nn.BCELoss()(pred,target),imgs.size(0))\n            preds.extend(pred.detach().cpu())\n            targets.extend(target.detach().cpu())\n    \n    acc = accuracy_score(targets,np.round(preds))\n    auc = roc_auc_score(targets,preds)\n    return {'loss':avg_loss.avg,'acc':acc,'auc':auc}\n    \ndef train_model(trn_loader,val_loader,model,optimizer,criterion,fold):\n    \n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n        optimizer,\n        patience=2,\n        threshold=0.0001,\n        mode=\"min\"\n    )\n    optimizer.zero_grad()\n    \n    metrics = defaultdict(list)\n    best_model = None\n    best_auc = 0\n    \n    es = 0\n    for epoch in range(config.epochs):\n        model.train()\n        avg_loss,avg_auc = AverageMeter(),AverageMeter()\n        \n        print('Epoch : {}'.format(epoch + 1))\n        for it,(imgs,target) in enumerate(trn_loader):\n            imgs,target = imgs.cuda(),target.cuda().float().unsqueeze(1)\n            \n            pred = model(imgs)\n            loss = criterion(pred,target)\n            loss.backward()\n            \n            optimizer.step()\n            optimizer.zero_grad()\n            \n            avg_loss.update(loss,imgs.size(0))\n            \n            if it % 40==0:\n                print('It: {}/{} | Loss: {}'.format(it,len(trn_loader),avg_loss.avg))\n        \n        val_score = eval_model(val_loader,model)\n        scheduler.step(avg_loss.avg)\n        metrics['train_loss'].append(avg_loss.avg)\n        for k,v in val_score.items():\n            metrics[k].append(v)\n            \n        \n        if val_score['auc'] > best_auc:\n            best_auc = val_score['auc']\n            best_model = model\n            es = 0\n        else:\n            es += 1\n            if es >= 3:\n                break\n            \n        print('It: {}/{} | Loss: {}'.format(it,len(trn_loader),avg_loss.avg))\n        \n        print('| Val loss: {} | Val auc: {} | Best auc: {}'.format(val_score['loss'],val_score['auc'],best_auc))\n        \n    return best_model,best_auc,metrics\n            \ndef train_kfolds(create_version):\n    fold = 0\n    cv_aucs = []\n    for trn_ind,val_ind in kf.split(train_df.image_name,train_df.target):\n        fold += 1\n        model,optimizer,criterion = create_version()\n        print('---- Fold {}'.format(fold))\n        trn_df = train_df.iloc[trn_ind]\n        val_df = train_df.iloc[val_ind]\n        trn_data = Data(trn_df,is_train=True,transform=train_transform)\n        val_data = Data(val_df,is_train=True,transform=test_transform)\n        trn_loader = DataLoader(trn_data,batch_size=config.train_batch_size,shuffle=True,num_workers=6)\n        val_loader = DataLoader(val_data,batch_size=config.train_batch_size,shuffle=False,num_workers=6)\n        model,auc,metrics = train_model(trn_loader,val_loader,model,optimizer,criterion,fold)\n        torch.save(model.state_dict(),'weight_{}.pt'.format(fold))\n        cv_aucs.append(auc)\n    return cv_aucs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 7. Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef create_baseline():\n    model = MyModel().cuda()\n    model.train()\n    params = list(model.named_parameters())\n    def is_fc(n):\n        return 'fc' in n\n    optimizer_grouped_parameters = [\n        {\"params\": [p for n, p in params if not is_fc(n)], \"lr\": 1e-4},\n        {\"params\": [p for n, p in params if is_fc(n)], \"lr\": 1e-4 * 200},\n    ]\n    optimizer = optim.AdamW(optimizer_grouped_parameters,lr=config.lr,weight_decay=0)\n    #optimizer = optim.AdamW(model.parameters(),lr=config.lr,weight_decay=0)\n    criterion = nn.BCELoss()\n    return model,optimizer,criterion\n\ncvs = train_kfolds(create_baseline)\nprint(cvs,np.mean(cvs))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef model_pred(model,test_loader):\n    model.eval()\n    prediction = []\n    with torch.no_grad():\n        for imgs in tqdm(test_loader):\n            imgs = imgs.cuda()\n            pred = list(model(imgs).squeeze(-1).cpu().detach().numpy())\n            prediction.extend(pred)\n    return prediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = []\nfor i in range(5):\n    model = MyModel().cuda()\n    model.load_state_dict(torch.load('weight_{}.pt'.format(i+1)))\n    pred = model_pred(model,test_loader)\n    preds.append(pred)\n    \npreds = np.array(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = np.mean(preds,axis=0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 8. Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\nsample_df.target = test_preds\nsample_df.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}