{"cells":[{"metadata":{},"cell_type":"markdown","source":"Thanks to [this kernel](https://www.kaggle.com/bibek777/training-pytorch-starter) I used for reference.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport glob\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tqdm\nimport torch\nimport torch.nn.functional as F\nimport torch.nn as nn\nfrom torch.utils.data.dataset import Dataset\nfrom torch.utils.data import DataLoader\nimport albumentations as A\nfrom albumentations.pytorch import ToTensor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nfrom transformers import get_cosine_schedule_with_warmup\nfrom efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 2019\ndevice = 'cuda:0'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"data_path ='../input/siim-isic-melanoma-classification'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(f'{data_path}/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imagelist = glob.glob(f'{data_path}/jpeg/train/*')\nimagelist[0], len(imagelist)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SIIMDataset(Dataset):\n    def __init__(self, df, data_path , mode= 'train', transform = None , size=256):\n        self.df = df\n        self.image_ids = df['image_name'].tolist()\n        self.data_path = data_path\n        self.mode= mode\n        self.transform = transform\n        self.size = size\n        \n    def __len__(self):\n        return len(self.image_ids)\n    \n    def __getitem__(self , idx):\n        image_id = self.image_ids[idx]\n        image_path = os.path.join(self.data_path , image_id + '.jpg')\n        image = cv2.imread(image_path)\n        image = cv2.resize(image, (self.size,self.size))\n        image = cv2.cvtColor(image , cv2.COLOR_BGR2RGB)\n        \n        if self.transform:\n            aug = self.transform(image=image)\n            image= aug['image']\n            \n        data = {}\n        data['image'] = image\n        data['image_id'] = image_id\n        \n        \n        if self.mode == 'test':\n            return data\n        else:\n            label = self.df.loc[self.df['image_name'] == image_id , 'target'].values[0]\n            data['label'] = torch.tensor(label)\n            return data\n        \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plotimgs(dataset):\n    f , ax = plt.subplots(1,3)\n    for p in range(3):\n        idx = np.random.randint(0 , len(dataset))\n        data = dataset[idx]\n        img = data['image']\n        ax[p].imshow(np.transpose(img,(1,2,0)), interpolation = 'nearest')\n        ax[p].set_title(idx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transforms = A.Compose([A.Flip(p=0.6),\n                              A.ShiftScaleRotate(p=0.7),\n                              A.Normalize(),\n                              ToTensor()])\n    \nvalid_transforms = A.Compose([A.Normalize(),\n                             ToTensor()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_dir = '../input/siim-isic-melanoma-classification/jpeg/train'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataloader(df , img_dir ,mode, size, batch_size):\n    dataset = SIIMDataset( df, img_dir ,mode ,transform = train_transforms if mode=='train' else valid_transforms , size=size)\n    istrain = mode == 'train'\n    dataloader = DataLoader(dataset , batch_size = batch_size , num_workers = 4 , shuffle = istrain ,drop_last = istrain)\n    return dataloader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_0 = train[train['target'] == 0]\ntrain_1 = train[train['target']== 1]\ntrain_0.shape , train_1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_0_new =train_0.sample(584)\ntrain_0_new.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"balanced = pd.concat([train_0_new ,train_1])\nbalanced['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df , valid_df = train_test_split(balanced,  test_size = 0.1 , stratify = balanced['target'].values ,random_state = seed)\ntrain_df.shape , valid_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.nn.parameter import Parameter\n\ndef gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM,self).__init__()\n        self.p = Parameter(torch.ones(1)*p)\n        self.eps = eps\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)       \n    def __repr__(self):\n        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def GlobalAveragePooling(x):\n    return x.mean(axis=-1).mean(axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomNet(nn.Module):\n    def __init__(self , num_classes):\n        super().__init__()\n        self.model = EfficientNet.from_pretrained('efficientnet-b3')\n        #self.gem = GeM()\n        self.pool= GlobalAveragePooling\n        self.out = nn.Linear(1536, num_classes)\n    \n    def forward(self,x):\n        x = self.model.extract_features(x)\n        #x = F.avg_pool2d(x, x.size()[2:]).reshape(-1, 1536)\n        x = self.pool(x)\n        return self.out(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def  update_avg(curr_avg , val , idx):\n    return (curr_avg * idx + val )/ (idx+1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_epoch(model , dataloader , criterion , optimizer , scheduler ):\n    model.train()\n    curr_avg_loss = 0\n    #train_losses = []\n    #avg_losses = []\n    t = tqdm.tqdm_notebook(dataloader , total = len(dataloader))\n    for bi , data in enumerate(t):\n        images = data['image'].cuda()\n        labels = data['label'].cuda()\n        scheduler.step()\n        logits = model(images)\n        loss = criterion(logits.squeeze() , labels.float())\n        loss.backward()\n        optimizer.step()\n        optimizer.zero_grad()\n        curr_avg_loss = update_avg(curr_avg_loss , loss , bi)\n        t.set_description('loss : %.5f , lr : %.6f' % (curr_avg_loss.item(), optimizer.param_groups[0]['lr']))\n        \n        return curr_avg_loss.item()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def validate(model , dataloader , criterion, ):\n    model.eval()\n    curr_avg_loss = 0\n    #val_avg_losses = []\n    #val_losses = []\n    val_preds , val_targets = [] , []\n    t =tqdm.tqdm_notebook(dataloader , total = len(dataloader))\n    with torch.no_grad():\n        for bi , data in enumerate(t):\n            images = data['image'].cuda()\n            labels = data['label'].cuda()\n            logits = model(images)\n            probs = torch.sigmoid(logits)\n            val_preds.append(probs.detach().cpu().numpy())\n            val_targets.append(labels.detach().cpu().numpy())\n            loss = criterion(logits.squeeze() , labels.float())\n            curr_avg_loss = update_avg(curr_avg_loss , loss , bi)\n                    \n            t.set_description('val loss: {:.4}'.format(curr_avg_loss.item()))\n    val_preds = np.concatenate(val_preds)\n    val_targets = np.concatenate(val_targets)\n    metric = roc_auc_score(val_targets , val_preds)\n    \n    \n    return curr_avg_loss.item() , metric\n    \n        \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = get_dataloader(train_df , img_dir , mode='train' , size=256 , batch_size = 32)\nvalid_loader = get_dataloader(valid_df ,img_dir , mode = 'valid' , size=256 , batch_size = 64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = CustomNet(num_classes = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_epochs = 30\nlr = 1e-3\ncriterion = torch.nn.BCEWithLogitsLoss()\noptimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-5)\nscheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=len(train_loader), num_training_steps=5000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_val_score = 0.0\nes = 0 #early stopping\ntrain_losses = []\nval_losses =[]\nmetrics =[]\n\nfor epoch in range(num_epochs):\n    train_loss = train_epoch(model, train_loader , criterion , optimizer ,scheduler )\n    valid_loss , roc = validate(model ,valid_loader , criterion )\n    train_losses.append(train_loss)\n    val_losses.append(valid_loss)\n    metrics.append(roc)\n    print(f'Epoch: {epoch+1} | Train_loss: {train_loss:.5f} | Val loss: {valid_loss:.5f} | roc_metric : {roc:.5f}')\n    \n    if roc > best_val_score:\n        es = 0\n        best_val_score = roc\n        torch.save(model.state_dict(), 'model.pth')\n        \n    else:\n        es += 1\n        if es == 10:\n            break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"col_names = [ 'Train Loss' , 'Val Loss'  , 'Val ROC-AUC']\nstats = pd.DataFrame(np.stack([train_losses  ,val_losses  ,metrics], axis =1), columns = col_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stats","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}