{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport glob\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tqdm\nimport torch\nimport torch.nn.functional as F\nimport torch.nn as nn\nfrom torch.utils.data.dataset import Dataset\nfrom torch.utils.data import DataLoader\nimport albumentations as A\nfrom albumentations.pytorch import ToTensor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nfrom transformers import get_cosine_schedule_with_warmup","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!ls ../input/siim-isic-melanoma-classification","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = '../input/siim-isic-melanoma-classification'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(f'{data_dir}/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images = glob.glob(f'{data_dir}/jpeg/train/*')\nlen(train_images), train_images[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig=plt.figure(figsize=(25, 25))\ncolumns = 3\nrows = 3\nfor i in range(1, columns*rows +1):\n    name = train_images[i].split('/')[-1]\n    img = cv2.imread(train_images[i])\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    fig.add_subplot(rows, columns, i)\n    plt.imshow(img)\n    plt.title(name)\n    plt.axis('off')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_imgs(dataset_show):\n    from pylab import rcParams\n    rcParams['figure.figsize'] = 20,20\n    for i in range(2):\n        f, axarr = plt.subplots(1,3)\n        for p in range(3):\n            idx = np.random.randint(0, len(dataset_show))\n            data = dataset_show[idx]\n            npimg = data['image'].numpy()\n            axarr[p].imshow(np.transpose(npimg, (1,2,0)), interpolation='nearest')\n            axarr[p].set_title(idx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_transforms(img_size):\n    \n    train_transform = A.Compose([\n                                A.Resize(img_size, img_size),\n                                A.HorizontalFlip(),\n                                A.Normalize(),\n                                ToTensor()\n                                ])\n    \n    valid_transform = A.Compose([\n                                A.Resize(img_size, img_size),\n                                A.Normalize(),\n                                ToTensor()\n                                ])\n    return train_transform, valid_transform","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SIIM_Dataset(Dataset):\n    def __init__(self, df, data_dir, mode ='train', transform=None):\n\n        self.df = df \n        self.image_ids = df['image_name'].tolist()\n        self.data_dir = data_dir\n        self.mode = mode\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def __getitem__(self, idx):\n        image_id = self.image_ids[idx]\n        image_dir = os.path.join(self.data_dir, image_id + '.jpg')\n        image = cv2.imread(image_dir)  \n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)     \n        \n        if self.transform is not None:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n            \n        data = {}\n        data['image'] =image\n        data['image_id'] = image_id\n        \n        if self.mode == 'test':    \n            return data\n        else:\n            label = self.df.loc[self.df['image_name']==image_id, 'target'].values[0]\n            data['label'] = torch.tensor(label)\n            return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_size = 512\ntrain_transform , valid_transform = generate_transforms(img_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_dir = '../input/siim-isic-melanoma-classification/jpeg/train'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_show = train.iloc[:1000]\ndataset_show = SIIM_Dataset(df_show, img_dir, transform=train_transform)\n# plotting transformed version of dataset\nplot_imgs(dataset_show)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataloader(df, img_dir, mode, img_size = 512, batch_size=64):\n\n    train_transform, valid_transform = generate_transforms(img_size)\n\n    datasets = SIIM_Dataset(df, img_dir, mode, transform= train_transform if mode =='train' else valid_transform)\n    \n    is_train = mode =='train'\n    dataloader = DataLoader(datasets,\n                            shuffle=is_train,\n                            batch_size=batch_size if is_train else 2*batch_size,\n                            drop_last=is_train,\n                            num_workers=4,\n                            pin_memory=False)\n    return dataloader","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Since the size of the dataset is 33126, and the dataset is very unbalanced. So I will use subset of the dataset for this kernel and the subset will be a achieved by taking equal number of `0` and `1` class. This is one trick that I find useful in Kaggle competitions: when the dataset is big, work on a subset of it in the beginning of the competition; hence iterating faster.\nAlso, experiment with smaller image sizes. These are two handful tricks that can significantly improve your iteration speed","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_0 = train[train['target']==0]\ntrain_1 = train[train['target']==1]\ntrain_0.shape, train_1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_0_sample = train_0.sample(584)\ntrain_0_sample.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_balanced = pd.concat([train_0_sample, train_1])\ndf_balanced['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, valid_df = train_test_split(df_balanced, test_size=0.1, stratify=df_balanced['target'].values, random_state=1234)\ntrain_df.shape, valid_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet_pytorch\nfrom efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self, num_classes):\n        super().__init__()\n        self.model = EfficientNet.from_pretrained('efficientnet-b0')\n        # 1280 is the number of neurons in last layer. is diff for diff. architecture\n        self.dense_output = nn.Linear(1280, num_classes)\n\n    def forward(self, x):\n        feat = self.model.extract_features(x)\n        feat = F.avg_pool2d(feat, feat.size()[2:]).reshape(-1, 1280)\n        return self.dense_output(feat)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = get_dataloader(train_df, img_dir, mode='train', img_size = 256, batch_size=16)\nvalid_loader = get_dataloader(valid_df, img_dir, mode='valid', img_size = 256, batch_size=16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Net(num_classes=1).cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_avg(curr_avg, val, idx):\n    return (curr_avg * idx + val) / (idx + 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate_single_epoch(model, dataloader, criterion):\n\n    model.eval()\n    curr_loss_avg = 0\n    valid_preds, valid_targets = [], [] \n    \n    tbar = tqdm.tqdm(dataloader,  total=len(dataloader))\n    with torch.no_grad():\n        for batch_idx, data in enumerate(tbar):\n            images = data['image'].cuda()\n            labels = data['label'].cuda()\n\n            logits = model(images)\n            probs =torch.sigmoid(logits)\n\n            valid_preds.append(probs.detach().cpu().numpy())\n            valid_targets.append(labels.detach().cpu().numpy())\n            \n            loss = criterion(logits.squeeze(), labels.float())\n            \n            curr_loss_avg = update_avg(curr_loss_avg, loss, batch_idx)        \n            tbar.set_description('loss: {:.4}'.format(curr_loss_avg.item()))\n\n        valid_preds = np.concatenate(valid_preds)\n        valid_targets = np.concatenate(valid_targets)\n        roc_metric =  roc_auc_score(valid_targets, valid_preds) \n        #roc = roc_auc_score(targs[:,i], preds[:,i])\n\n            \n    return curr_loss_avg.item(), roc_metric","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_single_epoch(model, dataloader, criterion, optimizer, scheduler):\n    \n    model.train()\n    curr_loss_avg = 0\n    tbar = tqdm.tqdm(dataloader,  total=len(dataloader))\n    for batch_idx, data in enumerate(tbar):\n        images = data['image'].cuda()\n        labels = data['label'].cuda()\n        \n        scheduler.step()\n        logits = model(images)     \n        loss = criterion(logits.squeeze(), labels.float())\n    \n        loss.backward()\n        optimizer.step()\n        optimizer.zero_grad()\n        \n        curr_loss_avg = update_avg(curr_loss_avg, loss, batch_idx)        \n        tbar.set_description('loss: %.5f, lr: %.6f' % (curr_loss_avg.item(), optimizer.param_groups[0]['lr']))\n    return curr_loss_avg.item()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# hyperparameters for training\nnum_epochs = 30\nlr = 0.001\ncriterion = torch.nn.BCEWithLogitsLoss()\noptimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4)\nscheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=len(train_loader), num_training_steps=5000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_val_score = 0.0 \nearly_stop = 0\nfor epoch in range(num_epochs):\n        \n    # train phase \n    train_loss = train_single_epoch(model, train_loader, criterion, optimizer, scheduler)\n       \n    # valid phase\n    val_loss, roc_metric = evaluate_single_epoch(model, valid_loader, criterion)\n\n    print(f'Epoch: {epoch+1} | Train_loss: {train_loss:.5f} | Val loss: {val_loss:.5f} | roc_metric : {roc_metric:.5f}')\n    \n    if roc_metric > best_val_score:\n        early_stop = 0\n        best_val_score = roc_metric\n        # save best model\n        torch.save(model.state_dict(), 'best_checkpoint.pth')\n    else:\n        early_stop += 1\n        if early_stop == 7: # stopping condition for training\n            break\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}