{"cells":[{"metadata":{},"cell_type":"markdown","source":"#### Versions:\n* v9: ColorJitter transformation added **[0.896]**\n* v10: Changed the dataset to [this one](https://www.kaggle.com/shonenkov/melanoma-merged-external-data-512x512-jpeg) with external data. **[0.894]**\n* v11: Switched to [another dataset](https://www.kaggle.com/nroman/melanoma-external-malignant-256/) which I've created by myself. Also switched from StratifiedKFold to GroupKFold **[0.916]**\n* v12: Switched to efficientnet-b1 **[0.919]**\n* v13: Using meta featues: sex and age **[0.918]**\n* v14: anatom_site_general_challenge meta feature added as one-hot encoded matrix **[0.923]**\n* v16: Fixed OOF - now it contains only data from original training dataset, without extarnal data. Also switched back to StratifiedKFold. Added DrawHair augmentation. **[0.909]**\n* v18: Too many things were changed at the same time. All experiments should have only one small change each, so it would be easy to understand how changes affect the result. Said that I rolled back everything, keeping only OOF fix, to make sure it work.\n* v19: Added 'Hair' augmentation. OOF rework posponed untill the best time, since there is some bug in my code for it. **[0.925]**\n* v20: Advanced Hair Augmentation technique used. Read more about it here: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/159176 **[0.923]**\n* v21: Microscope augmentation added instead of Cutout. Read more here: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/159476","execution_count":null},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!pip install efficientnet_pytorch torchtoolbox","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import torch\nimport torchvision\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torchtoolbox.transform as transforms\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nimport pandas as pd\nimport numpy as np\nimport gc\nimport os\nimport cv2\nimport time\nimport datetime\nimport warnings\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom efficientnet_pytorch import EfficientNet\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# At least fixing some random seeds. \n# It is still impossible to make results 100% reproducible when using GPU\nwarnings.simplefilter('ignore')\ntorch.manual_seed(47)\nnp.random.seed(47)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None):\n        \"\"\"\n        Class initialization\n        Args:\n            df (pd.DataFrame): DataFrame with data description\n            imfolder (str): folder with images\n            train (bool): flag of whether a training dataset is being initialized or testing one\n            transforms: image transformation method to be applied\n            meta_features (list): list of features with meta information, such as sex and age\n            \n        \"\"\"\n        self.df = df\n        self.imfolder = imfolder\n        self.transforms = transforms\n        self.train = train\n        self.meta_features = meta_features\n        \n    def __getitem__(self, index):\n        im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')\n        x = cv2.imread(im_path)\n        meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)\n\n        if self.transforms:\n            x = self.transforms(x)\n            \n        if self.train:\n            y = self.df.iloc[index]['target']\n            return (x, meta), y\n        else:\n            return (x, meta)\n    \n    def __len__(self):\n        return len(self.df)\n    \n    \nclass Net(nn.Module):\n    def __init__(self, arch, n_meta_features: int):\n        super(Net, self).__init__()\n        self.arch = arch\n        if 'ResNet' in str(arch.__class__):\n            self.arch.fc = nn.Linear(in_features=512, out_features=500, bias=True)\n        if 'EfficientNet' in str(arch.__class__):\n            self.arch._fc = nn.Linear(in_features=1280, out_features=500, bias=True)\n        self.meta = nn.Sequential(nn.Linear(n_meta_features, 500),\n                                  nn.BatchNorm1d(500),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2),\n                                  nn.Linear(500, 250),  # FC layer output will have 250 features\n                                  nn.BatchNorm1d(250),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2))\n        self.ouput = nn.Linear(500 + 250, 1)\n        \n    def forward(self, inputs):\n        \"\"\"\n        No sigmoid in forward because we are going to use BCEWithLogitsLoss\n        Which applies sigmoid for us when calculating a loss\n        \"\"\"\n        x, meta = inputs\n        cnn_features = self.arch(x)\n        meta_features = self.meta(meta)\n        features = torch.cat((cnn_features, meta_features), dim=1)\n        output = self.ouput(features)\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AdvancedHairAugmentation:\n    \"\"\"\n    Impose an image of a hair to the target image\n\n    Args:\n        hairs (int): maximum number of hairs to impose\n        hairs_folder (str): path to the folder with hairs images\n    \"\"\"\n\n    def __init__(self, hairs: int = 5, hairs_folder: str = \"\"):\n        self.hairs = hairs\n        self.hairs_folder = hairs_folder\n\n    def __call__(self, img):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to draw hairs on.\n\n        Returns:\n            PIL Image: Image with drawn hairs.\n        \"\"\"\n        n_hairs = random.randint(0, self.hairs)\n        \n        if not n_hairs:\n            return img\n        \n        height, width, _ = img.shape  # target image width and height\n        hair_images = [im for im in os.listdir(self.hairs_folder) if 'png' in im]\n        \n        for _ in range(n_hairs):\n            hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))\n            hair = cv2.flip(hair, random.choice([-1, 0, 1]))\n            hair = cv2.rotate(hair, random.choice([0, 1, 2]))\n\n            h_height, h_width, _ = hair.shape  # hair image width and height\n            roi_ho = random.randint(0, img.shape[0] - hair.shape[0])\n            roi_wo = random.randint(0, img.shape[1] - hair.shape[1])\n            roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]\n\n            # Creating a mask and inverse mask\n            img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)\n            ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)\n            mask_inv = cv2.bitwise_not(mask)\n\n            # Now black-out the area of hair in ROI\n            img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n\n            # Take only region of hair from hair image.\n            hair_fg = cv2.bitwise_and(hair, hair, mask=mask)\n\n            # Put hair in ROI and modify the target image\n            dst = cv2.add(img_bg, hair_fg)\n\n            img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst\n                \n        return img\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder=\"{self.hairs_folder}\")'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Microscope:\n    \"\"\"\n    Cutting out the edges around the center circle of the image\n    Imitating a picture, taken through the microscope\n\n    Args:\n        p (float): probability of applying an augmentation\n    \"\"\"\n\n    def __init__(self, p: float = 0.5):\n        self.p = p\n\n    def __call__(self, img):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to apply transformation to.\n\n        Returns:\n            PIL Image: Image with transformation.\n        \"\"\"\n        if random.random() < self.p:\n            circle = cv2.circle((np.ones(img.shape) * 255).astype(np.uint8), # image placeholder\n                        (img.shape[0]//2, img.shape[1]//2), # center point of circle\n                        random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15), # radius\n                        (0, 0, 0), # color\n                        -1)\n\n            mask = circle - 255\n            img = np.multiply(img, mask)\n        \n        return img\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(p={self.p})'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transform = transforms.Compose([\n    AdvancedHairAugmentation(hairs_folder='/kaggle/input/melanoma-hairs/'),\n    transforms.RandomResizedCrop(size=256, scale=(0.7, 1.0)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.ColorJitter(brightness=32. / 255.,saturation=0.5),\n    Microscope(p=0.6),\n#     transforms.Cutout(scale=(0.05, 0.007), value=(0, 0)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])\ntest_transform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"arch = EfficientNet.from_pretrained('efficientnet-b1')  # Going to use efficientnet-b1 NN architecture\n# skf = StratifiedKFold(n_splits=5, random_state=999, shuffle=True)\nskf = GroupKFold(n_splits=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/melanoma-external-malignant-256/train_concat.csv')\ntest_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# One-hot encoding of anatom_site_general_challenge feature\nconcat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)\ndummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')\ntrain_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)\ntest_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)\n\n# Sex features\ntrain_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})\ntest_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})\ntrain_df['sex'] = train_df['sex'].fillna(-1)\ntest_df['sex'] = test_df['sex'].fillna(-1)\n\n# Age features\n# train_df['age_approx'] /= train_df['age_approx'].max()\n# test_df['age_approx'] /= test_df['age_approx'].max()\n# train_df['age_approx'] = train_df['age_approx'].fillna(0)\n# test_df['age_approx'] = test_df['age_approx'].fillna(0)\n\n# standardize\nmean_age, std_age = train_df['age_approx'].mean(), train_df['age_approx'].std()\ntrain_df['age_approx'] = (train_df['age_approx'] - mean_age) / (1e-7 + std_age)\ntest_df['age_approx'] = (test_df['age_approx'] - mean_age) / (1e-7 + std_age)\ntrain_df['age_approx'] = train_df['age_approx'].fillna(0)\ntest_df['age_approx'] = test_df['age_approx'].fillna(0)\n\ntrain_df['patient_id'] = train_df['patient_id'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]\nmeta_features.remove('anatom_site_general_challenge')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = MelanomaDataset(df=test_df,\n                       imfolder='/kaggle/input/melanoma-external-malignant-256/test/test/', \n                       train=False,\n                       transforms=train_transform,\n                       meta_features=meta_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 10  # Number of epochs to run\nmodel_path = 'model.pth'  # Path and filename to save model to\nes_patience = 3  # Early Stopping patience - for how many epochs with no improvements to wait\nTTA = 3 # Test Time Augmentation rounds\n\noof = np.zeros((len(train_df), 1))  # Out Of Fold predictions\npreds = torch.zeros((len(test), 1), dtype=torch.float32, device=device)  # Predictions for test test\n\n# We stratify by target value, thus, according to sklearn StratifiedKFold documentation\n# We can fill `X` with zeroes of corresponding length to use it as a placeholder\n# since we only need `y` to stratify the data\n# for fold, (train_idx, val_idx) in enumerate(skf.split(X=np.zeros(len(train_df)), y=train_df['target']), 1):\nfor fold, (train_idx, val_idx) in enumerate(skf.split(X=np.zeros(len(train_df)), y=train_df['target'], groups=train_df['patient_id'].tolist()), 1):\n    print('=' * 20, 'Fold', fold, '=' * 20)\n    \n    best_val = None  # Best validation score within this fold\n    patience = es_patience  # Current patience counter\n    arch = EfficientNet.from_pretrained('efficientnet-b1')\n    model = Net(arch=arch, n_meta_features=len(meta_features))  # New model for each fold\n    model = model.to(device)\n    \n    \n    optim = torch.optim.Adam(model.parameters(), lr=0.001)\n    scheduler = ReduceLROnPlateau(optimizer=optim, mode='max', patience=1, verbose=True, factor=0.2)\n    criterion = nn.BCEWithLogitsLoss()\n    \n    train = MelanomaDataset(df=train_df.iloc[train_idx].reset_index(drop=True), \n                            imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/', \n                            train=True, \n                            transforms=train_transform,\n                            meta_features=meta_features)\n    val = MelanomaDataset(df=train_df.iloc[val_idx].reset_index(drop=True), \n                            imfolder='/kaggle/input/melanoma-external-malignant-256/train/train/', \n                            train=True, \n                            transforms=test_transform,\n                            meta_features=meta_features)\n    \n    train_loader = DataLoader(dataset=train, batch_size=64, shuffle=True, num_workers=2)\n    val_loader = DataLoader(dataset=val, batch_size=16, shuffle=False, num_workers=2)\n    test_loader = DataLoader(dataset=test, batch_size=16, shuffle=False, num_workers=2)\n    \n    for epoch in range(epochs):\n        start_time = time.time()\n        correct = 0\n        epoch_loss = 0\n        model.train()\n        \n        for x, y in train_loader:\n            x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)\n            x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)\n            y = torch.tensor(y, device=device, dtype=torch.float32)\n            optim.zero_grad()\n            z = model(x)\n            loss = criterion(z, y.unsqueeze(1))\n            loss.backward()\n            optim.step()\n            pred = torch.round(torch.sigmoid(z))  # round off sigmoid to obtain predictions\n            correct += (pred.cpu() == y.cpu().unsqueeze(1)).sum().item()  # tracking number of correctly predicted samples\n            epoch_loss += loss.item()\n        train_acc = correct / len(train_idx)\n\n        model.eval()  # switch model to the evaluation mode\n        val_preds = torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n        with torch.no_grad():  # Do not calculate gradient since we are only predicting\n            # Predicting on validation set\n            for j, (x_val, y_val) in enumerate(val_loader):\n                x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)\n                x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)\n                y_val = torch.tensor(y_val, device=device, dtype=torch.float32)\n                z_val = model(x_val)\n                val_pred = torch.sigmoid(z_val)\n                val_preds[j*x_val[0].shape[0]:j*x_val[0].shape[0] + x_val[0].shape[0]] = val_pred\n            val_acc = accuracy_score(train_df.iloc[val_idx]['target'].values, torch.round(val_preds.cpu()))\n            val_roc = roc_auc_score(train_df.iloc[val_idx]['target'].values, val_preds.cpu())\n            \n            print('Epoch {:03}: | Loss: {:.3f} | Train acc: {:.3f} | Val acc: {:.3f} | Val roc_auc: {:.3f} | Training time: {}'.format(\n            epoch + 1, \n            epoch_loss, \n            train_acc, \n            val_acc, \n            val_roc, \n            str(datetime.timedelta(seconds=time.time() - start_time))[:7]))\n            \n            scheduler.step(val_roc)\n            # During the first iteration (first epoch) best validation is set to None\n            if not best_val:\n                best_val = val_roc  # So any validation roc_auc we have is the best one for now\n                torch.save(model, model_path)  # Saving the model\n                continue\n                \n            if val_roc >= best_val:\n                best_val = val_roc\n                patience = es_patience  # Resetting patience since we have new best validation accuracy\n                torch.save(model, model_path)  # Saving current best model\n            else:\n                patience -= 1\n                if patience == 0:\n                    print('Early stopping. Best Val roc_auc: {:.3f}'.format(best_val))\n                    break\n                \n    model = torch.load(model_path)  # Loading best model of this fold\n    model.eval()  # switch model to the evaluation mode\n    val_preds = torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n    with torch.no_grad():\n        # Predicting on validation set once again to obtain data for OOF\n        for j, (x_val, y_val) in enumerate(val_loader):\n            x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)\n            x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)\n            y_val = torch.tensor(y_val, device=device, dtype=torch.float32)\n            z_val = model(x_val)\n            val_pred = torch.sigmoid(z_val)\n            val_preds[j*x_val[0].shape[0]:j*x_val[0].shape[0] + x_val[0].shape[0]] = val_pred\n        oof[val_idx] = val_preds.cpu().numpy()\n        \n        # Predicting on test set\n        for _ in range(TTA):\n            for i, x_test in enumerate(test_loader):\n                x_test[0] = torch.tensor(x_test[0], device=device, dtype=torch.float32)\n                x_test[1] = torch.tensor(x_test[1], device=device, dtype=torch.float32)\n                z_test = model(x_test)\n                z_test = torch.sigmoid(z_test)\n                preds[i*x_test[0].shape[0]:i*x_test[0].shape[0] + x_test[0].shape[0]] += z_test\n        preds /= TTA\n        \n    del train, val, train_loader, val_loader, x, y, x_val, y_val\n    gc.collect()\n    \npreds /= skf.n_splits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('OOF: {:.3f}'.format(roc_auc_score(train_df['target'], oof)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.kdeplot(pd.Series(preds.cpu().numpy().reshape(-1,)));","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Saving OOF predictions so stacking would be easier\npd.Series(oof.reshape(-1,)).to_csv('oof.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\nsub['target'] = preds.cpu().numpy().reshape(-1,)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}