{"cells":[{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!pip install efficientnet_pytorch torchtoolbox","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import torch\nimport torchvision\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torchtoolbox.transform as transforms\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nimport pandas as pd\nimport numpy as np\nimport gc\nimport os\nimport cv2\nimport time\nimport datetime\nimport warnings\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom efficientnet_pytorch import EfficientNet\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"warnings.simplefilter('ignore')\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(47)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None, meta_features = None):\n        \"\"\"\n        Class initialization\n        Args:\n            df (pd.DataFrame): DataFrame with data description\n            imfolder (str): folder with images\n            train (bool): flag of whether a training dataset is being initialized or testing one\n            transforms: image transformation method to be applied\n            meta_features (list): list of features with meta information, such as sex and age\n            \n        \"\"\"\n        self.df = df\n        self.imfolder = imfolder\n        self.transforms = transforms\n        self.train = train\n        self.meta_features = meta_features\n        \n    def __getitem__(self, index):\n        im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_name'] + '.jpg')\n        x = cv2.imread(im_path)\n        meta = np.array(self.df.iloc[index][self.meta_features].values, dtype=np.float32)\n\n        if self.transforms:\n            x = self.transforms(x)\n            \n        if self.train:\n            y = self.df.iloc[index]['target']\n            return (x, meta), y\n        else:\n            return (x, meta)\n    \n    def __len__(self):\n        return len(self.df)\n    \n    \nclass Net(nn.Module):\n    def __init__(self, arch, n_meta_features: int):\n        super(Net, self).__init__()\n        self.arch = arch\n        if 'ResNet' in str(arch.__class__):\n            self.arch.fc = nn.Linear(in_features=512, out_features=500, bias=True)\n        if 'EfficientNet' in str(arch.__class__):\n            self.arch._fc = nn.Linear(in_features=1280, out_features=500, bias=True)\n        self.meta = nn.Sequential(nn.Linear(n_meta_features, 500),\n                                  nn.BatchNorm1d(500),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2),\n                                  nn.Linear(500, 250),  # FC layer output will have 250 features\n                                  nn.BatchNorm1d(250),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2))\n        self.ouput = nn.Linear(500 + 250, 1)\n        \n    def forward(self, inputs):\n        \"\"\"\n        No sigmoid in forward because we are going to use BCEWithLogitsLoss\n        Which applies sigmoid for us when calculating a loss\n        \"\"\"\n        x, meta = inputs\n        cnn_features = self.arch(x)\n        meta_features = self.meta(meta)\n        features = torch.cat((cnn_features, meta_features), dim=1)\n        output = self.ouput(features)\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AdvancedHairAugmentation:\n    \"\"\"\n    Impose an image of a hair to the target image\n\n    Args:\n        hairs (int): maximum number of hairs to impose\n        hairs_folder (str): path to the folder with hairs images\n    \"\"\"\n\n    def __init__(self, hairs: int = 5, hairs_folder: str = \"\"):\n        self.hairs = hairs\n        self.hairs_folder = hairs_folder\n\n    def __call__(self, img):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to draw hairs on.\n\n        Returns:\n            PIL Image: Image with drawn hairs.\n        \"\"\"\n        n_hairs = random.randint(0, self.hairs)\n        \n        if not n_hairs:\n            return img\n        \n        height, width, _ = img.shape  # target image width and height\n        hair_images = [im for im in os.listdir(self.hairs_folder) if 'png' in im]\n        \n        for _ in range(n_hairs):\n            hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))\n            hair = cv2.flip(hair, random.choice([-1, 0, 1]))\n            hair = cv2.rotate(hair, random.choice([0, 1, 2]))\n\n            h_height, h_width, _ = hair.shape  # hair image width and height\n            roi_ho = random.randint(0, img.shape[0] - hair.shape[0])\n            roi_wo = random.randint(0, img.shape[1] - hair.shape[1])\n            roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]\n\n            # Creating a mask and inverse mask\n            img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)\n            ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)\n            mask_inv = cv2.bitwise_not(mask)\n\n            # Now black-out the area of hair in ROI\n            img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n\n            # Take only region of hair from hair image.\n            hair_fg = cv2.bitwise_and(hair, hair, mask=mask)\n\n            # Put hair in ROI and modify the target image\n            dst = cv2.add(img_bg, hair_fg)\n\n            img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst\n                \n        return img\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder=\"{self.hairs_folder}\")'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DrawHair:\n    \"\"\"\n    Draw a random number of pseudo hairs\n\n    Args:\n        hairs (int): maximum number of hairs to draw\n        width (tuple): possible width of the hair in pixels\n    \"\"\"\n\n    def __init__(self, hairs:int = 4, width:tuple = (1, 2)):\n        self.hairs = hairs\n        self.width = width\n\n    def __call__(self, img):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to draw hairs on.\n\n        Returns:\n            PIL Image: Image with drawn hairs.\n        \"\"\"\n        if not self.hairs:\n            return img\n        \n        width, height, _ = img.shape\n        \n        for _ in range(random.randint(0, self.hairs)):\n            # The origin point of the line will always be at the top half of the image\n            origin = (random.randint(0, width), random.randint(0, height // 2))\n            # The end of the line \n            end = (random.randint(0, width), random.randint(0, height))\n            color = (0, 0, 0)  # color of the hair. Black.\n            cv2.line(img, origin, end, color, random.randint(self.width[0], self.width[1]))\n        \n        return img\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(hairs={self.hairs}, width={self.width})'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Microscope:\n    \"\"\"\n    Cutting out the edges around the center circle of the image\n    Imitating a picture, taken through the microscope\n\n    Args:\n        p (float): probability of applying an augmentation\n    \"\"\"\n\n    def __init__(self, p: float = 0.5):\n        self.p = p\n\n    def __call__(self, img):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to apply transformation to.\n\n        Returns:\n            PIL Image: Image with transformation.\n        \"\"\"\n        if random.random() < self.p:\n            circle = cv2.circle((np.ones(img.shape) * 255).astype(np.uint8), # image placeholder\n                        (img.shape[0]//2, img.shape[1]//2), # center point of circle\n                        random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15), # radius\n                        (0, 0, 0), # color\n                        -1)\n\n            mask = circle - 255\n            img = np.multiply(img, mask)\n        \n        return img\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(p={self.p})'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transform = transforms.Compose([\n    DrawHair(),\n    transforms.RandomResizedCrop(size=256, scale=(0.8, 1.0)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    Microscope(p=0.4),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])\ntest_transform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"arch = EfficientNet.from_pretrained('efficientnet-b1')  # Going to use efficientnet-b1 NN architecture\n# skf = StratifiedKFold(n_splits=5, random_state=999, shuffle=True)\n# skf = GroupKFold(n_splits=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/train.csv')\ntest_df = pd.read_csv('/kaggle/input/jpeg-melanoma-256x256/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Using triple stratified KFolds\ntmp = pd.read_csv('/kaggle/input/melanoma-256x256/train.csv')\ntrain_df['fold'] = tmp['tfrecord']\ndel tmp\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# One-hot encoding of anatom_site_general_challenge feature\nconcat = pd.concat([train_df['anatom_site_general_challenge'], test_df['anatom_site_general_challenge']], ignore_index=True)\ndummies = pd.get_dummies(concat, dummy_na=True, dtype=np.uint8, prefix='site')\ntrain_df = pd.concat([train_df, dummies.iloc[:train_df.shape[0]]], axis=1)\ntest_df = pd.concat([test_df, dummies.iloc[train_df.shape[0]:].reset_index(drop=True)], axis=1)\n\n# Sex features\ntrain_df['sex'] = train_df['sex'].map({'male': 1, 'female': 0})\ntest_df['sex'] = test_df['sex'].map({'male': 1, 'female': 0})\ntrain_df['sex'] = train_df['sex'].fillna(-1)\ntest_df['sex'] = test_df['sex'].fillna(-1)\n\n# Age features\ntrain_df['age_approx'] /= train_df['age_approx'].max()\ntest_df['age_approx'] /= test_df['age_approx'].max()\ntrain_df['age_approx'] = train_df['age_approx'].fillna(0)\ntest_df['age_approx'] = test_df['age_approx'].fillna(0)\n\ntrain_df['patient_id'] = train_df['patient_id'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"meta_features = ['sex', 'age_approx'] + [col for col in train_df.columns if 'site_' in col]\nmeta_features.remove('anatom_site_general_challenge')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = MelanomaDataset(df=test_df,\n                       imfolder='/kaggle/input/jpeg-melanoma-256x256/test/', \n                       train=False,\n                       transforms=train_transform,  # For TTA\n                       meta_features=meta_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 10  # Number of epochs to run\nes_patience = 3  # Early Stopping patience - for how many epochs with no improvements to wait\nTTA = 3 # Test Time Augmentation rounds\n\noof = np.zeros((len(train_df), 1))  # Out Of Fold predictions\npreds = torch.zeros((len(test), 1), dtype=torch.float32, device=device)  # Predictions for test test\n\nskf = KFold(n_splits=5, shuffle=True, random_state=47)\nfor fold,(idxT, idxV) in enumerate(skf.split(np.arange(15)), 1):\n    print('=' * 20, 'Fold', fold, '=' * 20)\n    \n    train_idx = train_df.loc[train_df['fold'].isin(idxT)].index\n    val_idx = train_df.loc[train_df['fold'].isin(idxV)].index\n    \n    \n    model_path = f'model_{fold}.pth'  # Path and filename to save model to\n    best_val = 0  # Best validation score within this fold\n    patience = es_patience  # Current patience counter\n    arch = EfficientNet.from_pretrained('efficientnet-b1')\n    model = Net(arch=arch, n_meta_features=len(meta_features))  # New model for each fold\n    model = model.to(device)\n    \n    \n    optim = torch.optim.Adam(model.parameters(), lr=0.001)\n    scheduler = ReduceLROnPlateau(optimizer=optim, mode='max', patience=1, verbose=True, factor=0.2)\n    criterion = nn.BCEWithLogitsLoss()\n    \n    train = MelanomaDataset(df=train_df.iloc[train_idx].reset_index(drop=True), \n                            imfolder='/kaggle/input/jpeg-melanoma-256x256/train/', \n                            train=True, \n                            transforms=train_transform,\n                            meta_features=meta_features)\n    val = MelanomaDataset(df=train_df.iloc[val_idx].reset_index(drop=True), \n                            imfolder='/kaggle/input/jpeg-melanoma-256x256/train/', \n                            train=True, \n                            transforms=test_transform,\n                            meta_features=meta_features)\n    \n    train_loader = DataLoader(dataset=train, batch_size=64, shuffle=True, num_workers=2)\n    val_loader = DataLoader(dataset=val, batch_size=16, shuffle=False, num_workers=2)\n    test_loader = DataLoader(dataset=test, batch_size=16, shuffle=False, num_workers=2)\n    \n    for epoch in range(epochs):\n        start_time = time.time()\n        correct = 0\n        epoch_loss = 0\n        model.train()\n        \n        for x, y in train_loader:\n            x[0] = torch.tensor(x[0], device=device, dtype=torch.float32)\n            x[1] = torch.tensor(x[1], device=device, dtype=torch.float32)\n            y = torch.tensor(y, device=device, dtype=torch.float32)\n            optim.zero_grad()\n            z = model(x)\n            loss = criterion(z, y.unsqueeze(1))\n            loss.backward()\n            optim.step()\n            pred = torch.round(torch.sigmoid(z))  # round off sigmoid to obtain predictions\n            correct += (pred.cpu() == y.cpu().unsqueeze(1)).sum().item()  # tracking number of correctly predicted samples\n            epoch_loss += loss.item()\n        train_acc = correct / len(train_idx)\n        \n        model.eval()  # switch model to the evaluation mode\n        val_preds = torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n        with torch.no_grad():  # Do not calculate gradient since we are only predicting\n            # Predicting on validation set\n            for j, (x_val, y_val) in enumerate(val_loader):\n                x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)\n                x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)\n                y_val = torch.tensor(y_val, device=device, dtype=torch.float32)\n                z_val = model(x_val)\n                val_pred = torch.sigmoid(z_val)\n                val_preds[j*x_val[0].shape[0]:j*x_val[0].shape[0] + x_val[0].shape[0]] = val_pred\n            val_acc = accuracy_score(train_df.iloc[val_idx]['target'].values, torch.round(val_preds.cpu()))\n            val_roc = roc_auc_score(train_df.iloc[val_idx]['target'].values, val_preds.cpu())\n            \n            print('Epoch {:03}: | Loss: {:.3f} | Train acc: {:.3f} | Val acc: {:.3f} | Val roc_auc: {:.3f} | Training time: {}'.format(\n            epoch + 1, \n            epoch_loss, \n            train_acc, \n            val_acc, \n            val_roc, \n            str(datetime.timedelta(seconds=time.time() - start_time))[:7]))\n            \n            scheduler.step(val_roc)\n                \n            if val_roc >= best_val:\n                best_val = val_roc\n                patience = es_patience  # Resetting patience since we have new best validation accuracy\n                torch.save(model, model_path)  # Saving current best model\n            else:\n                patience -= 1\n                if patience == 0:\n                    print('Early stopping. Best Val roc_auc: {:.3f}'.format(best_val))\n                    break\n                \n    model = torch.load(model_path)  # Loading best model of this fold\n    model.eval()  # switch model to the evaluation mode\n    val_preds = torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n    with torch.no_grad():\n        # Predicting on validation set once again to obtain data for OOF\n        for j, (x_val, y_val) in enumerate(val_loader):\n            x_val[0] = torch.tensor(x_val[0], device=device, dtype=torch.float32)\n            x_val[1] = torch.tensor(x_val[1], device=device, dtype=torch.float32)\n            y_val = torch.tensor(y_val, device=device, dtype=torch.float32)\n            z_val = model(x_val)\n            val_pred = torch.sigmoid(z_val)\n            val_preds[j*x_val[0].shape[0]:j*x_val[0].shape[0] + x_val[0].shape[0]] = val_pred\n        oof[val_idx] = val_preds.cpu().numpy()\n        \n        # Predicting on test set\n        for _ in range(TTA):\n            for i, x_test in enumerate(test_loader):\n                x_test[0] = torch.tensor(x_test[0], device=device, dtype=torch.float32)\n                x_test[1] = torch.tensor(x_test[1], device=device, dtype=torch.float32)\n                z_test = model(x_test)\n                z_test = torch.sigmoid(z_test)\n                preds[i*x_test[0].shape[0]:i*x_test[0].shape[0] + x_test[0].shape[0]] += z_test\n        preds /= TTA\n        \n    del train, val, train_loader, val_loader, x, y, x_val, y_val\n    gc.collect()\n    \npreds /= skf.n_splits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('OOF: {:.3f}'.format(roc_auc_score(train_df['target'], oof)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.kdeplot(pd.Series(preds.cpu().numpy().reshape(-1,)));","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Saving OOF predictions so stacking would be easier\npd.Series(oof.reshape(-1,)).to_csv('oof.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\nsub['target'] = preds.cpu().numpy().reshape(-1,)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}