{"cells":[{"metadata":{"trusted":true,"id":"em2wsQILRcgw"},"cell_type":"code","source":"!pip install --upgrade albumentations > /dev/null\n!pip install efficientnet_pytorch pytorch_toolbelt > /dev/null\n!pip install --upgrade wandb > /dev/null\n!pip install torchcontrib > /dev/null\n!pip install timm > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"hB8Dk3hARcgn"},"cell_type":"code","source":"import torch\ntorch.cuda.get_device_name(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"CZ38eMu4Rcgy"},"cell_type":"code","source":"def get_data():\n    # mount kaggle\n    !mkdir /kaggle\n    !mkdir /kaggle/input\n    !mkdir /kaggle/input/siim-isic-melanoma-classification/\n    !mkdir /kaggle/input/melanoma-merged-external-data-512x512-jpeg\n    !pip install -q kaggle\n    from google.colab import files\n    files.upload()\n    !mv train.csv /kaggle/input/siim-isic-melanoma-classification/\n    !mv test.csv /kaggle/input/siim-isic-melanoma-classification/\n    !mv sample_submission.csv /kaggle/input/siim-isic-melanoma-classification/\n    !mkdir -p ~/.kaggle\n    !cp kaggle.json ~/.kaggle/\n    !ls ~/.kaggle\n    !chmod 600 /root/.kaggle/kaggle.json\n\n    #\n\n    !kaggle datasets download -d shonenkov/melanoma-merged-external-data-512x512-jpeg\n    !unzip melanoma-merged-external-data-512x512-jpeg.zip -d /kaggle/input/melanoma-merged-external-data-512x512-jpeg > /dev/null\n    !rm melanoma-merged-external-data-512x512-jpeg.zip","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"JXmaB_tsRcg2"},"cell_type":"code","source":"#get_data()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"id":"kHurv9rxRcg5"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"R7lF2NrcRcg7"},"cell_type":"code","source":"from tqdm.notebook import tqdm\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"6aEXFKoqRcg-"},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, average_precision_score\nfrom sklearn.preprocessing import StandardScaler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"KA3C_OfVRchB"},"cell_type":"code","source":"import gc\nimport time\nimport random","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"toJCF04mRchD"},"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader, Sampler\nimport torch.optim as optim\nimport torch.nn as nn\nimport torch.nn.functional as F","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"f-l7B2uRRchI"},"cell_type":"code","source":"import matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"VW4j0DXbRchO"},"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"id":"jc7SxgFTvvD0","trusted":true},"cell_type":"code","source":"from pytorch_toolbelt.losses.focal import BinaryFocalLoss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"1uawzOcoRchR"},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"id":"J55T3-MIWKe-","trusted":true},"cell_type":"code","source":"from torchcontrib.optim import SWA","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"2_ZiOAAoRchT"},"cell_type":"code","source":"import os","execution_count":null,"outputs":[]},{"metadata":{"id":"AStAkP0X4Q-N","trusted":true},"cell_type":"code","source":"import wandb","execution_count":null,"outputs":[]},{"metadata":{"id":"mg0CtVd1DZf-","trusted":true},"cell_type":"code","source":"import albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2","execution_count":null,"outputs":[]},{"metadata":{"id":"eqcNLm02zxV9","trusted":true},"cell_type":"code","source":"import timm","execution_count":null,"outputs":[]},{"metadata":{"id":"z8-pJZwnEeRx","trusted":true},"cell_type":"code","source":"hyperparameters = dict(\n        IMG_SIZE = 384,\n        TEST_FOLD = 1,\n        VALID_FOLD = 0,\n        BATCH_SIZE = 16,\n        EPOCHS = 20,\n        EARLY_STOPPING = 15,\n        LR = 0.00005,\n        LOSS_FN = 'bce',\n        BACKBONE = 'noisy-b3',\n        ALPHA = 0.5,\n        GAMMA = 2,\n        BEST_AUC = 0.0,\n        SEED = 1110\n    )","execution_count":null,"outputs":[]},{"metadata":{"id":"kR_2rqGNo9rx","trusted":true},"cell_type":"code","source":"SEED = hyperparameters['SEED']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"q3EZtloiRchV"},"cell_type":"code","source":"os.environ['PYTHONHASHSEED'] = str(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\nrandom.seed(SEED)\ntorch.cuda.manual_seed(SEED)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"DAFh1oceRchY"},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"wfX6zMk2Rcha"},"cell_type":"code","source":"PATH = '/kaggle/input/siim-isic-melanoma-classification/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"iIu8BnReRchc"},"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    def __init__(self, df, test, root_dir, transforms=None):\n        self.df = df\n        self.test = test\n        self.root_dir = root_dir\n        self.transforms = transforms  \n    \n    def __len__(self):\n        return len(self.df)\n\n    def get_labels(self):\n        return list(self.df.target)\n    \n    def __getitem__(self, idx):\n        img = cv2.imread(self.root_dir + self.df.iloc[idx].image_id + '.jpg', cv2.IMREAD_COLOR)\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        label = 0 if self.test else self.df.iloc[idx].target\n        sample = {'image': img, 'label': label}\n        return sample","execution_count":null,"outputs":[]},{"metadata":{"id":"YOG9VQj0SiM9","trusted":false},"cell_type":"code","source":"class UnbalancedBatchSampler(Sampler):\n    def __init__(self, pos_indices, neg_indices, batch_size):\n        self.pos_indices = pos_indices\n        self.neg_indices = neg_indices\n        self.batch_size = batch_size\n\n    def __iter__(self):\n        batch = []\n        i = 0\n        np.random.shuffle(self.pos_indices)\n        np.random.shuffle(self.neg_indices)\n        batch.append(self.pos_indices[i])\n        for neg in self.neg_indices:\n            batch.append(neg)\n            if len(batch) == self.batch_size:\n                yield batch\n                batch = []\n                i = i + 1 if i < len(self.pos_indices)-1 else 0\n                batch.append(self.pos_indices[i])\n\n    def __len__(self):\n        return (len(self.neg_indices)) // (self.batch_size-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"p6bQYH-7Rchh"},"cell_type":"code","source":"TRAIN_DIR = '/kaggle/input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/512x512-dataset-melanoma/'\nTEST_DIR = '/kaggle/input/melanoma-merged-external-data-512x512-jpeg/512x512-test/512x512-test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"eyLe8cQlRchj"},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/melanoma-merged-external-data-512x512-jpeg/folds_13062020.csv')\ntest_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')\n\ntest_df.rename(columns={'image_name': 'image_id'}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"6z-jVdul4OfR","trusted":false},"cell_type":"code","source":"def train_valid_split(valid_fold, test_fold, train_df):\n    X_train = train_df[((~train_df.fold.isin([valid_fold, test_fold])) | (train_df.source != 'ISIC20'))]\n    X_valid = train_df[(train_df.fold == valid_fold) & (train_df.source == 'ISIC20')]\n    X_test = train_df[(train_df.fold == test_fold) & (train_df.source == 'ISIC20')]\n    X_train.reset_index(inplace=True, drop=True)\n    X_valid.reset_index(inplace=True, drop=True)\n    X_test.reset_index(inplace=True, drop=True)\n    return X_train, X_test, X_valid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"gj0DZkZWRch5"},"cell_type":"code","source":"class MelanomaModel(nn.Module):\n    def __init__(self, basemodel):\n        super(MelanomaModel, self).__init__()\n\n        #self.backbone = EfficientNet.from_pretrained(basemodel)\n        self.backbone = timm.create_model('tf_efficientnet_b3_ns', pretrained=True)\n        self.output = nn.Sequential(nn.Linear(1000, 128),\n                                  nn.LeakyReLU(),\n                                  nn.Dropout(p=0.4),\n                                  nn.Linear(128, 1))\n\n    def forward(self, x):\n        x = self.backbone(x)\n        return  self.output(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"fzmcgf-fRch7"},"cell_type":"code","source":"def validation(model, device, loader, loss_fn):\n    model.eval()\n    valid_loss = 0\n    n_batches = 0\n    y_pred = np.array([])\n    target = np.array([])\n    for i, batch in tqdm(enumerate(loader), total=len(loader)):\n        with torch.no_grad():\n            inputs = batch['image'].to(device=device, dtype=torch.float32)\n            labels = batch['label'].to(device=device, dtype=torch.float32)\n\n            target = np.append(target, labels.cpu().numpy(), axis=0)\n\n            outputs = model(inputs)\n            loss = loss_fn(outputs, labels.unsqueeze(1))\n            valid_loss += loss.detach().item()\n            n_batches += 1\n            y_pred = np.append(y_pred, nn.Sigmoid()(outputs).detach().reshape(-1).cpu().numpy(), axis=0)\n    valid_auc = roc_auc_score(target, y_pred)\n    valid_pre = average_precision_score(target, y_pred)\n    return valid_loss/n_batches, valid_auc, valid_pre","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"gHIvei_wRch9"},"cell_type":"code","source":"def train(model, device, loss_fn, scheduler, optimizer, \n          train_loader, valid_loader, epochs, early_stopping):\n    y_pred = np.array([])\n    target = np.array([])\n    n_batches = 0\n    epoch_loss = 0.0\n    best_valid_auc = 0\n    patience = 0\n    inter = 0\n    for epoch_n in range(0, epochs):\n        print(f'EPOCH: {epoch_n}, PATIENCE: {patience}')\n        for i, batch in tqdm(enumerate(train_loader), total=len(train_loader)):\n            model.train()\n            inputs = batch['image'].to(device=device, dtype=torch.float32)\n            labels = batch['label'].to(device=device, dtype=torch.float32)\n            target = np.append(target, labels.cpu().numpy(), axis=0)\n            \n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = loss_fn(outputs, labels.unsqueeze(1))\n            loss.backward()\n            optimizer.step()\n            \n            scheduler.step()\n            epoch_loss += loss.item()\n            n_batches += 1\n            y_pred = np.append(y_pred, nn.Sigmoid()(outputs).detach().reshape(-1).cpu().numpy(), axis=0)\n            if n_batches % 1000 == 0:   \n                inter += 1\n                optimizer.update_swa()\n                optimizer.swap_swa_sgd()\n                valid_loss, valid_auc, valid_pre = validation(model, device, valid_loader, loss_fn)\n                train_auc = roc_auc_score(target, y_pred)\n                epoch_loss = epoch_loss/n_batches\n                n_batches = 0\n                y_pred = np.array([])\n                target = np.array([])\n                if best_valid_auc < valid_auc:\n                    patience = 0\n                    best_valid_auc = valid_auc\n                    torch.save(model.state_dict(), hyperparameters['MODEL_NAME'])\n                    wandb.config.update({'BEST_AUC': best_valid_auc}, allow_val_change=True) \n                    wandb.save(hyperparameters['MODEL_NAME'])\n                else:\n                    patience += 1\n                optimizer.swap_swa_sgd()\n                wandb.log({'train_loss': epoch_loss, 'valid_loss': valid_loss, 'train_auc': train_auc, \n                           'valid_auc': valid_auc, 'valid_pre': valid_pre, 'lr': scheduler.get_last_lr()[0]})\n                epoch_loss = 0\n                if patience >= early_stopping:\n                    return model\n    optimizer.swap_swa_sgd()\n    return model","execution_count":null,"outputs":[]},{"metadata":{"id":"rIbqx7ISPjdS","trusted":true},"cell_type":"code","source":"!wandb login","execution_count":null,"outputs":[]},{"metadata":{"id":"bRRs7Bl8JJq3","trusted":true},"cell_type":"code","source":"hyperparameters['MODEL_NAME'] = 'checkpoint_' + str(random.randint(0, 2000)) + '_' + str(hyperparameters['VALID_FOLD']) + '.bin'","execution_count":null,"outputs":[]},{"metadata":{"id":"VIUflbq-RInp","trusted":false},"cell_type":"code","source":"hyperparameters","execution_count":null,"outputs":[]},{"metadata":{"id":"fWkPYam_Diui","trusted":true},"cell_type":"code","source":"train_transforms = A.Compose([\n    #A.ToGray(p=1.0),\n    A.RandomCrop(hyperparameters['IMG_SIZE'], hyperparameters['IMG_SIZE']),\n    #A.Resize(hyperparameters['IMG_SIZE'], hyperparameters['IMG_SIZE']),                           \n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.HueSaturationValue(hue_shift_limit=20, sat_shift_limit=30, val_shift_limit=20, p=0.5),\n    A.ShiftScaleRotate(p=0.5),\n    A.GaussNoise(p=0.1),\n    A.CoarseDropout(max_holes=10, min_holes=1, max_height=64, max_width=64, p=0.8),\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225],),\n    ToTensorV2()\n])\n\nvalid_transforms = A.Compose([\n    #A.ToGray(p=1.0),\n    A.CenterCrop(hyperparameters['IMG_SIZE'], hyperparameters['IMG_SIZE']),\n    #A.Resize(hyperparameters['IMG_SIZE'], hyperparameters['IMG_SIZE']),\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225],),\n    ToTensorV2()\n])","execution_count":null,"outputs":[]},{"metadata":{"id":"-oqlpKMLDAAo","trusted":true},"cell_type":"code","source":"X_train, X_test, X_valid = train_valid_split(hyperparameters['VALID_FOLD'], hyperparameters['TEST_FOLD'], train_df)","execution_count":null,"outputs":[]},{"metadata":{"id":"w9HniL9fHa_4","trusted":true},"cell_type":"code","source":"pos_indices = np.asarray(X_train[X_train.target == 1].index)\nneg_indices = np.asarray(X_train[X_train.target == 0].index)","execution_count":null,"outputs":[]},{"metadata":{"id":"CKbOoHQ932bj","trusted":true},"cell_type":"code","source":"train_sampler = UnbalancedBatchSampler(pos_indices, neg_indices, batch_size=hyperparameters['BATCH_SIZE'])","execution_count":null,"outputs":[]},{"metadata":{"id":"4jIG1d8331a1","trusted":true},"cell_type":"code","source":"train_dataset = MelanomaDataset(X_train, False, TRAIN_DIR, train_transforms)\nvalid_dataset = MelanomaDataset(X_valid, False, TRAIN_DIR, valid_transforms)","execution_count":null,"outputs":[]},{"metadata":{"id":"AdU8R9xJ3zy6","trusted":true},"cell_type":"code","source":"train_loader = DataLoader(train_dataset, batch_sampler=train_sampler, num_workers=4)\nvalid_loader = DataLoader(valid_dataset, batch_size=32, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"id":"OwPzysTi-aB3","trusted":true},"cell_type":"code","source":"model = MelanomaModel(hyperparameters['BACKBONE']).to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"n3sP-zA7RciD"},"cell_type":"code","source":"if hyperparameters['LOSS_FN'] == 'focal':\n    loss_fn = BinaryFocalLoss(alpha=hyperparameters['ALPHA'], gamma=hyperparameters['GAMMA'])\nelse:\n    loss_fn = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([15]).to(device))\n\noptimizer_1 = optim.Adam(model.parameters(), lr=hyperparameters['LR'])","execution_count":null,"outputs":[]},{"metadata":{"id":"zhAJoC5XGYxR","trusted":false},"cell_type":"code","source":"optimizer = SWA(optimizer_1)\nscheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=hyperparameters['LR'], max_lr=hyperparameters['LR']*2, step_size_up=len(train_loader)/2, cycle_momentum=False)","execution_count":null,"outputs":[]},{"metadata":{"id":"nSdBSZuyAD28","trusted":false},"cell_type":"code","source":"wandb.init(project=\"melanoma-training\", config=hyperparameters)\nwandb.watch(model)\nmodel = train(model, device, loss_fn, scheduler, optimizer,\n              train_loader, valid_loader, epochs=hyperparameters['EPOCHS'], \n              early_stopping=hyperparameters['EARLY_STOPPING'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Melanoma tabular data"},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"TRAIN_DIR = '/kaggle/input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/512x512-dataset-melanoma/'\nTEST_DIR = '/kaggle/input/melanoma-merged-external-data-512x512-jpeg/512x512-test/512x512-test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/melanoma-merged-external-data-512x512-jpeg/folds_13062020.csv')\nsubmit_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit_df.rename(columns={'image_name': 'image_id'}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.drop(columns=['patient_id', 'stratify_group'], inplace=True)\nsubmit_df.drop(columns=['patient_id'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_target = train_df[['image_id', 'target', 'fold', 'source']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.drop(columns=['target', 'fold', 'source'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['SPLIT'] = 'TRAIN'\nsubmit_df['SPLIT'] = 'TEST'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = train_df.append(submit_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"one_hot = pd.get_dummies(df[['sex', 'anatom_site_general_challenge']], dummy_na=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[one_hot.columns] = one_hot","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.drop(columns=['sex', 'anatom_site_general_challenge'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.drop(columns=['sex_unknown', 'anatom_site_general_challenge_unknown'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = df.drop(columns=['image_id', 'SPLIT']).columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = df[df.SPLIT == 'TRAIN']\nsubmit_df = df[df.SPLIT == 'TEST']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.drop(columns=['SPLIT'], inplace=True)\nsubmit_df.drop(columns=['SPLIT'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.merge(train_target, on='image_id', how='outer')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit_df.drop(columns=['image_id'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = train_df[((train_df.fold != 1) | (train_df.source != 'ISIC20'))]\nX_test = train_df[(train_df.fold == 1) & (train_df.source == 'ISIC20')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.drop(columns=['source'], inplace=True)\nX_test.drop(columns=['source'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.reset_index(inplace=True, drop=True)\nX_test.reset_index(inplace=True, drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = X_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test = X_test.target\nX_test = X_test.drop(columns=['fold', 'target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_random_params():\n    params = {\n        'boosting_type': 'gbdt',\n        'metric': 'auc',\n        'num_leaves': random.randint(35, 60),\n        'max_depth': random.randint(15, 25),\n        'learning_rate': random.choice([0.001, 0.003, 0.005, 0.01]),\n        'n_estimators': random.randint(5000, 15000),\n        'objective': 'binary',\n        'reg_alpha': random.choice([0.001, 0.005, 0.01, 0.05]),\n        'reg_lambda': random.choice([0.001, 0.005, 0.01, 0.05]),       \n        'colsample_bytree': random.choice([0.5, 0.6, 0.7, 0.8]),\n        'min_child_samples': random.randint(50, 100),\n        'subsample_for_bin': random.randint(100000, 300000)\n    }\n    return params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_params = {'boosting_type': 'gbdt',\n 'metric': 'auc',\n 'num_leaves': 42,\n 'max_depth': 24,\n 'learning_rate': 0.01,\n 'n_estimators': 11876,\n 'objective': 'binary',\n 'reg_alpha': 0.01,\n 'reg_lambda': 0.001,\n 'colsample_bytree': 0.6,\n 'min_child_samples': 73,\n 'subsample_for_bin': 174110}\nbest_auc = 0.7058965701293586","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm.notebook import tqdm\nimport random, time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_best_params(hyper_rounds, n_folds, best_params=None, best_auc=0):\n    best_params = best_params\n    best_auc = best_auc\n    lgb_train = lgb.Dataset(data=train_df.drop(columns=['image_id', 'target', 'fold']), label=train_df.target)\n    for i in tqdm(range(hyper_rounds)):\n        curr_params = get_random_params()\n        start = time.time()\n        print(curr_params)\n        eval_hist = lgb.cv(curr_params, lgb_train, early_stopping_rounds = 200, nfold = n_folds, seed = 42, verbose_eval = 100)\n        end = time.time()\n        print('TIME:', end-start)\n        curr_auc = eval_hist['auc-mean'][-1]\n        if curr_auc > best_auc:\n            best_params = curr_params\n            best_auc = curr_auc\n    return best_params, best_auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"HYPER_ROUNDS = 10\nFOLDS = 5\n#best_params, best_auc = get_best_params(HYPER_ROUNDS, FOLDS, best_params, best_auc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_preds = np.zeros(len(submit_df))\nt1_preds = np.zeros(len(X_test))\n\navg_valid_auc = 0\nfor n_fold in [0, 2, 3, 4]:\n    print(\"FOLD N:\", n_fold)\n    X_train = train_df[train_df.fold != n_fold].drop(columns=['fold', 'target', 'image_id'])\n    y_train = train_df[train_df.fold != n_fold].target\n    X_valid = train_df[train_df.fold == n_fold].drop(columns=['fold', 'target', 'image_id'])\n    y_valid = train_df[train_df.fold == n_fold].target\n\n    lgb_train = lgb.Dataset(data=X_train, label=y_train)\n    lgb_eval = lgb.Dataset(data=X_valid, label=y_valid)\n    model = lgb.train(best_params, lgb_train, valid_sets=lgb_eval, early_stopping_rounds=1500, verbose_eval=100)\n    \n    y_pred = model.predict(X_test.drop(columns=['image_id']))\n    t1_preds += y_pred / 4\n    \n    sub_preds += model.predict(submit_df) / 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roc_auc_score(y_test, t1_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t1 = X_test[['image_id']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t1['target'] = y_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t1['TAB_PRED'] = t1_preds\nt1.to_csv('t1_tab.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/' + 'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit['target'] = sub_preds\nsubmit.to_csv('submission_tab.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit.sort_values(by='target')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Melanoma Inference and Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = '/kaggle/input/siim-isic-melanoma-classification/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.read_csv(PATH + 'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    def __init__(self, df, test, root_dir, transforms=None):\n        self.df = df\n        self.test = test\n        self.root_dir = root_dir\n        self.transforms = transforms  \n    \n    def __len__(self):\n        return len(self.df)\n\n    def get_labels(self):\n        return list(self.df.target)\n    \n    def __getitem__(self, idx):\n        img = cv2.imread(self.root_dir + self.df.iloc[idx].image_id + '.jpg', cv2.IMREAD_COLOR)\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        label = 0 if self.test else self.df.iloc[idx].target\n        fv = self.df.drop(columns=['image_id']).iloc[idx] if self.test else self.df.drop(columns=['image_id', 'target']).iloc[idx]\n        fv = np.array(fv)\n        sample = {'image': img, 'label': label}\n        return sample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_DIR = '/kaggle/input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/512x512-dataset-melanoma/'\nTEST_DIR = '/kaggle/input/melanoma-merged-external-data-512x512-jpeg/512x512-test/512x512-test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/melanoma-merged-external-data-512x512-jpeg/folds_13062020.csv')\ntest_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp = pd.read_csv('../input/melanoma-last-try/valid_1.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.drop(columns=['patient_id', 'source', 'stratify_group'], inplace=True)\n\ntest_df.rename(columns={'image_name': 'image_id'}, inplace=True)\n\ntest_df.drop(columns=['patient_id'], inplace=True)\n\ntrain_target = train_df[['image_id', 'target', 'fold']]\n\ntrain_df.drop(columns=['target', 'fold'], inplace=True)\n\ntrain_df['SPLIT'] = 'TRAIN'\ntest_df['SPLIT'] = 'TEST'\n\ndf = train_df.append(test_df)\n\none_hot = pd.get_dummies(df[['sex', 'anatom_site_general_challenge']], dummy_na=False)\n\ndf[one_hot.columns] = one_hot\n\ndf.drop(columns=['sex', 'anatom_site_general_challenge'], inplace=True)\n\ncols = df.drop(columns=['image_id', 'SPLIT']).columns\n\ndf[cols] = StandardScaler().fit_transform(df.drop(columns=['image_id', 'SPLIT']))\n\ntest_df = df[df.SPLIT == 'TEST']\ntrain_df = df[df.SPLIT == 'TRAIN']\n\n\ntest_df.drop(columns=['SPLIT'], inplace=True)\ntrain_df.drop(columns=['SPLIT'], inplace=True)\n\ntrain_df = train_df.merge(train_target, on='image_id', how='outer')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_valid = train_df[train_df.fold == 1]\nX_valid.drop(columns='fold', inplace=True)\nX_valid.reset_index(inplace=True, drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(models, device, loader, tta_rounds):\n    tta_preds = torch.Tensor([])\n    with torch.no_grad():\n        for t in range(tta_rounds):\n            print('TTA round:', t)\n            y_pred = torch.Tensor([])\n            for i, batch in tqdm(enumerate(loader), total=len(loader)):    \n                inputs = batch['image'].to(device=device, dtype=torch.float32)\n                batch_preds = torch.zeros(inputs.shape[0])\n                for model in models:\n                    model.eval()\n\n                    outputs = model(inputs)\n                    batch_preds += nn.Sigmoid()(outputs).detach().reshape(-1).cpu()\n                batch_preds /= len(models)\n\n                y_pred = torch.cat((y_pred, batch_preds))\n            tta_preds = torch.cat((tta_preds, y_pred.unsqueeze(1)), dim=1)\n    return tta_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!wandb login \nwandb.init(project=\"melanoma-training\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_transforms = A.Compose([\n    A.Resize(384, 384),\n    #A.HorizontalFlip(p=0.5),\n    #A.VerticalFlip(p=0.5),\n    #A.ShiftScaleRotate(p=0.5),\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225],),\n    ToTensorV2()\n])\n\nclass MelanomaModel_0(nn.Module):\n    def __init__(self, basemodel):\n        super(MelanomaModel_0, self).__init__()\n        self.efficient = EfficientNet.from_pretrained(basemodel)\n        self.output = nn.Linear(1000, 1)        \n\n    def forward(self, x):\n        x = self.efficient(x)\n        return  self.output(x)\n\nbest_model = wandb.restore('checkpoint_1309_0.bin', run_path='meraxes10/melanoma-training/3ibunqgp')\nmodel_0_0 = torch.load(best_model.name)\n\nbest_model = wandb.restore('checkpoint_1587_2.bin', run_path='meraxes10/melanoma-training/1tphq2cw')\nmodel_0_2 = torch.load(best_model.name)\n\nbest_model = wandb.restore('checkpoint_232_3.bin', run_path='meraxes10/melanoma-training/yfmsz1se')\nmodel_0_3 = torch.load(best_model.name)\n\nbest_model = wandb.restore('checkpoint_1889_4.bin', run_path='meraxes10/melanoma-training/3buhx3iw')\nmodel_0_4 = torch.load(best_model.name)\n\nweights_0 = [model_0_0, model_0_2, model_0_3, model_0_4]\n\nmodels_0 =  []\n\nfor weight in tqdm(weights_0):\n    model = MelanomaModel_0('efficientnet-b3')\n    model = model.to(device)\n    model.load_state_dict(weight)    \n    models_0.append(model)\n\nbest_model = wandb.restore('checkpoint_829_0.bin', run_path='meraxes10/melanoma-training/1lel9snt')\nmodel_1_0 = torch.load(best_model.name)\n\nbest_model = wandb.restore('checkpoint_527_2.bin', run_path='meraxes10/melanoma-training/1ug3iqji')\nmodel_1_2 = torch.load(best_model.name)\n\nbest_model = wandb.restore('checkpoint_527_3.bin', run_path='meraxes10/melanoma-training/2342rjre')\nmodel_1_3 = torch.load(best_model.name)\n\nbest_model = wandb.restore('checkpoint_1777_4.bin', run_path='meraxes10/melanoma-training/184ypebh')\nmodel_1_4 = torch.load(best_model.name)\n\nweights_1 = [model_1_0, model_1_2, model_1_3, model_1_4]\n\nmodels_1 =  []\n\nfor weight in tqdm(weights_1):\n    model = MelanomaModel_0('efficientnet-b3')\n    model = model.to(device)\n    model.load_state_dict(weight)    \n    models_1.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_1 = A.Compose([\n    A.RandomCrop(456, 456),\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225],),\n    ToTensorV2()\n])\n\nclass MelanomaModel_1(nn.Module):\n    def __init__(self, basemodel):\n        super(MelanomaModel_1, self).__init__()\n        self.backbone = timm.create_model('tf_efficientnet_b5_ns', pretrained=True)\n        self.output = nn.Sequential(nn.Linear(1000, 64),\n                                  nn.LeakyReLU(),\n                                  nn.Dropout(p=0.2),\n                                  nn.Linear(64, 1))\n\n    def forward(self, x):\n        x = self.backbone(x)\n        return  self.output(x)\n\nbest_model = wandb.restore('checkpoint_926_0.bin', run_path='meraxes10/melanoma-training/1tebit2x')\nmodel_2_0 = torch.load(best_model.name)\n\nbest_model = wandb.restore('checkpoint_121_2.bin', run_path='meraxes10/melanoma-training/37r1caar')\nmodel_2_2 = torch.load(best_model.name)\n\nbest_model = wandb.restore('checkpoint_827_3.bin', run_path='meraxes10/melanoma-training/2mxhjfef')\nmodel_2_3 = torch.load(best_model.name)\n\nbest_model = wandb.restore('checkpoint_1853_4.bin', run_path='meraxes10/melanoma-training/un47ujkc')\nmodel_2_4 = torch.load(best_model.name)\n\nweights_2 = [model_2_0, model_2_2, model_2_3, model_2_4]\n\nmodels_2 =  []\n\nfor weight in tqdm(weights_2):\n    model = MelanomaModel_1('efficientnet-b3')\n    model = model.to(device)\n    model.load_state_dict(weight)    \n    models_2.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_2 = A.Compose([\n    A.RandomCrop(424, 424),\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225],),\n    ToTensorV2()\n])\n\nclass MelanomaModel_2(nn.Module):\n    def __init__(self, basemodel):\n        super(MelanomaModel_2, self).__init__()\n\n        #self.backbone = EfficientNet.from_pretrained(basemodel)\n        self.backbone = timm.create_model('tf_efficientnet_b4_ns', pretrained=True)\n        self.output = nn.Sequential(nn.Linear(1000, 128),\n                                  nn.LeakyReLU(),\n                                  nn.Dropout(p=0.4),\n                                  nn.Linear(128, 1))\n\n    def forward(self, x):\n        x = self.backbone(x)\n        return  self.output(x)\n\nbest_model = wandb.restore('checkpoint_1172_0.bin', run_path='meraxes10/melanoma-training/3eg3brbf')\nmodel_3_0 = torch.load(best_model.name)\n\nbest_model = wandb.restore('checkpoint_1152_2.bin', run_path='meraxes10/melanoma-training/3hly12g9')\nmodel_3_2 = torch.load(best_model.name)\n\nbest_model = wandb.restore('checkpoint_1608_3.bin', run_path='meraxes10/melanoma-training/2q72g2vp')\nmodel_3_3 = torch.load(best_model.name)\n\nbest_model = wandb.restore('checkpoint_985_4.bin', run_path='meraxes10/melanoma-training/gtruril0')\nmodel_3_4 = torch.load(best_model.name)\n\nweights_3 = [model_3_0, model_3_2, model_3_3, model_3_4]\n\nmodels_3 =  []\n\nfor weight in tqdm(weights_3):\n    model = MelanomaModel_2('efficientnet-b3')\n    model = model.to(device)\n    model.load_state_dict(weight)    \n    models_3.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_dataset = MelanomaDataset(X_valid, True, TRAIN_DIR, transforms_2)\nvalid_loader = DataLoader(valid_dataset, batch_size=32, num_workers=4, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tta_preds_3 = predict(models_3, device, valid_loader, 1)\ntta_preds_3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp['PRED_3'] = tta_preds_3.cpu().numpy().flatten()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tab = pd.read_csv('../input/melanoma-last-try/t1_tab.csv')\ntab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/melanoma-merged-external-data-512x512-jpeg/folds_13062020.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp = temp.merge(train_df, how='left', on=['image_id', 'target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp = temp.merge(tab, how='left', on=['image_id', 'target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp = temp[temp.source == 'ISIC20']\ntemp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = 0.3*temp.PRED_0 + 0.6*temp.PRED_3 + 0.05*temp.PRED_1 + 0.05*temp.PRED_2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roc_auc_score(temp.target, 0.8*y_pred + 0.2*temp.TAB_PRED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = MelanomaDataset(test_df, True, TEST_DIR, transforms_2)\ntest_loader = DataLoader(test_dataset, batch_size=64, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_preds_3 = predict(models_3, device, test_loader, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp = pd.read_csv('../input/melanoma-last-try/test_sub.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp['SUB_3'] = sub_preds_3.cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp.to_csv('test_sub.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"0.3*temp.SUB_0 + 0.6*temp.SUB_3 + 0.05*temp.SUB_1 + 0.05*temp.SUB_2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp = pd.read_csv('../input/melanoma-last-try/submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp2 = pd.read_csv('../input/melanoma-last-try/submission_tab (1).csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit['target'] = 0.8*temp.target + 0.2*temp2.target\nsubmit.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit","execution_count":null,"outputs":[]},{"metadata":{"id":"gJwaJP3wR1Db","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}