{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib.pyplot as plt ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ohe = pd.read_csv('../input/combined-train/test.csv')\ntest = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\ntrain_ohe = pd.read_csv('../input/combined-train/train0.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ohe.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ohe.drop(['target', 'strat'], axis = 1).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom skimage import io, transform\nfrom torchvision import models, transforms\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport time\nimport copy \nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split as ttp\nfrom skimage.filters import threshold_otsu\nfrom skimage.color import rgb2gray\nimport cv2 as cv \nimport pickle\nimport random \nimport albumentations\nimport gc ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''if mask:\n    TRAIN_FOLDER ='../input/channel1/train_masks'\n    TEST_FOLDER = '../input/channel1/test_masks'\n\nelif nohair:\n    TRAIN_FOLDER = '../input/nohair/train_nohair'\n    TEST_FOLDER = '../input/nohair/test_nohair'\nelse:\n    TRAIN_FOLDER = '/kaggle/input/melanoma-external-malignant-256/train/train'\n    TEST_FOLDER = '/kaggle/input/melanoma-external-malignant-256/test/test'\nTRAIN_CSV = '/kaggle/input/melanoma-external-malignant-256/train_concat.csv'\nTEST_CSV = '../input/siim-isic-melanoma-classification/test.csv'''\n\nMODELS_PATH = '../input/melanoma-models/'\n#MODELS2 = '../input/practice-models/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DIR0 = '../input/jpeg-melanoma-384x384/train'\nDIR1 = '../input/jpeg-isic2019-384x384/train'\n\nTEST_FOLDER = '../input/jpeg-melanoma-384x384/test'\nTEST_CSV = '../input/combined-train/test.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLD_CSVS ={0:'../input/combined-train/train0.csv',1:'../input/combined-train/train1.csv',2:'../input/combined-train/train2.csv',\n            3:'../input/combined-train/train3.csv',4:'../input/combined-train/train4.csv'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean=[0.485, 0.456, 0.406]\nstd=[0.229, 0.224, 0.225]\n\ntransform_train = albumentations.Compose([\n    albumentations.ShiftScaleRotate(shift_limit = 0.15, scale_limit = 0.1, rotate_limit = 25, p = 0.75),\n    albumentations.VerticalFlip(p=0.5),\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.RandomBrightness(limit=0.2, p=0.65),\n    albumentations.RandomContrast(limit=0.2, p=0.65),\n    albumentations.OneOf([\n            albumentations.OpticalDistortion(distort_limit=1.0),\n            albumentations.GridDistortion(num_steps=5, distort_limit=1.),\n        ], p=0.7),\n    albumentations.OneOf([\n            albumentations.MotionBlur(blur_limit=5),\n            albumentations.MedianBlur(blur_limit=5),\n            albumentations.GaussianBlur(blur_limit=5),\n            albumentations.GaussNoise(var_limit=(5.0, 30.0)),\n        ], p=0.7),\n    albumentations.Normalize(mean, std, always_apply = True),\n\n])\n\ntransform_valid = albumentations.Compose([\n    albumentations.Normalize(mean, std, always_apply = True),\n])\n\ntransform_test = albumentations.Compose([\n    albumentations.Normalize(mean, std, always_apply = True),\n])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"binary is (256,256), want to make that (256,256,3)\nhave (256,256,1), that 1 is currently "},{"metadata":{"trusted":true},"cell_type":"code","source":"class melanoma_dataset(Dataset):\n    def __init__(self, root_dir, transform, df = pd.DataFrame() , csv_file = False, train = True):\n        \n        \n        self.df = df\n        \n        if csv_file:\n            self.csv = pd.read_csv(csv_file)\n        \n        self.directory = root_dir\n        \n        self.transform = transform\n        \n        self.train = train\n        \n        \n        \n    def __getitem__(self,idx):\n        \n        if not self.df.empty:\n            tab = self.df\n        else:\n            tab = self.csv\n        \n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        \n        if tab.iloc[idx,1] == '-1':\n            directory = self.directory[1]\n        else:\n            directory = self.directory[0]\n        \n        if not self.train:\n            directory = TEST_FOLDER\n        img_name = os.path.join(directory, tab.iloc[idx, 0]) + '.jpg'\n        idd =  tab.iloc[idx, 0]\n        \n        img = cv.imread(img_name)\n\n        target= tab.iloc[idx, 3] if self.train else 0\n        \n        if self.transform:\n            #sample= self.transform(image = self['image'], target = self['target'])\n            image = self.transform(image = img)\n            flipped = image['image']\n            image = np.transpose(flipped, (2, 0, 1)).astype(np.float32)\n        if self.train:\n            meta = np.asarray(tab.iloc[idx, [2,5,6,7,8,9,10,11,12,13,14]].values, dtype = np.float32)\n        else:\n            meta = np.asarray(tab.iloc[idx,[2,3,4,5,6,7,8,9,10,11,12]].values, dtype = np.float32)\n        if self.train:\n            return (image, meta), target\n        else:\n            return (image, meta)\n        \n        \n        \n        \n    \n    def __len__(self):\n        if not self.df.empty:\n            return len(self.df)\n        else:\n            return len(self.csv)\n        \n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train = melanoma_dataset(pd.DataFrame(), TRAIN_CSV, TRAIN_FOLDER, transform_train)\n#train_size = int(0.8*len(train))\n#valid_size = len(train) - train_size\n\n#train1, valid1 = torch.utils.data.random_split(train, [train_size, valid_size])\n#dataset_sizes = {x:len(x) for x in [train1, valid1]}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(47)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet \n#model0 = EfficientNet.from_pretrained('efficientnet-b1')\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nfeature_extract = True\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self, arch, n_meta_features = 11):\n        super(Net, self).__init__()\n        self.arch = arch\n\n        self.arch._fc = nn.Linear(in_features=2304, out_features=500, bias=True)\n        self.meta = nn.Sequential(nn.Linear(n_meta_features, 1000),\n                                  nn.BatchNorm1d(1000),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.3),\n                                  nn.Linear(1000, 500),  # FC layer output will have 250 features\n                                  nn.BatchNorm1d(500),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2))\n        self.ouput = nn.Linear(500 + 500, 1)\n        \n    def forward(self, inputs):\n\n        x, meta = inputs\n        cnn_features = self.arch(x)\n        meta_features = self.meta(meta)\n        features = torch.cat((cnn_features, meta_features), dim=1)\n        output = self.ouput(features)\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def kfold_train(folds=3, num_epochs = 3,train = True):\n\n\n    #general variables\n    since = time.time()\n    val_acc_history = []\n    roc = 0\n    \n    if train == True:\n        for fold in range(folds):\n            best_roc_auc = 0.0\n            #model = EfficientNet.from_pretrained('efficientnet-b6')\n            model = Net(EfficientNet.from_pretrained('efficientnet-b6'))\n            #model.cuda()\n            model_path = f'model_{fold}.pth'\n            model._fc = nn.Linear(2304, 1)\n            model.to(device)\n            #each epoch has k folds\n            print('-'*10)\n            acc_dic = {}\n            loss_dic = {}\n            CSV = pd.read_csv(FOLD_CSVS[fold])\n            train_df, valid_df, _,_ =  ttp(CSV, np.zeros(len(CSV)), train_size = .75)\n        \n            print('Fold {}/{}'.format(fold, folds-1))\n\n            optimizer = optim.SGD(model.parameters(), lr = 0.001, momentum = 0.9)\n            scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\n            criterion = nn.BCEWithLogitsLoss()\n\n\n\n            for epoch in range(num_epochs):\n                print('Epoch {}/{}'.format(epoch, num_epochs-1))\n                epoch_roc_auc = 0.0\n                epoch_loss = 0.0\n                for phase in ['train', 'val']:\n                    if phase == 'train':\n                        model.train()\n                        dataset = melanoma_dataset([DIR0,DIR1], transform_train, train_df)\n                        dataloader = DataLoader(dataset, batch_size = 8, shuffle = True, num_workers = 2) \n                        count = 0\n                        ts = time.time()\n                    else:\n                        model.eval()\n                        dataset = melanoma_dataset([DIR0,DIR1], transform_valid, valid_df)\n                        dataloader = DataLoader(dataset, batch_size = 8, shuffle = False, num_workers = 2)\n                        count =0\n                    dataset_size = len(dataset)\n                    running_loss = 0.0\n                    running_outputs = []\n\n                    for inputs, labels in dataloader:\n                        inputs[0] = torch.tensor(inputs[0], device = device, dtype = torch.float32)\n                        inputs[1] = torch.tensor(inputs[1], device = device, dtype = torch.float32)\n                        labels = torch.tensor(labels, device = device, dtype = torch.float32)\n                        #print(len(labels))\n                        if ((phase == 'train' )& (len(labels)!= 8)):\n                            break\n                        optimizer.zero_grad()\n                        inputs_size = inputs[0].size(0)\n                        #forward\n\n                        with torch.set_grad_enabled(phase == 'train'):\n                            outputs = model(inputs[:2])\n                            running_outputs.append(outputs.sigmoid().cpu())\n                            #preds = outputs.sigmoid()\n                            loss = criterion(outputs, labels.unsqueeze(1))\n                            del inputs, labels\n                            gc.collect()\n                            if phase == 'train':\n                                loss.backward()\n                                optimizer.step()\n                        count += inputs_size\n                        #print(count)\n                        #if count%80 == 0:\n                        #   print(time.time() -ts)\n                        running_loss += loss.item()*inputs_size\n                        gc.collect()  \n\n\n                    if phase == 'train':\n                        scheduler.step()\n\n                    epoch_loss = running_loss/dataset_size\n                    if phase == 'val':\n                        epoch_roc_auc = roc_auc_score((np.asarray(valid_df.iloc[:len(np.concatenate(running_outputs).ravel()),3])) , np.concatenate(running_outputs).ravel())\n                    \n                    \n                    if phase == 'val' and epoch_roc_auc > best_roc_auc:\n                        best_roc_auc = epoch_roc_auc\n                        best_epoch = epoch\n                        best_model_wts = copy.deepcopy(model.state_dict())\n                        torch.save(model,model_path)\n\n\n                    print('{} Epoch: {} Loss: {:.4f} ROC-AUC: {:.4f}'.format(phase,epoch, epoch_loss, 0 if phase == 'train' else epoch_roc_auc))  \n\n\n\n\n            print('Fold {} Best Val ROC-AUC: {:4f}, ({})'.format(fold, best_roc_auc, best_epoch))\n            roc += best_roc_auc\n        \n        print('FINAL ROC-AUC: {}'.format(roc/folds))\n            \n    if train == False:\n        dataset = melanoma_dataset(TEST_FOLDER, transform_test, csv_file = TEST_CSV, train = False)\n        dataloader = DataLoader(dataset, batch_size = 64, shuffle = False, num_workers = 2)\n\n        outputs_df = pd.DataFrame()\n        outputs_dic = {}\n        means = []\n        for fold in range(folds):\n            ''''if fold < 4:\n                model = torch.load(f'{MODELS_PATH}model_{fold}.pth')\n            else:\n                model = torch.load('../input/fold4-mod/model_0.pth')'''\n            model = torch.load(f'{MODELS_PATH}model_{fold}.pth')\n            model.eval()\n            model.to(device)\n            torch.set_grad_enabled(False)\n            outputs_dic[fold] = []\n            for inputs in dataloader:\n                inputs[0] = torch.tensor(inputs[0], device = device, dtype = torch.float32)\n                inputs[1] = torch.tensor(inputs[1], device = device, dtype = torch.float32)\n                outputs = model(inputs).cpu().numpy()\n                #preds = torch.round(outputs.sigmoid())\n                outputs_dic[fold].append(outputs)\n            outputs_df[fold] = np.concatenate(outputs_dic[fold]).ravel()\n\n        for i in range(len(outputs_df)):\n            means.append(np.mean(outputs_df.iloc[i,:]))\n        means2 = torch.from_numpy(np.asarray(means))\n        preds = means2.sigmoid().numpy()\n        \n        sub = pd.DataFrame()\n        sub['image_name'] = pd.read_csv(TEST_CSV)['image_name']\n        sub['target'] = preds\n        sub.to_csv('melanoma_preds.csv', index = False)\n    \n                ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CSV = pd.read_csv(FOLD_CSVS[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kfold_train(folds = 1, num_epochs = 15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#kfold_train(folds = 1, num_epochs = 5, train = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}