{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"},{"sourceId":1243687,"sourceType":"datasetVersion","datasetId":690737},{"sourceId":37251692,"sourceType":"kernelVersion"}],"dockerImageVersionId":29926,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"[https://www2.cs.sfu.ca/~hamarneh/ecopy/ipmi2019.pdf](https://www2.cs.sfu.ca/~hamarneh/ecopy/ipmi2019.pdf)","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n!pip install -q efficientnet_pytorch\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport cv2\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader,Dataset\nfrom torchvision import datasets, transforms, models # add models to the list\nfrom torchvision.utils import make_grid\nimport os\nfrom sklearn.model_selection import train_test_split\nfrom PIL import Image\nimport os\nfrom sklearn.metrics import auc,roc_auc_score\ndevice = torch.device(\"cpu\")\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom efficientnet_pytorch import EfficientNet\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nimport time\nimport datetime\nimport warnings\nimport random\nimport gc\nimport pickle","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir='../input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/512x512-dataset-melanoma/'\ntest_dir='/kaggle/input/siim-isic-melanoma-classification/jpeg/test/'\ntrain=pd.read_csv('../input/melanoma-merged-external-data-512x512-jpeg/marking.csv')\ntest=pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')\nsubmission=pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['target'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_0=train[train['target']==0].sample(6000)\ndf_1=train[train['target']==1]\ntrain=pd.concat([df_0,df_1])\ntrain=train.reset_index()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels=[]\ndata=[]\nfor i in range(train.shape[0]):\n    data.append(train_dir + train['image_id'].iloc[i]+'.jpg')\n    labels.append(train['target'].iloc[i])\ndf=pd.DataFrame(data)\ndf.columns=['images']\ndf['target']=labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data=[]\nfor i in range(test.shape[0]):\n    test_data.append(test_dir + test['image_name'].iloc[i]+'.jpg')\ndf_test=pd.DataFrame(test_data)\ndf_test.columns=['images']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = transforms.Compose([\n        transforms.RandomRotation(10),      # rotate +/- 10 degrees\n        transforms.RandomHorizontalFlip(),  # reverse 50% of images\n        transforms.Resize(224),             # resize shortest side to 224 pixels\n        transforms.CenterCrop(224),         # crop longest side to 224 pixels at center\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225])\n    ])\n\ntest_transform = transforms.Compose([\n        transforms.Resize(224),\n        transforms.CenterCrop(224),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225])\n    ])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImageDataset(Dataset):\n    \n    def __init__(self,data_paths,labels,transform=None,mode='train'):\n         self.data=data_paths\n         self.labels=labels\n         self.transform=transform\n         self.mode=mode\n    def __len__(self):\n       return len(self.data)\n    \n    def __getitem__(self,idx):\n        img_name = self.data[idx]\n        img = cv2.imread(img_name)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img=Image.fromarray(img)\n        if self.transform is not None:\n          img = self.transform(img)\n        img=img.cuda()\n        \n        if self.mode=='test':\n            return img\n        else:\n           \n            labels = torch.tensor(self.labels[idx]).cuda()\n\n            return img, labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, alpha=0.25, gamma=2.0, logits=False, reduce=True):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.logits = logits\n        self.reduce = reduce\n\n    def forward(self, inputs, targets):\n        if self.logits:\n            BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduce=False)\n        else:\n            BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)\n        pt = torch.exp(-BCE_loss)\n        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n\n        if self.reduce:\n            return torch.mean(F_loss)\n        else:\n            return F_loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Out of Fold Predictions\ntrain_len=train.shape[0]\ntest_len=df_test.shape[0]\noof = np.zeros(shape = (train_len, 1))\n\n# Predictions\npreds_submission = torch.zeros(size = (test_len, 1), dtype=torch.float32, device=device)\n\nprint('oof shape:', oof.shape, '\\n' +\n      'predictions shape:', preds_submission.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Object\ngroup_fold = GroupKFold(n_splits = 5)\n\n# Generate indices to split data into training and test set.\nfolds = group_fold.split(X = np.zeros(train_len), \n                         y = train['target'], \n                         groups = train['patient_id'].tolist())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 7\npatience = 2\n# TTA = 3\nnum_workers = 6\nlearning_rate = 0.001\nweight_decay = 0.0\nlr_patience = 1            # 1 model not improving until lr is decreasing\nlr_factor = 0.2            # by how much the lr is decreasing\n\nbatch_size1 = 32\nbatch_size2 = 16\n\n# version = 'v3'             # to keep tabs if versions\n# -------------------","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nfrom tqdm import tqdm\ndef train_folds(preds_submission, model):\n    # Creates a .txt file that will contain the logs\n#     f = open(f\"logs_{version}.txt\", \"w+\")\n    \n    oofs=[]\n    for fold, (train_index, valid_index) in enumerate(folds):\n\n        print('-'*10, 'Fold:', fold+1, '-'*10)\n\n        # --- Create Instances ---\n        # Best ROC score in this fold\n        best_roc = None\n        # Reset patience before every fold\n        patience_f = patience\n        \n        # Initiate the model\n        model = model\n\n        optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate, weight_decay=weight_decay)\n        scheduler = ReduceLROnPlateau(optimizer=optimizer, mode='max', \n                                      patience=lr_patience, verbose=True, factor=lr_factor)\n        criterion = FocalLoss()\n\n\n        # --- Read in Data ---\n        train_data = df.iloc[train_index].reset_index(drop=True)\n        valid_data = df.iloc[valid_index].reset_index(drop=True)\n\n        # Create Data instances\n        train=ImageDataset(data_paths=train_data['images'].values,labels=train_data['target'].values,transform=train_transform)\n        valid=ImageDataset(data_paths=valid_data['images'].values,labels=train_data['target'].values,transform=test_transform)\n        # Read in test data | Remember! We're using data augmentation like we use for Train data.\n        test=ImageDataset(data_paths=df_test['images'].values,labels=None,transform=test_transform,mode='test')\n\n        # Dataloaders\n        train_loader = DataLoader(train, batch_size=batch_size1, shuffle=True)\n        # shuffle=False! Otherwise function won't work!!!\n                # how do I know? ^^\n        valid_loader = DataLoader(valid, batch_size=batch_size2, shuffle=False)\n        test_loader = DataLoader(test, batch_size=batch_size2, shuffle=False)\n\n\n        # === EPOCHS ===\n        for epoch in range(epochs):\n            start_time = time.time()\n            correct = 0\n            train_losses = 0\n\n#             # === TRAIN ===\n            # Sets the module in training mode.\n            model.train()\n\n            for images, labels in train_loader:\n                # Save them to device\n\n                # Clear gradients first; very important, usually done BEFORE prediction\n                optimizer.zero_grad()\n\n                # Log Probabilities & Backpropagation\n                out = model(images)\n                loss = criterion(torch.sigmoid(out.type(torch.FloatTensor)), labels.type(torch.FloatTensor))\n                loss.backward()\n                optimizer.step()\n\n                # --- Save information after this batch ---\n                # Save loss\n                train_losses += loss.item()\n                # From log probabilities to actual probabilities\n                train_preds = torch.round(torch.sigmoid(out)) # 0 and 1\n                # Number of correct predictions\n                correct += (train_preds.cpu() == labels.cpu()).sum().item()\n\n            # Compute Train Accuracy\n            train_acc = correct / len(train_index)\n\n\n            # === EVAL ===\n            # Sets the model in evaluation mode\n            model.eval()\n\n            # Create matrix to store evaluation predictions (for accuracy)\n            valid_preds = torch.zeros(size = (len(valid_index), 1), device=device, dtype=torch.float32)\n\n\n            # Disables gradients (we need to be sure no optimization happens)\n            with torch.no_grad():\n                for k, (images, labels) in enumerate(valid_loader):\n\n\n                    out = model(images)\n                    pred = torch.sigmoid(out)\n                    valid_preds[k*images.shape[0] : k*images.shape[0] + images.shape[0]] = pred\n\n                # Compute accuracy\n                valid_acc = accuracy_score(valid_data['target'].values, \n                                           torch.round(valid_preds.cpu()))\n                # Compute ROC\n                valid_roc = roc_auc_score(valid_data['target'].values, \n                                          valid_preds.cpu())\n\n                # Compute time on Train + Eval\n                duration = str(datetime.timedelta(seconds=time.time() - start_time))[:7]\n\n            \n                # Print to console\n                print('{} | Epoch: {}/{} | Loss: {:.4} | Train Acc: {:.3} | Valid Acc: {:.3} | ROC: {:.3}'.\\\n                     format(duration, epoch+1, epochs, train_losses, train_acc, valid_acc, valid_roc))\n\n\n                # === SAVE MODEL ===\n\n                # Update scheduler (for learning_rate)\n                scheduler.step(valid_roc)\n\n                # Update best_roc\n                if not best_roc: # If best_roc = None\n                    best_roc = valid_roc\n                    torch.save(model.state_dict(),\n                               f\"Fold{fold+1}_Epoch{epoch+1}_ValidAcc_{valid_acc:.3f}_ROC_{valid_roc:.3f}.pth\")\n                    continue\n\n                if valid_roc > best_roc:\n                    best_roc = valid_roc\n                    # Reset patience (because we have improvement)\n                    patience_f = patience\n                    torch.save(model.state_dict(),\n                               f\"Fold{fold+1}_Epoch{epoch+1}_ValidAcc_{valid_acc:.3f}_ROC_{valid_roc:.3f}.pth\")\n                else:\n                    # Decrease patience (no improvement in ROC)\n                    patience_f = patience_f - 1\n                    if patience_f == 0:\n                        print('Early stopping (no improvement since 2 models) | Best ROC: {}'.\\\n                              format(best_roc))\n                        break\n\n\n        # === INFERENCE ===\n        # Choose model with best_roc in this fold\n        best_model_path = '../working/' + [file for file in os.listdir('../working') if str(round(best_roc, 3)) in file and 'Fold'+str(fold+1) in file][0]\n        # Using best model from Epoch Train\n#         best_model_path='../input/melanoma-pytorch/Fold1_Epoch8_ValidAcc_0.803_ROC_0.874.pth'\n        model = EfficientNet.from_pretrained('efficientnet-b3')\n        model._fc=nn.Linear(in_features=1536, out_features=1, bias=True)\n        model=model.cuda()\n        model.load_state_dict(torch.load(best_model_path))\n        # Set the model in evaluation mode\n        model.eval()\n\n\n        with torch.no_grad():\n            # --- EVAL ---\n            # Predicting again on Validation data to get preds for OOF\n            valid_preds = torch.zeros(size = (len(valid_index), 1), device=device, dtype=torch.float32)\n\n            for k, (images, _) in enumerate(valid_loader):\n                out = model(images)\n                pred = torch.sigmoid(out)\n                valid_preds[k*images.shape[0] : k*images.shape[0] + images.shape[0]] = pred\n\n            # Save info to OOF\n            oofs.append(valid_preds.cpu().numpy())\n            oof[valid_index] = valid_preds.cpu().numpy()\n\n\n            # --- TEST ---\n            # Now (Finally) prediction for our TEST data\n            \n            for k, images in enumerate(test_loader):\n               \n\n                out = model(images)\n                # Covert to probablities\n                out = torch.sigmoid(out).cpu().numpy()\n\n                # ADDS! the prediction to the matrix we already created\n                \n                preds_submission[k*images.shape[0] : k*images.shape[0] + images.shape[0]] += out\n\n\n            # Divide Predictions by TTA (to average the results during TTA)\n            \n        # === CLEANING ===\n        # Clear memory\n        del train, valid, train_loader, valid_loader, images, labels\n        # Garbage collector\n        gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.manual_seed(42)\nmodel = EfficientNet.from_pretrained('efficientnet-b3')\nmodel._fc=nn.Linear(in_features=1536, out_features=1, bias=True)\nmodel=model.cuda()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_folds(preds_submission = preds_submission, model = model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('OOF ROC: {:.3f}'.format(roc_auc_score(train['target'], oof)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file=open('oofs.pickle','w+')\npickle.dump(oofs,file)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_submission /= 5\npreds_submission = preds_submission.cpu().numpy().reshape(-1,)\n\nsubmission['target'] = preds_submission\nsubmission.to_csv(f'submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}