{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-12T05:52:31.457409Z","iopub.execute_input":"2021-12-12T05:52:31.457715Z","iopub.status.idle":"2021-12-12T05:52:35.982778Z","shell.execute_reply.started":"2021-12-12T05:52:31.457636Z","shell.execute_reply":"2021-12-12T05:52:35.972944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing\nimport matplotlib.pyplot as plt # Plotting\nimport seaborn as sns # Plotting\n\n# Import Image Libraries - Pillow and OpenCV\nfrom PIL import Image\nimport cv2\n\n# Import PyTorch and useful fuctions\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import TensorDataset, DataLoader, Dataset\nimport torchvision.transforms as transforms\nfrom torch.utils.data.sampler import SubsetRandomSampler\nimport torchvision\nimport torch.optim as optim\nimport torchvision.models as models # Pre-Trained models\n\n# Import useful sklearn functions\nimport sklearn\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score\n\nimport time\nimport os\nfrom tqdm import tqdm_notebook\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu') #Use GPU if it's available or else use CPU.\nprint(device) #Prints the device we're using.\n\ndata = pd.read_csv('/kaggle/input/aptos2019-blindness-detection/train.csv')\nprint('Train Size = {}'.format(len(data)))\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:52:36.000764Z","iopub.execute_input":"2021-12-12T05:52:36.000976Z","iopub.status.idle":"2021-12-12T05:52:38.922072Z","shell.execute_reply.started":"2021-12-12T05:52:36.000950Z","shell.execute_reply":"2021-12-12T05:52:38.921351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counts = data['diagnosis'].value_counts()\nclass_list = ['No DR', 'Mild', 'Moderate', 'Severe', 'Proliferate']\nfor i,x in enumerate(class_list):\n    counts[x] = counts.pop(i)\n\nplt.figure(figsize=(10,5))\nsns.barplot(x=counts.index, y=counts.values, alpha=0.8, palette='bright')\nplt.title('Distribution of Output Classes')\nplt.ylabel('Number of Occurrences', fontsize=12)\nplt.xlabel('Target Classes', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:52:38.923638Z","iopub.execute_input":"2021-12-12T05:52:38.924127Z","iopub.status.idle":"2021-12-12T05:52:39.158479Z","shell.execute_reply.started":"2021-12-12T05:52:38.924073Z","shell.execute_reply":"2021-12-12T05:52:39.157779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#As you can see,the data is imbalanced.\n#So we've to calculate weights for each class,which can be used in calculating loss.\n\nfrom sklearn.utils import class_weight #For calculating weights for each class.\nclass_weights = class_weight.compute_class_weight(class_weight='balanced',classes=np.array([0,1,2,3,4]),y=data['diagnosis'].values)\nclass_weights = torch.tensor(class_weights,dtype=torch.float).to(device)\n \nprint(class_weights) #Prints the calculated weights for the classes.","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:52:39.159925Z","iopub.execute_input":"2021-12-12T05:52:39.160371Z","iopub.status.idle":"2021-12-12T05:52:42.068917Z","shell.execute_reply.started":"2021-12-12T05:52:39.160331Z","shell.execute_reply":"2021-12-12T05:52:42.067356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(12, 12))\n# display random 16 images\ntrain_imgs = os.listdir(\"/kaggle/input/aptos2019-blindness-detection/train_images/\")\nfor idx, img in enumerate(np.random.choice(train_imgs, 16)):\n    ax = fig.add_subplot(4, 4, idx+1, xticks=[], yticks=[])\n    im = Image.open('/kaggle/input/aptos2019-blindness-detection/train_images/' + img)\n    plt.imshow(im)\n    lab = data.loc[data['id_code'] == img.split('.')[0], 'diagnosis'].values[0]\n    ax.set_title('Severity: %s'%lab)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:52:42.071345Z","iopub.execute_input":"2021-12-12T05:52:42.071602Z","iopub.status.idle":"2021-12-12T05:52:53.338454Z","shell.execute_reply.started":"2021-12-12T05:52:42.071566Z","shell.execute_reply":"2021-12-12T05:52:53.337120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class dataset(Dataset): # Inherits from the Dataset class.\n    '''\n    dataset class overloads the __init__, __len__, __getitem__ methods of the Dataset class. \n    \n    Attributes :\n        df:  DataFrame object for the csv file.\n        data_path: Location of the dataset.\n        image_transform: Transformations to apply to the image.\n        train: A boolean indicating whether it is a training_set or not.\n    '''\n    \n    def __init__(self,df,data_path,image_transform=None,train=True): # Constructor.\n        super(Dataset,self).__init__() #Calls the constructor of the Dataset class.\n        self.df = df\n        self.data_path = data_path\n        self.image_transform = image_transform\n        self.train = train\n        \n    def __len__(self):\n        return len(self.df) #Returns the number of samples in the dataset.\n    \n    def __getitem__(self,index):\n        image_id = self.df['id_code'][index]\n        image = Image.open(f'{self.data_path}/{image_id}.png') #Image.\n        if self.image_transform :\n            image = self.image_transform(image) #Applies transformation to the image.\n        \n        if self.train :\n            label = self.df['diagnosis'][index] #Label.\n            return image,label #If train == True, return image & label.\n        \n        else:\n            return image #If train != True, return image.","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:52:53.339756Z","iopub.execute_input":"2021-12-12T05:52:53.340310Z","iopub.status.idle":"2021-12-12T05:52:53.350525Z","shell.execute_reply.started":"2021-12-12T05:52:53.340271Z","shell.execute_reply":"2021-12-12T05:52:53.349785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_transforms = torchvision.transforms.Compose([\n#     torchvision.transforms.ToPILImage(),\n#     torchvision.transforms.Resize((224, 224)),\n#     #torchvision.transforms.ColorJitter(brightness=2, contrast=2),\n#     torchvision.transforms.RandomHorizontalFlip(p=0.5),\n#     torchvision.transforms.ToTensor(),\n#     torchvision.transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))\n# ])\n\nimage_transform = transforms.Compose([transforms.Resize([512,512]),\n                                      transforms.ToTensor(),\n                                      torchvision.transforms.RandomHorizontalFlip(p=0.5),\n                                      transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))]) #Transformations to apply to the image.\ndata_set = dataset(data,f'/kaggle/input/aptos2019-blindness-detection/train_images',image_transform=image_transform)\n\n#Split the data_set so that valid_set contains 0.1 samples of the data_set. \ntrain_set,valid_set = torch.utils.data.random_split(data_set,[3302,360])","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:52:53.351720Z","iopub.execute_input":"2021-12-12T05:52:53.354660Z","iopub.status.idle":"2021-12-12T05:52:53.373085Z","shell.execute_reply.started":"2021-12-12T05:52:53.354624Z","shell.execute_reply":"2021-12-12T05:52:53.371972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(train_set,batch_size=32,shuffle=True) #DataLoader for train_set.\nvalid_dataloader = DataLoader(valid_set,batch_size=32,shuffle=False) #DataLoader for validation_set.","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:52:53.376561Z","iopub.execute_input":"2021-12-12T05:52:53.376808Z","iopub.status.idle":"2021-12-12T05:52:53.383378Z","shell.execute_reply.started":"2021-12-12T05:52:53.376775Z","shell.execute_reply":"2021-12-12T05:52:53.382592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Since we've less data, we'll use Transfer learning.\nmodel = models.resnet34(pretrained=True) #Downloads the resnet18 model which is pretrained on Imagenet dataset.\n\n#Replace the Final layer of pretrained resnet18 with 4 new layers.\nmodel.fc = nn.Sequential(nn.Linear(512,256), nn.Dropout(p=0.2), nn.Linear(256,128), nn.Dropout(p=0.5), nn.Linear(128,64), nn.Linear(64,5))","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:52:53.384421Z","iopub.execute_input":"2021-12-12T05:52:53.387242Z","iopub.status.idle":"2021-12-12T05:52:55.109233Z","shell.execute_reply.started":"2021-12-12T05:52:53.387200Z","shell.execute_reply":"2021-12-12T05:52:55.108508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = model.to(device) #Moves the model to the device.","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:52:55.110519Z","iopub.execute_input":"2021-12-12T05:52:55.110773Z","iopub.status.idle":"2021-12-12T05:52:55.147558Z","shell.execute_reply.started":"2021-12-12T05:52:55.110729Z","shell.execute_reply":"2021-12-12T05:52:55.146797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(dataloader,model,loss_fn,optimizer):\n    '''\n    train function updates the weights of the model based on the\n    loss using the optimizer in order to get a lower loss.\n    \n    Args :\n         dataloader: Iterator for the batches in the data_set.\n         model: Given an input produces an output by multiplying the input with the model weights.\n         loss_fn: Calculates the discrepancy between the label & the model's predictions.\n         optimizer: Updates the model weights.\n         \n    Returns :\n         Average loss per batch which is calculated by dividing the losses for all the batches\n         with the number of batches.\n    '''\n\n    model.train() #Sets the model for training.\n    \n    total = 0\n    correct = 0\n    running_loss = 0\n    \n    for batch,(x,y) in enumerate(dataloader): #Iterates through the batches.\n        \n        output = model(x.to(device)) #model's predictions.\n        loss   = loss_fn(output,y.to(device)) #loss calculation.\n       \n        running_loss += loss.item()\n        \n        total        += y.asize(0)\n        predictions   = output.argmax(dim=1).cpu().detach() #Index for the highest score for all the samples in the batch.\n        correct      += (predictions == y.cpu().detach()).sum().item() #No.of.cases where model's predictions are equal to the label.\n        \n        optimizer.zero_grad() #Gradient values are set to zero.\n        loss.backward() #Calculates the gradients.\n        optimizer.step() #Updates the model weights.\n             \n    \n    avg_loss = running_loss/len(dataloader) # Average loss for a single batch\n    avg_acc = 100*(correct/total)\n    \n    print(f'\\nTraining Loss per batch = {avg_loss:.6f}',end='\\t')\n    print(f'Accuracy on Training set = {100*(correct/total):.6f}% [{correct}/{total}]') #Prints the Accuracy.\n    \n    return avg_loss, avg_acc","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:52:55.148939Z","iopub.execute_input":"2021-12-12T05:52:55.149293Z","iopub.status.idle":"2021-12-12T05:52:55.158261Z","shell.execute_reply.started":"2021-12-12T05:52:55.149239Z","shell.execute_reply":"2021-12-12T05:52:55.157555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate(dataloader,model,loss_fn):\n    '''\n    validate function calculates the average loss per batch and the accuracy of the model's predictions.\n    \n    Args :\n         dataloader: Iterator for the batches in the data_set.\n         model: Given an input produces an output by multiplying the input with the model weights.\n         loss_fn: Calculates the discrepancy between the label & the model's predictions.\n    \n    Returns :\n         Average loss per batch which is calculated by dividing the losses for all the batches\n         with the number of batches.\n    '''\n    \n    model.eval() #Sets the model for evaluation.\n    \n    total = 0\n    correct = 0\n    running_loss = 0\n    \n    with torch.no_grad(): #No need to calculate the gradients.\n        \n        for x,y in dataloader:\n            \n            output        = model(x.to(device)) #model's output.\n            loss          = loss_fn(output,y.to(device)).item() #loss calculation.\n            running_loss += loss\n            \n            total        += y.size(0)\n            predictions   = output.argmax(dim=1).cpu().detach()\n            correct      += (predictions == y.cpu().detach()).sum().item()\n            \n    avg_loss = running_loss/len(dataloader) #Average loss per batch.  \n    avg_acc = 100*(correct/total)\n    \n    print(f'\\nValidation Loss per batch = {avg_loss:.6f}',end='\\t')\n    print(f'Accuracy on Validation set = {100*(correct/total):.6f}% [{correct}/{total}]') #Prints the Accuracy.\n    \n    return avg_loss, avg_acc","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:52:55.159594Z","iopub.execute_input":"2021-12-12T05:52:55.160095Z","iopub.status.idle":"2021-12-12T05:52:55.172200Z","shell.execute_reply.started":"2021-12-12T05:52:55.160057Z","shell.execute_reply":"2021-12-12T05:52:55.171549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def optimize(train_dataloader,valid_dataloader,model,loss_fn,optimizer,nb_epochs):\n    '''\n    optimize function calls the train & validate functions for (nb_epochs) times.\n    \n    Args :\n        train_dataloader: DataLoader for the train_set.\n        valid_dataloader: DataLoader for the valid_set.\n        model: Given an input produces an output by multiplying the input with the model weights.\n        loss_fn: Calculates the discrepancy between the label & the model's predictions.\n        optimizer: Updates the model weights.\n        nb_epochs: Number of epochs.\n        \n    Returns :\n        Tuple of lists containing losses for all the epochs.\n    '''\n    #Lists to store losses for all the epochs.\n    train_losses = []\n    valid_losses = []\n    train_acc = []\n    valid_acc = []\n\n    for epoch in range(nb_epochs):\n        print(f'\\nEpoch {epoch+1}/{nb_epochs}')\n        print('-------------------------------')\n        train_loss, train_a = train(train_dataloader,model,loss_fn,optimizer) #Calls the train function.\n        train_losses.append(train_loss)\n        train_acc.append(train_a)\n        valid_loss, valid_a = validate(valid_dataloader,model,loss_fn) #Calls the validate function.\n        valid_losses.append(valid_loss)\n        valid_acc.append(valid_a)\n    \n    print('\\nTraining has completed!')\n    return train_losses, valid_losses, train_acc, valid_acc\n    ","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:52:55.173448Z","iopub.execute_input":"2021-12-12T05:52:55.173897Z","iopub.status.idle":"2021-12-12T05:52:55.185354Z","shell.execute_reply.started":"2021-12-12T05:52:55.173862Z","shell.execute_reply":"2021-12-12T05:52:55.184666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_fn   = nn.CrossEntropyLoss(weight=class_weights) #CrossEntropyLoss with class_weights.\noptimizer = torch.optim.SGD(model.parameters(),lr=0.001) \nnb_epochs = 30\n#Call the optimize function.\ntrain_losses, valid_losses, train_acc, valid_acc = optimize(train_dataloader,valid_dataloader,model,loss_fn,optimizer,nb_epochs)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T05:52:55.189921Z","iopub.execute_input":"2021-12-12T05:52:55.191976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = range(30)\nplt.plot(epochs, train_acc, 'g', label='Training Accuracy')\nplt.plot(epochs, valid_acc, 'b', label='validation Accuracy')\nplt.title('Training and Validation Accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = range(30)\nplt.plot(epochs, train_losses, 'g', label='Training loss')\nplt.plot(epochs, valid_losses, 'b', label='validation loss')\nplt.title('Training and Validation loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model(data_set['x'])\nmodel.eval()\nresults = []\n# data_pred = dataset(data[:100], f'train_images',image_transform=image_transform)\ni = 0\nj = 0\nk = 0\nl = 0\nm = 0\nfor x, y in DataLoader(data_set):\n  if (y.numpy()[0] == 0 and i<20):\n    result = model(x.to(device)).argmax(dim=1).cpu().detach()\n    results.append((result.numpy()[0], y.numpy()[0]))\n    i = i+1\n  if (y.numpy()[0] == 1 and j<20):\n    result = model(x.to(device)).argmax(dim=1).cpu().detach()\n    results.append((result.numpy()[0], y.numpy()[0]))\n    j = j+1\n  if (y.numpy()[0] == 2 and k<20):\n    result = model(x.to(device)).argmax(dim=1).cpu().detach()\n    results.append((result.numpy()[0], y.numpy()[0]))\n    k = k+1\n  if (y.numpy()[0] == 3 and l<20):\n    result = model(x.to(device)).argmax(dim=1).cpu().detach()\n    results.append((result.numpy()[0], y.numpy()[0]))\n    l = l+1\n  if (y.numpy()[0] == 4 and m<20):\n    result = model(x.to(device)).argmax(dim=1).cpu().detach()\n    results.append((result.numpy()[0], y.numpy()[0]))\n    m = m+1\n  if(i+j+k+l+m >=100):\n    break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i, j, k, m","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport numpy as np\n\nlabelis = [val[1] for val in results]\npredis = [val[0] for val in results]\ncount = 0\nfor x,y in results:\n    if x == y:\n        count += 1\ncm = confusion_matrix(labelis, predis, labels=[0, 1, 2, 3, 4])\nprint('accuracy', 100*(count)/len(predis))\n# recall = np.diag(cm) / np.sum(cm, axis = 1)\n# precision = np.diag(cm) / np.sum(cm, axis = 0)\n# print(np.mean(recall), np.mean(precision))\ncm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm_df = pd.DataFrame(cm,\n                     index = ['0','1','2', '3', '4'], \n                     columns = ['0','1','2', '3', '4'])\ncm_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total = sum(cm_df.sum(axis=1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arr = np.array(cm_df)\ncnfm = []\nfor i in range(0,5):\n  TP = arr[:][i][i]\n  FN = sum(arr[:][i]) - arr[:][i][i]\n  FP = cm_df.sum(axis=0)[i] - arr[:][i][i]\n  TN = total - TP\n  precision = TP /(TP+FP)\n  recall  = TP/(TP+FN)\n  f1 = 2*precision*recall/(precision+recall)\n  cnfm.append([i, TP,FN,FP,TN, precision, recall, f1])\nf1_result = pd.DataFrame(cnfm, columns = ['severity', 'True Positive','False Negative','False Positive', 'True Negative', 'Precision', 'Recall', 'F1-score'])\nf1_result","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Plotting the confusion matrix\nplt.figure(figsize=(5,4))\nsns.heatmap(cm_df, annot=True)\nplt.title('Confusion Matrix')\nplt.ylabel('Actal Values')\nplt.xlabel('Predicted Values')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve\nfrom sklearn.metrics import roc_auc_score, auc\n\ndef test_class_probabilities(model, test_loader):\n    model.eval()\n    actuals = []\n    probabilities = []\n    i = 0\n    j = 0\n    k = 0\n    l = 0\n    f = 0\n    with torch.no_grad():\n        for x, y in test_loader:\n          y = y.numpy()[0]\n          outputs = model(x.to(device))\n          prediction = outputs.argmax(dim=1, keepdim=True).cpu().detach()\n          # if (prediction.numpy()[0] == y):\n          if (y == 0 and i<10):\n            actuals.append(y)\n            values = outputs.cpu().detach().numpy()[0]\n            m = -1*min(values)\n            values = [val+m for val in values]\n            prob = values/sum(values)\n            probabilities.append(np.exp(prob))\n            i = i+1\n          if (y == 1 and j<10):\n            actuals.append(y)\n            values = outputs.cpu().detach().numpy()[0]\n            m = -1*min(values)\n            values = [val+m for val in values]\n            prob = values/sum(values)\n            probabilities.append(np.exp(prob))\n            j = j+1\n          if (y == 2 and k<10):\n            actuals.append(y)\n            values = outputs.cpu().detach().numpy()[0]\n            m = -1*min(values)\n            values = [val+m for val in values]\n            prob = values/sum(values)\n            probabilities.append(np.exp(prob))\n            k = k+1\n          if (y == 3 and l<10):\n            actuals.append(y)\n            values = outputs.cpu().detach().numpy()[0]\n            m = -1*min(values)\n            values = [val+m for val in values]\n            prob = values/sum(values)\n            probabilities.append(np.exp(prob))\n            l = l+1\n          if (y == 4 and f<10):\n            actuals.append(y)\n            values = outputs.cpu().detach().numpy()[0]\n            m = -1*min(values)\n            values = [val+m for val in values]\n            prob = values/sum(values)\n            probabilities.append(np.exp(prob))\n            f = f+1\n          if(i+j+k+l+f >=50):\n            break\n    return actuals,probabilities\n\n\ndef plot_roc( actuals,  probabilities):\n    \"\"\"\n    compute ROC curve and ROC area for each class in each fold\n\n    \"\"\"\n    fpr = {}\n    tpr = {}\n    thresh ={}\n    roc_auc = {}\n    n_class = 5\n\n    for i in range(n_class):    \n        fpr[i], tpr[i], thresh[i] = roc_curve(actuals, [val[i] for val in probabilities], pos_label=i)\n        roc_auc[i] = auc(fpr[i], tpr[i])\n\n        \n    # plotting    \n    plt.plot(fpr[0], tpr[0], linestyle='--',color='orange', label=\"Class 0 vs Rest (area = %0.2f)\" % roc_auc[0])\n    plt.plot(fpr[1], tpr[1], linestyle='--',color='green', label=\"Class 1 vs Rest (area = %0.2f)\" % roc_auc[1])\n    plt.plot(fpr[2], tpr[2], linestyle='--',color='blue', label=\"Class 2 vs Rest (area = %0.2f)\" % roc_auc[2])\n    plt.plot(fpr[3], tpr[3], linestyle='--',color='red', label=\"Class 3 vs Rest (area = %0.2f)\" % roc_auc[3])\n    plt.plot(fpr[3], tpr[3], linestyle='--',color='purple', label=\"Class 4 vs Rest (area = %0.2f)\" % roc_auc[4])\n    plt.plot([0, 1], [0, 1], color=\"navy\", linestyle=\"--\")\n    plt.title('Multiclass ROC curve')\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive rate')\n    plt.legend(loc='best')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"act, prob = test_class_probabilities(model, DataLoader(data_set))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_roc(act, prob)\nprint(act)","metadata":{},"execution_count":null,"outputs":[]}]}