{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"## Check GPU Availability\n\nMake sure that GPU is available. If not turn the GPU state to on in Settings."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":1,"outputs":[{"output_type":"stream","text":"Sat Mar 30 15:04:21 2019 \r\n+-----------------------------------------------------------------------------+\r\n| NVIDIA-SMI 410.104 Driver Version: 410.104 CUDA Version: 10.0 |\r\n|-------------------------------+----------------------+----------------------+\r\n| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\r\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\r\n|===============================+======================+======================|\r\n| 0 Tesla P100-PCIE... On | 00000000:00:04.0 Off | 0 |\r\n| N/A 38C P0 27W / 250W | 0MiB / 16280MiB | 0% Default |\r\n+-------------------------------+----------------------+----------------------+\r\n \r\n+-----------------------------------------------------------------------------+\r\n| Processes: GPU Memory |\r\n| GPU PID Type Process name Usage |\r\n|=============================================================================|\r\n| No running processes found |\r\n+-----------------------------------------------------------------------------+\r\n","name":"stdout"}]},{"metadata":{"trusted":true,"_uuid":"30cd364133489f7aa8c9dbfb18a11a21a5b64f5f"},"cell_type":"markdown","source":"## Import Libraries\n\nImport all the required libraries."},{"metadata":{"trusted":true,"_uuid":"dd2b59d1dad8f2f1a34682ed4df15cd2a4f09f00"},"cell_type":"code","source":"import os\nimport time\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom matplotlib import pyplot as plt\nfrom PIL import Image\nfrom torch.optim import Adam, lr_scheduler\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn import Conv2d, Linear, CrossEntropyLoss\nfrom torchvision.models import densenet201, resnet152, vgg19_bn \nfrom torchvision.transforms import Compose, RandomAffine, RandomApply, ColorJitter, Normalize, ToTensor","execution_count":7,"outputs":[]},{"metadata":{"_uuid":"70a72ba28a827d584937a9b1a7070389b2d95382"},"cell_type":"markdown","source":"## Generate required folders\n\nGenerate the required folders to be able to\n* save the states\n* load from saved states\n* save plots\n* save results"},{"metadata":{"trusted":true,"_uuid":"3a5599522fd93efb3974bebd7d6fd5be9840d9de"},"cell_type":"code","source":"folders = {\n \"plots\": \"plots\",\n \"models\": \"models\",\n \"results\": \"results\"\n}\nfor key in folders.keys():\n try:\n os.makedirs(folders[key])\n except FileExistsError:\n # if file exists, pass\n pass","execution_count":8,"outputs":[]},{"metadata":{"_uuid":"5d3294d779ad2c64b17c71a913def94f8fc4578c"},"cell_type":"markdown","source":"## PCam Dataset\n\nCustom dataset definition to be able to use PyTorch style of efficient data loading.\n\n### Challenges Faced\n\nA deep neural network tries to get the best performance and so having relatively more number of examples in one class is making the network to have a biased view of it's world. So, have to come up with a way to have same number of examples for each category."},{"metadata":{"trusted":true,"_uuid":"d4a7cf38a8d8059cfc227f09c0f87f4efa85b02e"},"cell_type":"code","source":"class PCam(Dataset):\n \"\"\"Patch Camelyon dataset.\"\"\"\n\n def __init__(self, csv_file, root_dir, train=True, transform=None):\n \"\"\"\n Args:\n csv_file (string): Path to the csv file with labels.\n root_dir (string): Root directory.\n train (boolean): Whether loading training or testing data. \n This is required to have same number of examples in each \n classification to be able to train better.\n transform (callable, optional): Optional transform to be applied\n on a sample.\n \"\"\"\n if train:\n # make the number of example in each classification equal\n dataframe = pd.read_csv(os.path.join(root_dir, csv_file))\n min_value = dataframe['label'].value_counts().min()\n frames = []\n for label in dataframe['label'].unique():\n frames.append(dataframe[dataframe['label'] == label].sample(min_value))\n # .sample(frac=1) shuffles the data\n # .reset_index(drop=True) do not add index while shuffling\n self.labels = pd.DataFrame().append(frames).sample(frac=1).reset_index(drop=True)\n self.data_folder = \"train\"\n else:\n self.labels = pd.read_csv(os.path.join(root_dir, csv_file))\n self.data_folder = \"test\"\n \n self.root_dir = root_dir\n self.transform = transform\n\n def __len__(self):\n return len(self.labels)\n\n def __getitem__(self, idx):\n image_name = os.path.join(self.root_dir,\n \"%s/%s.tif\" % (self.data_folder, self.labels.iloc[idx, 0]))\n image = Image.open(image_name)\n # reduce image size to be able to train fast\n image.thumbnail((40, 40), Image.ANTIALIAS)\n if self.transform is not None:\n image = self.transform(image)\n\n return self.labels.iloc[idx, 0], image, self.labels.iloc[idx, 1]","execution_count":9,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hyperparameters\n\nSetting the hyperparameters."},{"metadata":{"trusted":true,"_uuid":"a74f67187868f0bd6ddcd83d4d56e8acc74cf66d"},"cell_type":"code","source":"NUM_CLASSES = 2 # number of classes\nBATCH_SIZE = 32 # mini_batch size\nMAX_EPOCH = 10 # maximum epoch to train\nSTEP_SIZE = 2 # decrease in learning rate after epochs\nLEARNING_RATE = 0.00007 # learning rate\nGAMMA = 0.1 # used in decreasing the gamma","execution_count":10,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loading Data\n\nAugment training data and not testing data and them in their respective data loaders."},{"metadata":{"trusted":true,"_uuid":"c50ff6299c7f62aa203bbe26fe0e17eaf1db4da3"},"cell_type":"code","source":"# other transformations are not included because they are included in these or those are not required in real life\ntrain_transform = Compose([\n RandomAffine(45, translate=(0.15, 0.15), shear=45),\n RandomApply([ColorJitter(saturation=0.5, hue=0.5)]),\n ToTensor(),\n Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n])\ntest_transform = Compose(\n [ToTensor(),\n Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n\ntrainset = PCam(csv_file='train_labels.csv', root_dir='../input', train=True, transform=train_transform)\ntrainloader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\n\ntestset = PCam(csv_file='sample_submission.csv', root_dir='../input', train=False, transform=test_transform)\ntestloader = DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)","execution_count":11,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Evaluation\n\nFunction to evaluate the model during training. A prediction is done based on the average value of the predictions using all the models."},{"metadata":{"trusted":true,"_uuid":"a7e032fe58bab371a3781dcac9428b1cee283dce"},"cell_type":"code","source":"def eval_ensemble(nets, criterion, dataloader):\n correct = 0\n total = 0\n total_loss = 0\n\n for data in dataloader:\n _, images, labels = data\n # images, labels = Variable(images), Variable(labels)\n images, labels = Variable(images).cuda(), Variable(labels).cuda()\n\n predictions = torch.zeros([images.size(0), NUM_CLASSES]).cuda()\n for net in nets:\n net.eval()\n outputs = net(images)\n predictions = predictions.add(outputs)\n\n # predictions = predictions / len(nets) # redundant division\n _, predicted = torch.max(predictions.data, 1)\n \n total += labels.size(0)\n correct += (predicted == labels.data).sum().item()\n \n loss = criterion(predictions, labels)\n total_loss += loss.item()\n return total_loss / total, correct / total","execution_count":12,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Model\n\nFunction to train the ensemble model. Loss is calculated based on the average prediction during training and gradients of each model in ensemble is calculated using this loss."},{"metadata":{"trusted":true,"_uuid":"42a533bdc3500fe54bad59ee21f1c81a9e1904d5"},"cell_type":"code","source":"def train_ensemble(nets, optimizers, schedulers, criterion, eval_criterion):\n ensemble_name = 'ensemble'\n train_loss_array = []\n test_loss_array = []\n train_accuracy_array = []\n test_accuracy_array = []\n\n print('Start training...')\n for epoch in range(MAX_EPOCH): # loop over the dataset multiple times\n for scheduler in schedulers:\n scheduler.step()\n \n running_loss = 0.0\n for i, data in enumerate(trainloader):\n _, images, labels = data\n # inputs, labels = Variable(inputs), Variable(labels)\n images, labels = Variable(images).cuda(), Variable(labels).cuda()\n\n predictions = torch.zeros([images.size(0), NUM_CLASSES]).cuda()\n for net, optimizer in zip(nets, optimizers):\n net.train()\n optimizer.zero_grad()\n outputs = net(images)\n predictions = predictions.add(outputs)\n\n # predictions = predictions / len(nets) # redundant division\n \n # back prop\n loss = criterion(predictions, labels)\n loss.backward()\n for optimizer in optimizers:\n optimizer.step()\n running_loss += loss.item()\n \n if i % 500 == 499: # print every 2000 mini-batches\n print('Step: %5d avg_batch_loss: %.5f' % (i + 1, running_loss / 500))\n running_loss = 0.0\n \n print('Finish training this EPOCH, start evaluating...')\n train_loss, train_acc = eval_ensemble(nets, eval_criterion, trainloader)\n test_loss, test_acc = eval_ensemble(nets, eval_criterion, testloader)\n print('EPOCH: %d train_loss: %.5f train_acc: %.5f test_loss: %.5f test_acc %.5f' %\n (epoch + 1, train_loss, train_acc, test_loss, test_acc))\n\n train_loss_array.append(train_loss)\n test_loss_array.append(test_loss)\n\n train_accuracy_array.append(train_acc)\n test_accuracy_array.append(test_acc)\n print('Finished Training')\n\n # plot loss\n plt.clf()\n plt.plot(list(range(1, MAX_EPOCH + 1)), train_loss_array, label='Train')\n plt.plot(list(range(1, MAX_EPOCH + 1)), test_loss_array, label='Test')\n plt.xlabel('Epochs')\n plt.ylabel('Loss')\n plt.legend()\n plt.title('Loss vs Epochs [%s]' % net.name)\n plt.savefig('./%s/loss-%s.png' % (folders['plots'], ensemble_name))\n\n # plot accuracy\n plt.clf()\n plt.plot(list(range(1, MAX_EPOCH + 1)), train_accuracy_array, label='Train')\n plt.plot(list(range(1, MAX_EPOCH + 1)), test_accuracy_array, label='Test')\n plt.xlabel('Epochs')\n plt.ylabel('Accuracy')\n plt.legend()\n plt.title('Accuracy vs Epochs [%s]' % net.name)\n plt.savefig('./%s/accuracy-%s.png' % (folders['plots'], ensemble_name))","execution_count":13,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Classify Image\n\nGet predictions on the image using each of the model in ensemble. Prediction by a model impacts the overall prediction by a factor of its true positive count. This is because accuracy of a model is decided by sensitivity for data where false negative can cost a huge loss (here human life)."},{"metadata":{"trusted":true,"_uuid":"388e6abd4393e481b7b8b80e4da09f4591248ce1"},"cell_type":"code","source":"def dump_ensemble_results(dataloader, nets):\n ensemble_name = 'ensemble'\n \n alphas = []\n for net in nets:\n net.eval()\n \n true_positives = 0\n for data in trainloader:\n _, images, labels = data\n images, labels = Variable(images).cuda(), Variable(labels).cuda()\n outputs = net(images)\n _, outputs = torch.max(outputs.data, 1)\n index = (labels == 1)\n true_positives += (outputs[index] == labels[index].data).sum().item()\n alphas.append(true_positives)\n# total_true_positives = sum(alphas)\n \n results = pd.DataFrame()\n for data in dataloader:\n image_names, images, labels = data\n# images, labels = Variable(images), Variable(labels)\n images, labels = Variable(images).cuda(), Variable(labels).cuda()\n \n predictions = torch.zeros([images.size(0), NUM_CLASSES]).cuda()\n for index, net in enumerate(nets):\n net.eval()\n outputs = net(images) * alphas[index] # / total_true_positives\n predictions = predictions.add(outputs)\n\n# predictions = predictions / total_true_positives\n _, predictions = torch.max(predictions.data, 1)\n results = results.append(pd.DataFrame({\"id\": image_names, \"label\": predictions.cpu().numpy()}))\n results.to_csv(\"%s/%s.csv\" % (folders['results'], ensemble_name), index=False)","execution_count":14,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create Ensemble\n\nCreate a list of DenseNet201, ResNet152 and VGG19_BN. Each model has it's own criterion, optimizer and learning rate scheduler."},{"metadata":{"trusted":true,"_uuid":"bf9ddc673d24b6c98cbc462592398bc6e60daa75","scrolled":true},"cell_type":"code","source":"start = time.time()\nnet_list = []\noptimizer_list = []\nscheduler_list = []\n\n# DenseNet201\ndense_net = densenet201()\ndense_num_ftrs = dense_net.classifier.in_features\ndense_net.classifier = Linear(dense_num_ftrs, NUM_CLASSES)\ndense_net.name = \"DenseNet201\"\ndense_net = dense_net.cuda()\ndense_optimizer = Adam(dense_net.parameters(), lr=LEARNING_RATE)\ndense_exp_lr_scheduler = lr_scheduler.StepLR(dense_optimizer, step_size=STEP_SIZE, gamma=GAMMA)\nnet_list.append(dense_net)\noptimizer_list.append(dense_optimizer)\nscheduler_list.append(dense_exp_lr_scheduler)\n\n# ResNet152\nres_net = resnet152()\nres_num_ftrs = res_net.fc.in_features\nres_net.fc = Linear(res_num_ftrs, NUM_CLASSES)\nres_net.name = \"ResNet152\"\nres_net = res_net.cuda()\nres_optimizer = Adam(res_net.parameters(), lr=LEARNING_RATE)\nres_exp_lr_scheduler = lr_scheduler.StepLR(res_optimizer, step_size=STEP_SIZE, gamma=GAMMA)\nnet_list.append(res_net)\noptimizer_list.append(res_optimizer)\nscheduler_list.append(res_exp_lr_scheduler)\n\n# VGG19\nvgg_net = vgg19_bn()\nvgg_num_ftrs = vgg_net.classifier._modules['6'].in_features\nvgg_net.classifier._modules['6'] = Linear(vgg_num_ftrs, NUM_CLASSES)\nvgg_net.name = \"VGG11\"\nvgg_net = vgg_net.cuda()\nvgg_optimizer = Adam(vgg_net.parameters(), lr=LEARNING_RATE)\nvgg_exp_lr_scheduler = lr_scheduler.StepLR(vgg_optimizer, step_size=STEP_SIZE, gamma=GAMMA)\nnet_list.append(vgg_net)\noptimizer_list.append(vgg_optimizer)\nscheduler_list.append(vgg_exp_lr_scheduler)\n\ntrain_criterion = CrossEntropyLoss()\nval_criterion = CrossEntropyLoss(reduction='sum')\ntrain_ensemble(net_list, optimizer_list, scheduler_list, train_criterion, val_criterion)\nprint(\"Time taken: %d secs\" % int(time.time() - start))","execution_count":15,"outputs":[{"output_type":"stream","text":"Start training...\nStep: 500 avg_batch_loss: 0.57295\n","name":"stdout"},{"output_type":"error","ename":"KeyboardInterrupt","evalue":"","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[0mtrain_criterion\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCrossEntropyLoss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0mval_criterion\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCrossEntropyLoss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreduction\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'sum'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m \u001b[0mtrain_ensemble\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnet_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheduler_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_criterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_criterion\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 45\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Time taken: %d secs\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m\u001b[0m in \u001b[0;36mtrain_ensemble\u001b[0;34m(nets, optimizers, schedulers, criterion, eval_criterion)\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0moptimizer\u001b[0m \u001b[0;32min\u001b[0m \u001b[0moptimizers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 33\u001b[0m \u001b[0mrunning_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax_exp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 100\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 103\u001b[0m \u001b[0mbias_correction1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m \u001b[0;34m**\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'step'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]},{"metadata":{},"cell_type":"markdown","source":"## Classify Images\n\nUse the above trained ensemble to classify each of the testing image."},{"metadata":{"trusted":true,"_uuid":"b762139f58f67af6a949045ba4d84a759ff15485"},"cell_type":"code","source":"dump_ensemble_results(testloader, net_list)","execution_count":16,"outputs":[{"output_type":"error","ename":"KeyboardInterrupt","evalue":"","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdump_ensemble_results\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtestloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnet_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m\u001b[0m in \u001b[0;36mdump_ensemble_results\u001b[0;34m(dataloader, nets)\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m \u001b[0mtrue_positives\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 16\u001b[0m \u001b[0malphas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrue_positives\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;31m# total_true_positives = sum(alphas)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}