{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b7d2fca0fe35e9f5cf951ef679c147806b8cc84b"},"cell_type":"markdown","source":"## Reading in the training data table"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"../input/train_labels.csv\")\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2ce9b4408ba51bbfa55b0154d7dc9571271ad94e"},"cell_type":"markdown","source":"## Viewing an example image"},{"metadata":{"trusted":true,"_uuid":"c171164642c85ae30d1aea94ae81575c5914c5d5"},"cell_type":"code","source":"from PIL import Image\nimport matplotlib.pyplot as plt\n\nexample_image = Image.open(\"../input/train/\" + df['id'][0] + \".tif\")\nexample_image_numpy = np.array(example_image.getdata())\n\n#Finding the dimensions of the image\nprint(example_image_numpy.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"878a018642a96de6f469f2908c88cea386166d34"},"cell_type":"markdown","source":"### Image Dimension determination"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"72b0b76bfb0eac0de68e468d6488ffd1300f2a05"},"cell_type":"code","source":"print(np.sqrt(example_image_numpy.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1339f3d5bc72ff45d8a709db649cf88d04c7ee69"},"cell_type":"code","source":"example_image_numpy = np.array(example_image.getdata()).reshape(96, 96, 3)\n\nplt.imshow(example_image_numpy)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2f687d3381396d6fb32d9dc3853beb3b623505ed"},"cell_type":"markdown","source":"## Viewing few more examples"},{"metadata":{"trusted":true,"_uuid":"1adfd79261de975c51c18065856c27b95af61532","scrolled":false},"cell_type":"code","source":"example_images_numpy = []\nexample_labels = []\nimage_num = np.random.choice(df.shape[0] - 21)\n\n# We will view around 20 consecutive examples\n\nfor i in range(image_num, image_num + 20):\n    example_image = Image.open(\"../input/train/\" + df['id'][i] + \".tif\")\n    example_images_numpy.append(np.array(example_image.getdata()).reshape(96, 96, 3))\n    example_labels.append(str(df['label'][i]))\n\n# Setting up matplotlib\nfig, ax = plt.subplots(4, 5, figsize = (25, 20))\n\nk = 0\n# Setting up the axes objects\nfor i in range(4):\n    for j in range(5):\n        ax[i, j].imshow(example_images_numpy[k])\n        ax[i, j].axis('off')\n        ax[i, j].set_title(example_labels[k])\n        k += 1\n# Viewing the plots\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9ea58b93dd5f86c6978598cb84fbe0181cba22ce"},"cell_type":"markdown","source":"## Making the model"},{"metadata":{"_uuid":"80e7ea40d5d446ce1a1b24276d1f231ce76a8d97"},"cell_type":"markdown","source":"### Importing all the necasarry pytorch functions and packages"},{"metadata":{"trusted":true,"_uuid":"ed6b6d42a857ad2f91c0cb74c5e13ccbf380f524"},"cell_type":"code","source":"import torch\nimport torchvision.transforms as transforms\nimport torch.nn as nn\nfrom torch import optim, save\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.data.sampler import Sampler\nfrom torch.autograd import Variable\nprint(torch.__version__)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"13ef858b4fefd33cc396a2736cbfb70cd219574a"},"cell_type":"markdown","source":"### Making the dataset and dataloader objects\nPyTorch has its own dataset and dataloader objects which makes it very easy to load in the data from a hard drive and run the models. This is specially useful if the training data are huge in size and cannot fit in computer memory"},{"metadata":{"trusted":true,"_uuid":"da1b2dbadb5c61776ca0dcc21ead68469f3e7cd9"},"cell_type":"code","source":"class HS_Dataset(Dataset):\n    \n    def __init__(self, csv_file, root_dir, transform = None):\n        self.df = pd.read_csv(csv_file)\n        self.dir = root_dir\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, i):\n        \"\"\"This function should return the ith example from the training set.\n        The example should be returned in the form of a dictionary: \n        {'image': image_data, 'label': label_data}\"\"\"\n        \n        file = df['id'][i]\n        \n        label = np.array(df['label'][i])\n        if label == 0:\n            label == 0.0\n        else:\n            label == 1.0\n            \n        \"\"\"Reshape needed to make the output of shape [1]\"\"\"\n        label = label.reshape((1))\n                \n        image = Image.open(\"../input/train/\" + file + \".tif\")\n        image = np.array(image.getdata()).reshape(96, 96, 3)\n        \n        sample = {'image': image, 'label': label}\n        \n        if self.transform:\n            sample = self.transform(sample)\n            \n        return sample\n        \n        \nclass ToTensor(object):\n    \n    def __call__(self, sample):\n        image, label = sample['image'], sample['label']\n        \"\"\"This transposition is very important as PyTorch take in the image data in the current shape:\n        Number of Channels, Height, Width; So the third axis(channels) in the original image has to \n        be made the first axis.\"\"\"\n        image = image.transpose(2, 0, 1)        \n        image = torch.from_numpy(image)\n        image = image.type(torch.FloatTensor)\n        \n        label = torch.from_numpy(label)\n        label = label.type(torch.FloatTensor)\n        \"\"\"The optimizer takes in FloatTensor type data. Hence the data has to be converted from any other format\n        to FloatTensor type\"\"\"\n        return {'image': image, 'label': label}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f1ffa4a2674e14e47584f08d458b34e8c395443"},"cell_type":"code","source":"from torch.utils.data import random_split\n\nhs_dataset = HS_Dataset(\"../input/train_labels.csv\", \"../input/train/\", transform = transforms.Compose([ToTensor()]))\n\ntrain_size = int(0.995 * len(hs_dataset))\nval_size = int((len(hs_dataset) - train_size) / 8)\ntest_size = int(val_size * 7 / 8)\ntest_size += len(hs_dataset) - train_size - val_size - test_size\n\nprint(\"train size: \", train_size)\nprint(\"val size: \", val_size)\nprint(\"test size: \", test_size)\n\ntrain_data, val_data, test_data = random_split(hs_dataset, [train_size, val_size, test_size])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a156ff8a21b45a2a2fa75704a63c79fe1c652711"},"cell_type":"markdown","source":"### Testing the dataset object"},{"metadata":{"trusted":true,"_uuid":"95d59ae736ee55175cb3b863e105a16e0133f2c0"},"cell_type":"code","source":"print(\"training\")\nfor i in range(len(train_data)):\n    sample = train_data[i]\n    print(i, sample['image'].size(), sample['label'].size())\n    if i == 5:\n        break\n\nprint(\"validation\")\nfor i in range(len(val_data)):\n    sample = val_data[i]\n    print(i, sample['image'].size(), sample['label'].size())\n    if i == 5:\n        break\n        \nprint(\"testing\")\nfor i in range(len(test_data)):\n    sample = test_data[i]\n    print(i, sample['image'].size(), sample['label'].size())\n    if i == 5:\n        break","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5822a4b7e7f44911e0c254e85e44019ba6427532"},"cell_type":"markdown","source":"### Making the dataloader object"},{"metadata":{"trusted":true,"_uuid":"d74447248dd80aefee0f19c78ffdd8d7741c0fe9"},"cell_type":"code","source":"train_loader = DataLoader(dataset = train_data, batch_size = 128, shuffle=True, num_workers=0)\nval_loader = DataLoader(dataset= val_data, batch_size = val_size, num_workers=0)\ntest_loader = DataLoader(dataset= test_data, batch_size = 128, num_workers=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1295036fd2e95bdf77c5c472d756bc00e56620c7"},"cell_type":"code","source":"print(len(train_loader))\nprint(len(val_loader))\nprint(len(test_loader))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d0c327b0bb3d46852ecc0e432347d8a8cad2e004"},"cell_type":"markdown","source":"## Making the model architecture"},{"metadata":{"trusted":true,"_uuid":"e8c36bb92067cf486eb84d7317bd7857b01206c7"},"cell_type":"code","source":"class SkipConvNet(nn.Module):\n    def __init__(self, num_channels, num_classes):\n        super(SkipConvNet, self).__init__()\n        \n        self.conv1 = nn.Sequential(nn.Conv2d(num_channels, 8, kernel_size=3, padding = 1),\n                                  nn.BatchNorm2d(8),\n                                  nn.ReLU(),\n                                  nn.MaxPool2d(kernel_size=2, stride=2))\n        \n        self.skip1_1 = nn.Sequential(nn.Conv2d(8, 8, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(8),\n                                    nn.ReLU(),\n                                    nn.Conv2d(8, 8, kernel_size=3, padding = 1))\n        \n        self.skip1_2 = nn.Sequential(nn.Conv2d(8, 8, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(8),\n                                    nn.ReLU(),\n                                    nn.Conv2d(8, 8, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(8),\n                                    nn.ReLU(),\n                                    nn.Conv2d(8, 8, kernel_size=3, padding = 1))\n        \n        self.conv2 = nn.Sequential(nn.Conv2d(8, 16, kernel_size=3, padding = 1),\n                                  nn.BatchNorm2d(16),\n                                  nn.ReLU(),\n                                  nn.MaxPool2d(kernel_size=2, stride=2))\n        \n        self.skip2_1 = nn.Sequential(nn.Conv2d(16, 16, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(16),\n                                    nn.ReLU(),\n                                    nn.Conv2d(16, 16, kernel_size=3, padding = 1))\n        \n        self.skip2_2 = nn.Sequential(nn.Conv2d(16, 16, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(16),\n                                    nn.ReLU(),\n                                    nn.Conv2d(16, 16, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(16),\n                                    nn.ReLU(),\n                                    nn.Conv2d(16, 16, kernel_size=3, padding = 1))\n        \n        self.conv3 = nn.Sequential(nn.Conv2d(16, 32, kernel_size=3, padding=1),\n                                  nn.BatchNorm2d(32),\n                                  nn.ReLU(),\n                                  nn.MaxPool2d(kernel_size=2, stride=2))\n        \n        self.skip3_1 = nn.Sequential(nn.Conv2d(32, 32, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(32),\n                                    nn.ReLU(),\n                                    nn.Conv2d(32, 32, kernel_size=3, padding = 1))\n        \n        self.skip3_2 = nn.Sequential(nn.Conv2d(32, 32, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(32),\n                                    nn.ReLU(),\n                                    nn.Conv2d(32, 32, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(32),\n                                    nn.ReLU(),\n                                    nn.Conv2d(32, 32, kernel_size=3, padding = 1))\n        \n        self.conv4 = nn.Sequential(nn.Conv2d(32, 64, kernel_size=3, padding = 1),\n                                  nn.BatchNorm2d(64),\n                                  nn.ReLU(),\n                                  nn.MaxPool2d(kernel_size=2, stride=2))\n        \n        self.skip4_1 = nn.Sequential(nn.Conv2d(64, 64, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(64),\n                                    nn.ReLU(),\n                                    nn.Conv2d(64, 64, kernel_size=3, padding = 1))\n        \n        self.skip4_2 = nn.Sequential(nn.Conv2d(64, 64, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(64),\n                                    nn.ReLU(),\n                                    nn.Conv2d(64, 64, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(64),\n                                    nn.ReLU(),\n                                    nn.Conv2d(64, 64, kernel_size=3, padding = 1))\n        \n        self.conv5 = nn.Sequential(nn.Conv2d(64, 128, kernel_size=3, padding=1),\n                                  nn.BatchNorm2d(128),\n                                  nn.ReLU(),\n                                  nn.MaxPool2d(kernel_size=2, stride=2))\n        \n        self.skip5_1 = nn.Sequential(nn.Conv2d(128, 128, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(128),\n                                    nn.ReLU(),\n                                    nn.Conv2d(128, 128, kernel_size=3, padding = 1))\n        \n        self.skip5_2 = nn.Sequential(nn.Conv2d(128, 128, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(128),\n                                    nn.ReLU(),\n                                    nn.Conv2d(128, 128, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(128),\n                                    nn.ReLU(),\n                                    nn.Conv2d(128, 128, kernel_size=3, padding = 1))\n        \n         \n        self.ff1 = nn.Linear(3 * 3 * 128, 128)\n        self.ff2 = nn.Linear(128, 32)\n        \n        self.output = nn.Linear(32, num_classes)\n        \n    def forward(self, x):\n        out = self.conv1(x)\n        out = out + self.skip1_1(out) + self.skip1_2(out)\n        out = self.conv2(out)\n        out = out + self.skip2_1(out) + self.skip2_2(out)\n        out = self.conv3(out)\n        out = out + self.skip3_1(out) + self.skip3_2(out)\n        out = self.conv4(out)\n        out = out + self.skip4_1(out) + self.skip4_2(out)\n        out = self.conv5(out)\n        out = out + self.skip5_1(out) + self.skip5_2(out)\n        out = out.reshape(-1, 3 * 3 * 128)\n        out = self.ff1(out)\n        out = self.ff2(out)\n        out = self.output(out)\n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93e62494ea493fd5b14b72bd7a677ba9e8eeac3b"},"cell_type":"code","source":"model = SkipConvNet(3, 1)\nmodel = model.cuda()\n\noptimizer = torch.optim.Adam(model.parameters())\ncriterion = torch.nn.BCEWithLogitsLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa6c8dee38a66e8949fe3459ecb3b3bbc2996a40"},"cell_type":"code","source":"def sigmoid(x):\n    return 1.0/(1.0 + np.exp(-x))\n\ndef accuracy_mini_batch(predicted, true, i, acc, tpr, tnr):\n    \n    predicted = predicted.cpu()\n    true = true.cpu()\n    \n    predicted = (sigmoid(predicted.data.numpy()) > 0.5)\n    true = true.data.numpy()\n    \n    accuracy = np.sum(predicted == true) / true.shape[0]\n    true_positive_rate = np.sum((predicted == 1) * (true == 1)) / np.sum(true == 1)\n    true_negative_rate = np.sum((predicted == 0) * (true == 0)) / np.sum(true == 0)\n    \n    acc = acc * (i) / (i + 1)  + accuracy / (i + 1)\n    tpr = tpr * (i) / (i + 1)  + true_positive_rate / (i + 1)\n    tnr = tnr * (i) / (i + 1) + true_negative_rate / (i + 1)\n    \n    return acc, tpr, tnr\n\n\ndef accuracy(predicted, true):\n    predicted = predicted.cpu()\n    true = true.cpu()\n    \n    predicted = (sigmoid(predicted.data.numpy()) > 0.5)\n    true = true.data.numpy()\n    \n    accuracy = np.sum(predicted == true) / true.shape[0]\n    true_positive_rate = np.sum((predicted == 1) * (true == 1)) / np.sum(true == 1)\n    true_negative_rate = np.sum((predicted == 0) * (true == 0)) / np.sum(true == 0)\n\n    return accuracy, true_positive_rate, true_negative_rate","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"580c8c8c9fb13d0d989f65c75ce24499923dc613"},"cell_type":"code","source":"import time\nimport matplotlib.pyplot as plt\n\nepochs = 10\n\naccuracy_array = []\ntpr_array = []\ntnr_array = []\nloss_array = []\n\nval_loss_array = []\nval_acc_array = []\nval_tpr_array = []\nval_tnr_array = []\n\nuse_cuda = torch.cuda.is_available()\ndevice = \"cuda:0\"\n\nfor epoch in range(epochs):\n    start_time = time.time() \n    \n    loss_temp = []\n    \n    acc, tpr, tnr = 0., 0., 0.\n    \n    for mini_batch_num, data in enumerate(train_loader):\n        images, labels = data['image'], data['label']\n        images, labels = images.to(device), labels.to(device)\n        \n        preds = model(images)\n        \n        loss = criterion(preds, labels)\n        acc, tpr, tnr = accuracy_mini_batch(preds, labels, i, acc, tpr, tnr)\n        \n        optimizer.zero_grad()\n        loss.backward()\n        loss_temp.append(loss.item())\n        \n        optimizer.step()\n        if (mini_batch_num) % 4 == 0:\n            print ('Epoch {}/{}; Iter {}/{}; Loss: {:.4f}; Acc: {:.3f}; True Pos: {:.3f}; True Neg: {:.3f}'\n                   .format(epoch+1, epochs, mini_batch_num + 1, len(train_loader), loss.item(), acc, tpr, tnr), end = \"\\r\", flush = True)\n    \n    end_time = time.time()\n    \n    with torch.no_grad():\n        for i, data in enumerate(test_loader):\n            images, labels = data['image'], data['label']\n            images, labels = images.to(device), labels.to(device)\n            preds = model(images)\n            loss_test = criterion(preds, labels)\n            t_acc, t_tpr, t_tnr = accuracy(preds, labels)\n    \n    val_loss_array.append(loss_test)\n    val_acc_array.append(t_acc)\n    val_tpr_array.append(t_tpr)\n    val_tnr_array.append(t_tnr)\n    \n    print ('Epoch {}/{}; Loss: {:.4f}; Train Acc: {:.3f}; Train TPR: {:.3f}; Train TNR: {:.3f}; Epoch Time: {} mins; \\nTest Loss: {:.4f}; Test Acc: {:.3f}; Test TPR: {:.3f}; Test TNR: {:.3f}\\n'\n           .format(epoch+1, epochs, loss.item(),acc, tpr, tnr, round((end_time - start_time)/ 60., 2), loss_test, t_acc, t_tpr, t_tnr))\n    \n    loss_array.append(np.mean(np.array(loss_temp)))\n    accuracy_array.append(acc)\n    tpr_array.append(tpr)\n    tnr_array.append(tnr)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"833ed48e9326fd07a579ad4c39d24b05be634112"},"cell_type":"markdown","source":"## Viewing the resultant measures plots"},{"metadata":{"_uuid":"2d10e95939f2ea4d9242f3e69e1f39d351ac75b0"},"cell_type":"markdown","source":"### The training measures plot"},{"metadata":{"trusted":true,"_uuid":"379b3339445b7ba20c57933347da9d0459dc3c3f"},"cell_type":"code","source":"plt.plot(loss_array, color=\"red\")\nplt.plot(accuracy_array, color=\"blue\")\nplt.plot(tpr_array, color=\"green\")\nplt.plot(tnr_array, color=\"orange\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"992c033c5f9fac142fc578f70f7685a380dfa36b"},"cell_type":"markdown","source":"### The testing measures plot"},{"metadata":{"trusted":true,"_uuid":"088d95492643bc7e16aa6acc22c8a6882349f1fc"},"cell_type":"code","source":"plt.plot(val_loss_array, color=\"red\")\nplt.plot(val_acc_array, color=\"blue\")\nplt.plot(val_tpr_array, color=\"green\")\nplt.plot(val_tnr_array, color=\"orange\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"472aee8501b6c165c37a53cdc4f8ccdc45cd5fa4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}