{"cells":[{"metadata":{},"cell_type":"markdown","source":"### **Import Packages**"},{"metadata":{"_uuid":"ad079fc8-4670-4562-b07a-cccb40f38e8b","_cell_guid":"595a76c1-234c-45e0-86a2-1b8c659d1948","trusted":true},"cell_type":"code","source":"# Libraries\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\n%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nfrom sklearn.model_selection import train_test_split\n\nimport torch \nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch.utils.data import TensorDataset, DataLoader, Dataset\nfrom torch import optim, save\nfrom torchvision import datasets, transforms, models\nfrom torch.utils.data.sampler import Sampler\nfrom torch.autograd import Variable\n\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Parameters for the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = pd.read_csv('../input/histopathologic-cancer-detection/train_labels.csv')\ntest = pd.read_csv('../input/histopathologic-cancer-detection/sample_submission.csv')\n\ntrain_path = '../input/histopathologic-cancer-detection/train/'\ntest_path = '../input/histopathologic-cancer-detection/test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"This is number of files in the train: \", len(os.listdir(train_path)))\nprint(\"This is number of files in the test: \", len(os.listdir(test_path)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Hyper parameters\nnum_epochs = 5\nnum_classes = 2\nbatch_size = 128\nlearning_rate = 0.002\n\n# Device configuration\n#device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Preview Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(25, 4))\n# display 20 images\ntrain_imgs = os.listdir(\"../input/histopathologic-cancer-detection/train\")\nfor idx, img in enumerate(np.random.choice(train_imgs, 20)):\n    ax = fig.add_subplot(2, 20//2, idx+1, xticks=[], yticks=[])\n    im = Image.open(\"../input/histopathologic-cancer-detection/train/\" + img)\n    plt.imshow(im)\n    lab = labels.loc[labels['id'] == img.split('.')[0], 'label'].values[0]\n    ax.set_title(f'Label: {lab}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"example_image = Image.open(\"../input/histopathologic-cancer-detection/train/\" + labels['id'][0] + \".tif\")\nexample_image_numpy = np.array(example_image.getdata())\n\n#Finding the dimensions of the image\nprint(example_image_numpy.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Image dimension determination\nprint(np.sqrt(example_image_numpy.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Label Percentages"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels.label.value_counts() / len(labels.label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class HS_Dataset(Dataset):\n    \n    def __init__(self, csv_file, root_dir, transform = None):\n        self.df = pd.read_csv(csv_file)\n        self.dir = root_dir\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, i):\n        \"\"\"This function should return the ith example from the training set.\n        The example should be returned in the form of a dictionary: \n        {'image': image_data, 'label': label_data}\"\"\"\n        \n        file = labels['id'][i]\n        \n        label = np.array(labels['label'][i])\n        if label == 0:\n            label == 0.0\n        else:\n            label == 1.0\n            \n        \"\"\"Reshape needed to make the output of shape [1]\"\"\"\n        label = label.reshape((1))\n                \n        image = Image.open(\"../input/histopathologic-cancer-detection/train/\" + file + \".tif\")\n        image = np.array(image.getdata()).reshape(96, 96, 3)\n        \n        sample = {'image': image, 'label': label}\n        \n        if self.transform:\n            sample = self.transform(sample)\n            \n        return sample\n        \n        \nclass ToTensor(object):\n    \n    def __call__(self, sample):\n        image, label = sample['image'], sample['label']\n        \"\"\"This transposition is very important as PyTorch take in the image data in the current shape:\n        Number of Channels, Height, Width; So the third axis(channels) in the original image has to \n        be made the first axis.\"\"\"\n        image = image.transpose(2, 0, 1)        \n        image = torch.from_numpy(image)\n        image = image.type(torch.FloatTensor)\n        \n        label = torch.from_numpy(label)\n        label = label.type(torch.FloatTensor)\n        \"\"\"The optimizer takes in FloatTensor type data. Hence the data has to be converted from any other format\n        to FloatTensor type\"\"\"\n        return {'image': image, 'label': label}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import random_split\n\nhs_dataset = HS_Dataset(\"../input/histopathologic-cancer-detection/train_labels.csv\", \"../input/histopathologic-cancer-detection/train/\", transform = transforms.Compose([ToTensor()]))\n\ntrain_size = int(0.995 * len(hs_dataset))\nval_size = int((len(hs_dataset) - train_size) / 8)\ntest_size = int(val_size * 7 / 8)\ntest_size += len(hs_dataset) - train_size - val_size - test_size\n\nprint(\"train size: \", train_size)\nprint(\"val size: \", val_size)\nprint(\"test size: \", test_size)\n\ntrain_data, val_data, test_data = random_split(hs_dataset, [train_size, val_size, test_size])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#testing the datasete obj\n\nprint(\"training\")\nfor i in range(len(train_data)):\n    sample = train_data[i]\n    print(i, sample['image'].size(), sample['label'].size())\n    if i == 5:\n        break\n\nprint(\"validation\")\nfor i in range(len(val_data)):\n    sample = val_data[i]\n    print(i, sample['image'].size(), sample['label'].size())\n    if i == 5:\n        break\n        \nprint(\"testing\")\nfor i in range(len(test_data)):\n    sample = test_data[i]\n    print(i, sample['image'].size(), sample['label'].size())\n    if i == 5:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Making the dataloader object\n\ntrain_loader = DataLoader(dataset = train_data, batch_size = 128, shuffle=True, num_workers=0)\nval_loader = DataLoader(dataset= val_data, batch_size = val_size, num_workers=0)\ntest_loader = DataLoader(dataset= test_data, batch_size = 128, num_workers=0)\n\nprint(len(train_loader))\nprint(len(val_loader))\nprint(len(test_loader))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Making the model architecture\n\nclass SkipConvNet(nn.Module):\n    def __init__(self, num_channels, num_classes):\n        super(SkipConvNet, self).__init__()\n        \n        self.conv1 = nn.Sequential(nn.Conv2d(num_channels, 8, kernel_size=3, padding = 1),\n                                  nn.BatchNorm2d(8),\n                                  nn.ReLU(),\n                                  nn.MaxPool2d(kernel_size=2, stride=2))\n        \n        self.skip1_1 = nn.Sequential(nn.Conv2d(8, 8, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(8),\n                                    nn.ReLU(),\n                                    nn.Conv2d(8, 8, kernel_size=3, padding = 1))\n        \n        self.skip1_2 = nn.Sequential(nn.Conv2d(8, 8, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(8),\n                                    nn.ReLU(),\n                                    nn.Conv2d(8, 8, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(8),\n                                    nn.ReLU(),\n                                    nn.Conv2d(8, 8, kernel_size=3, padding = 1))\n        \n        self.conv2 = nn.Sequential(nn.Conv2d(8, 16, kernel_size=3, padding = 1),\n                                  nn.BatchNorm2d(16),\n                                  nn.ReLU(),\n                                  nn.MaxPool2d(kernel_size=2, stride=2))\n        \n        self.skip2_1 = nn.Sequential(nn.Conv2d(16, 16, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(16),\n                                    nn.ReLU(),\n                                    nn.Conv2d(16, 16, kernel_size=3, padding = 1))\n        \n        self.skip2_2 = nn.Sequential(nn.Conv2d(16, 16, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(16),\n                                    nn.ReLU(),\n                                    nn.Conv2d(16, 16, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(16),\n                                    nn.ReLU(),\n                                    nn.Conv2d(16, 16, kernel_size=3, padding = 1))\n        \n        self.conv3 = nn.Sequential(nn.Conv2d(16, 32, kernel_size=3, padding=1),\n                                  nn.BatchNorm2d(32),\n                                  nn.ReLU(),\n                                  nn.MaxPool2d(kernel_size=2, stride=2))\n        \n        self.skip3_1 = nn.Sequential(nn.Conv2d(32, 32, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(32),\n                                    nn.ReLU(),\n                                    nn.Conv2d(32, 32, kernel_size=3, padding = 1))\n        \n        self.skip3_2 = nn.Sequential(nn.Conv2d(32, 32, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(32),\n                                    nn.ReLU(),\n                                    nn.Conv2d(32, 32, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(32),\n                                    nn.ReLU(),\n                                    nn.Conv2d(32, 32, kernel_size=3, padding = 1))\n        \n        self.conv4 = nn.Sequential(nn.Conv2d(32, 64, kernel_size=3, padding = 1),\n                                  nn.BatchNorm2d(64),\n                                  nn.ReLU(),\n                                  nn.MaxPool2d(kernel_size=2, stride=2))\n        \n        self.skip4_1 = nn.Sequential(nn.Conv2d(64, 64, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(64),\n                                    nn.ReLU(),\n                                    nn.Conv2d(64, 64, kernel_size=3, padding = 1))\n        \n        self.skip4_2 = nn.Sequential(nn.Conv2d(64, 64, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(64),\n                                    nn.ReLU(),\n                                    nn.Conv2d(64, 64, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(64),\n                                    nn.ReLU(),\n                                    nn.Conv2d(64, 64, kernel_size=3, padding = 1))\n        \n        self.conv5 = nn.Sequential(nn.Conv2d(64, 128, kernel_size=3, padding=1),\n                                  nn.BatchNorm2d(128),\n                                  nn.ReLU(),\n                                  nn.MaxPool2d(kernel_size=2, stride=2))\n        \n        self.skip5_1 = nn.Sequential(nn.Conv2d(128, 128, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(128),\n                                    nn.ReLU(),\n                                    nn.Conv2d(128, 128, kernel_size=3, padding = 1))\n        \n        self.skip5_2 = nn.Sequential(nn.Conv2d(128, 128, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(128),\n                                    nn.ReLU(),\n                                    nn.Conv2d(128, 128, kernel_size=3, padding = 1),\n                                    nn.BatchNorm2d(128),\n                                    nn.ReLU(),\n                                    nn.Conv2d(128, 128, kernel_size=3, padding = 1))\n        \n         \n        self.ff1 = nn.Linear(3 * 3 * 128, 128)\n        self.ff2 = nn.Linear(128, 32)\n        \n        self.output = nn.Linear(32, num_classes)\n        \n    def forward(self, x):\n        out = self.conv1(x)\n        out = out + self.skip1_1(out) + self.skip1_2(out)\n        out = self.conv2(out)\n        out = out + self.skip2_1(out) + self.skip2_2(out)\n        out = self.conv3(out)\n        out = out + self.skip3_1(out) + self.skip3_2(out)\n        out = self.conv4(out)\n        out = out + self.skip4_1(out) + self.skip4_2(out)\n        out = self.conv5(out)\n        out = out + self.skip5_1(out) + self.skip5_2(out)\n        out = out.reshape(-1, 3 * 3 * 128)\n        out = self.ff1(out)\n        out = self.ff2(out)\n        out = self.output(out)\n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = SkipConvNet(3, 1)\nmodel = model.cuda()\n\noptimizer = torch.optim.Adam(model.parameters())\ncriterion = torch.nn.BCEWithLogitsLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sigmoid(x):\n    return 1.0/(1.0 + np.exp(-x))\n\ndef accuracy_mini_batch(predicted, true, i, acc, tpr, tnr):\n    \n    predicted = predicted.cpu()\n    true = true.cpu()\n    \n    predicted = (sigmoid(predicted.data.numpy()) > 0.5)\n    true = true.data.numpy()\n    \n    accuracy = np.sum(predicted == true) / true.shape[0]\n    true_positive_rate = np.sum((predicted == 1) * (true == 1)) / np.sum(true == 1)\n    true_negative_rate = np.sum((predicted == 0) * (true == 0)) / np.sum(true == 0)\n    \n    acc = acc * (i) / (i + 1)  + accuracy / (i + 1)\n    tpr = tpr * (i) / (i + 1)  + true_positive_rate / (i + 1)\n    tnr = tnr * (i) / (i + 1) + true_negative_rate / (i + 1)\n    \n    return acc, tpr, tnr\n\n\ndef accuracy(predicted, true):\n    predicted = predicted.cpu()\n    true = true.cpu()\n    \n    predicted = (sigmoid(predicted.data.numpy()) > 0.5)\n    true = true.data.numpy()\n    \n    accuracy = np.sum(predicted == true) / true.shape[0]\n    true_positive_rate = np.sum((predicted == 1) * (true == 1)) / np.sum(true == 1)\n    true_negative_rate = np.sum((predicted == 0) * (true == 0)) / np.sum(true == 0)\n\n    return accuracy, true_positive_rate, true_negative_rate","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nimport matplotlib.pyplot as plt\n\nepochs = num_epochs\n\naccuracy_array = []\ntpr_array = []\ntnr_array = []\nloss_array = []\n\nval_loss_array = []\nval_acc_array = []\nval_tpr_array = []\nval_tnr_array = []\n\n## Try without cuda and use GPU \nuse_cuda = torch.cuda.is_available()\ndevice = \"cuda:0\"\n\nfor epoch in range(epochs):\n    start_time = time.time() \n    \n    loss_temp = []\n    \n    acc, tpr, tnr = 0., 0., 0.\n    \n    for mini_batch_num, data in enumerate(train_loader):\n        images, labels_pic = data['image'], data['label']\n        images, labels_pic = images.to(device), labels_pic.to(device)\n        \n        preds = model(images)\n        \n        loss = criterion(preds, labels_pic)\n        acc, tpr, tnr = accuracy_mini_batch(preds, labels_pic, i, acc, tpr, tnr)\n        \n        optimizer.zero_grad()\n        loss.backward()\n        loss_temp.append(loss.item())\n        \n        optimizer.step()\n        if (mini_batch_num) % 4 == 0:\n            print ('Epoch {}/{}; Iter {}/{}; Loss: {:.4f}; Acc: {:.3f}; True Pos: {:.3f}; True Neg: {:.3f}'\n                   .format(epoch+1, epochs, mini_batch_num + 1, len(train_loader), loss.item(), acc, tpr, tnr), end = \"\\r\", flush = True)\n    \n    end_time = time.time()\n    \n    with torch.no_grad():\n        for i, data in enumerate(test_loader):\n            images, labels_pic = data['image'], data['label']\n            images, labels_pic = images.to(device), labels_pic.to(device)\n            preds = model(images)\n            loss_test = criterion(preds, labels_pic)\n            t_acc, t_tpr, t_tnr = accuracy(preds, labels_pic)\n    \n    val_loss_array.append(loss_test)\n    val_acc_array.append(t_acc)\n    val_tpr_array.append(t_tpr)\n    val_tnr_array.append(t_tnr)\n    \n    print ('Epoch {}/{}; Loss: {:.4f}; Train Acc: {:.3f}; Train TPR: {:.3f}; Train TNR: {:.3f}; Epoch Time: {} mins; \\nTest Loss: {:.4f}; Test Acc: {:.3f}; Test TPR: {:.3f}; Test TNR: {:.3f}\\n'\n           .format(epoch+1, epochs, loss.item(),acc, tpr, tnr, round((end_time - start_time)/ 60., 2), loss_test, t_acc, t_tpr, t_tnr))\n    \n    loss_array.append(np.mean(np.array(loss_temp)))\n    accuracy_array.append(acc)\n    tpr_array.append(tpr)\n    tnr_array.append(tnr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training measures plot\n\nplt.plot(loss_array, color=\"red\")\nplt.plot(accuracy_array, color=\"blue\")\nplt.plot(tpr_array, color=\"green\")\nplt.plot(tnr_array, color=\"orange\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Testing measures plot \n\nplt.plot(val_loss_array, color=\"red\")\nplt.plot(val_acc_array, color=\"blue\")\nplt.plot(val_tpr_array, color=\"green\")\nplt.plot(val_tnr_array, color=\"orange\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"end of code. Below is test code.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"class MyDataset(Dataset):\n    def __init__(self, df_data, data_dir = './', transform=None):\n        super().__init__()\n        self.df = df_data.values\n        self.data_dir = data_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_name,label = self.df[index]\n        img_path = os.path.join(self.data_dir, img_name+'.tif')\n        image = cv2.imread(img_path)\n        if self.transform is not None:\n            image = self.transform(image)\n        return image, label"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# RandomCrop, RandomAffine, etc can't be used because they may cause the pixels of tumor tissue\n# move away from center\n# transforms that changed colours (such as GreyScale) can't be used too\ntrans_train = transforms.Compose([transforms.ToPILImage(),\n                                  transforms.Pad(64, padding_mode='reflect'),\n                                  transforms.RandomHorizontalFlip(), \n                                  transforms.RandomVerticalFlip(),\n                                  transforms.RandomRotation(30), \n                                  transforms.ToTensor(),\n                                  transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])\n\ntrans_valid = transforms.Compose([transforms.ToPILImage(),\n                                  transforms.Pad(64, padding_mode='reflect'),\n                                  transforms.ToTensor(),\n                                  transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])\n\ndataset_train = MyDataset(df_data=train, data_dir=train_path, transform=trans_train)\ndataset_valid = MyDataset(df_data=val, data_dir=train_path, transform=trans_valid)\n\nloader_train = DataLoader(dataset = dataset_train, batch_size=batch_size, shuffle=True, num_workers=0)\nloader_valid = DataLoader(dataset = dataset_valid, batch_size=batch_size//2, shuffle=False, num_workers=0)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"class SimpleCNN(nn.Module):\n    def __init__(self):\n        # This constructor will initialize the model architecture\n        # 512 256 3 3\n        # [128, 128, 15, 15] \n        super(SimpleCNN, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding=2)\n        self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, padding=2)\n        self.conv3 = nn.Conv2d(in_channels=64, out_channels=96, kernel_size=3, padding=2)\n        self.conv4 = nn.Conv2d(in_channels=96, out_channels=128, kernel_size=3, padding=2)\n        self.conv5 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, padding=2)\n        self.conv5 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, padding=2)\n        self.bn1 = nn.BatchNorm2d(32)\n        self.bn2 = nn.BatchNorm2d(64)\n        self.bn3 = nn.BatchNorm2d(96)\n        self.bn4 = nn.BatchNorm2d(128)\n        self.bn5 = nn.BatchNorm2d(256)\n        self.bn6 = nn.BatchNorm2d(512) \n        self.pool = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.avg = nn.AvgPool2d(8)\n        self.fc = nn.Linear(512 * 1 * 1, 2) \n        self.fc = nn.Sigmoid()\n    def forward(self, x):\n        x = self.pool(F.leaky_relu(self.bn1(self.conv1(x)))) # first convolutional layer then batchnorm, then activation then pooling layer.\n        x = self.pool(F.leaky_relu(self.bn2(self.conv2(x))))\n        x = self.pool(F.leaky_relu(self.bn3(self.conv3(x))))\n        x = self.pool(F.leaky_relu(self.bn4(self.conv4(x))))\n        x = self.pool(F.leaky_relu(self.bn5(self.conv5(x))))\n        x = self.pool(F.leaky_relu(self.bn6(self.conv6(x))))\n        x = self.avg(x)\n        # print(x.shape) # lifehack to find out the correct dimension for the Linear Layer\n        x = x.view(-1, 512 * 1 * 1)\n        x = self.fc(x)\n        return x"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"model = SimpleCNN().to(device)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Loss and optimizer\ncriterion = nn.CrossEntropyLoss()\n#criterion = nn.BCELoss() \noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) \n# torch.optim.Adam was also used and showed poorer result during test accuracy"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Train the model\ntotal_step = len(loader_train)\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(loader_train):\n        images = images.to(device)\n        labels = labels.to(device)\n        \n        # Forward pass\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        \n        # Backward and optimize\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n        if (i+1) % 100 == 0:\n            print(f\"Epoch {epoch+1}/{epochs}.. \"\n                  f\"Train loss: {running_loss/print_every:.3f}.. \"\n                  f\"Test loss: {test_loss/len(testloader):.3f}.. \"\n                  f\"Test accuracy: {accuracy/len(testloader):.3f}\")\n           # print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' \n           #        .format(epoch+1, num_epochs, i+1, total_step, loss.item()))"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Test the model\nmodel.eval()  # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)\nwith torch.no_grad():\n    correct = 0\n    total = 0\n    for images, labels in loader_valid:\n        images = images.to(device)\n        labels = labels.to(device)\n        outputs = model(images)\n        _, predicted = torch.max(outputs.data, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n          \n    print('Test Accuracy of the model on the 22003 test images: {} %'.format(100 * correct / total))\n\n# Save the model checkpoint\ntorch.save(model.state_dict(), 'model.ckpt')"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"labels.id = labels.id + '.tif'\ntest_id = test.id # original ids for submission\ntest.id = test.id + '.tif'\nprint(labels.head())\nprint(test.head())"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"n = 80000\nnegative_sample = labels.loc[labels.label == '0', :].sample(n, random_state=1)\npositive_sample = labels.loc[labels.label == '1', :].sample(n, random_state=1)\nlabeled_sample = pd.concat([negative_sample, positive_sample], axis=0).reset_index(drop=True)\n\ntrain_df, valid_df = train_test_split(labeled_sample, test_size=0.2, random_state=1, stratify=labeled_sample.label)\n\nprint(train_df.shape)\nprint(valid_df.shape)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"base_dir = 'images/'\ntrain_dir = 'images/train/'\nvalid_dir = 'images/valid/'\n\n\nos.mkdir(base_dir)\nos.mkdir(train_dir)\nos.mkdir(valid_dir)\n\nos.mkdir(train_dir + 'negative')\nos.mkdir(train_dir + 'positive')\nos.mkdir(valid_dir + 'negative')\nos.mkdir(valid_dir + 'positive')"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"%%time \n\nfor i in range(len(train_df.id)):\n    \n    src = train_path + train_df.id.iloc[i]\n        \n    if train_df.label.iloc[i] == '0':    \n        dest = train_dir + 'negative/' + train_df.id.iloc[i]\n    else: \n        dest = train_dir + 'positive/' + train_df.id.iloc[i]\n        \n    shutil.copyfile(src, dest)\n\nprint(len(os.listdir(train_dir + 'negative')))\nprint(len(os.listdir(train_dir + 'positive')))"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"%%time \n\nfor i in range(len(valid_df.id)):\n    \n    src = train_path + valid_df.id.iloc[i]\n        \n    if valid_df.label.iloc[i] == '0':    \n        dest = valid_dir + 'negative/' + valid_df.id.iloc[i]\n    else: \n        dest = valid_dir + 'positive/' + valid_df.id.iloc[i]\n        \n    shutil.copyfile(src, dest)\n\nprint(len(os.listdir(valid_dir + 'negative')))\nprint(len(os.listdir(valid_dir + 'positive')))"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"bs = 64\n\ntrain_datagen = ImageDataGenerator(rescale=1/255)\nvalid_datagen = ImageDataGenerator(rescale=1/255)\ntest_datagen = ImageDataGenerator(rescale=1/255)\n\ntrain_generator = train_datagen.flow_from_directory(\n    directory = train_dir,\n    batch_size = bs,\n    shuffle = True,\n    class_mode = \"binary\",\n    target_size = (96,96))\n\nvalid_generator = train_datagen.flow_from_directory(\n    directory = valid_dir,\n    batch_size = bs,\n    shuffle = True,\n    class_mode = \"binary\",\n    target_size = (96,96))"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"def training_images(seed):\n    np.random.seed(seed)\n    train_generator.reset()\n    imgs, labels = next(train_generator)\n        \n    plt.figure(figsize=(12,12))\n    for i in range(16):\n        plt.subplot(4,4,i+1)\n        plt.imshow(imgs[i,:,:,:])\n        if(labels[i] == 1):\n            plt.text(0, -5, 'Positive', color='r')\n        else:\n            plt.text(0, -5, 'Negative', color='b')\n        plt.axis('off')\n    plt.show()\n\ntraining_images(1)"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Split Train and Test datasets"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"train, val = train_test_split(labels, stratify=labels.label, test_size=0.1)\nlen(train), len(val)"},{"metadata":{},"cell_type":"markdown","source":"#### Simple Custom Generator"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"class MyDataset(Dataset):\n    def __init__(self, df_data, data_dir = './', transform=None):\n        super().__init__()\n        self.df = df_data.values\n        self.data_dir = data_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_name,label = self.df[index]\n        img_path = os.path.join(self.data_dir, img_name+'.tif')\n        image = cv2.imread(img_path)\n        if self.transform is not None:\n            image = self.transform(image)\n        return image, label"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"trans_train = transforms.Compose([transforms.ToPILImage(),\n                                  transforms.Pad(64, padding_mode='reflect'),\n                                  transforms.RandomHorizontalFlip(), \n                                  transforms.RandomVerticalFlip(),\n                                  transforms.RandomRotation(20), \n                                  transforms.ToTensor(),\n                                  transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])\n\ntrans_valid = transforms.Compose([transforms.ToPILImage(),\n                                  transforms.Pad(64, padding_mode='reflect'),\n                                  transforms.ToTensor(),\n                                  transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])\n\ndataset_train = MyDataset(df_data=train, data_dir=train_path, transform=trans_train)\ndataset_valid = MyDataset(df_data=val, data_dir=train_path, transform=trans_valid)\n\nloader_train = DataLoader(dataset = dataset_train, batch_size=batch_size, shuffle=True, num_workers=0)\nloader_valid = DataLoader(dataset = dataset_valid, batch_size=batch_size//2, shuffle=False, num_workers=0)"},{"metadata":{},"cell_type":"markdown","source":"im = Image.open(trans_train[0])\n"},{"metadata":{},"cell_type":"markdown","source":"#### CNN Model"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"class SimpleCNN(nn.Module):\n    def __init__(self):\n        # ancestor constructor call\n        super(SimpleCNN, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding=2)\n        self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, padding=2)\n        self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, padding=2)\n        self.conv4 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, padding=2)\n        self.conv5 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, padding=2)\n        self.bn1 = nn.BatchNorm2d(32)\n        self.bn2 = nn.BatchNorm2d(64)\n        self.bn3 = nn.BatchNorm2d(128)\n        self.bn4 = nn.BatchNorm2d(256)\n        self.bn5 = nn.BatchNorm2d(512)\n        self.pool = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.avg = nn.AvgPool2d(8)\n        self.fc = nn.Linear(512 * 1 * 1, 2) # !!!\n    def forward(self, x):\n        x = self.pool(F.leaky_relu(self.bn1(self.conv1(x)))) # first convolutional layer then batchnorm, then activation then pooling layer.\n        x = self.pool(F.leaky_relu(self.bn2(self.conv2(x))))\n        x = self.pool(F.leaky_relu(self.bn3(self.conv3(x))))\n        x = self.pool(F.leaky_relu(self.bn4(self.conv4(x))))\n        x = self.pool(F.leaky_relu(self.bn5(self.conv5(x))))\n        x = self.avg(x)\n        #print(x.shape) # lifehack to find out the correct dimension for the Linear Layer\n        x = x.view(-1, 512 * 1 * 1) # !!!\n        x = self.fc(x)\n        return x"},{"metadata":{},"cell_type":"markdown","source":"**Important note:** \n\nYou may notice that in lines with # !!! there is not very clear 512 x 1 x 1. This is the dimension of the picture before the FC layers (H x W x C), then you have to calculate it manually (in Keras, for example, .Flatten () does everything for you). However, there is one life hack — just make print (x.shape) in forward () (commented out line). You will see the size (batch_size, C, H, W) - you need to multiply everything except the first (batch_size), this will be the first dimension of Linear (), and it is in C H W that you need to \"expand\" x before feeding to Linear ()."},{"metadata":{"trusted":true},"cell_type":"markdown","source":"model = SimpleCNN().to(device)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Loss and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adamax(model.parameters(), lr=learning_rate)\n"},{"metadata":{},"cell_type":"markdown","source":"#### Train the model"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Train the model\ntotal_step = len(loader_train)\nfor epoch in range(num_epochs):\n    for i, (images, labels) in enumerate(loader_train):\n        images = images.to(device)\n        labels = labels.to(device)\n        \n        # Forward pass\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        \n        # Backward and optimize\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n        if (i+1) % 100 == 0:\n            print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' \n                   .format(epoch+1, num_epochs, i+1, total_step, loss.item()))"},{"metadata":{},"cell_type":"markdown","source":"#### Accuracy Check"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Test the model\nmodel.eval()  # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)\nwith torch.no_grad():\n    correct = 0\n    total = 0\n    for images, labels in loader_valid:\n        images = images.to(device)\n        labels = labels.to(device)\n        outputs = model(images)\n        _, predicted = torch.max(outputs.data, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n          \n    print('Test Accuracy of the model on the 22003 test images: {} %'.format(100 * correct / total))\n\n# Save the model checkpoint\ntorch.save(model.state_dict(), 'model.ckpt')"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"plt.plot(train_loss, label='Training loss')\nplt.plot(test_loss, label='Validation loss')\nplt.legend(frameon=False)\nplt.show()"}],"metadata":{"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":4}