{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install torchsummary\n!pip install tqdm\nimport os\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"\nimport pandas as pd\nimport cv2\nfrom PIL import Image\nimport os\nimport numpy as np\nfrom matplotlib import pyplot as plt\nfrom tqdm import notebook\n\nimport torch\nimport torchvision.models as models\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import TensorDataset, DataLoader, Dataset\nfrom torchsummary import summary\n\nimport math\n\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load images"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"labels = pd.read_csv(\"../input/histopathologic-cancer-detection/train_labels.csv\")\nsub = pd.read_csv('../input/histopathologic-cancer-detection/sample_submission.csv')\ntrain_path = '../input/histopathologic-cancer-detection/train/'\ntest_path = '../input/histopathologic-cancer-detection/test/'\n\n%matplotlib inline \n\nprint(labels.shape)\n#train['label'].mean() # means 40% of data is positive\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\ndevice","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Split data into training and testing set, create dataset object"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#Splitting data into train and val\ntrain, test = train_test_split(labels, stratify=labels.label, test_size=0.1)\nlen(train), len(test)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self, df_data, data_dir = './', transform=None):\n        super().__init__()\n        self.df = df_data.values\n        self.data_dir = data_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_name,label = self.df[index]\n        img_path = os.path.join(self.data_dir, img_name+'.tif')\n        image = cv2.imread(img_path)\n        if self.transform is not None:\n            image = self.transform(image)\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create dataloaders"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torchvision\nimport torchvision.transforms as transforms\n\nbatch_size = 512\n\ntrans = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.Resize(100),\n    transforms.RandomCrop(96),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n])\n\ndataset_train = MyDataset(df_data=train, data_dir=train_path, transform=trans)\ndataset_test = MyDataset(df_data=test, data_dir=train_path, transform=trans)\n\ntrainloader = DataLoader(dataset = dataset_train, batch_size=batch_size, shuffle=True, num_workers=8, pin_memory=True)\ntestloader = DataLoader(dataset = dataset_test, batch_size=batch_size//2, shuffle=True, num_workers=8, pin_memory=True )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img, label = dataset_train[0]\nprint('Shape of img:', img.shape)\nprint('Range of img:', img.min().item(), 'to', img.max().item())\nprint('\\nType of img:', type(img))\nprint('Type of items in img:', img.dtype)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## ** Define Squeeeze and Excitation layer**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Referenced works:\n#https://github.com/moskomule/senet.pytorch/blob/master/senet/se_module.py\nclass SELayer(nn.Module):\n    def __init__(self, channel, reduction=16):\n        super(SELayer, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Sequential(\n            nn.Linear(channel, channel // reduction, bias=False),\n            nn.ReLU(inplace=True),\n            nn.Linear(channel // reduction, channel, bias=False),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        b, c, _, _ = x.size()\n        y = self.avg_pool(x).view(b, c)\n        y = self.fc(y).view(b, c, 1, 1)\n        return x * y.expand_as(x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model definition (custom architecture)"},{"metadata":{"trusted":true},"cell_type":"code","source":"class custom_arch(nn.Module):\n    def __init__(self):\n        super().__init__()        \n        \n        self.conv1=nn.Sequential(\n            nn.Conv2d(in_channels=3,out_channels=32,kernel_size=1,stride=1,padding=1),\n            nn.BatchNorm2d(32),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(2,2))\n\n        self.conv2=nn.Sequential(\n            nn.Conv2d(in_channels=32,out_channels=64,kernel_size=3,stride=1,padding=1),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(2,2))\n        \n        self.conv3=nn.Sequential(\n            nn.Conv2d(in_channels=64,out_channels=128,kernel_size=3,stride=1,padding=1),\n            nn.BatchNorm2d(128),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(2,2))\n        \n        self.conv4=nn.Sequential(\n            nn.Conv2d(in_channels=128,out_channels=256,kernel_size=3,stride=1,padding=1),\n            nn.BatchNorm2d(256),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(2,2))\n        \n        self.conv5=nn.Sequential(\n            nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(512),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(2,2))\n        \n        self.dropout2d = nn.Dropout2d()\n        \n        \n        self.fc=nn.Sequential(\n            nn.Linear(512*3*3,1024),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.4),\n            nn.Linear(1024,512),\n            nn.Dropout(0.4),\n            nn.Linear(512, 2),\n            nn.Sigmoid())\n\n                \n    def forward(self, x):\n        x=self.conv1(x)\n        x=self.conv2(x)\n        x=self.conv3(x)\n        x=self.conv4(x)\n        x=self.conv5(x)\n        #print(x.shape) #<-- Life saving debugging step :D\n        x=x.view(x.shape[0],-1)\n        x=self.fc(x)\n        return x\n\n#summary(Network1(),(3,96,96), device=\"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper definition for se_resnet50"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Referenced works\n#https://github.com/moskomule/senet.pytorch/blob/master/senet/se_resnet.py\n\nfrom torch.utils.model_zoo import load_url as load_state_dict_from_url\nfrom torchvision.models import ResNet\n\n\n\ndef conv3x3(in_planes, out_planes, stride=1):\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False)\n\n\nclass SEBasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None,\n                 *, reduction=16):\n        super(SEBasicBlock, self).__init__()\n        self.conv1 = conv3x3(inplanes, planes, stride)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = conv3x3(planes, planes, 1)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.se = SELayer(planes, reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        residual = x\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.se(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n        out = self.relu(out)\n\n        return out\n\n\nclass SEBottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None,\n                 *, reduction=16):\n        super(SEBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,\n                               padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se = SELayer(planes * 4, reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n        out = self.se(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n        out = self.relu(out)\n\n        return out\n\n\ndef se_resnet18(num_classes=1_000):\n    \"\"\"Constructs a ResNet-18 model.\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNet(SEBasicBlock, [2, 2, 2, 2], num_classes=num_classes)\n    model.avgpool = nn.AdaptiveAvgPool2d(1)\n    return model\n\n\ndef se_resnet34(num_classes=1_000):\n    \"\"\"Constructs a ResNet-34 model.\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNet(SEBasicBlock, [3, 4, 6, 3], num_classes=num_classes)\n    model.avgpool = nn.AdaptiveAvgPool2d(1)\n    return model\n\n\ndef se_resnet50(num_classes=1_000, pretrained=False):\n    \"\"\"Constructs a ResNet-50 model.\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNet(SEBottleneck, [3, 4, 6, 3], num_classes=num_classes)\n    model.avgpool = nn.AdaptiveAvgPool2d(1)\n    if pretrained:\n        model.load_state_dict(load_state_dict_from_url(\n            \"https://github.com/moskomule/senet.pytorch/releases/download/archive/seresnet50-60a8950a85b2b.pkl\"))\n    return model\n\n\ndef se_resnet101(num_classes=1_000):\n    \"\"\"Constructs a ResNet-101 model.\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNet(SEBottleneck, [3, 4, 23, 3], num_classes=num_classes)\n    model.avgpool = nn.AdaptiveAvgPool2d(1)\n    return model\n\n\ndef se_resnet152(num_classes=1_000):\n    \"\"\"Constructs a ResNet-152 model.\n\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNet(SEBottleneck, [3, 8, 36, 3], num_classes=num_classes)\n    model.avgpool = nn.AdaptiveAvgPool2d(1)\n    return model\n\n\nclass CifarSEBasicBlock(nn.Module):\n    def __init__(self, inplanes, planes, stride=1, reduction=16):\n        super(CifarSEBasicBlock, self).__init__()\n        self.conv1 = conv3x3(inplanes, planes, stride)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = conv3x3(planes, planes)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.se = SELayer(planes, reduction)\n        if inplanes != planes:\n            self.downsample = nn.Sequential(nn.Conv2d(inplanes, planes, kernel_size=1, stride=stride, bias=False),\n                                            nn.BatchNorm2d(planes))\n        else:\n            self.downsample = lambda x: x\n        self.stride = stride\n\n    def forward(self, x):\n        residual = self.downsample(x)\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.se(out)\n\n        out += residual\n        out = self.relu(out)\n\n        return out\n\n\nclass CifarSEResNet(nn.Module):\n    def __init__(self, block, n_size, num_classes=10, reduction=16):\n        super(CifarSEResNet, self).__init__()\n        self.inplane = 16\n        self.conv1 = nn.Conv2d(\n            3, self.inplane, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(self.inplane)\n        self.relu = nn.ReLU(inplace=True)\n        self.layer1 = self._make_layer(\n            block, 16, blocks=n_size, stride=1, reduction=reduction)\n        self.layer2 = self._make_layer(\n            block, 32, blocks=n_size, stride=2, reduction=reduction)\n        self.layer3 = self._make_layer(\n            block, 64, blocks=n_size, stride=2, reduction=reduction)\n        self.avgpool = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(64, num_classes)\n        self.initialize()\n\n    def initialize(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight)\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n    def _make_layer(self, block, planes, blocks, stride, reduction):\n        strides = [stride] + [1] * (blocks - 1)\n        layers = []\n        for stride in strides:\n            layers.append(block(self.inplane, planes, stride, reduction))\n            self.inplane = planes\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n\n        x = self.avgpool(x)\n        x = x.view(x.size(0), -1)\n        x = self.fc(x)\n\n        return x\n\n\nclass CifarSEPreActResNet(CifarSEResNet):\n    def __init__(self, block, n_size, num_classes=10, reduction=16):\n        super(CifarSEPreActResNet, self).__init__(\n            block, n_size, num_classes, reduction)\n        self.bn1 = nn.BatchNorm2d(self.inplane)\n        self.initialize()\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n\n        x = self.bn1(x)\n        x = self.relu(x)\n\n        x = self.avgpool(x)\n        x = x.view(x.size(0), -1)\n        x = self.fc(x)\n\n\ndef se_resnet20(**kwargs):\n    \"\"\"Constructs a ResNet-18 model.\n\n    \"\"\"\n    model = CifarSEResNet(CifarSEBasicBlock, 3, **kwargs)\n    return model\n\n\ndef se_resnet32(**kwargs):\n    \"\"\"Constructs a ResNet-34 model.\n\n    \"\"\"\n    model = CifarSEResNet(CifarSEBasicBlock, 5, **kwargs)\n    return model\n\n\ndef se_resnet56(**kwargs):\n    \"\"\"Constructs a ResNet-34 model.\n\n    \"\"\"\n    model = CifarSEResNet(CifarSEBasicBlock, 9, **kwargs)\n    return model\n\n\ndef se_preactresnet20(**kwargs):\n    \"\"\"Constructs a ResNet-18 model.\n\n    \"\"\"\n    model = CifarSEPreActResNet(CifarSEBasicBlock, 3, **kwargs)\n    return model\n\n\ndef se_preactresnet32(**kwargs):\n    \"\"\"Constructs a ResNet-34 model.\n\n    \"\"\"\n    model = CifarSEPreActResNet(CifarSEBasicBlock, 5, **kwargs)\n    return model\n\n\ndef se_preactresnet56(**kwargs):\n    \"\"\"Constructs a ResNet-34 model.\n\n    \"\"\"\n    model = CifarSEPreActResNet(CifarSEBasicBlock, 9, **kwargs)\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Default model definition (custom, resnet18, GoogLeNet, resnet 50) \n## with SE and frozen variants"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_network_custom(freeze_se = False):\n    custom = custom_arch()\n    if freeze_se == True:\n        #Add SELayers\n        custom.conv1.add_module(\"SELayer\", SELayer(32, 16))\n        custom.conv2.add_module(\"SELayer\",SELayer(64, 16))\n        custom.conv3.add_module(\"SELayer\",SELayer(128, 16))\n        custom.conv4.add_module(\"SELayer\",SELayer(256, 16))\n        custom.conv5.add_module(\"SELayer\",SELayer(512, 16))\n        #Freeze half of layers\n        for name, param in custom.named_parameters():  \n            if any(name.startswith(ext) for ext in ['conv1', 'conv2', 'conv3.0', 'conv3.1']):\n                param.requires_grad = False\n    return custom\n\ndef build_network_resnet18(pretrained = True, freeze_se = False):\n    #Note, cannot load state dict for se_resnet18 \n    if freeze_se == False:  \n        resnet18 = models.resnet18(pretrained = pretrained)\n        in_c = resnet18.fc.in_features\n        resnet18.fc = nn.Linear(in_c, 2)\n    else:\n        #Create se_resnet18, customize classification layer\n        resnet18 = se_resnet18(num_classes = 2)\n        #Freeze first half of layers\n        for name, param in resnet18.named_parameters():\n            if not any(name.startswith(ext) for ext in ['layer3', 'layer4', 'fc']):\n                param.requires_grad = False\n    return resnet18\n\n#Referenced works:\n#https://github.com/pytorch/vision/blob/master/torchvision/models/googlenet.py\n#https://github.com/moskomule/senet.pytorch/blob/master/senet/se_inception.py\ndef build_network_googlenet(pretrained = True, freeze_se = False):\n    #Pretrain and customize classifier\n    googlenet = models.googlenet(pretrained = pretrained)\n    in_c = googlenet.fc.in_features\n    googlenet.fc = nn.Linear(in_c, 2)\n    \n    #Add SELayers and freeze first half of layers\n    if(freeze_se == True):\n        for name, param in googlenet.named_parameters():\n            if not any(name.startswith(ext) for ext in ['inception4c.branch3','inception4c.branch4', 'inception4d', 'inception4e', 'inception5', 'fc']):\n                param.requires_grad = False\n                \n        #Add SELayers to each inception module when freeze_se is true\n        googlenet.inception3a.add_module(\"SELayer\", SELayer(32, 16))\n        googlenet.inception3b.add_module(\"SELayer\", SELayer(64, 16))\n        googlenet.inception4a.add_module(\"SELayer\", SELayer(64, 16))\n        googlenet.inception4b.add_module(\"SELayer\", SELayer(64, 16))\n        googlenet.inception4c.add_module(\"SELayer\", SELayer(64, 16))\n        googlenet.inception4d.add_module(\"SELayer\", SELayer(64, 16))\n        googlenet.inception4e.add_module(\"SELayer\", SELayer(128, 16))\n        googlenet.inception5a.add_module(\"SELayer\", SELayer(128, 16))\n        googlenet.inception5b.add_module(\"SELayer\", SELayer(128, 16))\n        if googlenet.aux_logits:\n            googlenet.aux1.add_module(\"SELayer\", SELayer(2))\n            googlenet.aux2.add_module(\"SELayer\", SELayer(2))\n    return googlenet\n\ndef build_network_resnet50(pretrained = True, freeze_se = False):\n    if freeze_se == False:  \n        resnet50 = models.resnet50(pretrained = pretrained)\n        in_c = resnet50.fc.in_features\n        resnet50.fc = nn.Linear(in_c, 2)\n    else:\n        #Note, although se_resnet50 can be pretrained, it's for 1K classes in the fc layer\n        #We can only change our fc layer after we have pretrained\n        resnet50 = se_resnet50(num_classes = 1000, pretrained = pretrained)\n        #Change classification layer\n        in_c = resnet50.fc.in_features\n        resnet50.fc = nn.Linear(in_c, 2)\n        #Freeze layers\n        freeze_list = ['layer3.0.downsample', 'layer3.1', 'layer3.2', 'layer3.3', 'layer3.4', 'layer3.5', 'layer4', 'fc']\n        for name, param in resnet50.named_parameters():  \n              if not any(name.startswith(ext) for ext in freeze_list):\n                param.requires_grad = False\n        \n    return resnet50","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check model definitions for error"},{"metadata":{"trusted":true},"cell_type":"code","source":"summary(build_network_custom(),(3,96,96), device=\"cpu\")\nsummary(build_network_custom(freeze_se = True),(3,96,96), device=\"cpu\")\nsummary(build_network_resnet18(pretrained = True),(3,96,96), device=\"cpu\")\nsummary(build_network_resnet18(pretrained = True, freeze_se = True),(3,96,96), device=\"cpu\")\nsummary(build_network_googlenet(pretrained = True),(3,96,96), device=\"cpu\")\nsummary(build_network_googlenet(pretrained = True,freeze_se = True),(3,96,96), device=\"cpu\")\nsummary(build_network_resnet50(pretrained = True), (3,96,96), device=\"cpu\")\nsummary(build_network_resnet50(pretrained = True, freeze_se = True),(3,96,96), device=\"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train helper function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(net, epochs=5, lr=0.1, momentum=0.9):\n    history_step=int(len(trainloader)/(150/epochs))\n    train_loss_history = []\n    \n    # transfer model to GPU\n    if torch.cuda.is_available():\n       net = net.cuda()\n    \n    # set to training mode\n    net.train()\n    \n    # train the network\n    for e in notebook.tqdm(range(epochs),desc=\"Epoch\"):   \n        lr = lr/10\n        total_loss = 0\n        loss_count = 0\n\n        for i, (inputs, labels) in notebook.tqdm(enumerate(trainloader),total=len(trainloader), desc=\"Epoch \"+ str(e+1),leave=False):\n              \n            # set the optimizer. Use SGD with momentum\n            optimizer = torch.optim.SGD(net.parameters(), lr=lr,momentum=momentum)\n            \n            # Clear all the gradient to 0\n            optimizer.zero_grad()\n\n            # transfer data to GPU\n            inputs = inputs.to(device)\n            labels = labels.to(device)\n\n            # forward propagation to get h\n            outs = net(inputs)\n\n            # compute loss \n            #change to loss = F.cross_entropy(outs.logits, labels) for se_googlenet if having problems \n            loss = F.cross_entropy(outs, labels)\n\n            # backpropagation to get gradients of all parameters\n            loss.backward()\n\n            # update parameters\n            optimizer.step()\n\n            # get the loss\n            total_loss += loss.item()\n            loss_count += 1\n            \n            # display the averaged loss value \n            if i % history_step == 0 and i != 0:\n                train_loss_history.append(total_loss/loss_count)\n                total_loss=0\n                loss_count=0\n\n    return train_loss_history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training the model (custom architecture)"},{"metadata":{"trusted":true},"cell_type":"code","source":"custom = build_network_custom()\ncustom.name='custom'\nhist_custom = train(custom)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"se_custom = build_network_custom(freeze_se = True)\nse_custom.name='se_custom'\nhist_custom_se = train(se_custom)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training the model (resnet18)"},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet18 = build_network_resnet18(pretrained = True)\nresnet18.name='resnet18'\nhist_resnet18 = train(resnet18)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training the model (se_resnet18)"},{"metadata":{"trusted":true},"cell_type":"code","source":"se_resnet18 = build_network_resnet18(pretrained = True, freeze_se = True)\nse_resnet18.name='se_resnet18'\nhist_se_resnet18 = train(se_resnet18)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training the model (GoogLeNet)"},{"metadata":{"trusted":true},"cell_type":"code","source":"googlenet = build_network_googlenet(pretrained = True)\ngooglenet.name='googlenet'\nhist_googlenet = train(googlenet)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training the model (se_googlenet)"},{"metadata":{"trusted":true},"cell_type":"code","source":"se_googlenet= build_network_googlenet(pretrained = True, freeze_se = True)\nse_googlenet.name='se_googlenet'\nhist_se_googlenet = train(se_googlenet)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training the model (resnet 50)"},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet50 = build_network_resnet50(pretrained = True)\nresnet50.name='resnet50'\nhist_resnet50 = train(resnet50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training the model (se_resnet50)"},{"metadata":{"trusted":true},"cell_type":"code","source":"se_resnet50 = build_network_resnet50(pretrained = True, freeze_se = True)\nse_resnet50.name='se_resnet50'\nhist_se_resnet50= train(se_resnet50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define list of models"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_list = [custom, se_custom, resnet18, se_resnet18, googlenet, se_googlenet, resnet50, se_resnet50]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test function"},{"metadata":{"trusted":true},"cell_type":"code","source":"#test models without threshold, calc_plot_beta will calculate F1 and F2 scores for all thresholds\ndef test(model, threshold=None):\n    model.eval()\n    with torch.no_grad(): #set all the requires_grad flag to false\n        pred = []\n        test_y = []\n        for images, labels in notebook.tqdm(testloader):\n            images = images.to(device)\n            labels = labels.to(device)\n            outputs = model(images)\n            pred.append(outputs)\n            test_y.append(labels)\n\n    pred = torch.cat(pred, dim=0)#convert list to torch and flatten\n    pred2 = pred.detach().cpu().numpy()\n    pred_prob = pred2[:,1]#flatten and convert to array\n\n    \n    test_y = torch.cat(test_y, dim=0).cpu()\n    if threshold==None:\n        return pred_prob,test_y\n    else:\n        pred_binary = np.where(pred_prob > threshold,1,0)#set prediction become binary\n        return pred_prob,pred_binary,test_y\n\ndef get_binary(pred_prob, threshold):\n    pred_binary = np.where(pred_prob > threshold,1,0)#set prediction become binary\n    return pred_binary","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper function to plot F beta score vs threshold "},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_fbeta_threshold(beta, fbeta_scores, threshold, model_name_str):\n    max_fbeta = np.sort(fbeta_scores)[len(fbeta_scores)-1]\n    max_fbeta_idx = fbeta_scores.tolist().index(max_fbeta)\n    xcoord = threshold[max_fbeta_idx]\n    \n   \n    title1_str = model_name_str + \" F-\" + str(beta) +  \" scores vs. Threshold\"\n    ylabel_str = \"F-\" + str(beta) + \" scores\"\n    f_score_str = \"F-\" + str(beta) + \" score vs. threshold\"\n    plt.figure()\n    plt.title(title1_str)\n    plt.plot(fbeta_scores,  'b', label =f_score_str)\n    \n    #plt.plot([0,1], [0,1], 'r--')\n   \n    plt.ylabel(ylabel_str)\n    plt.xlabel('Threshold')\n    threshold_label = \"Threshold: \" + str(np.round(xcoord,3))\n    max_fbeta_label = \"Max F-\" + str(beta) + \" score: \" +  str(np.round(max_fbeta,3))\n    plt.axvline(x = xcoord, c = 'r', label = threshold_label)\n    plt.axhline(y = max_fbeta, c = 'y', label =max_fbeta_label )\n  \n    plt.legend(loc='lower right')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper function to calculate F1 & F2 scores for thresholds"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import fbeta_score\n\ndef calc_plot_fbeta(model, size):\n    threshold = np.linspace(0,1, size)\n    fbeta1_scores = []\n    fbeta2_scores = []\n    \n    #For each threshold from 0 to 1, calculate the model's F1 & F2 score\n    model_ans=test(model)\n    for i in notebook.tqdm(threshold):\n        y_pred_class = get_binary(model_ans[0], i)\n        beta = 1  #calculate F1 score\n        fbeta1 = fbeta_score(model_ans[1], y_pred_class, beta)\n        fbeta1_scores = np.append(fbeta1_scores, fbeta1)\n        beta = 2  #calculate F2 score\n        fbeta2 = fbeta_score(model_ans[1], y_pred_class, beta)\n        fbeta2_scores = np.append(fbeta2_scores, fbeta2)\n    \n    fbeta_scores = [(1, fbeta1_scores), (2, fbeta2_scores)]\n    \n    #Plot F1 vs. each threshold and F2 vs. each threshold for this model\n    for i, fb_scores in fbeta_scores: \n        max_fbeta = np.sort(fb_scores)[len(fb_scores)-1]#get max score\n        max_fbeta_idx = fb_scores.tolist().index(max_fbeta)# use max score to get current threshold\n        xcoord = threshold[max_fbeta_idx]\n        best_thresholds[str(model.name) + \"_F\" + str(i)] = xcoord, max_fbeta#record best thresholds and score\n        plot_fbeta_threshold(i, fb_scores,  threshold, model.name)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## For each model, plot F1 & F2 vs threshold"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"best_thresholds = {} #store best thresholds (for F1 and F2 score) for each model\n\nfor architecture in model_list:\n    #For each model in model_list, calculate F1 & F2 scores for various thresholds, and plot graph of scores vs. thresholds\n    calc_plot_fbeta(architecture, 1000)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define function to plot confusion matrix and ROC-curve, calculate AU-ROC curve"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Accuracy measurements\nfrom sklearn.metrics import confusion_matrix,accuracy_score, roc_curve, auc\nfrom sklearn.metrics import plot_confusion_matrix\nimport seaborn as sns\n\ndef show_acc_mea(result, model_name, t=0.5,plot=True):\n    pred_prob=result[0]\n    pred_binary=get_binary(pred_prob, t)\n    test_y=result[1]\n    \n    #Print accuracy\n    acc_NN = accuracy_score(test_y, pred_binary)\n    print('Overall accuracy of Neural Network model:', acc_NN)\n\n    #Print confusion matrix\n    print('Confusion matrix:')\n    cf_matrix = confusion_matrix(test_y, pred_binary)\n    sns.heatmap(cf_matrix/np.sum(cf_matrix), annot=True, fmt='.2%', cmap='Blues')\n    \n    if(plot):\n        #Print Area Under Curve for Receiver Operating Characteristics AUROC/AUCROC\n        false_positive_rate, recall, _ = roc_curve(test_y, pred_prob)\n        roc_auc = auc(false_positive_rate, recall)\n        plt.figure()\n        plt.title(str(model_name) +  ', threshold = ' +  str(t) +  ' Receiver Operating Characteristic (ROC)')\n        plt.plot(false_positive_rate, recall, 'b', label = 'AUC = %0.3f' %roc_auc)\n        plt.legend(loc='lower right')\n        plt.plot([0,1], [0,1], 'r--')\n        plt.xlim([0.0,1.0])\n        plt.ylim([0.0,1.0])\n        plt.ylabel('Recall/True-Positve rate')\n        plt.xlabel('Fall-out (1-Specificity)/False-Positive rate')\n    plt.show()\n\n    #Comments\n    #Higher AUC means better predictions\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Testing the models (custom, resnet18, GoogLeNet, resnet 50)\n## Plot confusion matrix and ROC curve\n## Calcualte AU-ROC curve"},{"metadata":{"trusted":true},"cell_type":"code","source":"answer = {}\n\ndef print_acc_mea(net):\n    print('----------------------------'+ net.name+'----------------------------------')\n    answer[net.name]=test(net)#get prediction and y\n    print(\"Thresholds: 0.5\")\n    show_acc_mea(answer[net.name], net.name) \n    for i in range(1, 3):\n        t,s= best_thresholds[str(net.name)+ \"_F\" + str(i)]\n        print(\"Best thresholds for F\"+ str(i) +\" :\"+ str(t))\n        print(\"Max F\"+ str(i) +\" score :\"+ str(s))\n        show_acc_mea(answer[net.name], net.name, t, False)\n\nfor architecture in model_list:\n    print_acc_mea(architecture)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define function for precision-recall curve"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import precision_recall_curve\nfrom matplotlib import pyplot\n\ndef show_precision_recall(result, model_name):\n    pred_prob=result[0]\n    pred_binary=get_binary(pred_prob, 0.5)\n    test_y=result[1]\n    \n    precision, recall, _ = precision_recall_curve(test_y, pred_binary)\n    pr_curve_auc = auc(recall, precision)\n    \n    # plot the precision-recall curves\n    no_skill = len(test_y[test_y==1]) / len(test_y)\n    pyplot.plot([0, 1], [no_skill, no_skill], linestyle='--', label='No Skill classifier')\n    pyplot.plot(recall, precision, marker='.', label=model_name)\n    # axis labels\n    pyplot.xlabel('Recall')\n    pyplot.ylabel('Precision')\n    # show the legend\n    pyplot.legend()\n    # show the plot\n    pyplot.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot precision-recall curves"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Calculate precision recall for all classification thresholds \nfor architecture in model_list:\n    show_precision_recall(answer[architecture.name], architecture.name)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plotting Training Loss for all models"},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ntraining_history_list = [hist_custom, hist_custom_se, hist_resnet18, hist_se_resnet18, hist_googlenet, hist_se_googlenet, hist_resnet50, hist_se_resnet50]\nj=0\nfor hist_training in training_history_list:\n    plt.plot(hist_training, label= model_list[j].name)\n    j+=1\nplt.legend()\nplt.show()\n    \n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}