{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Residual Attention Network"},{"metadata":{},"cell_type":"markdown","source":"Hello everyone! I'm trying attention based mechanism on this dataset to concentrate only the necessary parts of the data, I will train it for only 1 epoch and basic augmentations so that you guys can do much more with that. \n\nPlease note, these parameters I'm using are not tested and can be improved further, Image size is 448 which is according to the model architecture also I'm using AUC ROC for Multiclass because I don't trust accuracy :-P.\n\n\n\nThanks"},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose, HueSaturationValue, CenterCrop, CoarseDropout\n)\n\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch\nfrom torch.nn import init\nimport functools\nfrom torch.autograd import Variable\nimport numpy as np\nimport cv2\nfrom PIL import Image\nfrom PIL import ImageFile \nfrom tqdm import tqdm\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\nimport os\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn import metrics\n\nImageFile.LOAD_TRUNCATED_IMAGES = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResidualBlock(nn.Module):\n    def __init__(self, input_channels, output_channels, stride=1):\n        super(ResidualBlock, self).__init__()\n        self.input_channels = input_channels\n        self.output_channels = output_channels\n        self.stride = stride\n        self.bn1 = nn.BatchNorm2d(input_channels)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv1 = nn.Conv2d(input_channels, int(output_channels/4), 1, 1, bias = False)\n        self.bn2 = nn.BatchNorm2d(int(output_channels/4))\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = nn.Conv2d(int(output_channels/4), int(output_channels/4), 3, stride, padding = 1, bias = False)\n        self.bn3 = nn.BatchNorm2d(int(output_channels/4))\n        self.relu = nn.ReLU(inplace=True)\n        self.conv3 = nn.Conv2d(int(output_channels/4), output_channels, 1, 1, bias = False)\n        self.conv4 = nn.Conv2d(input_channels, output_channels , 1, stride, bias = False)\n        \n    def forward(self, x):\n        residual = x\n        out = self.bn1(x)\n        out1 = self.relu(out)\n        out = self.conv1(out1)\n        out = self.bn2(out)\n        out = self.relu(out)\n        out = self.conv2(out)\n        out = self.bn3(out)\n        out = self.relu(out)\n        out = self.conv3(out)\n        if (self.input_channels != self.output_channels) or (self.stride !=1 ):\n            residual = self.conv4(out1)\n        out += residual\n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AttentionModule_pre(nn.Module):\n\n    def __init__(self, in_channels, out_channels, size1, size2, size3):\n        super(AttentionModule_pre, self).__init__()\n        self.first_residual_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.trunk_branches = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n         )\n\n        self.mpool1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n\n        self.softmax1_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.skip1_connection_residual_block = ResidualBlock(in_channels, out_channels)\n\n        self.mpool2 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n\n        self.softmax2_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.skip2_connection_residual_block = ResidualBlock(in_channels, out_channels)\n\n        self.mpool3 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n\n        self.softmax3_blocks = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n        )\n\n        self.interpolation3 = nn.UpsamplingBilinear2d(size=size3)\n\n        self.softmax4_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.interpolation2 = nn.UpsamplingBilinear2d(size=size2)\n\n        self.softmax5_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.interpolation1 = nn.UpsamplingBilinear2d(size=size1)\n\n        self.softmax6_blocks = nn.Sequential(\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels , kernel_size = 1, stride = 1, bias = False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels , kernel_size = 1, stride = 1, bias = False),\n            nn.Sigmoid()\n        )\n\n        self.last_blocks = ResidualBlock(in_channels, out_channels)\n\n    def forward(self, x):\n        x = self.first_residual_blocks(x)\n        out_trunk = self.trunk_branches(x)\n        out_mpool1 = self.mpool1(x)\n        out_softmax1 = self.softmax1_blocks(out_mpool1)\n        out_skip1_connection = self.skip1_connection_residual_block(out_softmax1)\n        out_mpool2 = self.mpool2(out_softmax1)\n        out_softmax2 = self.softmax2_blocks(out_mpool2)\n        out_skip2_connection = self.skip2_connection_residual_block(out_softmax2)\n        out_mpool3 = self.mpool3(out_softmax2)\n        out_softmax3 = self.softmax3_blocks(out_mpool3)\n        #\n        out_interp3 = self.interpolation3(out_softmax3)\n        # print(out_skip2_connection.data)\n        # print(out_interp3.data)\n        out = out_interp3 + out_skip2_connection\n        out_softmax4 = self.softmax4_blocks(out)\n        out_interp2 = self.interpolation2(out_softmax4)\n        out = out_interp2 + out_skip1_connection\n        out_softmax5 = self.softmax5_blocks(out)\n        out_interp1 = self.interpolation1(out_softmax5)\n        out_softmax6 = self.softmax6_blocks(out_interp1)\n        out = (1 + out_softmax6) * out_trunk\n        out_last = self.last_blocks(out)\n\n        return out_last\n\n\nclass AttentionModule_stage0(nn.Module):\n    # input size is 112*112\n    def __init__(self, in_channels, out_channels, size1=(112, 112), size2=(56, 56), size3=(28, 28), size4=(14, 14)):\n        super(AttentionModule_stage0, self).__init__()\n        self.first_residual_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.trunk_branches = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n         )\n\n        self.mpool1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        # 56*56\n        self.softmax1_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.skip1_connection_residual_block = ResidualBlock(in_channels, out_channels)\n\n        self.mpool2 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        # 28*28\n        self.softmax2_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.skip2_connection_residual_block = ResidualBlock(in_channels, out_channels)\n\n        self.mpool3 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        # 14*14\n        self.softmax3_blocks = ResidualBlock(in_channels, out_channels)\n        self.skip3_connection_residual_block = ResidualBlock(in_channels, out_channels)\n        self.mpool4 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        # 7*7\n        self.softmax4_blocks = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n        )\n        self.interpolation4 = nn.UpsamplingBilinear2d(size=size4)\n        self.softmax5_blocks = ResidualBlock(in_channels, out_channels)\n        self.interpolation3 = nn.UpsamplingBilinear2d(size=size3)\n        self.softmax6_blocks = ResidualBlock(in_channels, out_channels)\n        self.interpolation2 = nn.UpsamplingBilinear2d(size=size2)\n        self.softmax7_blocks = ResidualBlock(in_channels, out_channels)\n        self.interpolation1 = nn.UpsamplingBilinear2d(size=size1)\n\n        self.softmax8_blocks = nn.Sequential(\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels, kernel_size=1, stride=1, bias = False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels , kernel_size=1, stride=1, bias = False),\n            nn.Sigmoid()\n        )\n\n        self.last_blocks = ResidualBlock(in_channels, out_channels)\n\n    def forward(self, x):\n        # 112*112\n        x = self.first_residual_blocks(x)\n        out_trunk = self.trunk_branches(x)\n        out_mpool1 = self.mpool1(x)\n        # 56*56\n        out_softmax1 = self.softmax1_blocks(out_mpool1)\n        out_skip1_connection = self.skip1_connection_residual_block(out_softmax1)\n        out_mpool2 = self.mpool2(out_softmax1)\n        # 28*28\n        out_softmax2 = self.softmax2_blocks(out_mpool2)\n        out_skip2_connection = self.skip2_connection_residual_block(out_softmax2)\n        out_mpool3 = self.mpool3(out_softmax2)\n        # 14*14\n        out_softmax3 = self.softmax3_blocks(out_mpool3)\n        out_skip3_connection = self.skip3_connection_residual_block(out_softmax3)\n        out_mpool4 = self.mpool4(out_softmax3)\n        # 7*7\n        out_softmax4 = self.softmax4_blocks(out_mpool4)\n        out_interp4 = self.interpolation4(out_softmax4) + out_softmax3\n        out = out_interp4 + out_skip3_connection\n        out_softmax5 = self.softmax5_blocks(out)\n        out_interp3 = self.interpolation3(out_softmax5) + out_softmax2\n        # print(out_skip2_connection.data)\n        # print(out_interp3.data)\n        out = out_interp3 + out_skip2_connection\n        out_softmax6 = self.softmax6_blocks(out)\n        out_interp2 = self.interpolation2(out_softmax6) + out_softmax1\n        out = out_interp2 + out_skip1_connection\n        out_softmax7 = self.softmax7_blocks(out)\n        out_interp1 = self.interpolation1(out_softmax7) + out_trunk\n        out_softmax8 = self.softmax8_blocks(out_interp1)\n        out = (1 + out_softmax8) * out_trunk\n        out_last = self.last_blocks(out)\n\n        return out_last\n\n\nclass AttentionModule_stage1(nn.Module):\n    # input size is 56*56\n    def __init__(self, in_channels, out_channels, size1=(56, 56), size2=(28, 28), size3=(14, 14)):\n        super(AttentionModule_stage1, self).__init__()\n        self.first_residual_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.trunk_branches = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n         )\n\n        self.mpool1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n\n        self.softmax1_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.skip1_connection_residual_block = ResidualBlock(in_channels, out_channels)\n\n        self.mpool2 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n\n        self.softmax2_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.skip2_connection_residual_block = ResidualBlock(in_channels, out_channels)\n\n        self.mpool3 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n\n        self.softmax3_blocks = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n        )\n\n        self.interpolation3 = nn.UpsamplingBilinear2d(size=size3)\n\n        self.softmax4_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.interpolation2 = nn.UpsamplingBilinear2d(size=size2)\n\n        self.softmax5_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.interpolation1 = nn.UpsamplingBilinear2d(size=size1)\n\n        self.softmax6_blocks = nn.Sequential(\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels , kernel_size = 1, stride = 1, bias = False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels , kernel_size = 1, stride = 1, bias = False),\n            nn.Sigmoid()\n        )\n\n        self.last_blocks = ResidualBlock(in_channels, out_channels)\n\n    def forward(self, x):\n        x = self.first_residual_blocks(x)\n        out_trunk = self.trunk_branches(x)\n        out_mpool1 = self.mpool1(x)\n        out_softmax1 = self.softmax1_blocks(out_mpool1)\n        out_skip1_connection = self.skip1_connection_residual_block(out_softmax1)\n        out_mpool2 = self.mpool2(out_softmax1)\n        out_softmax2 = self.softmax2_blocks(out_mpool2)\n        out_skip2_connection = self.skip2_connection_residual_block(out_softmax2)\n        out_mpool3 = self.mpool3(out_softmax2)\n        out_softmax3 = self.softmax3_blocks(out_mpool3)\n        #\n        out_interp3 = self.interpolation3(out_softmax3) + out_softmax2\n        # print(out_skip2_connection.data)\n        # print(out_interp3.data)\n        out = out_interp3 + out_skip2_connection\n        out_softmax4 = self.softmax4_blocks(out)\n        out_interp2 = self.interpolation2(out_softmax4) + out_softmax1\n        out = out_interp2 + out_skip1_connection\n        out_softmax5 = self.softmax5_blocks(out)\n        out_interp1 = self.interpolation1(out_softmax5) + out_trunk\n        out_softmax6 = self.softmax6_blocks(out_interp1)\n        out = (1 + out_softmax6) * out_trunk\n        out_last = self.last_blocks(out)\n\n        return out_last\n\n\nclass AttentionModule_stage2(nn.Module):\n    # input image size is 28*28\n    def __init__(self, in_channels, out_channels, size1=(28, 28), size2=(14, 14)):\n        super(AttentionModule_stage2, self).__init__()\n        self.first_residual_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.trunk_branches = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n         )\n\n        self.mpool1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n\n        self.softmax1_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.skip1_connection_residual_block = ResidualBlock(in_channels, out_channels)\n\n        self.mpool2 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n\n        self.softmax2_blocks = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n        )\n\n        self.interpolation2 = nn.UpsamplingBilinear2d(size=size2)\n\n        self.softmax3_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.interpolation1 = nn.UpsamplingBilinear2d(size=size1)\n\n        self.softmax4_blocks = nn.Sequential(\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels, kernel_size=1, stride=1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels, kernel_size=1, stride=1, bias=False),\n            nn.Sigmoid()\n        )\n\n        self.last_blocks = ResidualBlock(in_channels, out_channels)\n\n    def forward(self, x):\n        x = self.first_residual_blocks(x)\n        out_trunk = self.trunk_branches(x)\n        out_mpool1 = self.mpool1(x)\n        out_softmax1 = self.softmax1_blocks(out_mpool1)\n        out_skip1_connection = self.skip1_connection_residual_block(out_softmax1)\n        out_mpool2 = self.mpool2(out_softmax1)\n        out_softmax2 = self.softmax2_blocks(out_mpool2)\n\n        out_interp2 = self.interpolation2(out_softmax2) + out_softmax1\n        # print(out_skip2_connection.data)\n        # print(out_interp3.data)\n        out = out_interp2 + out_skip1_connection\n        out_softmax3 = self.softmax3_blocks(out)\n        out_interp1 = self.interpolation1(out_softmax3) + out_trunk\n        out_softmax4 = self.softmax4_blocks(out_interp1)\n        out = (1 + out_softmax4) * out_trunk\n        out_last = self.last_blocks(out)\n\n        return out_last\n\n\nclass AttentionModule_stage3(nn.Module):\n    # input image size is 14*14\n    def __init__(self, in_channels, out_channels, size1=(14, 14)):\n        super(AttentionModule_stage3, self).__init__()\n        self.first_residual_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.trunk_branches = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n         )\n\n        self.mpool1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.softmax1_blocks = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n        )\n\n        self.interpolation1 = nn.UpsamplingBilinear2d(size=size1)\n\n        self.softmax2_blocks = nn.Sequential(\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels, kernel_size=1, stride=1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels, kernel_size=1, stride=1, bias=False),\n            nn.Sigmoid()\n        )\n\n        self.last_blocks = ResidualBlock(in_channels, out_channels)\n\n    def forward(self, x):\n        x = self.first_residual_blocks(x)\n        out_trunk = self.trunk_branches(x)\n        out_mpool1 = self.mpool1(x)\n        out_softmax1 = self.softmax1_blocks(out_mpool1)\n\n        out_interp1 = self.interpolation1(out_softmax1) + out_trunk\n        out_softmax2 = self.softmax2_blocks(out_interp1)\n        out = (1 + out_softmax2) * out_trunk\n        out_last = self.last_blocks(out)\n\n        return out_last\n\n\nclass AttentionModule_stage1_cifar(nn.Module):\n    # input size is 16*16\n    def __init__(self, in_channels, out_channels, size1=(16, 16), size2=(8, 8)):\n        super(AttentionModule_stage1_cifar, self).__init__()\n        self.first_residual_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.trunk_branches = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n         )\n\n        self.mpool1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)  # 8*8\n\n        self.down_residual_blocks1 = ResidualBlock(in_channels, out_channels)\n\n        self.skip1_connection_residual_block = ResidualBlock(in_channels, out_channels)\n\n        self.mpool2 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)  # 4*4\n\n        self.middle_2r_blocks = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n        )\n\n        self.interpolation1 = nn.UpsamplingBilinear2d(size=size2)  # 8*8\n\n        self.up_residual_blocks1 = ResidualBlock(in_channels, out_channels)\n\n        self.interpolation2 = nn.UpsamplingBilinear2d(size=size1)  # 16*16\n\n        self.conv1_1_blocks = nn.Sequential(\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels, kernel_size=1, stride=1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels, kernel_size=1, stride=1, bias = False),\n            nn.Sigmoid()\n        )\n\n        self.last_blocks = ResidualBlock(in_channels, out_channels)\n\n    def forward(self, x):\n        x = self.first_residual_blocks(x)\n        out_trunk = self.trunk_branches(x)\n        out_mpool1 = self.mpool1(x)\n        out_down_residual_blocks1 = self.down_residual_blocks1(out_mpool1)\n        out_skip1_connection = self.skip1_connection_residual_block(out_down_residual_blocks1)\n        out_mpool2 = self.mpool2(out_down_residual_blocks1)\n        out_middle_2r_blocks = self.middle_2r_blocks(out_mpool2)\n        #\n        out_interp = self.interpolation1(out_middle_2r_blocks) + out_down_residual_blocks1\n        # print(out_skip2_connection.data)\n        # print(out_interp3.data)\n        out = out_interp + out_skip1_connection\n        out_up_residual_blocks1 = self.up_residual_blocks1(out)\n        out_interp2 = self.interpolation2(out_up_residual_blocks1) + out_trunk\n        out_conv1_1_blocks = self.conv1_1_blocks(out_interp2)\n        out = (1 + out_conv1_1_blocks) * out_trunk\n        out_last = self.last_blocks(out)\n\n        return out_last\n\n\nclass AttentionModule_stage2_cifar(nn.Module):\n    # input size is 8*8\n    def __init__(self, in_channels, out_channels, size=(8, 8)):\n        super(AttentionModule_stage2_cifar, self).__init__()\n        self.first_residual_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.trunk_branches = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n         )\n\n        self.mpool1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)  # 4*4\n\n        self.middle_2r_blocks = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n        )\n\n        self.interpolation1 = nn.UpsamplingBilinear2d(size=size)  # 8*8\n\n        self.conv1_1_blocks = nn.Sequential(\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels, kernel_size=1, stride=1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels, kernel_size=1, stride=1, bias = False),\n            nn.Sigmoid()\n        )\n\n        self.last_blocks = ResidualBlock(in_channels, out_channels)\n\n    def forward(self, x):\n        x = self.first_residual_blocks(x)\n        out_trunk = self.trunk_branches(x)\n        out_mpool1 = self.mpool1(x)\n        out_middle_2r_blocks = self.middle_2r_blocks(out_mpool1)\n        #\n        out_interp = self.interpolation1(out_middle_2r_blocks) + out_trunk\n        # print(out_skip2_connection.data)\n        # print(out_interp3.data)\n        out_conv1_1_blocks = self.conv1_1_blocks(out_interp)\n        out = (1 + out_conv1_1_blocks) * out_trunk\n        out_last = self.last_blocks(out)\n\n        return out_last\n\n\nclass AttentionModule_stage3_cifar(nn.Module):\n    # input size is 4*4\n    def __init__(self, in_channels, out_channels, size=(8, 8)):\n        super(AttentionModule_stage3_cifar, self).__init__()\n        self.first_residual_blocks = ResidualBlock(in_channels, out_channels)\n\n        self.trunk_branches = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n         )\n\n        self.middle_2r_blocks = nn.Sequential(\n            ResidualBlock(in_channels, out_channels),\n            ResidualBlock(in_channels, out_channels)\n        )\n\n        self.conv1_1_blocks = nn.Sequential(\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels, kernel_size=1, stride=1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels, kernel_size=1, stride=1, bias = False),\n            nn.Sigmoid()\n        )\n\n        self.last_blocks = ResidualBlock(in_channels, out_channels)\n\n    def forward(self, x):\n        x = self.first_residual_blocks(x)\n        out_trunk = self.trunk_branches(x)\n        out_middle_2r_blocks = self.middle_2r_blocks(x)\n        #\n        out_conv1_1_blocks = self.conv1_1_blocks(out_middle_2r_blocks)\n        out = (1 + out_conv1_1_blocks) * out_trunk\n        out_last = self.last_blocks(out)\n\n        return out_last","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResidualAttentionModel_448input(nn.Module):\n    # for input size 448\n    def __init__(self):\n        super(ResidualAttentionModel_448input, self).__init__()\n        self.conv1 = nn.Sequential(\n            nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias = False),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True)\n        )\n        self.mpool1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        # tbq add\n        # 112*112\n        self.residual_block0 = ResidualBlock(64, 128)\n        self.attention_module0 = AttentionModule_stage0(128, 128)\n        # tbq add end\n        self.residual_block1 = ResidualBlock(128, 256, 2)\n        # 56*56\n        self.attention_module1 = AttentionModule_stage1(256, 256)\n        self.residual_block2 = ResidualBlock(256, 512, 2)\n        self.attention_module2 = AttentionModule_stage2(512, 512)\n        self.attention_module2_2 = AttentionModule_stage2(512, 512)  # tbq add\n        self.residual_block3 = ResidualBlock(512, 1024, 2)\n        self.attention_module3 = AttentionModule_stage3(1024, 1024)\n        self.attention_module3_2 = AttentionModule_stage3(1024, 1024)  # tbq add\n        self.attention_module3_3 = AttentionModule_stage3(1024, 1024)  # tbq add\n        self.residual_block4 = ResidualBlock(1024, 2048, 2)\n        self.residual_block5 = ResidualBlock(2048, 2048)\n        self.residual_block6 = ResidualBlock(2048, 2048)\n        self.mpool2 = nn.Sequential(\n            nn.BatchNorm2d(2048),\n            nn.ReLU(inplace=True),\n            nn.AvgPool2d(kernel_size=7, stride=1)\n        )\n        self.fc = nn.Linear(2048,10)\n\n    def forward(self, x):\n        out = self.conv1(x)\n        out = self.mpool1(out)\n        out = self.residual_block0(out)\n        out = self.attention_module0(out)\n        # print(out.data)\n        out = self.residual_block1(out)\n        out = self.attention_module1(out)\n        out = self.residual_block2(out)\n        out = self.attention_module2(out)\n        out = self.attention_module2_2(out)\n        out = self.residual_block3(out)\n        # print(out.data)\n        out = self.attention_module3(out)\n        out = self.attention_module3_2(out)\n        out = self.attention_module3_3(out)\n        out = self.residual_block4(out)\n        out = self.residual_block5(out)\n        out = self.residual_block6(out)\n        out = self.mpool2(out)\n        out = out.view(out.size(0), -1)\n        out = self.fc(out)\n\n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResAttention(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.model = ResidualAttentionModel_448input()\n        # self.model.fc.classifier = nn.Linear(n_features, 5)\n        self.model.fc = nn.Linear(2048, 5)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class ClassificationDataset:\n  def __init__(\n    self, \n    image_paths, \n    targets, \n    resize=None, \n    augmentations=None\n  ):\n    \"\"\"\n    :param image_paths: list of path to images\n    :param targets: numpy array\n    :param resize: tuple, e.g. (256, 256), resizes image if not None\n    :param augmentations: albumentation augmentations\n    \"\"\"\n    self.image_paths = image_paths\n    self.targets = targets\n    self.resize = resize\n    self.augmentations = augmentations\n    \n    self.fmix_params = {\n     'alpha': 1., \n     'decay_power': 3., \n     'shape': (512, 512),\n     'max_soft': True, \n     'reformulate': False\n    },\n\n    self.cutmix_params = {\n     'alpha': 1,\n    }\n    \n  def __len__(self):\n    \"\"\"\n    Return the total number of samples in the dataset\n    \"\"\"\n    return len(self.image_paths)\n  \n  def __getitem__(self,item):\n    \"\"\"\n    For a given \"item\" index, return everything we needto train a given model\n    \"\"\"\n    # use PIL to open the image\n    image = Image.open(self.image_paths[item]) \n    # convert image to RGB, we have single channel images\n    image = image.convert(\"RGB\")\n    # grab correct targets\n    targets = self.targets[item]\n    \n    # resize if needed\n    if self.resize is not None:\n      image = image.resize(\n        (self.resize[1], self.resize[0]), \n        resample=Image.BILINEAR\n      )\n    # convert image to numpy array\n    image = np.array(image)\n    \n    # if we have albumentation augmentations\n    # add them to the image\n    if self.augmentations is not None:\n      augmented = self.augmentations(image=image)\n      image = augmented[\"image\"]\n      \n#       if np.random.uniform(0., 1., size=1)[0] > 0.5:\n#         #print(img.sum(), img.shape)\n#         with torch.no_grad():\n#             cmix_ix = np.random.choice(self.image_paths.shape[0], size=1)[0]\n#             cmix_img  = get_img(self.image_paths[cmix_ix])\n#             cmix_img = self.augmentations(image=cmix_img)['image']\n\n#             lam = np.clip(np.random.beta(1, 1),0.3,0.4)\n#             bbx1, bby1, bbx2, bby2 = rand_bbox((512, 512), lam)\n\n#             image[:, bbx1:bbx2, bby1:bby2] = cmix_img[:, bbx1:bbx2, bby1:bby2]\n\n#             rate = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / 512 * 512)\n#             targets = rate*targets + (1.-rate)*self.targets[cmix_ix]\n    \n    # pytorch expects CHW instead of HWC\n    image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n    \n    # return tensors of image and targets\n    # take a look at the types!\n    # for regression tasks, \n    # dtype of targets will change to torch.float\n    return {\n      \"image\": torch.tensor(image, dtype=torch.float),\n      \"targets\": torch.tensor(targets, dtype=torch.long),\n    }\n\ndef get_scheduler(optimizer, scheduler):\n    if scheduler=='ReduceLROnPlateau':\n        scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.2, patience=4, verbose=True, eps=1e-6)\n    elif scheduler=='CosineAnnealingLR':\n        scheduler = CosineAnnealingLR(optimizer, T_max=10, eta_min=1e-6, last_epoch=-1)\n    elif scheduler=='CosineAnnealingWarmRestarts':\n        scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1, eta_min=1e-6, last_epoch=-1)\n    return scheduler\n\ndef train_model(data_loader, model, optimizer, scheduler, device):\n  \"\"\"\n  This function does training for one epoch\n  :param data_loader: this is the pytorch dataloader\n  :param model: pytorch model\n  :param optimizer: optimizer, for e.g. adam, sgd, etc\n  :param device: cuda/cpu\n  \"\"\"\n  # put the model in train mode\n  model.train()\n  # go over every batch of data in data loader\n  for data in data_loader:\n    # remember, we have image and targets\n    # in our dataset class\n    inputs = data[\"image\"]\n    targets = data[\"targets\"]\n    \n    # move inputs/targets to cuda/cpu device\n    inputs = inputs.to(device, dtype=torch.float)\n    targets = targets.to(device, dtype=torch.long)\n    \n    # zero grad the optimizer\n    optimizer.zero_grad()\n    #do the forward step of model\n    outputs = model(inputs)\n    \n    criterion = nn.CrossEntropyLoss().to(device)\n    outputs = model(inputs)\n    loss = criterion(outputs, targets)\n\n    # backward step the loss\n    loss.backward()\n    # step optimizer\n    optimizer.step()\n    # if you have a scheduler, you either need to\n    # step it here or you have to step it after\n    # the epoch. here, we are not using any learning\n    # rate scheduler\n  scheduler.step()\n    \ndef evaluate_model(data_loader, model, device):\n  \"\"\"\n  This function does evaluation for one epoch\n  :param data_loader: this is the pytorch dataloader\n  :param model: pytorch model\n  :param device: cuda/cpu\n  \"\"\"\n  # put model in evaluation mode\n  model.eval()\n  \n  # init lists to store targets and outputs\n  final_targets = []\n  final_outputs = []\n  \n  # we use no_grad context\n  with torch.no_grad():\n    for data in data_loader:\n      inputs = data[\"image\"]\n      targets = data[\"targets\"]\n      inputs = inputs.to(device, dtype=torch.float)\n      targets = targets.to(device, dtype=torch.float)\n      \n      # do the forward step to generate prediction\n      output = model(inputs)\n    \n      # convert targets and outputs to lists\n      targets = targets.detach().cpu().numpy().tolist()\n      output = output.detach().cpu().numpy().tolist()\n      \n      # extend the original list\n      final_targets.extend(targets)\n      final_outputs.extend(output)\n      \n  # return final output and final targets\n  return final_outputs, final_targets\n\ndef get_model(pretrained = True):\n    model = ResAttention()\n    \n#     model.last_linear = nn.Sequential(\n#         nn.BatchNorm1d(2048),\n#         nn.Dropout(p=0.25),\n#         nn.Linear(in_features=204800, out_features=2048),\n#         nn.ReLU(),\n#         nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1),\n#         nn.Dropout(p=0.5),\n#         nn.Linear(in_features=2048, out_features=5), # out features \n#     )\n    return model\n\ndef save_checkpoint(model, optimizer, path):\n    if not os.path.exists(os.path.dirname(path)):\n        print(\"Creating directories on path: `{}`\".format(path))\n        os.makedirs(os.path.dirname(path))\n\n    torch.save({\n        \"model_state_dict\": model.state_dict(),\n        \"optimizer_state_dict\": optimizer.state_dict(),\n    }, path)\n\n\ndef load_checkpoint(model, path):\n    checkpoint = torch.load(path)\n\n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n\n    optimizer = torch.optim.Adam(model.parameters())\n    optimizer.load_state_dict(checkpoint[\"optimizer_state_dict\"])\n\n    return model, optimizer\n\n\ndef save_model(model, path):\n  if not os.path.exists(os.path.dirname(path)):\n      print(\"Creating directories on path: `{}`\".format(path))\n      os.makedirs(os.path.dirname(path))\n\n  torch.save({\n      \"model_state_dict\": model.state_dict(),\n  }, path)\n\n\ndef load_model(moodel, path):\n  restore_dict = torch.load(path)\n\n  model.load_state_dict(restore_dict[\"model_state_dict\"])\n  model.eval()\n\n  return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = cv2.imread('../input/cassavapreprocessed/train_images/train_images/1001320321.jpg')\nplt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassavapreprocessed/merged_data.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, (tr_in, val_in) in enumerate(StratifiedKFold(random_state=42, shuffle = True).split(train['image_id'],train['label'])):\n    train[f'fold_{i}'] = 0\n    train.at[tr_in,f'fold_{i}'] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.to_csv('folds_training.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('./folds_training.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fold = 0\nX_train, y_train, X_test, y_test = (train[train[f'fold_{fold}']==1].loc[:,'image_id'], \n                                    train[train[f'fold_{fold}']==1].loc[:,'label'],\n                                    train[train[f'fold_{fold}']==0].loc[:,'image_id'],\n                                    train[train[f'fold_{fold}']==0].loc[:,'label'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.apply(lambda x: '../input/cassavapreprocessed/train_images/train_images/'+x)\nX_test = X_test.apply(lambda x: '../input/cassavapreprocessed/train_images/train_images/'+x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug = Compose(\n    [\n        RandomResizedCrop(448, 448),\n        Transpose(p=0.5),\n        HorizontalFlip(p=0.5),\n        VerticalFlip(p=0.5),\n        HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n        RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n        #CenterCrop(512, 512, p=1.0),\n        CoarseDropout(p=0.5),\n        Cutout(p=0.5),\n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n    ]\n)\nTrain_dataset = ClassificationDataset(X_train.values, y_train.values, resize = (448,448), \n                                      augmentations=aug)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug = Compose(\n    [\n        Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225],\n        ),\n    ]\n)\n\n# aug = Compose(\n#     [\n#         RandomResizedCrop(448, 448),\n#         Transpose(p=0.5),\n#         HorizontalFlip(p=0.5),\n#         VerticalFlip(p=0.5),\n#         HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n#         RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n#         CenterCrop(448, 448, p=1.0),\n#         CoarseDropout(p=0.5),\n#         Cutout(p=0.5),\n#         Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n#     ]\n# )\nVal_dataset = ClassificationDataset(X_test.values, y_test.values, resize = (448,448),\n                                   augmentations=aug)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = 'cuda'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomResNext(nn.Module):\n    def __init__(self, model_name='resnext50_32x4d', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, 5)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = get_model(pretrained=False)\nmodel.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loader = torch.utils.data.DataLoader(\n    Train_dataset, batch_size=8, shuffle=True, num_workers=4,pin_memory=True\n)\n\nvalid_loader = torch.utils.data.DataLoader(\n    Val_dataset, batch_size=8, shuffle=False, num_workers=4,pin_memory=True\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = torch.optim.Adam(model.parameters(), lr=5e-4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def multi_class_roc_auc(true, pred_probs_arr, labels):\n    auc_all = []\n    for label_number in labels:\n        true_labels = true.loc[:,label_number].copy()\n        pred_probs = pred_probs_arr.loc[:, label_number].copy()\n        \n       #AUROC and AP (sliding across multiple decision thresholds)\n        fpr, tpr, thresholds = metrics.roc_curve(y_true = true_labels,\n                                         y_score = pred_probs,\n                                         pos_label = 1)\n        auc = metrics.auc(fpr, tpr)\n        auc_all.append(auc)\n    print(f'AUC of each class: {auc_all}')\n    return np.mean(auc_all)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = [0.0, 1.0, 2.0, 3.0, 4.0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scheduler = get_scheduler(optimizer, 'CosineAnnealingWarmRestarts')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for epoch in range(epochs):\n    train_model(train_loader, model, optimizer, scheduler, device=device)\n    \n    predictions, valid_targets = evaluate_model(\n      valid_loader, model, device=device\n    )\n    \n    preds = pd.DataFrame(predictions, columns = labels)\n    targets = pd.get_dummies(valid_targets, columns = labels)\n    \n    # roc_auc = metrics.roc_auc_score(valid_targets, predictions.argmax(axis=1))\n    roc_auc = multi_class_roc_auc(targets.copy(), preds.copy(), labels)\n    \n    print(\n      f\"Epoch={epoch}, Valid ROC AUC={roc_auc}\"\n    )\n    save_model(model, './model.h5')\n    save_checkpoint(model, optimizer, f'./model_{fold}_{epoch}.pth')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}