{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Importing Libraries","metadata":{}},{"cell_type":"code","source":"!pip install einops --no-index --find-links=file:///kaggle/input/einopspackage/ ","metadata":{"execution":{"iopub.status.busy":"2021-06-18T09:30:52.983805Z","iopub.execute_input":"2021-06-18T09:30:52.984132Z","iopub.status.idle":"2021-06-18T09:30:58.505224Z","shell.execute_reply.started":"2021-06-18T09:30:52.984103Z","shell.execute_reply":"2021-06-18T09:30:58.504199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nimport torchvision.models as models\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom PIL import Image\n\nimport matplotlib.pyplot as plt\n\nfrom einops import rearrange, repeat\nfrom einops.layers.torch import Rearrange\n\nimport time\nimport copy\nimport glob","metadata":{"execution":{"iopub.status.busy":"2021-06-18T09:30:58.507149Z","iopub.execute_input":"2021-06-18T09:30:58.507515Z","iopub.status.idle":"2021-06-18T09:30:58.516291Z","shell.execute_reply.started":"2021-06-18T09:30:58.507475Z","shell.execute_reply":"2021-06-18T09:30:58.515486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Architecture","metadata":{}},{"cell_type":"code","source":"'''\nResnet\n'''\n\n# Resnet\nclass ResNet(nn.Module):\n    def __init__(self, layers, dropout=0.0):\n        super(ResNet, self).__init__()\n        self.inplanes = 64\n        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, padding=3, stride=2, bias=False)  ## this is stride2 in the original implementation\n        self.bn1 = nn.BatchNorm2d(self.inplanes)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self.make_layer(64, layers[0])\n        self.layer2 = self.make_layer(128, layers[1], stride=2)\n        self.layer3 = self.make_layer(256, layers[2], stride=2)\n        self.layer4 = self.make_layer(512, layers[3], stride=2)\n        self.avgpool = nn.AdaptiveAvgPool2d((1,1))\n        self.fc = nn.Linear(512, 10)\n        self.dropout = nn.Dropout(dropout) if dropout > 0.0 else None\n\n\n\n    def make_layer(self, planes, blocks, stride=1):\n        downsample = None\n        if stride != 1:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes, kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(planes)\n            )\n\n        layers = []\n        layers.append(ResBlock(self.inplanes, planes, stride, downsample))\n        self.inplanes = planes\n        for _ in range(1, blocks):\n            layers.append(ResBlock(self.inplanes, planes))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        out = self.relu(self.bn1(self.conv1(x)))\n        out = self.maxpool(out)\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = self.layer4(out)\n        out = self.avgpool(out)\n        out = torch.flatten(out, 1)\n        out = self.fc(out)\n        if self.dropout is not None:\n            out = self.dropout(out)\n\n        return out\n\n\nclass ResBlock(nn.Module):\n    def __init__(self, inplanes, planes, stride=1, downsample=None):\n        super().__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.downsample = downsample\n    \n    def forward(self, x):\n        identity = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = self.conv2(out)\n        out = self.bn2(out)\n\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\nclass MobileNetV2(nn.Module):\n    def __init__(self, width_mult=1.0, dropout=0.0):\n        super(MobileNetV2, self).__init__()\n        inverted_residual_setting = [\n            [1, 16, 1, 1],\n            [6, 24, 2, 2],\n            [6, 32, 3, 2],\n            [6, 64, 4, 2],\n            [6, 96, 3, 1],\n            [6, 160, 3, 2],\n            [6, 320, 1, 1]\n        ]\n\n        input_channel = 32\n        last_channel = 1280\n        \n        input_channel = _make_divisible(input_channel * width_mult, 8)\n        last_channel = _make_divisible(last_channel * max(1.0, width_mult) * width_mult, 8)\n        features = [ConvBNReLU(3, input_channel, stride=2)]\n\n        for t,c,n,s in inverted_residual_setting:\n            output_channel = _make_divisible(c * width_mult, 8)\n            for i in range(n):\n                stride = s if i == 0 else 1\n                features.append(InvertedResidual(input_channel, output_channel, stride, expand_ratio=t))\n                input_channel = output_channel\n\n        features.append(ConvBNReLU(input_channel, last_channel, kernel_size=1))\n        self.features = nn.Sequential(*features)\n\n        self.classifier = nn.Sequential(\n            nn.Dropout(dropout),\n            nn.Linear(last_channel, 10)\n        )\n\n    def forward(self, x):\n        out = self.features(x)\n        out = F.adaptive_avg_pool2d(out, (1,1))\n        out = torch.flatten(out, 1)\n        out = self.classifier(out)\n        return out\n\nclass InvertedResidual(nn.Module):\n    def __init__(self, in_planes, out_planes, stride, expand_ratio):\n        super().__init__()\n        hidden_dim = int(round(in_planes * expand_ratio))\n        self.use_res_connect = stride == 1 and in_planes == out_planes\n\n        layers = []  # depthwise separable convolution with bottleneck\n        if expand_ratio != 1:\n            layers.append(ConvBNReLU(in_planes, hidden_dim, kernel_size=1))\n        layers.extend([\n            ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim),\n            nn.Conv2d(hidden_dim, out_planes, kernel_size=1, bias=False),\n            nn.BatchNorm2d(out_planes)\n        ])\n        self.conv = nn.Sequential(*layers)\n\n    def forward(self, x):\n        if self.use_res_connect:\n            return x + self.conv(x)\n        else:\n            return self.conv(x)\n\n\nclass ConvBNReLU(nn.Module):\n    def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1):\n        super().__init__()\n        padding = (kernel_size-1) // 2\n        self.layers = nn.Sequential(\n            nn.Conv2d(in_planes, out_planes, kernel_size, stride, padding, groups=groups, bias=False),\n            nn.BatchNorm2d(out_planes),\n            nn.ReLU6(inplace=True) # necessary? consider switching to relu\n        )\n\n    def forward(self, x):\n        out = self.layers(x)\n        return out\n\n\ndef _make_divisible(v, divisor, min_value=None):\n    if min_value is None:\n        min_value = divisor\n\n    new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)  # rounds numbers in range [num-divisor/2, num+divisor/2-1] to num, where num is a multiple of divisor\n\n    if new_v < 0.9 * v:\n        new_v += divisor  # ensures that round down does not decrease v by more than 10%\n\n    return new_v\n\n'''\nVIT\n'''\n# helpers\n\ndef pair(t):\n    return t if isinstance(t, tuple) else (t, t)\n\n# classes\n\nclass PreNorm(nn.Module):\n    def __init__(self, dim, fn):\n        super().__init__()\n        self.norm = nn.LayerNorm(dim)\n        self.fn = fn\n    def forward(self, x, **kwargs):\n        return self.fn(self.norm(x), **kwargs)\n\nclass FeedForward(nn.Module):\n    def __init__(self, dim, hidden_dim, dropout = 0.):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(dim, hidden_dim),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, dim),\n            nn.Dropout(dropout)\n        )\n    def forward(self, x):\n        return self.net(x)\n\nclass Attention(nn.Module):\n    def __init__(self, dim, heads = 8, dim_head = 64, dropout = 0.):\n        super().__init__()\n        inner_dim = dim_head *  heads\n        project_out = not (heads == 1 and dim_head == dim)\n\n        self.heads = heads\n        self.scale = dim_head ** -0.5\n\n        self.attend = nn.Softmax(dim = -1)\n        self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)\n\n        self.to_out = nn.Sequential(\n            nn.Linear(inner_dim, dim),\n            nn.Dropout(dropout)\n        ) if project_out else nn.Identity()\n\n    def forward(self, x):\n        b, n, _, h = *x.shape, self.heads\n        qkv = self.to_qkv(x).chunk(3, dim = -1)\n        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = h), qkv)\n\n        dots = torch.einsum('b h i d, b h j d -> b h i j', q, k) * self.scale\n\n        attn = self.attend(dots)\n\n        out = torch.einsum('b h i j, b h j d -> b h i d', attn, v)\n        out = rearrange(out, 'b h n d -> b n (h d)')\n        return self.to_out(out)\n\nclass Transformer(nn.Module):\n    def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout = 0.):\n        super().__init__()\n        self.layers = nn.ModuleList([])\n        for _ in range(depth):\n            self.layers.append(nn.ModuleList([\n                PreNorm(dim, Attention(dim, heads = heads, dim_head = dim_head, dropout = dropout)),\n                PreNorm(dim, FeedForward(dim, mlp_dim, dropout = dropout))\n            ]))\n    def forward(self, x):\n        for attn, ff in self.layers:\n            x = attn(x) + x\n            x = ff(x) + x\n        return x\n\n\nclass VIT(nn.Module):\n    def __init__(self, *, image_size, patch_size, num_classes, dim, depth, heads, mlp_dim, pool = 'cls', channels = 3, dim_head = 64, dropout = 0., emb_dropout = 0.):\n        super().__init__()\n        image_height, image_width = pair(image_size)\n        patch_height, patch_width = pair(patch_size)\n\n        assert image_height % patch_height == 0 and image_width % patch_width == 0, 'Image dimensions must be divisible by the patch size.'\n\n        num_patches = (image_height // patch_height) * (image_width // patch_width)\n        patch_dim = channels * patch_height * patch_width\n        assert pool in {'cls', 'mean'}, 'pool type must be either cls (cls token) or mean (mean pooling)'\n\n        self.to_patch_embedding = nn.Sequential(\n            Rearrange('b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1 = patch_height, p2 = patch_width),\n            nn.Linear(patch_dim, dim),\n        )\n\n        self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, dim))\n        self.cls_token = nn.Parameter(torch.randn(1, 1, dim))\n        self.dropout = nn.Dropout(emb_dropout)\n\n        self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim, dropout)\n\n        self.pool = pool\n        self.to_latent = nn.Identity()\n\n        self.mlp_head = nn.Sequential(\n            nn.LayerNorm(dim),\n            nn.Linear(dim, num_classes)\n        )\n\n    def forward(self, img):\n        x = self.to_patch_embedding(img)\n        b, n, _ = x.shape\n\n        cls_tokens = repeat(self.cls_token, '() n d -> b n d', b = b)\n        x = torch.cat((cls_tokens, x), dim=1)\n        x += self.pos_embedding[:, :(n + 1)]\n        x = self.dropout(x)\n\n        x = self.transformer(x)\n\n        x = x.mean(dim = 1) if self.pool == 'mean' else x[:, 0]\n\n        x = self.to_latent(x)\n        return self.mlp_head(x)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T09:30:58.518599Z","iopub.execute_input":"2021-06-18T09:30:58.519071Z","iopub.status.idle":"2021-06-18T09:30:58.571745Z","shell.execute_reply.started":"2021-06-18T09:30:58.519033Z","shell.execute_reply":"2021-06-18T09:30:58.570931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test Data Loading","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T09:30:58.573342Z","iopub.execute_input":"2021-06-18T09:30:58.573732Z","iopub.status.idle":"2021-06-18T09:30:58.594522Z","shell.execute_reply.started":"2021-06-18T09:30:58.573677Z","shell.execute_reply":"2021-06-18T09:30:58.593752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, img_dir, transform=None, target_transform=None, aug=True):\n        self.img_dir = img_dir\n        self.transform = transform\n        self.target_transform = target_transform\n        self.aug = aug\n\n    def __len__(self):\n        img_path = os.path.join(self.img_dir)\n        images = os.listdir(img_path)\n        return len(images) \n\n    def __getitem__(self, idx):\n        img_dir = os.path.join(self.img_dir)\n        img_path = os.path.join(img_dir, os.listdir(img_dir)[idx])\n        image = Image.open(img_path).convert('RGB')\n        if self.transform:\n            if self.aug:\n                image = np.array(image)\n                image = self.transform(image=image)\n                image = image['image']\n            else:\n                image = self.transform(image)\n\n        if self.target_transform:\n            label = self.target_transform(label)\n        sample = {\"image\": image}\n        return sample","metadata":{"execution":{"iopub.status.busy":"2021-06-18T09:30:58.595791Z","iopub.execute_input":"2021-06-18T09:30:58.596106Z","iopub.status.idle":"2021-06-18T09:30:58.606223Z","shell.execute_reply.started":"2021-06-18T09:30:58.596073Z","shell.execute_reply":"2021-06-18T09:30:58.605438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transforms(aug=True, p=0.3):\n    test_transforms = None\n    if aug:\n        test_transforms = A.Compose(\n            [\n                A.CenterCrop(288, 288),\n                A.Resize(384, 384),\n                A.Normalize(mean=(0.5,0.5,0.5), std=(0.5,0.5,0.5)),\n                ToTensorV2(),\n            ]\n        )\n    else:\n        test_transforms = transforms.Compose(\n            [transforms.ToTensor(),\n            transforms.CenterCrop((384,384)),\n            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]\n        )\n    return test_transforms","metadata":{"execution":{"iopub.status.busy":"2021-06-18T09:30:58.609282Z","iopub.execute_input":"2021-06-18T09:30:58.609539Z","iopub.status.idle":"2021-06-18T09:30:58.617606Z","shell.execute_reply.started":"2021-06-18T09:30:58.609516Z","shell.execute_reply":"2021-06-18T09:30:58.616760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root = '../'\ntest_data_dir = os.path.join(root, 'input/cassava-leaf-disease-classification/')\ntest_img_dir = os.path.join(test_data_dir, 'test_images')\n\ntest_transforms = get_transforms(True)\n\ntest_dataset = TestDataset(test_img_dir, transform=test_transforms, aug=True)\n\ntest_dataloader = DataLoader(test_dataset, batch_size=1, num_workers=0, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T09:31:16.625461Z","iopub.execute_input":"2021-06-18T09:31:16.625796Z","iopub.status.idle":"2021-06-18T09:31:16.630598Z","shell.execute_reply.started":"2021-06-18T09:31:16.625765Z","shell.execute_reply":"2021-06-18T09:31:16.629809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading the Model","metadata":{}},{"cell_type":"code","source":"MODEL_PATH = '../input/bestmodel/best_net.pth'\n\nmodel = VIT(image_size=(384,384), patch_size=16, num_classes=5, dim=1024, depth=6, heads=16, mlp_dim=2048)\nmodel.load_state_dict(torch.load(MODEL_PATH))\n#model.eval()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T09:31:18.973895Z","iopub.execute_input":"2021-06-18T09:31:18.974213Z","iopub.status.idle":"2021-06-18T09:31:19.501067Z","shell.execute_reply.started":"2021-06-18T09:31:18.974183Z","shell.execute_reply":"2021-06-18T09:31:19.500277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_test = []\n\nfor i, data in enumerate(test_dataloader):\n    imgs = data['image'].float()\n    pred_test += model(imgs)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T09:31:21.514059Z","iopub.execute_input":"2021-06-18T09:31:21.514391Z","iopub.status.idle":"2021-06-18T09:31:22.652032Z","shell.execute_reply.started":"2021-06-18T09:31:21.514359Z","shell.execute_reply":"2021-06-18T09:31:22.651205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generating Predictions File","metadata":{}},{"cell_type":"code","source":"pred_test_labels = np.argmax(pred_test, axis = -1)\ntest_images = glob.glob('../input/cassava-leaf-disease-classification/test_images/*.jpg')\n\nfinal_submission = pd.DataFrame(test_images, columns = ['path'])\nfinal_submission['image_id'] = final_submission.path.str.split('/').str[-1]\nfinal_submission['label'] = pred_test_labels\n\nfinal_csv = final_submission[['image_id', 'label']]\nfinal_csv.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T09:31:23.799649Z","iopub.execute_input":"2021-06-18T09:31:23.800011Z","iopub.status.idle":"2021-06-18T09:31:23.815998Z","shell.execute_reply.started":"2021-06-18T09:31:23.799979Z","shell.execute_reply":"2021-06-18T09:31:23.815165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_csv.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T09:31:26.415844Z","iopub.execute_input":"2021-06-18T09:31:26.416194Z","iopub.status.idle":"2021-06-18T09:31:26.423538Z","shell.execute_reply.started":"2021-06-18T09:31:26.416162Z","shell.execute_reply":"2021-06-18T09:31:26.422759Z"},"trusted":true},"execution_count":null,"outputs":[]}]}