{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\n\nroot = '/kaggle/'\nsys.path.append(root)\n\n'''\nImport Libraries\n'''\n\nimport os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nimport torchvision.models as models\nfrom torch.utils.tensorboard import SummaryWriter\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom PIL import Image\n\n# from einops import rearrange, repeat\n# from einops.layers.torch import Rearrange\n\nimport matplotlib.pyplot as plt\n\nimport time\nimport copy\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-18T11:45:07.329084Z","iopub.execute_input":"2021-06-18T11:45:07.329474Z","iopub.status.idle":"2021-06-18T11:45:07.341059Z","shell.execute_reply.started":"2021-06-18T11:45:07.329398Z","shell.execute_reply":"2021-06-18T11:45:07.339614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' \nDataset Class\n'''\n\nclass CSVDataset(Dataset):\n    def __init__(self, annotations_df, img_dir, transform=None, target_transform=None, aug=True):\n        self.img_labels = annotations_df\n        self.img_dir = img_dir\n        self.transform = transform\n        self.target_transform = target_transform\n        self.aug = aug\n\n    def __len__(self):\n        return len(self.img_labels)\n\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_dir, self.img_labels.iloc[idx, 0])\n        image = Image.open(img_path).convert('RGB')\n        #label = self.img_labels.iloc[idx, 1]\n        if self.transform:\n            if self.aug:\n                image = np.array(image)\n                image = self.transform(image=image)\n                image = image['image']\n            else:\n                image = self.transform(image)\n\n#         if self.target_transform:\n#             label = self.target_transform(label)\n        sample = {\"image\": image}\n        return sample\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T11:45:07.491875Z","iopub.execute_input":"2021-06-18T11:45:07.492183Z","iopub.status.idle":"2021-06-18T11:45:07.501196Z","shell.execute_reply.started":"2021-06-18T11:45:07.492141Z","shell.execute_reply":"2021-06-18T11:45:07.499679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nResnet\n'''\n\n# Resnet\nclass ResNet(nn.Module):\n    def __init__(self, layers, dropout=0.0):\n        super(ResNet, self).__init__()\n        self.inplanes = 64\n        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, padding=3, stride=2, bias=False)  ## this is stride2 in the original implementation\n        self.bn1 = nn.BatchNorm2d(self.inplanes)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self.make_layer(64, layers[0])\n        self.layer2 = self.make_layer(128, layers[1], stride=2)\n        self.layer3 = self.make_layer(256, layers[2], stride=2)\n        self.layer4 = self.make_layer(512, layers[3], stride=2)\n        self.avgpool = nn.AdaptiveAvgPool2d((1,1))\n        self.fc = nn.Linear(512, 10)\n        self.dropout = nn.Dropout(dropout) if dropout > 0.0 else None\n\n\n\n    def make_layer(self, planes, blocks, stride=1):\n        downsample = None\n        if stride != 1:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes, kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(planes)\n            )\n\n        layers = []\n        layers.append(ResBlock(self.inplanes, planes, stride, downsample))\n        self.inplanes = planes\n        for _ in range(1, blocks):\n            layers.append(ResBlock(self.inplanes, planes))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        out = self.relu(self.bn1(self.conv1(x)))\n        out = self.maxpool(out)\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = self.layer4(out)\n        out = self.avgpool(out)\n        out = torch.flatten(out, 1)\n        out = self.fc(out)\n        if self.dropout is not None:\n            out = self.dropout(out)\n\n        return out\n\n\nclass ResBlock(nn.Module):\n    def __init__(self, inplanes, planes, stride=1, downsample=None):\n        super().__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.downsample = downsample\n    \n    def forward(self, x):\n        identity = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = self.conv2(out)\n        out = self.bn2(out)\n\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\nclass MobileNetV2(nn.Module):\n    def __init__(self, width_mult=1.0, dropout=0.0):\n        super(MobileNetV2, self).__init__()\n        inverted_residual_setting = [\n            [1, 16, 1, 1],\n            [6, 24, 2, 2],\n            [6, 32, 3, 2],\n            [6, 64, 4, 2],\n            [6, 96, 3, 1],\n            [6, 160, 3, 2],\n            [6, 320, 1, 1]\n        ]\n\n        input_channel = 32\n        last_channel = 1280\n        \n        input_channel = _make_divisible(input_channel * width_mult, 8)\n        last_channel = _make_divisible(last_channel * max(1.0, width_mult) * width_mult, 8)\n        features = [ConvBNReLU(3, input_channel, stride=2)]\n\n        for t,c,n,s in inverted_residual_setting:\n            output_channel = _make_divisible(c * width_mult, 8)\n            for i in range(n):\n                stride = s if i == 0 else 1\n                features.append(InvertedResidual(input_channel, output_channel, stride, expand_ratio=t))\n                input_channel = output_channel\n\n        features.append(ConvBNReLU(input_channel, last_channel, kernel_size=1))\n        self.features = nn.Sequential(*features)\n\n        self.classifier = nn.Sequential(\n            nn.Dropout(dropout),\n            nn.Linear(last_channel, 10)\n        )\n\n    def forward(self, x):\n        out = self.features(x)\n        out = F.adaptive_avg_pool2d(out, (1,1))\n        out = torch.flatten(out, 1)\n        out = self.classifier(out)\n        return out\n\nclass InvertedResidual(nn.Module):\n    def __init__(self, in_planes, out_planes, stride, expand_ratio):\n        super().__init__()\n        hidden_dim = int(round(in_planes * expand_ratio))\n        self.use_res_connect = stride == 1 and in_planes == out_planes\n\n        layers = []  # depthwise separable convolution with bottleneck\n        if expand_ratio != 1:\n            layers.append(ConvBNReLU(in_planes, hidden_dim, kernel_size=1))\n        layers.extend([\n            ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim),\n            nn.Conv2d(hidden_dim, out_planes, kernel_size=1, bias=False),\n            nn.BatchNorm2d(out_planes)\n        ])\n        self.conv = nn.Sequential(*layers)\n\n    def forward(self, x):\n        if self.use_res_connect:\n            return x + self.conv(x)\n        else:\n            return self.conv(x)\n\n\nclass ConvBNReLU(nn.Module):\n    def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1):\n        super().__init__()\n        padding = (kernel_size-1) // 2\n        self.layers = nn.Sequential(\n            nn.Conv2d(in_planes, out_planes, kernel_size, stride, padding, groups=groups, bias=False),\n            nn.BatchNorm2d(out_planes),\n            nn.ReLU6(inplace=True) # necessary? consider switching to relu\n        )\n\n    def forward(self, x):\n        out = self.layers(x)\n        return out\n\n\ndef _make_divisible(v, divisor, min_value=None):\n    if min_value is None:\n        min_value = divisor\n\n    new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)  # rounds numbers in range [num-divisor/2, num+divisor/2-1] to num, where num is a multiple of divisor\n\n    if new_v < 0.9 * v:\n        new_v += divisor  # ensures that round down does not decrease v by more than 10%\n\n    return new_v\n\n'''\nVIT\n'''\n# helpers\n\ndef pair(t):\n    return t if isinstance(t, tuple) else (t, t)\n\n# classes\n\nclass PreNorm(nn.Module):\n    def __init__(self, dim, fn):\n        super().__init__()\n        self.norm = nn.LayerNorm(dim)\n        self.fn = fn\n    def forward(self, x, **kwargs):\n        return self.fn(self.norm(x), **kwargs)\n\nclass FeedForward(nn.Module):\n    def __init__(self, dim, hidden_dim, dropout = 0.):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(dim, hidden_dim),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, dim),\n            nn.Dropout(dropout)\n        )\n    def forward(self, x):\n        return self.net(x)\n\nclass Attention(nn.Module):\n    def __init__(self, dim, heads = 8, dim_head = 64, dropout = 0.):\n        super().__init__()\n        inner_dim = dim_head *  heads\n        project_out = not (heads == 1 and dim_head == dim)\n\n        self.heads = heads\n        self.scale = dim_head ** -0.5\n\n        self.attend = nn.Softmax(dim = -1)\n        self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)\n\n        self.to_out = nn.Sequential(\n            nn.Linear(inner_dim, dim),\n            nn.Dropout(dropout)\n        ) if project_out else nn.Identity()\n\n    def forward(self, x):\n        b, n, _, h = *x.shape, self.heads\n        qkv = self.to_qkv(x).chunk(3, dim = -1)\n        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = h), qkv)\n\n        dots = torch.einsum('b h i d, b h j d -> b h i j', q, k) * self.scale\n\n        attn = self.attend(dots)\n\n        out = torch.einsum('b h i j, b h j d -> b h i d', attn, v)\n        out = rearrange(out, 'b h n d -> b n (h d)')\n        return self.to_out(out)\n\nclass Transformer(nn.Module):\n    def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout = 0.):\n        super().__init__()\n        self.layers = nn.ModuleList([])\n        for _ in range(depth):\n            self.layers.append(nn.ModuleList([\n                PreNorm(dim, Attention(dim, heads = heads, dim_head = dim_head, dropout = dropout)),\n                PreNorm(dim, FeedForward(dim, mlp_dim, dropout = dropout))\n            ]))\n    def forward(self, x):\n        for attn, ff in self.layers:\n            x = attn(x) + x\n            x = ff(x) + x\n        return x\n\n\nclass VIT(nn.Module):\n    def __init__(self, *, image_size, patch_size, num_classes, dim, depth, heads, mlp_dim, pool = 'cls', channels = 3, dim_head = 64, dropout = 0., emb_dropout = 0.):\n        super().__init__()\n        image_height, image_width = pair(image_size)\n        patch_height, patch_width = pair(patch_size)\n\n        assert image_height % patch_height == 0 and image_width % patch_width == 0, 'Image dimensions must be divisible by the patch size.'\n\n        num_patches = (image_height // patch_height) * (image_width // patch_width)\n        patch_dim = channels * patch_height * patch_width\n        assert pool in {'cls', 'mean'}, 'pool type must be either cls (cls token) or mean (mean pooling)'\n\n        self.to_patch_embedding = nn.Sequential(\n            Rearrange('b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1 = patch_height, p2 = patch_width),\n            nn.Linear(patch_dim, dim),\n        )\n\n        self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, dim))\n        self.cls_token = nn.Parameter(torch.randn(1, 1, dim))\n        self.dropout = nn.Dropout(emb_dropout)\n\n        self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim, dropout)\n\n        self.pool = pool\n        self.to_latent = nn.Identity()\n\n        self.mlp_head = nn.Sequential(\n            nn.LayerNorm(dim),\n            nn.Linear(dim, num_classes)\n        )\n\n    def forward(self, img):\n        x = self.to_patch_embedding(img)\n        b, n, _ = x.shape\n\n        cls_tokens = repeat(self.cls_token, '() n d -> b n d', b = b)\n        x = torch.cat((cls_tokens, x), dim=1)\n        x += self.pos_embedding[:, :(n + 1)]\n        x = self.dropout(x)\n\n        x = self.transformer(x)\n\n        x = x.mean(dim = 1) if self.pool == 'mean' else x[:, 0]\n\n        x = self.to_latent(x)\n        return self.mlp_head(x)\n\nclass Ensemble(nn.Module):\n    def __init__(self, modelA, modelB, modelC):\n        super().__init__()\n        self.modelA = modelA\n        self.modelB = modelB\n        self.modelC = modelC\n        \n    def forward(self, x):\n        outA = self.modelA(x)\n        outB = self.modelB(x)\n        outC = self.modelC(x)\n        out = outA + outB + outC\n        return out\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T12:03:08.797424Z","iopub.execute_input":"2021-06-18T12:03:08.797823Z","iopub.status.idle":"2021-06-18T12:03:08.982951Z","shell.execute_reply.started":"2021-06-18T12:03:08.797791Z","shell.execute_reply":"2021-06-18T12:03:08.981797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nAuxiliary Functions\n'''\n\ndef get_model(model, width_mult=1.0, dropout=0.2):\n    if model == 'base':\n        return models.resnet18()\n    elif model == 'resnet':\n        return ResNet([2,2,2,2], dropout)\n    elif model == 'mobilenet':\n        return MobileNetV2(width_mult=width_mult, dropout=dropout)\n    elif model == 'VIT':\n        return VIT(image_size=(384,384), patch_size=16, num_classes=5, dim=512, depth=6, heads=12, mlp_dim=1024)\n    else:\n        raise NotImplementedError(f'Model [{model}] not implemented')\n\ndef get_transforms(aug=True, p=0.3):\n    train_transforms = None\n    val_transforms = None\n    if aug:\n        train_transforms = A.Compose(\n            [\n                A.RandomCrop(288, 288),\n                A.Resize(384, 384),\n                A.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.05, rotate_limit=15, p=p),\n                A.RandomBrightnessContrast(p=p),\n                A.HorizontalFlip(p=p),\n                A.Normalize(mean=(0.5,0.5,0.5), std=(0.5,0.5,0.5)),\n                ToTensorV2(),\n            ]\n        )\n        val_transforms = A.Compose(\n            [\n                A.CenterCrop(288, 288),\n                A.Resize(384, 384),\n                A.Normalize(mean=(0.5,0.5,0.5), std=(0.5,0.5,0.5)),\n                ToTensorV2(),\n            ]\n        )\n    else:\n        train_transforms = transforms.Compose(\n            [transforms.ToTensor(),\n            transforms.RandomCrop((384,384)),\n            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]\n        )\n        val_transforms = transforms.Compose(\n            [transforms.ToTensor(),\n            transforms.CenterCrop((384,384)),\n            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]\n        )\n    return train_transforms, val_transforms\n    \ndef save_model(net, name, epoch, save_dir):\n    path = os.path.join(save_dir, f'{epoch}_net.pth')\n    torch.save(net.state_dict(), path)\n    \ndef load_model(net, name, epoch, save_dir, device):\n    path = os.path.join(save_dir, f'{epoch}_net.pth')\n    net.load_state_dict(torch.load(path, map_location=device), strict=False)\n    return net\n    \ndef print_and_save_args(args, path):\n    message = \"\"\n    for k, v in args.items():\n        message += f'{str(k):>15}: {str(v):<10}\\n'\n    print(' '*20 + '[OPTIONS]' + ' '*20)\n    print(message)\n    with open(path, 'w') as f:\n        f.write(message)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T12:01:17.050807Z","iopub.execute_input":"2021-06-18T12:01:17.051136Z","iopub.status.idle":"2021-06-18T12:01:17.068064Z","shell.execute_reply.started":"2021-06-18T12:01:17.051106Z","shell.execute_reply":"2021-06-18T12:01:17.066589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test Parameters\nargs = {}\nargs['name'] = 'mobilenet_384_randomcrop_width_mult_1.8'  # MUST SET EXPERIMENT NAME BEFORE TRAINING (for saving model)\nargs['batch_size'] = 32\nargs['width_mult1'] = 1.8\nargs['width_mult2'] = 1.0\nargs['width_mult3'] = 1.0\nargs['dropout'] = 0.0\nargs['aug'] = False\nargs['model1'] = 'mobilenet'\nargs['model2'] = 'mobilenet'\nargs['model3'] = 'resnet'\nargs['gpu_id'] = 0\n\nassert args['name'] is not None, \"Must set experiment name before training\"\n\ndata_dir = os.path.join(root, 'input/cassava-leaf-disease-classification/')\nsave_dir1 = os.path.join(root, 'input/m-18-aug')\nsave_dir2 = os.path.join(root, 'input/mnoaug')\nsave_dir3 = os.path.join(root, 'input/r-augg')\n\nimg_dir = os.path.join(data_dir, 'test_images')\ntest_pd = pd.DataFrame()\ntest_pd['image_id'] = list(os.listdir(img_dir))\nnum_test = len(test_pd)\n\n_, test_transforms = get_transforms(args['aug'])\n\ntest_dataset = CSVDataset(test_pd, img_dir, transform=test_transforms, aug=args['aug'])\ntest_dataloader = DataLoader(test_dataset, batch_size=args['batch_size'], shuffle=False)\n\ndevice = 'cuda:' + str(args['gpu_id']) if torch.cuda.is_available() else 'cpu'\nprint(f'test images: {num_test} \\t device: {device}')\n\nnet1 = get_model(args['model1'], width_mult=args['width_mult1'], dropout=args['dropout']).to(device)\nnet1 = load_model(net1, args['name'], 'best', save_dir1, device)\n\nnet2 = get_model(args['model2'], width_mult=args['width_mult2'], dropout=args['dropout']).to(device)\nnet2 = load_model(net2, args['name'], 'best', save_dir2, device)\n\nnet3 = get_model(args['model3'], width_mult=args['width_mult3'], dropout=args['dropout']).to(device)\nnet3 = load_model(net3, args['name'], 'best', save_dir3, device)\n\nnet = Ensemble(net1, net2, net3)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T12:03:11.159399Z","iopub.execute_input":"2021-06-18T12:03:11.159838Z","iopub.status.idle":"2021-06-18T12:03:11.749840Z","shell.execute_reply.started":"2021-06-18T12:03:11.159790Z","shell.execute_reply":"2021-06-18T12:03:11.748692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nTest\n'''\nnum_params = sum(p.numel() for p in net.parameters() if p.requires_grad)\ndef human_format(num):\n    magnitude = 0\n    while abs(num) >= 1000:\n        magnitude += 1\n        num /= 1000.0\n    return '%.2f%s' % (num, ['', 'K', 'M', 'G', 'T', 'P'][magnitude])\n\nprint(f'Number of total parameters: {human_format(num_params)}')\n\n# Calculate validation accuracy and save best model\npred_list = []\nwith torch.no_grad():\n    for i, data in enumerate(test_dataloader):\n        imgs = data['image'].float().to(device)\n        outputs = net(imgs)\n        pred_list += outputs.argmax(dim=1).tolist()\n\ntest_pd['label'] = np.array(pred_list)\nprint(test_pd.head())\ntest_pd.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T12:03:13.271662Z","iopub.execute_input":"2021-06-18T12:03:13.272024Z","iopub.status.idle":"2021-06-18T12:03:13.395139Z","shell.execute_reply.started":"2021-06-18T12:03:13.271993Z","shell.execute_reply":"2021-06-18T12:03:13.393884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}