{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nimport os\nimport random\nimport json\nimport gc\nimport cv2\nimport pandas as pd\nimport numpy as np\n\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom sklearn.metrics import accuracy_score\nfrom functools import partial\nfrom albumentations import (Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, CenterCrop, \n                            HorizontalFlip, VerticalFlip, Rotate, ShiftScaleRotate, Transpose)\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\nimport tensorflow as tf\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as models\nfrom torch.utils.data import DataLoader, Dataset\n\nimport torchvision.transforms as transforms\nfrom transformers import ViTForImageClassification,ViTConfig,AutoModel\nfrom transformers import ViTModel\n\nimport math","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-04-27T01:48:37.470877Z","iopub.execute_input":"2023-04-27T01:48:37.471263Z","iopub.status.idle":"2023-04-27T01:48:43.414307Z","shell.execute_reply.started":"2023-04-27T01:48:37.471223Z","shell.execute_reply":"2023-04-27T01:48:43.413058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"/kaggle/input/cassava-leaf-disease-classification/\"\nimage_path = path+\"test_images/\"\n\nIMAGE_SIZE = (512,512)\nsubmission_df = pd.DataFrame(columns={\"image_id\",\"label\"})\nsubmission_df[\"image_id\"] = os.listdir(image_path)\nsubmission_df[\"label\"] = 0","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-04-27T01:48:47.364418Z","iopub.execute_input":"2023-04-27T01:48:47.365447Z","iopub.status.idle":"2023-04-27T01:48:47.377759Z","shell.execute_reply.started":"2023-04-27T01:48:47.365387Z","shell.execute_reply":"2023-04-27T01:48:47.376520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# used_models_pytorch = {\"resnext\": [f'../input/models/resnext50_32x4d_fold{fold}_best.pth' for fold in [1]]}\n# used_models_pytorch = {\"vit\": f'../input/model-vit/original_save_pretrained'}\n# used_models_pytorch = {\"mobilenet\": f'../input/model-mobilenet/mn3_bt20_ep5_lr1.pth'}\nused_models_pytorch = {\"resnext\": [f'../input/models/resnext50_32x4d_fold{fold}_best.pth' for fold in [1]],\n                       \"vit\": f'../input/model-vit/original_save_pretrained',\n                       \"mobilenet\": f'../input/model-mobilenet/mn3_bt20_ep5_lr1.pth'}","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-04-27T01:48:50.932584Z","iopub.execute_input":"2023-04-27T01:48:50.933105Z","iopub.status.idle":"2023-04-27T01:48:50.941983Z","shell.execute_reply.started":"2023-04-27T01:48:50.933058Z","shell.execute_reply":"2023-04-27T01:48:50.940885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ResNext50_32x4d","metadata":{}},{"cell_type":"code","source":"class CustomResNext(nn.Module):\n        def __init__(self, model_name='resnext50_32x4d', pretrained=False):\n            super().__init__()\n            self.model = timm.create_model(model_name, pretrained=pretrained)\n            n_features = self.model.fc.in_features\n            self.model.fc = nn.Linear(n_features, 5)\n\n        def forward(self, x):\n            x = self.model(x)\n            return x\n\nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_path_id'].values\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        image = cv2.imread(file_name)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image\n\nif \"resnext\" in used_models_pytorch:\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    def get_transforms():\n        return Compose([Resize(512, 512),\n                        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n                        ToTensorV2()])\n\n    def inference(model, states, test_loader, device):\n        model.to(device)\n\n        probabilities = []\n        for i, (images) in enumerate(test_loader):\n            images = images.to(device)\n            avg_preds = []\n            for state in states:\n                model.load_state_dict(state['model'])\n                model.eval()\n                with torch.no_grad():\n                    y_preds = model(images)\n                avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n            avg_preds = np.mean(avg_preds, axis=0)\n            probabilities.append(avg_preds)\n        return np.concatenate(probabilities)\n    \n\n    predictions_resnext = pd.DataFrame(columns={\"image_id\"})\n    predictions_resnext[\"image_id\"] = submission_df[\"image_id\"].values\n    predictions_resnext['image_path_id'] = image_path + predictions_resnext['image_id'].astype(str)\n\n    model = CustomResNext('resnext50_32x4d', pretrained=False)\n    states = [torch.load(f) for f in used_models_pytorch[\"resnext\"]]\n\n    test_dataset = TestDataset(predictions_resnext, transform=get_transforms())\n    test_loader = DataLoader(test_dataset, batch_size=16, shuffle=False, num_workers=4, pin_memory=True)\n    predictions = inference(model, states, test_loader, device)\n    print(predictions)\n\n    predictions_resnext['resnext'] = [np.squeeze(p) for p in predictions]\n    predictions_resnext = predictions_resnext.drop([\"image_path_id\"], axis=1)\n\n    torch.cuda.empty_cache()\n    try:\n        del(model)\n        del(states)\n    except:\n        pass\n    gc.collect()","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-04-27T01:48:53.093609Z","iopub.execute_input":"2023-04-27T01:48:53.094141Z","iopub.status.idle":"2023-04-27T01:48:58.977569Z","shell.execute_reply.started":"2023-04-27T01:48:53.094105Z","shell.execute_reply":"2023-04-27T01:48:58.976530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ViT","metadata":{}},{"cell_type":"code","source":"if \"vit\" in used_models_pytorch:\n    \n    IMG_SIZE = 224\n    BATCH_SIZE = 16\n    num_classes = 5\n\n    mean = [0.485, 0.456, 0.406]\n    std = [0.229, 0.224, 0.225]\n        \n    class LeafDataset(torch.utils.data.Dataset):\n    \n        def __init__(self, df, data_path, mode='train', transforms=None):\n            super().__init__()\n            self.df_data = df.values\n            self.data_path = data_path\n            self.transforms = transforms \n            self.mode = mode\n            self.data_dir = 'train_images' if mode == 'train' else 'test_images'\n\n        def __len__(self):\n            return len(self.df_data)\n\n        def __getitem__(self, index):\n            img_name = self.df_data[index][0]\n            img_path = os.path.join(self.data_path, self.data_dir, img_name)\n            img = Image.open(img_path).convert('RGB')\n\n            if self.transforms is not None:\n                img = self.transforms(img)\n\n            return img\n\n    transforms_val = transforms.Compose([\n        transforms.Resize((IMG_SIZE, IMG_SIZE)),\n        transforms.ToTensor(),\n        transforms.Normalize(mean, std)\n    ])\n\n    def predict(model, test_dataset):\n        preds = []\n        test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=BATCH_SIZE)\n\n        for test_images in tqdm(test_dataloader):\n            test_images = test_images.to(device)\n            model.eval()\n            with torch.no_grad():\n                output = model(test_images)\n            preds.extend(output.logits.data.softmax(1).cpu().data.numpy() * 0.5)\n\n        return preds    \n\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    predictions_vit = pd.DataFrame(columns={\"image_id\"})\n    predictions_vit[\"image_id\"] = submission_df[\"image_id\"].values\n\n    model = ViTForImageClassification.from_pretrained(used_models_pytorch[\"vit\"], num_labels = num_classes)\n    model.to(device)\n    \n    test_dataset = LeafDataset(df=predictions_vit, data_path=path, mode='test', transforms=transforms_val)\n\n    predictions_raw_vit = predict(model, test_dataset)\n    print(predictions_raw_vit)\n\n    predictions_vit['vit'] = [np.squeeze(p) for p in predictions_raw_vit]\n    \n    torch.cuda.empty_cache()\n    try:\n        del(model)\n    except:\n        pass\n\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-04-27T01:49:34.656373Z","iopub.execute_input":"2023-04-27T01:49:34.656900Z","iopub.status.idle":"2023-04-27T01:49:36.128949Z","shell.execute_reply.started":"2023-04-27T01:49:34.656840Z","shell.execute_reply":"2023-04-27T01:49:36.127755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mobilenet V3 (CropNet)","metadata":{}},{"cell_type":"code","source":"if \"mobilenet\" in used_models_pytorch:   \n    __all__ = ['mobilenetv3_large', 'mobilenetv3_small']\n\n    def _make_divisible(v, divisor, min_value=None):\n        \"\"\"\n        This function is taken from the original tf repo.\n        It ensures that all layers have a channel number that is divisible by 8\n        It can be seen here:\n        https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py\n        :param v:\n        :param divisor:\n        :param min_value:\n        :return:\n        \"\"\"\n        if min_value is None:\n            min_value = divisor\n        new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)\n        # Make sure that round down does not go down by more than 10%.\n        if new_v < 0.9 * v:\n            new_v += divisor\n        return new_v\n\n    class h_sigmoid(nn.Module):\n        def __init__(self, inplace=True):\n            super(h_sigmoid, self).__init__()\n            self.relu = nn.ReLU6(inplace=inplace)\n\n        def forward(self, x):\n            return self.relu(x + 3) / 6\n\n    class h_swish(nn.Module):\n        def __init__(self, inplace=True):\n            super(h_swish, self).__init__()\n            self.sigmoid = h_sigmoid(inplace=inplace)\n\n        def forward(self, x):\n            return x * self.sigmoid(x)\n\n    class SELayer(nn.Module):\n        def __init__(self, channel, reduction=4):\n            super(SELayer, self).__init__()\n            self.avg_pool = nn.AdaptiveAvgPool2d(1)\n            self.fc = nn.Sequential(\n                    nn.Linear(channel, _make_divisible(channel // reduction, 8)),\n                    nn.ReLU(inplace=True),\n                    nn.Linear(_make_divisible(channel // reduction, 8), channel),\n                    h_sigmoid()\n            )\n\n        def forward(self, x):\n            b, c, _, _ = x.size()\n            y = self.avg_pool(x).view(b, c)\n            y = self.fc(y).view(b, c, 1, 1)\n            return x * y\n\n    def conv_3x3_bn(inp, oup, stride):\n        return nn.Sequential(\n            nn.Conv2d(inp, oup, 3, stride, 1, bias=False),\n            nn.BatchNorm2d(oup),\n            h_swish()\n        )\n\n    def conv_1x1_bn(inp, oup):\n        return nn.Sequential(\n            nn.Conv2d(inp, oup, 1, 1, 0, bias=False),\n            nn.BatchNorm2d(oup),\n            h_swish()\n        )\n\n    class InvertedResidual(nn.Module):\n        def __init__(self, inp, hidden_dim, oup, kernel_size, stride, use_se, use_hs):\n            super(InvertedResidual, self).__init__()\n            assert stride in [1, 2]\n\n            self.identity = stride == 1 and inp == oup\n\n            if inp == hidden_dim:\n                self.conv = nn.Sequential(\n                    # dw\n                    nn.Conv2d(hidden_dim, hidden_dim, kernel_size, stride, (kernel_size - 1) // 2, groups=hidden_dim, bias=False),\n                    nn.BatchNorm2d(hidden_dim),\n                    h_swish() if use_hs else nn.ReLU(inplace=True),\n                    # Squeeze-and-Excite\n                    SELayer(hidden_dim) if use_se else nn.Identity(),\n                    # pw-linear\n                    nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),\n                    nn.BatchNorm2d(oup),\n                )\n            else:\n                self.conv = nn.Sequential(\n                    # pw\n                    nn.Conv2d(inp, hidden_dim, 1, 1, 0, bias=False),\n                    nn.BatchNorm2d(hidden_dim),\n                    h_swish() if use_hs else nn.ReLU(inplace=True),\n                    # dw\n                    nn.Conv2d(hidden_dim, hidden_dim, kernel_size, stride, (kernel_size - 1) // 2, groups=hidden_dim, bias=False),\n                    nn.BatchNorm2d(hidden_dim),\n                    # Squeeze-and-Excite\n                    SELayer(hidden_dim) if use_se else nn.Identity(),\n                    h_swish() if use_hs else nn.ReLU(inplace=True),\n                    # pw-linear\n                    nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),\n                    nn.BatchNorm2d(oup),\n                )\n\n        def forward(self, x):\n            if self.identity:\n                return x + self.conv(x)\n            else:\n                return self.conv(x)\n\n    class MobileNetV3(nn.Module):\n        def __init__(self, cfgs, mode, num_classes=1000, width_mult=1.):\n            super(MobileNetV3, self).__init__()\n            # setting of inverted residual blocks\n            self.cfgs = cfgs\n            assert mode in ['large', 'small']\n\n            # building first layer\n            input_channel = _make_divisible(16 * width_mult, 8)\n            layers = [conv_3x3_bn(3, input_channel, 2)]\n            # building inverted residual blocks\n            block = InvertedResidual\n            for k, t, c, use_se, use_hs, s in self.cfgs:\n                output_channel = _make_divisible(c * width_mult, 8)\n                exp_size = _make_divisible(input_channel * t, 8)\n                layers.append(block(input_channel, exp_size, output_channel, k, s, use_se, use_hs))\n                input_channel = output_channel\n            self.features = nn.Sequential(*layers)\n            # building last several layers\n            self.conv = conv_1x1_bn(input_channel, exp_size)\n            self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n            output_channel = {'large': 1280, 'small': 1024}\n            output_channel = _make_divisible(output_channel[mode] * width_mult, 8) if width_mult > 1.0 else output_channel[mode]\n            self.classifier = nn.Sequential(\n                nn.Linear(exp_size, output_channel),\n                h_swish(),\n                nn.Dropout(0.2),\n                nn.Linear(output_channel, num_classes),\n            )\n\n            self._initialize_weights()\n\n        def forward(self, x):\n            x = self.features(x)\n            x = self.conv(x)\n            x = self.avgpool(x)\n            x = x.view(x.size(0), -1)\n            x = self.classifier(x)\n            return x\n\n        def _initialize_weights(self):\n            for m in self.modules():\n                if isinstance(m, nn.Conv2d):\n                    n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n                    m.weight.data.normal_(0, math.sqrt(2. / n))\n                    if m.bias is not None:\n                        m.bias.data.zero_()\n                elif isinstance(m, nn.BatchNorm2d):\n                    m.weight.data.fill_(1)\n                    m.bias.data.zero_()\n                elif isinstance(m, nn.Linear):\n                    m.weight.data.normal_(0, 0.01)\n                    m.bias.data.zero_()\n\n    def mobilenetv3_large(**kwargs):\n        \"\"\"\n        Constructs a MobileNetV3-Large model\n        \"\"\"\n        cfgs = [\n            # k, t, c, SE, HS, s \n            [3,   1,  16, 0, 0, 1],\n            [3,   4,  24, 0, 0, 2],\n            [3,   3,  24, 0, 0, 1],\n            [5,   3,  40, 1, 0, 2],\n            [5,   3,  40, 1, 0, 1],\n            [5,   3,  40, 1, 0, 1],\n            [3,   6,  80, 0, 1, 2],\n            [3, 2.5,  80, 0, 1, 1],\n            [3, 2.3,  80, 0, 1, 1],\n            [3, 2.3,  80, 0, 1, 1],\n            [3,   6, 112, 1, 1, 1],\n            [3,   6, 112, 1, 1, 1],\n            [5,   6, 160, 1, 1, 2],\n            [5,   6, 160, 1, 1, 1],\n            [5,   6, 160, 1, 1, 1]\n        ]\n        return MobileNetV3(cfgs, mode='large', **kwargs)\n\n    def mobilenetv3_small(**kwargs):\n        \"\"\"\n        Constructs a MobileNetV3-Small model\n        \"\"\"\n        cfgs = [\n            # k, t, c, SE, HS, s \n            [3,    1,  16, 1, 0, 2],\n            [3,  4.5,  24, 0, 0, 2],\n            [3, 3.67,  24, 0, 0, 1],\n            [5,    4,  40, 1, 1, 2],\n            [5,    6,  40, 1, 1, 1],\n            [5,    6,  40, 1, 1, 1],\n            [5,    3,  48, 1, 1, 1],\n            [5,    3,  48, 1, 1, 1],\n            [5,    6,  96, 1, 1, 2],\n            [5,    6,  96, 1, 1, 1],\n            [5,    6,  96, 1, 1, 1],\n        ]\n\n        return MobileNetV3(cfgs, mode='small', **kwargs)\n\n    class LeafDataset(torch.utils.data.Dataset):\n    \n        def __init__(self, df, data_path, mode='train', transforms=None):\n            super().__init__()\n            self.df_data = df.values\n            self.data_path = data_path\n            self.transforms = transforms \n            self.mode = mode\n            self.data_dir = 'train_images' if mode == 'train' else 'test_images'\n\n        def __len__(self):\n            return len(self.df_data)\n\n        def __getitem__(self, index):\n            img_name = self.df_data[index][0]\n            img_path = os.path.join(self.data_path, self.data_dir, img_name)\n            img = Image.open(img_path).convert('RGB')\n\n            if self.transforms is not None:\n                img = self.transforms(img)\n\n            return img\n\n    def predict(model, test_dataset):\n        preds = []\n        test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=20)\n\n        for test_images in tqdm(test_dataloader):\n            test_images = test_images.to(device)\n            model.eval()\n            with torch.no_grad():\n                output = model(test_images)\n            preds.extend(output[:, :5].softmax(1).cpu().data.numpy() * 0.5)\n\n        return preds\n\n    transforms_val = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n    ])\n\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    predictions_mobilenet = pd.DataFrame(columns={\"image_id\"})\n    predictions_mobilenet[\"image_id\"] = submission_df[\"image_id\"].values \n\n    model = mobilenetv3_large()\n    model.to(device)\n    model.load_state_dict(torch.load(used_models_pytorch[\"mobilenet\"]))\n\n    test_dataset=LeafDataset(df=predictions_mobilenet, data_path=path, mode='test', transforms=transforms_val)\n\n    predictions_raw_mobilenet = predict(model, test_dataset)\n    print(predictions_raw_mobilenet)\n\n    predictions_mobilenet['mobilenet'] = [np.squeeze(p) for p in predictions_raw_mobilenet]\n\n    torch.cuda.empty_cache()\n    try:\n        del(model)\n    except:\n        pass\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-04-27T01:49:54.101302Z","iopub.execute_input":"2023-04-27T01:49:54.101779Z","iopub.status.idle":"2023-04-27T01:49:54.599652Z","shell.execute_reply.started":"2023-04-27T01:49:54.101742Z","shell.execute_reply":"2023-04-27T01:49:54.598581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final Ensembling","metadata":{}},{"cell_type":"code","source":"submission_df[\"label\"] = 0\n\nif \"resnext\" in used_models_pytorch:\n    submission_df = submission_df.merge(predictions_resnext, on=\"image_id\")\n    \nif \"mobilenet\" in used_models_pytorch:\n    submission_df = submission_df.merge(predictions_mobilenet, on=\"image_id\")\n    \nif \"vit\" in used_models_pytorch:\n    submission_df = submission_df.merge(predictions_vit, on=\"image_id\")","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-04-27T01:50:01.509607Z","iopub.execute_input":"2023-04-27T01:50:01.509994Z","iopub.status.idle":"2023-04-27T01:50:01.527535Z","shell.execute_reply.started":"2023-04-27T01:50:01.509961Z","shell.execute_reply":"2023-04-27T01:50:01.526354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df[\"label\"] = submission_df.apply(lambda row: np.argmax(\n    [np.sum(e) for e in zip(*[row[m] for m in list(used_models_pytorch.keys())])]), axis=1)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-04-27T01:50:03.762343Z","iopub.execute_input":"2023-04-27T01:50:03.763060Z","iopub.status.idle":"2023-04-27T01:50:03.772598Z","shell.execute_reply.started":"2023-04-27T01:50:03.763021Z","shell.execute_reply":"2023-04-27T01:50:03.769580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head(1)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-04-27T01:50:05.974690Z","iopub.execute_input":"2023-04-27T01:50:05.975402Z","iopub.status.idle":"2023-04-27T01:50:05.992097Z","shell.execute_reply.started":"2023-04-27T01:50:05.975363Z","shell.execute_reply":"2023-04-27T01:50:05.990860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df[[\"image_id\",\"label\"]].to_csv(\"submission.csv\", index=False)\n!head submission.csv","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-04-27T01:50:59.704757Z","iopub.execute_input":"2023-04-27T01:50:59.705802Z","iopub.status.idle":"2023-04-27T01:51:00.769625Z","shell.execute_reply.started":"2023-04-27T01:50:59.705762Z","shell.execute_reply":"2023-04-27T01:51:00.768349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}