{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":13836,"databundleVersionId":1718836,"sourceType":"competition"},{"sourceId":11681709,"sourceType":"datasetVersion","datasetId":7331722},{"sourceId":11715121,"sourceType":"datasetVersion","datasetId":7353604}],"dockerImageVersionId":31012,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.models.feature_extraction import create_feature_extractor\nfrom typing import Union\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport copy\nsys.path.insert(0, '/kaggle/input/timm-pytorch-image-model/pytorch-image-models-master')\nimport timm\nprint(timm.__file__)\nimport os\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-07T09:44:35.595409Z","iopub.execute_input":"2025-05-07T09:44:35.595655Z","iopub.status.idle":"2025-05-07T09:44:46.292633Z","shell.execute_reply.started":"2025-05-07T09:44:35.595636Z","shell.execute_reply":"2025-05-07T09:44:46.291745Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = (\n    \"cuda\"\n    if torch.cuda.is_available()\n    else \"mps\"\n    if torch.backends.mps.is_available()\n    else \"cpu\"\n)\nprint(f\"Using {device} device\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-07T09:44:46.293458Z","iopub.execute_input":"2025-05-07T09:44:46.293678Z","iopub.status.idle":"2025-05-07T09:44:46.379612Z","shell.execute_reply.started":"2025-05-07T09:44:46.293659Z","shell.execute_reply":"2025-05-07T09:44:46.378912Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_transforms = transforms.Compose([\n        transforms.Resize((510, 510), Image.BILINEAR),\n        transforms.CenterCrop((384, 384)),\n        transforms.ToTensor(),\n        transforms.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225]\n        )\n])\n\n\nclass CassavaDataset(Dataset):\n    def __init__(self, csv_file, img_dir, transform=None):\n        self.data = pd.read_csv(csv_file)\n        self.img_dir = img_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        row = self.data.iloc[idx]\n        img_path = os.path.join(self.img_dir, row[0])\n        image = Image.open(img_path).convert(\"RGB\")\n        label = int(row[1])\n\n        if self.transform:\n            image = self.transform(image)\n\n        return idx, image, label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-07T09:44:46.381156Z","iopub.execute_input":"2025-05-07T09:44:46.381379Z","iopub.status.idle":"2025-05-07T09:44:46.505863Z","shell.execute_reply.started":"2025-05-07T09:44:46.381361Z","shell.execute_reply":"2025-05-07T09:44:46.505136Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class GCNCombiner(nn.Module):\n\n    def __init__(self, \n                 total_num_selects: int,\n                 num_classes: int, \n                 inputs: Union[dict, None] = None, \n                 proj_size: Union[int, None] = None,\n                 fpn_size: Union[int, None] = None):\n        \"\"\"\n        If building backbone without FPN, set fpn_size to None and MUST give \n        'inputs' and 'proj_size', the reason of these setting is to constrain the \n        dimension of graph convolutional network input.\n        \"\"\"\n        super(GCNCombiner, self).__init__()\n\n        assert inputs is not None or fpn_size is not None, \\\n            \"To build GCN combiner, you must give one features dimension.\"\n\n        ### auto-proj\n        self.fpn_size = fpn_size\n        if fpn_size is None:\n            for name in inputs:\n                if len(name) == 4:\n                    in_size = inputs[name].size(1)\n                elif len(name) == 3:\n                    in_size = inputs[name].size(2)\n                else:\n                    raise ValueError(\"The size of output dimension of previous must be 3 or 4.\")\n                m = nn.Sequential(\n                    nn.Linear(in_size, proj_size),\n                    nn.ReLU(),\n                    nn.Linear(proj_size, proj_size)\n                )\n                self.add_module(\"proj_\"+name, m)\n            self.proj_size = proj_size\n        else:\n            self.proj_size = fpn_size\n\n        ### build one layer structure (with adaptive module)\n        num_joints = total_num_selects // 64\n\n        self.param_pool0 = nn.Linear(total_num_selects, num_joints)\n        \n        A = torch.eye(num_joints) / 100 + 1 / 100\n        self.adj1 = nn.Parameter(copy.deepcopy(A))\n        self.conv1 = nn.Conv1d(self.proj_size, self.proj_size, 1)\n        self.batch_norm1 = nn.BatchNorm1d(self.proj_size)\n        \n        self.conv_q1 = nn.Conv1d(self.proj_size, self.proj_size//4, 1)\n        self.conv_k1 = nn.Conv1d(self.proj_size, self.proj_size//4, 1)\n        self.alpha1 = nn.Parameter(torch.zeros(1))\n\n        ### merge information\n        self.param_pool1 = nn.Linear(num_joints, 1)\n        \n        #### class predict\n        self.dropout = nn.Dropout(p=0.1)\n        self.classifier = nn.Linear(self.proj_size, num_classes)\n\n        self.tanh = nn.Tanh()\n\n    def forward(self, x):\n        \"\"\"\n        \"\"\"\n        hs = []\n        names = []\n        for name in x:\n            if \"FPN1_\" in name:\n                continue\n            if self.fpn_size is None:\n                _tmp = getattr(self, \"proj_\"+name)(x[name])\n            else:\n                _tmp = x[name]\n            hs.append(_tmp)\n            names.append([name, _tmp.size()])\n\n        hs = torch.cat(hs, dim=1).transpose(1, 2).contiguous() # B, S', C --> B, C, S\n        hs = self.param_pool0(hs)\n        ### adaptive adjacency\n        q1 = self.conv_q1(hs).mean(1)\n        k1 = self.conv_k1(hs).mean(1)\n        A1 = self.tanh(q1.unsqueeze(-1) - k1.unsqueeze(1))\n        A1 = self.adj1 + A1 * self.alpha1\n        ### graph convolution\n        hs = self.conv1(hs)\n        hs = torch.matmul(hs, A1)\n        hs = self.batch_norm1(hs)\n        ### predict\n        hs = self.param_pool1(hs)\n        hs = self.dropout(hs)\n        hs = hs.flatten(1)\n        hs = self.classifier(hs)\n\n        return hs\n    \nclass WeaklySelector(nn.Module):\n\n    def __init__(self, inputs: dict, num_classes: int, num_select: dict, fpn_size: Union[int, None] = None):\n        \"\"\"\n        inputs: dictionary contain torch.Tensors, which comes from backbone\n                [Tensor1(hidden feature1), Tensor2(hidden feature2)...]\n                Please note that if len(features.size) equal to 3, the order of dimension must be [B,S,C],\n                S mean the spatial domain, and if len(features.size) equal to 4, the order must be [B,C,H,W]\n        \"\"\"\n        super(WeaklySelector, self).__init__()\n\n        self.num_select = num_select\n\n        self.fpn_size = fpn_size\n        ### build classifier\n        if self.fpn_size is None:\n            self.num_classes = num_classes\n            for name in inputs:\n                fs_size = inputs[name].size()\n                if len(fs_size) == 3:\n                    in_size = fs_size[2]\n                elif len(fs_size) == 4:\n                    in_size = fs_size[1]\n                m = nn.Linear(in_size, num_classes)\n                self.add_module(\"classifier_l_\"+name, m)\n\n        self.thresholds = {}\n        for name in inputs:\n            self.thresholds[name] = []\n\n    # def select(self, logits, l_name):\n    #     \"\"\"\n    #     logits: [B, S, num_classes]\n    #     \"\"\"\n    #     probs = torch.softmax(logits, dim=-1)\n    #     scores, _ = torch.max(probs, dim=-1)\n    #     _, ids = torch.sort(scores, -1, descending=True)\n    #     sn = self.num_select[l_name]\n    #     s_ids = ids[:, :sn]\n    #     not_s_ids = ids[:, sn:]\n    #     return s_ids.unsqueeze(-1), not_s_ids.unsqueeze(-1)\n\n    def forward(self, x, logits=None):\n        \"\"\"\n        x : \n            dictionary contain the features maps which \n            come from your choosen layers.\n            size must be [B, HxW, C] ([B, S, C]) or [B, C, H, W].\n            [B,C,H,W] will be transpose to [B, HxW, C] automatically.\n        \"\"\"\n        if self.fpn_size is None:\n            logits = {}\n        selections = {}\n        for name in x:\n            if \"FPN1_\" in name:\n                continue\n            if len(x[name].size()) == 4:\n                B, C, H, W = x[name].size()\n                x[name] = x[name].view(B, C, H*W).permute(0, 2, 1).contiguous()\n            C = x[name].size(-1)\n            if self.fpn_size is None:\n                logits[name] = getattr(self, \"classifier_l_\"+name)(x[name])\n            \n            probs = torch.softmax(logits[name], dim=-1)\n            sum_probs = torch.softmax(logits[name].mean(1), dim=-1)\n            selections[name] = []\n            preds_1 = []\n            preds_0 = []\n            num_select = self.num_select[name]\n            for bi in range(logits[name].size(0)):\n                _, max_ids = torch.max(sum_probs[bi], dim=-1)\n                confs, ranks = torch.sort(probs[bi, :, max_ids], descending=True)\n                sf = x[name][bi][ranks[:num_select]]\n                nf = x[name][bi][ranks[num_select:]]  # calculate\n                selections[name].append(sf) # [num_selected, C]\n                preds_1.append(logits[name][bi][ranks[:num_select]])\n                preds_0.append(logits[name][bi][ranks[num_select:]])\n\n                if bi >= len(self.thresholds[name]):\n                    self.thresholds[name].append(confs[num_select]) # for initialize\n                else:\n                    self.thresholds[name][bi] = confs[num_select]\n            \n            selections[name] = torch.stack(selections[name])\n            preds_1 = torch.stack(preds_1)\n            preds_0 = torch.stack(preds_0)\n\n            logits[\"select_\"+name] = preds_1\n            logits[\"drop_\"+name] = preds_0\n\n        return selections\n    \nclass FPN(nn.Module):\n\n    def __init__(self, inputs: dict, fpn_size: int, proj_type: str, upsample_type: str):\n        \"\"\"\n        inputs : dictionary contains torch.Tensor\n                 which comes from backbone output\n        fpn_size: integer, fpn \n        proj_type: \n            in [\"Conv\", \"Linear\"]\n        upsample_type:\n            in [\"Bilinear\", \"Conv\", \"Fc\"]\n            for convolution neural network (e.g. ResNet, EfficientNet), recommand 'Bilinear'. \n            for Vit, \"Fc\". and Swin-T, \"Conv\"\n        \"\"\"\n        super(FPN, self).__init__()\n        assert proj_type in [\"Conv\", \"Linear\"], \\\n            \"FPN projection type {} were not support yet, please choose type 'Conv' or 'Linear'\".format(proj_type)\n        assert upsample_type in [\"Bilinear\", \"Conv\"], \\\n            \"FPN upsample type {} were not support yet, please choose type 'Bilinear' or 'Conv'\".format(proj_type)\n\n        self.fpn_size = fpn_size\n        self.upsample_type = upsample_type\n        inp_names = [name for name in inputs]\n        for i, node_name in enumerate(inputs):\n            ### projection module\n            if proj_type == \"Conv\":\n                m = nn.Sequential(\n                    nn.Conv2d(inputs[node_name].size(1), inputs[node_name].size(1), 1),\n                    nn.ReLU(),\n                    nn.Conv2d(inputs[node_name].size(1), fpn_size, 1)\n                )\n            elif proj_type == \"Linear\":\n                m = nn.Sequential(\n                    nn.Linear(inputs[node_name].size(-1), inputs[node_name].size(-1)),\n                    nn.ReLU(),\n                    nn.Linear(inputs[node_name].size(-1), fpn_size),\n                )\n            self.add_module(\"Proj_\"+node_name, m)\n\n            ### upsample module\n            if upsample_type == \"Conv\" and i != 0:\n                assert len(inputs[node_name].size()) == 3 # B, S, C\n                in_dim = inputs[node_name].size(1)\n                out_dim = inputs[inp_names[i-1]].size(1)\n                # if in_dim != out_dim:\n                m = nn.Conv1d(in_dim, out_dim, 1) # for spatial domain\n                # else:\n                #     m = nn.Identity()\n                self.add_module(\"Up_\"+node_name, m)\n\n        if upsample_type == \"Bilinear\":\n            self.upsample = nn.Upsample(scale_factor=2, mode='bilinear')\n\n    def upsample_add(self, x0: torch.Tensor, x1: torch.Tensor, x1_name: str):\n        \"\"\"\n        return Upsample(x1) + x1\n        \"\"\"\n        if self.upsample_type == \"Bilinear\":\n            if x1.size(-1) != x0.size(-1):\n                x1 = self.upsample(x1)\n        else:\n            x1 = getattr(self, \"Up_\"+x1_name)(x1)\n        return x1 + x0\n\n    def forward(self, x):\n        \"\"\"\n        x : dictionary\n            {\n                \"node_name1\": feature1,\n                \"node_name2\": feature2, ...\n            }\n        \"\"\"\n        ### project to same dimension\n        hs = []\n        for i, name in enumerate(x):\n            if \"FPN1_\" in name:\n                continue\n            x[name] = getattr(self, \"Proj_\"+name)(x[name])\n            hs.append(name)\n        \n        x[\"FPN1_\" + \"layer4\"] = x[\"layer4\"]\n\n        for i in range(len(hs)-1, 0, -1):\n            x1_name = hs[i]\n            x0_name = hs[i-1]\n            x[x0_name] = self.upsample_add(x[x0_name], \n                                           x[x1_name], \n                                           x1_name)\n            x[\"FPN1_\" + x0_name] = x[x0_name]\n\n        return x\n\nclass FPN_UP(nn.Module):\n\n    def __init__(self, \n                 inputs: dict, \n                 fpn_size: int):\n        super(FPN_UP, self).__init__()\n\n        inp_names = [name for name in inputs]\n\n        for i, node_name in enumerate(inputs):\n            ### projection module\n            m = nn.Sequential(\n                nn.Linear(fpn_size, fpn_size),\n                nn.ReLU(),\n                nn.Linear(fpn_size, fpn_size),\n            )\n            self.add_module(\"Proj_\"+node_name, m)\n\n            ### upsample module\n            if i != (len(inputs) - 1):\n                assert len(inputs[node_name].size()) == 3 # B, S, C\n                in_dim = inputs[node_name].size(1)\n                out_dim = inputs[inp_names[i+1]].size(1)\n                m = nn.Conv1d(in_dim, out_dim, 1) # for spatial domain\n                self.add_module(\"Down_\"+node_name, m)\n                \"\"\"\n                Down_layer1 2304 576\n                Down_layer2 576 144\n                Down_layer3 144 144\n                \"\"\"\n\n    def downsample_add(self, x0: torch.Tensor, x1: torch.Tensor, x0_name: str):\n        \"\"\"\n        return Upsample(x1) + x1\n        \"\"\"\n        # print(\"[downsample_add] Down_\" + x0_name)\n        x0 = getattr(self, \"Down_\" + x0_name)(x0)\n        return x1 + x0\n\n    def forward(self, x):\n        \"\"\"\n        x : dictionary\n            {\n                \"node_name1\": feature1,\n                \"node_name2\": feature2, ...\n            }\n        \"\"\"\n        ### project to same dimension\n        hs = []\n        for i, name in enumerate(x):\n            if \"FPN1_\" in name:\n                continue\n            x[name] = getattr(self, \"Proj_\"+name)(x[name])\n            hs.append(name)\n\n        # print(hs)\n        for i in range(0, len(hs) - 1):\n            x0_name = hs[i]\n            x1_name = hs[i+1]\n            # print(x0_name, x1_name)\n            # print(x[x0_name].size(), x[x1_name].size())\n            x[x1_name] = self.downsample_add(x[x0_name], \n                                             x[x1_name], \n                                             x0_name)\n        return x\n\nclass PluginMoodel(nn.Module):\n\n    def __init__(self, \n                 backbone: torch.nn.Module,\n                 return_nodes: Union[dict, None],\n                 img_size: int,\n                 use_fpn: bool,\n                 fpn_size: Union[int, None],\n                 proj_type: str,\n                 upsample_type: str,\n                 use_selection: bool,\n                 num_classes: int,\n                 num_selects: dict, \n                 use_combiner: bool,\n                 comb_proj_size: Union[int, None]\n                 ):\n        \"\"\"\n        * backbone: \n            torch.nn.Module class (recommand pretrained on ImageNet or IG-3.5B-17k(provided by FAIR))\n        * return_nodes:\n            e.g.\n            return_nodes = {\n                # node_name: user-specified key for output dict\n                'layer1.2.relu_2': 'layer1',\n                'layer2.3.relu_2': 'layer2',\n                'layer3.5.relu_2': 'layer3',\n                'layer4.2.relu_2': 'layer4',\n            } # you can see the example on https://pytorch.org/vision/main/feature_extraction.html\n            !!! if using 'Swin-Transformer', please set return_nodes to None\n            !!! and please set use_fpn to True\n        * feat_sizes: \n            tuple or list contain features map size of each layers. \n            ((C, H, W)). e.g. ((1024, 14, 14), (2048, 7, 7))\n        * use_fpn: \n            boolean, use features pyramid network or not\n        * fpn_size: \n            integer, features pyramid network projection dimension\n        * num_selects:\n            num_selects = {\n                # match user-specified in return_nodes\n                \"layer1\": 2048,\n                \"layer2\": 512,\n                \"layer3\": 128,\n                \"layer4\": 32,\n            }\n        Note: after selector module (WeaklySelector) , the feature map's size is [B, S', C] which \n        contained by 'logits' or 'selections' dictionary (S' is selection number, different layer \n        could be different).\n        \"\"\"\n        super(PluginMoodel, self).__init__()\n        \n        ### = = = = = Backbone = = = = =\n        self.return_nodes = return_nodes\n        if return_nodes is not None:\n            self.backbone = create_feature_extractor(backbone, return_nodes=return_nodes)\n        else:\n            self.backbone = backbone\n        \n        ### get hidden feartues size\n        rand_in = torch.randn(1, 3, img_size, img_size)\n        outs = self.backbone(rand_in)\n        # print('outs')\n        # print(outs)\n\n        ### just original backbone\n        if not use_fpn and (not use_selection and not use_combiner):\n            for name in outs:\n                fs_size = outs[name].size()\n                if len(fs_size) == 3:\n                    out_size = fs_size.size(-1)\n                elif len(fs_size) == 4:\n                    out_size = fs_size.size(1)\n                else:\n                    raise ValueError(\"The size of output dimension of previous must be 3 or 4.\")\n            self.classifier = nn.Linear(out_size, num_classes)\n\n        ### = = = = = FPN = = = = =\n        self.use_fpn = use_fpn\n        if self.use_fpn:\n            self.fpn_down = FPN(outs, fpn_size, proj_type, upsample_type)\n            self.build_fpn_classifier_down(outs, fpn_size, num_classes)\n            self.fpn_up = FPN_UP(outs, fpn_size)\n            self.build_fpn_classifier_up(outs, fpn_size, num_classes)\n\n        self.fpn_size = fpn_size\n\n        ### = = = = = Selector = = = = =\n        self.use_selection = use_selection\n        if self.use_selection:\n            w_fpn_size = self.fpn_size if self.use_fpn else None # if not using fpn, build classifier in weakly selector\n            self.selector = WeaklySelector(outs, num_classes, num_selects, w_fpn_size)\n\n        ### = = = = = Combiner = = = = =\n        self.use_combiner = use_combiner\n        if self.use_combiner:\n            assert self.use_selection, \"Please use selection module before combiner\"\n            if self.use_fpn:\n                gcn_inputs, gcn_proj_size = None, None\n            else:\n                gcn_inputs, gcn_proj_size = outs, comb_proj_size # redundant, fix in future\n            total_num_selects = sum([num_selects[name] for name in num_selects]) # sum\n            self.combiner = GCNCombiner(total_num_selects, num_classes, gcn_inputs, gcn_proj_size, self.fpn_size)\n\n    def build_fpn_classifier_up(self, inputs: dict, fpn_size: int, num_classes: int):\n        \"\"\"\n        Teh results of our experiments show that linear classifier in this case may cause some problem.\n        \"\"\"\n        for name in inputs:\n            m = nn.Sequential(\n                    nn.Conv1d(fpn_size, fpn_size, 1),\n                    nn.BatchNorm1d(fpn_size),\n                    nn.ReLU(),\n                    nn.Conv1d(fpn_size, num_classes, 1)\n                )\n            self.add_module(\"fpn_classifier_up_\"+name, m)\n\n    def build_fpn_classifier_down(self, inputs: dict, fpn_size: int, num_classes: int):\n        \"\"\"\n        Teh results of our experiments show that linear classifier in this case may cause some problem.\n        \"\"\"\n        for name in inputs:\n            m = nn.Sequential(\n                    nn.Conv1d(fpn_size, fpn_size, 1),\n                    nn.BatchNorm1d(fpn_size),\n                    nn.ReLU(),\n                    nn.Conv1d(fpn_size, num_classes, 1)\n                )\n            self.add_module(\"fpn_classifier_down_\" + name, m)\n\n    def forward_backbone(self, x):\n        return self.backbone(x)\n\n    def fpn_predict_down(self, x: dict, logits: dict):\n        \"\"\"\n        x: [B, C, H, W] or [B, S, C]\n           [B, C, H, W] --> [B, H*W, C]\n        \"\"\"\n        for name in x:\n            if \"FPN1_\" not in name:\n                continue \n            ### predict on each features point\n            if len(x[name].size()) == 4:\n                B, C, H, W = x[name].size()\n                logit = x[name].view(B, C, H*W)\n            elif len(x[name].size()) == 3:\n                logit = x[name].transpose(1, 2).contiguous()\n            model_name = name.replace(\"FPN1_\", \"\")\n            logits[name] = getattr(self, \"fpn_classifier_down_\" + model_name)(logit)\n            logits[name] = logits[name].transpose(1, 2).contiguous() # transpose\n\n    def fpn_predict_up(self, x: dict, logits: dict):\n        \"\"\"\n        x: [B, C, H, W] or [B, S, C]\n           [B, C, H, W] --> [B, H*W, C]\n        \"\"\"\n        for name in x:\n            if \"FPN1_\" in name:\n                continue\n            ### predict on each features point\n            if len(x[name].size()) == 4:\n                B, C, H, W = x[name].size()\n                logit = x[name].view(B, C, H*W)\n            elif len(x[name].size()) == 3:\n                logit = x[name].transpose(1, 2).contiguous()\n            model_name = name.replace(\"FPN1_\", \"\")\n            logits[name] = getattr(self, \"fpn_classifier_up_\" + model_name)(logit)\n            logits[name] = logits[name].transpose(1, 2).contiguous() # transpose\n\n    def forward(self, x: torch.Tensor):\n\n        logits = {}\n\n        x = self.forward_backbone(x)\n\n        if self.use_fpn:\n            x = self.fpn_down(x)\n            # print([name for name in x])\n            self.fpn_predict_down(x, logits)\n            x = self.fpn_up(x)\n            self.fpn_predict_up(x, logits)\n\n        if self.use_selection:\n            selects = self.selector(x, logits)\n\n        if self.use_combiner:\n            comb_outs = self.combiner(selects)\n            logits['comb_outs'] = comb_outs\n            return logits\n        \n        if self.use_selection or self.fpn:\n            return logits\n\n        ### original backbone (only predict final selected layer)\n        for name in x:\n            hs = x[name]\n\n        if len(hs.size()) == 4:\n            hs = F.adaptive_avg_pool2d(hs, (1, 1))\n            hs = hs.flatten(1)\n        else:\n            hs = hs.mean(1)\n        out = self.classifier(hs)\n        logits['ori_out'] = logits\n\n        return logits\n\n@torch.no_grad()\ndef sum_all_out(out, sum_type=\"softmax\"):\n    target_layer_names = \\\n    ['layer1', 'layer2', 'layer3', 'layer4',\n    'FPN1_layer1', 'FPN1_layer2', 'FPN1_layer3', 'FPN1_layer4', \n    'comb_outs']\n\n    sum_out = None\n    for name in target_layer_names:\n        if name != \"comb_outs\":\n            tmp_out = out[name].mean(1)\n        else:\n            tmp_out = out[name]\n        \n        if sum_type == \"softmax\":\n            tmp_out = torch.softmax(tmp_out, dim=-1)\n        if sum_out is None:\n            sum_out = tmp_out\n        else:\n            sum_out = sum_out + tmp_out # note that use '+=' would cause inplace error\n    return sum_out\n\ndef build_model(pretrainewd_path: str,\n                img_size: int, \n                fpn_size: int, \n                num_classes: int,\n                num_selects: dict,\n                use_fpn: bool = True, \n                use_selection: bool = True,\n                use_combiner: bool = True, \n                comb_proj_size: int = None):\n    model = \\\n        PluginMoodel(backbone=timm.create_model('swin_large_patch4_window12_384_in22k'),\n                     return_nodes=None,\n                     img_size = img_size,\n                     use_fpn = use_fpn,\n                     fpn_size = fpn_size,\n                     proj_type = \"Linear\",\n                     upsample_type = \"Conv\",\n                     use_selection = use_selection,\n                     num_classes = num_classes,\n                     num_selects = num_selects, \n                     use_combiner = use_combiner,\n                     comb_proj_size = comb_proj_size)\n\n\n    return model\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-07T09:44:46.508997Z","iopub.execute_input":"2025-05-07T09:44:46.509501Z","iopub.status.idle":"2025-05-07T09:44:46.557925Z","shell.execute_reply.started":"2025-05-07T09:44:46.509473Z","shell.execute_reply":"2025-05-07T09:44:46.557210Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"used_models_pytorch = {\"HERB\": [f'../input/5-weights/backup/fold_{fold}_best.pt' for fold in [1,2,3,4,5]],}\nprint(used_models_pytorch['HERB'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-07T09:44:46.558860Z","iopub.execute_input":"2025-05-07T09:44:46.559074Z","iopub.status.idle":"2025-05-07T09:44:46.574980Z","shell.execute_reply.started":"2025-05-07T09:44:46.559050Z","shell.execute_reply":"2025-05-07T09:44:46.574470Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path = \"../input/cassava-leaf-disease-classification/\"\nimage_path = path+\"test_images/\"\n\nsubmission_df = pd.DataFrame(columns=[\"image_id\",\"label\"])\nsubmission_df[\"image_id\"] = os.listdir(image_path)\nsubmission_df[\"label\"] = 0\n\ndef inference(model, states, test_loader, device):\n        model.to(device)\n\n        probabilities = []\n        for i, (_, images, _) in enumerate(test_loader):\n            images = images.to(device)\n            avg_preds = []\n            for state in states:\n                model.load_state_dict(state['model'])\n                model.eval()\n                with torch.no_grad():\n                    outs = model(images)\n                    outs = sum_all_out(outs, sum_type=\"softmax\") # softmax\n                    # print(outs.shape)\n                avg_preds.append(outs.to('cpu').numpy())\n            avg_preds = np.mean(avg_preds, axis=0)\n            probabilities.append(avg_preds)\n        return np.concatenate(probabilities)\n    \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-07T09:44:46.575646Z","iopub.execute_input":"2025-05-07T09:44:46.575878Z","iopub.status.idle":"2025-05-07T09:44:46.614277Z","shell.execute_reply.started":"2025-05-07T09:44:46.575861Z","shell.execute_reply":"2025-05-07T09:44:46.613744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = build_model(\n    pretrainewd_path='',\n    img_size=384,\n    fpn_size=1536,\n    num_classes=5,\n    num_selects={'layer1': 256, 'layer2': 128, 'layer3': 64,'layer4': 32}\n)\n\nmodel.to(device)\n\nstates = [torch.load(f, weights_only=False, map_location=\"cuda:0\") for f in used_models_pytorch[\"HERB\"]]\n\n# 4. Predict on test data\ntest_df = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\ntest_dataset = CassavaDataset('../input/cassava-leaf-disease-classification/sample_submission.csv', '../input/cassava-leaf-disease-classification/test_images', transform=val_transforms)\ntest_loader = DataLoader(test_dataset, batch_size=16, shuffle=False)\n\n# myDataloader = DataLoader(myCassavaDataset, batch_size=32, shuffle=True, num_workers=4)\n\n# Predict on test data\npredictions = []\nfor _, images, _ in test_loader:\n    images = images.to(device)\n    avg_preds = []\n    for state in states:\n        model.load_state_dict(state['model'])\n        model.eval()\n        with torch.no_grad():\n            outs = model(images)\n            outs = sum_all_out(outs, sum_type=\"softmax\") # softmax\n        avg_preds.append(outs.to('cpu').numpy())\n    avg_preds = np.mean(avg_preds, axis=0)\n    avg_preds = torch.tensor(avg_preds)  \n    _, preds = torch.max(avg_preds, 1)\n    predictions.extend(preds.cpu().numpy())\n    \n# Convert predictions to a NumPy array\npredictions = np.array(predictions)\n\n# 5. Create submission file\ntest_df['label'] = predictions\ntest_df.to_csv('submission.csv', index=False)\nprint(\"Submission file created!\")\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-07T09:44:46.614885Z","iopub.execute_input":"2025-05-07T09:44:46.615160Z","iopub.status.idle":"2025-05-07T09:45:54.669203Z","shell.execute_reply.started":"2025-05-07T09:44:46.615137Z","shell.execute_reply":"2025-05-07T09:45:54.668412Z"}},"outputs":[],"execution_count":null}]}