{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.9","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":13836,"databundleVersionId":1718836,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":8232278,"sourceType":"datasetVersion","datasetId":1047930},{"sourceId":411083,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":335625,"modelId":356640}],"dockerImageVersionId":30061,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nOUTPUT_DIR = \"./\"\nTEST_PATH = '../input/cassava-leaf-disease-classification/test_images'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:43.586739Z","iopub.execute_input":"2025-05-26T02:44:43.587046Z","iopub.status.idle":"2025-05-26T02:44:43.590732Z","shell.execute_reply.started":"2025-05-26T02:44:43.587020Z","shell.execute_reply":"2025-05-26T02:44:43.589840Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nsys.path.append('../input/cassava_models/')\nsys.path.append('../input/pytorchimagemodels')\n\nimport time\nimport random\nfrom functools import partial\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam\nimport torch.nn.functional as F\nfrom torch.nn import BCEWithLogitsLoss, CrossEntropyLoss\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, ReduceLROnPlateau, LambdaLR\nfrom torch.utils.data import DataLoader, Dataset\nfrom albumentations import (\n    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n    IAAAdditiveGaussianNoise, Transpose, HueSaturationValue, \n    )\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\nfrom timm.models.vision_transformer import VisionTransformer\n\nimport warnings \nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:43.596341Z","iopub.execute_input":"2025-05-26T02:44:43.596658Z","iopub.status.idle":"2025-05-26T02:44:43.604171Z","shell.execute_reply.started":"2025-05-26T02:44:43.596634Z","shell.execute_reply":"2025-05-26T02:44:43.603468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:43.611177Z","iopub.execute_input":"2025-05-26T02:44:43.611438Z","iopub.status.idle":"2025-05-26T02:44:43.615272Z","shell.execute_reply.started":"2025-05-26T02:44:43.611386Z","shell.execute_reply":"2025-05-26T02:44:43.614569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed_torch(seed=1006):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_torch()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:43.621217Z","iopub.execute_input":"2025-05-26T02:44:43.621439Z","iopub.status.idle":"2025-05-26T02:44:43.625940Z","shell.execute_reply.started":"2025-05-26T02:44:43.621419Z","shell.execute_reply":"2025-05-26T02:44:43.625159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\ntest.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:43.637888Z","iopub.execute_input":"2025-05-26T02:44:43.638131Z","iopub.status.idle":"2025-05-26T02:44:43.650645Z","shell.execute_reply.started":"2025-05-26T02:44:43.638108Z","shell.execute_reply":"2025-05-26T02:44:43.649613Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:43.653051Z","iopub.execute_input":"2025-05-26T02:44:43.653281Z","iopub.status.idle":"2025-05-26T02:44:43.661368Z","shell.execute_reply.started":"2025-05-26T02:44:43.653258Z","shell.execute_reply":"2025-05-26T02:44:43.660675Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_transforms(*, data, vit=False):\n    \n    if vit:\n        MEAN = [0.5, 0.5, 0.5]\n        STD = [0.5, 0.5, 0.5]\n    else:\n        MEAN = [0.485, 0.456, 0.406]\n        STD = [0.229, 0.224, 0.225]\n    \n    if data == 'train':\n        return Compose([\n            RandomResizedCrop(IMG_SIZE, IMG_SIZE),\n            #RandomCrop(IMG_SIZE, IMG_SIZE),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            #HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            #RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            #Resize(IMG_SIZE, IMG_SIZE),\n            Normalize(\n                mean=MEAN,\n                std=STD,\n            ),\n            ToTensorV2(),\n        ])\n\n    elif data == 'valid':\n        return Compose([\n            Resize(IMG_SIZE, IMG_SIZE),\n            Normalize(\n                mean=MEAN,\n                std=STD,\n            ),\n            ToTensorV2(),\n        ])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:43.663127Z","iopub.execute_input":"2025-05-26T02:44:43.663334Z","iopub.status.idle":"2025-05-26T02:44:43.672913Z","shell.execute_reply.started":"2025-05-26T02:44:43.663313Z","shell.execute_reply":"2025-05-26T02:44:43.672374Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class NetVit(nn.Module):\n    def __init__(self, model_name, pretrained=False, n_class=5, att_activate=False, no_att=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.head.in_features\n        self.model.head = nn.Identity()\n        \n        if att_activate:\n            self.att_layer = nn.Sequential(\n                nn.Linear(n_features, 256),\n                nn.Tanh(),\n                nn.Linear(256, 1),\n            )\n        else:\n            if no_att:\n                pass\n            else:\n                self.att_layer = nn.Linear(n_features, 1)\n            \n        self.head = nn.Linear(n_features, n_class)\n\n    def forward(self, x):\n        x = self.model(x)\n        output = self.head(x)\n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:43.770032Z","iopub.execute_input":"2025-05-26T02:44:43.770261Z","iopub.status.idle":"2025-05-26T02:44:43.776026Z","shell.execute_reply.started":"2025-05-26T02:44:43.770242Z","shell.execute_reply":"2025-05-26T02:44:43.775315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class NetVit4(nn.Module):\n    def __init__(self, model_name, pretrained=False, n_class=5, att_activate=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.head.in_features\n        self.model.head = nn.Identity()\n        if att_activate:\n            self.att_layer = nn.Sequential(\n                nn.Linear(n_features, 256),\n                nn.Tanh(),\n                nn.Linear(256, 1),\n            )\n        else:\n            self.att_layer = nn.Linear(n_features, 1)\n            \n        self.head = nn.Linear(n_features, n_class)\n\n    def forward(self, x):\n        l = x.shape[2] // 2\n        h1 = self.model(x[:, :, :l, :l])\n        h2 = self.model(x[:, :, :l, l:])\n        h3 = self.model(x[:, :, l:, :l])\n        h4 = self.model(x[:, :, l:, l:])\n\n        a1 = self.att_layer(h1)\n        a2 = self.att_layer(h2)\n        a3 = self.att_layer(h3)\n        a4 = self.att_layer(h4)\n\n        w = F.softmax(torch.cat([a1, a2, a3, a4], dim=1), dim=1)\n\n        h = h1 * w[:, 0].unsqueeze(-1) + h2 * w[:, 1].unsqueeze(-1) + \\\n            h3 * w[:, 2].unsqueeze(-1) + h4 * w[:, 3].unsqueeze(-1)\n        output = self.head(h)\n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:43.778051Z","iopub.execute_input":"2025-05-26T02:44:43.778424Z","iopub.status.idle":"2025-05-26T02:44:43.788822Z","shell.execute_reply.started":"2025-05-26T02:44:43.778368Z","shell.execute_reply":"2025-05-26T02:44:43.787957Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from collections import OrderedDict\n\ndef inference(model, states, test_loader, device, temp=1):\n    model.to(device)\n    preds = []\n    for state in states:\n        pred = []\n        model.load_state_dict(state)\n        model.eval()\n        for i, image in enumerate(test_loader):\n            with torch.no_grad():\n                pred.append((model(image.to(device))*temp).softmax(1).to('cpu'))\n        pred = torch.cat(pred, dim=0)\n        preds.append(pred.numpy())\n    return np.mean(preds, axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:43.791119Z","iopub.execute_input":"2025-05-26T02:44:43.791460Z","iopub.status.idle":"2025-05-26T02:44:43.805276Z","shell.execute_reply.started":"2025-05-26T02:44:43.791428Z","shell.execute_reply":"2025-05-26T02:44:43.804513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def multi2single(path, se=False):\n    state_dict = torch.load(path, map_location=lambda storage, loc: storage)\n    new_state_dict = OrderedDict()\n    for k, v in state_dict.items():\n        if 'module' in k:\n            k = k.replace('se_module', 'dummy')\n            k = k.replace('module.', '')\n            k = k.replace('dummy', 'se_module')\n        if 'attention_linear' in k:\n            k = k.replace('attention_linear', 'att_layer')\n        new_state_dict[k] = v\n    return new_state_dict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:43.807211Z","iopub.execute_input":"2025-05-26T02:44:43.807603Z","iopub.status.idle":"2025-05-26T02:44:43.817562Z","shell.execute_reply.started":"2025-05-26T02:44:43.807566Z","shell.execute_reply":"2025-05-26T02:44:43.816801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temp = 1.0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:43.818753Z","iopub.execute_input":"2025-05-26T02:44:43.819125Z","iopub.status.idle":"2025-05-26T02:44:43.830122Z","shell.execute_reply.started":"2025-05-26T02:44:43.819087Z","shell.execute_reply":"2025-05-26T02:44:43.829494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"    MODEL_NAME = \"vit_base_patch16_384\"\n    MODEL_NUM = \"single_vit\"\n    MODEL_DIR = \"../input/cassava_models/pytorch/default/1/\"\n    IMG_SIZE = 384\n    TTA = 3\n    BATCH = 32\n\n    model = NetVit(MODEL_NAME, pretrained=False, no_att=True)\n    states = [multi2single(MODEL_DIR+f'{MODEL_NUM}_{fold+3}.pth') for fold in range(1)]\n    if TTA == 1:\n        test_dataset = TestDataset(test, transform=get_transforms(data='valid', vit=True))\n    else:\n        test_dataset = TestDataset(test, transform=get_transforms(data='train', vit=True))\n\n    test_loader = DataLoader(test_dataset, batch_size=BATCH, shuffle=False, num_workers=4, pin_memory=True)\n    vit_predictions = np.zeros((len(test), 5))\n    for _ in range(TTA):\n        vit_predictions += inference(model, states, test_loader, device, temp) / TTA","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:43.831069Z","iopub.execute_input":"2025-05-26T02:44:43.831301Z","iopub.status.idle":"2025-05-26T02:44:46.061680Z","shell.execute_reply.started":"2025-05-26T02:44:43.831279Z","shell.execute_reply":"2025-05-26T02:44:46.060565Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if True:\n    MODEL_NAME = \"vit_base_patch16_224\"\n    MODEL_NUM = \"single_vit4_type_A\"\n    MODEL_DIR = \"../input/cassava_models/pytorch/default/1/\"\n    IMG_SIZE = 448\n    TTA = 3\n    BATCH = 32\n    att_activate = True  # Changed to True to match pretrained weights\n\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model = NetVit4(MODEL_NAME, pretrained=False, att_activate=att_activate)    \n    states = [multi2single(MODEL_DIR+f'{MODEL_NUM}_{fold+4}.pth') for fold in range(1)]\n    if TTA == 1:\n        test_dataset = TestDataset(test, transform=get_transforms(data='valid', vit=True))\n    else:\n        test_dataset = TestDataset(test, transform=get_transforms(data='train', vit=True))\n\n    test_loader = DataLoader(test_dataset, batch_size=BATCH, shuffle=False, num_workers=4, pin_memory=True)\n    vit4_predictions_a = np.zeros((len(test), 5))\n    for _ in range(TTA):\n        vit4_predictions_a += inference(model, states, test_loader, device, temp=1.0) / TTA","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:46.063499Z","iopub.execute_input":"2025-05-26T02:44:46.063770Z","iopub.status.idle":"2025-05-26T02:44:48.376259Z","shell.execute_reply.started":"2025-05-26T02:44:46.063742Z","shell.execute_reply":"2025-05-26T02:44:48.375087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if True:\n    MODEL_NAME = \"vit_base_patch16_224\"\n    MODEL_NUM = \"single_vit4_type_B\"\n    MODEL_DIR = \"../input/cassava_models/pytorch/default/1/\"\n    IMG_SIZE = 448\n    TTA = 3\n    BATCH = 32\n    att_activate = False\n\n    model = NetVit4(MODEL_NAME, pretrained=False, att_activate=att_activate)    \n    states = [multi2single(MODEL_DIR+f'{MODEL_NUM}_{fold+4}.pth') for fold in range(1)]\n    if TTA == 1:\n        test_dataset = TestDataset(test, transform=get_transforms(data='valid', vit=True))\n    else:\n        test_dataset = TestDataset(test, transform=get_transforms(data='train', vit=True))\n\n    test_loader = DataLoader(test_dataset, batch_size=BATCH, shuffle=False, num_workers=4, pin_memory=True)\n    vit4_predictions_b = np.zeros((len(test), 5))\n    for _ in range(TTA):\n        vit4_predictions_b += inference(model, states, test_loader, device, temp) / TTA","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:48.378063Z","iopub.execute_input":"2025-05-26T02:44:48.378441Z","iopub.status.idle":"2025-05-26T02:44:50.686476Z","shell.execute_reply.started":"2025-05-26T02:44:48.378385Z","shell.execute_reply":"2025-05-26T02:44:50.685223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions = (vit_predictions * 0.45 + vit4_predictions_a * 0.55) / 9 * 10 + vit4_predictions_b * 0.08","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:50.688293Z","iopub.execute_input":"2025-05-26T02:44:50.688648Z","iopub.status.idle":"2025-05-26T02:44:50.693091Z","shell.execute_reply.started":"2025-05-26T02:44:50.688616Z","shell.execute_reply":"2025-05-26T02:44:50.692113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submission\ntest['label'] = predictions.argmax(1)\ntest[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)\ntest.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-26T02:44:50.694986Z","iopub.execute_input":"2025-05-26T02:44:50.695361Z","iopub.status.idle":"2025-05-26T02:44:50.712530Z","shell.execute_reply.started":"2025-05-26T02:44:50.695315Z","shell.execute_reply":"2025-05-26T02:44:50.711727Z"}},"outputs":[],"execution_count":null}]}