{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"}],"dockerImageVersionId":30887,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport torch\nimport torch.nn as nn\nimport torchvision.transforms as transforms\nfrom torchvision import models\nfrom torch.utils.data import Dataset, DataLoader\nimport timm  # For advanced models\nimport cv2\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, f1_score, roc_auc_score\nimport torch.optim as optim\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-02-07T12:59:17.145692Z","iopub.execute_input":"2025-02-07T12:59:17.146052Z","iopub.status.idle":"2025-02-07T12:59:31.953973Z","shell.execute_reply.started":"2025-02-07T12:59:17.146020Z","shell.execute_reply":"2025-02-07T12:59:31.952963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define dataset path\nDATA_DIR = \"/kaggle/input/aptos2019-blindness-detection\"\n\nprint(\"Loading dataset...\")\ndf = pd.read_csv(os.path.join(DATA_DIR, \"train.csv\"))\ndf[\"diagnosis\"] = df[\"diagnosis\"].astype(int)\n\n# Split dataset\nprint(\"Splitting dataset...\")\ntrain_df, val_df = train_test_split(df, test_size=0.2, random_state=42, stratify=df[\"diagnosis\"])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T12:59:31.955145Z","iopub.execute_input":"2025-02-07T12:59:31.955635Z","iopub.status.idle":"2025-02-07T12:59:31.997217Z","shell.execute_reply.started":"2025-02-07T12:59:31.955609Z","shell.execute_reply":"2025-02-07T12:59:31.996490Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class DRDataset(Dataset):\n    def __init__(self, dataframe, data_dir, transform=None):\n        self.dataframe = dataframe\n        self.data_dir = data_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join(self.data_dir, \"train_images\", self.dataframe.iloc[idx, 0] + \".png\")\n        image = cv2.imread(img_name)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (224, 224))\n\n        if self.transform:\n            image = self.transform(image)\n\n        label = self.dataframe.iloc[idx, 1]\n        return image, label\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T12:59:31.998553Z","iopub.execute_input":"2025-02-07T12:59:31.998799Z","iopub.status.idle":"2025-02-07T12:59:32.003954Z","shell.execute_reply.started":"2025-02-07T12:59:31.998779Z","shell.execute_reply":"2025-02-07T12:59:32.003074Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Applying transformations...\")\ntransform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation(20),\n    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n    transforms.RandomResizedCrop(224, scale=(0.8, 1.0)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\nprint(\"Creating datasets and dataloaders...\")\ntrain_dataset = DRDataset(train_df, DATA_DIR, transform=transform)\nval_dataset = DRDataset(val_df, DATA_DIR, transform=transform)\n\ntrain_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=16, shuffle=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T12:59:32.005074Z","iopub.execute_input":"2025-02-07T12:59:32.005369Z","iopub.status.idle":"2025-02-07T12:59:32.021785Z","shell.execute_reply.started":"2025-02-07T12:59:32.005338Z","shell.execute_reply":"2025-02-07T12:59:32.021205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Initializing models...\")\nclass DRModel(nn.Module):\n    def __init__(self, model_name):\n        super(DRModel, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=True, num_classes=5)\n        self.dropout = nn.Dropout(0.4)  # Added dropout to reduce overfitting\n    \n    def forward(self, x):\n        x = self.model(x)\n        x = self.dropout(x)\n        return x\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T12:59:32.022577Z","iopub.execute_input":"2025-02-07T12:59:32.022781Z","iopub.status.idle":"2025-02-07T12:59:32.044345Z","shell.execute_reply.started":"2025-02-07T12:59:32.022763Z","shell.execute_reply":"2025-02-07T12:59:32.043467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model1 = DRModel(\"swin_large_patch4_window7_224\").cuda()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T12:59:32.368972Z","iopub.execute_input":"2025-02-07T12:59:32.369282Z","iopub.status.idle":"2025-02-07T12:59:40.950401Z","shell.execute_reply.started":"2025-02-07T12:59:32.369258Z","shell.execute_reply":"2025-02-07T12:59:40.949625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model2 = DRModel(\"tf_efficientnet_b7\").cuda()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T12:59:40.951363Z","iopub.execute_input":"2025-02-07T12:59:40.951577Z","iopub.status.idle":"2025-02-07T12:59:43.834383Z","shell.execute_reply.started":"2025-02-07T12:59:40.951558Z","shell.execute_reply":"2025-02-07T12:59:43.833662Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model3 = DRModel(\"convnext_large\").cuda()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T12:59:43.835870Z","iopub.execute_input":"2025-02-07T12:59:43.836206Z","iopub.status.idle":"2025-02-07T12:59:51.422299Z","shell.execute_reply.started":"2025-02-07T12:59:43.836173Z","shell.execute_reply":"2025-02-07T12:59:51.421100Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Starting training...\")\n\ndef train_model(model, train_loader, val_loader, epochs=10):\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.AdamW(model.parameters(), lr=0.00005, weight_decay=1e-4)  # Lower LR and weight decay\n    best_val_loss = float('inf')\n    patience = 3\n    early_stop_count = 0\n    \n    for epoch in range(epochs):\n        model.train()\n        train_loss = 0\n        correct = 0\n        total = 0\n\n        for images, labels in train_loader:\n            images, labels = images.cuda(), labels.cuda()\n            optimizer.zero_grad()\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n\n            train_loss += loss.item()\n            _, predicted = outputs.max(1)\n            total += labels.size(0)\n            correct += predicted.eq(labels).sum().item()\n        \n        val_loss = 0\n        model.eval()\n        with torch.no_grad():\n            for images, labels in val_loader:\n                images, labels = images.cuda(), labels.cuda()\n                outputs = model(images)\n                loss = criterion(outputs, labels)\n                val_loss += loss.item()\n        \n        avg_train_loss = train_loss / len(train_loader)\n        avg_val_loss = val_loss / len(val_loader)\n        print(f\"Epoch {epoch+1}, Train Loss: {avg_train_loss:.4f}, Validation Loss: {avg_val_loss:.4f}, Accuracy: {100*correct/total:.2f}%\")\n        \n        # Early Stopping\n        if avg_val_loss < best_val_loss:\n            best_val_loss = avg_val_loss\n            early_stop_count = 0\n        else:\n            early_stop_count += 1\n            if early_stop_count >= patience:\n                print(\"Early stopping triggered.\")\n                break\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T12:59:51.423777Z","iopub.execute_input":"2025-02-07T12:59:51.424154Z","iopub.status.idle":"2025-02-07T12:59:51.434184Z","shell.execute_reply.started":"2025-02-07T12:59:51.424118Z","shell.execute_reply":"2025-02-07T12:59:51.433351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_model(model1, train_loader, val_loader)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T12:59:58.288390Z","iopub.execute_input":"2025-02-07T12:59:58.288763Z","iopub.status.idle":"2025-02-07T14:51:39.026376Z","shell.execute_reply.started":"2025-02-07T12:59:58.288736Z","shell.execute_reply":"2025-02-07T14:51:39.025328Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def test_model(model, dataloader):\n    model.eval()  # Set to evaluation mode\n    total_preds = []\n    total_labels = []\n    \n    with torch.no_grad():\n        for images, labels in dataloader:\n            images = images.cuda()\n            outputs = model(images)\n            preds = torch.argmax(outputs, dim=1).cpu().numpy()\n            total_preds.extend(preds)\n            total_labels.extend(labels.numpy())\n\n    accuracy = accuracy_score(total_labels, total_preds)\n    f1 = f1_score(total_labels, total_preds, average='weighted')\n    print(f\"Model Test Accuracy: {accuracy:.4f}, F1-score: {f1:.4f}\")\n    return accuracy, f1\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T15:11:24.092432Z","iopub.execute_input":"2025-02-07T15:11:24.092726Z","iopub.status.idle":"2025-02-07T15:11:24.097774Z","shell.execute_reply.started":"2025-02-07T15:11:24.092703Z","shell.execute_reply":"2025-02-07T15:11:24.097070Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Testing Model 1 (Swin Large)...\")\nacc1, f1_1 = test_model(model1, val_loader)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T15:11:27.001092Z","iopub.execute_input":"2025-02-07T15:11:27.001363Z","iopub.status.idle":"2025-02-07T15:13:03.474461Z","shell.execute_reply.started":"2025-02-07T15:11:27.001341Z","shell.execute_reply":"2025-02-07T15:13:03.473592Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_model(model2, train_loader, val_loader)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T15:17:48.248064Z","iopub.execute_input":"2025-02-07T15:17:48.248447Z","iopub.status.idle":"2025-02-07T16:43:20.072323Z","shell.execute_reply.started":"2025-02-07T15:17:48.248417Z","shell.execute_reply":"2025-02-07T16:43:20.071525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Testing Model 2 (EfficientNet B7)...\")\nacc2, f1_2 = test_model(model2, val_loader)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T16:44:29.937322Z","iopub.execute_input":"2025-02-07T16:44:29.937632Z","iopub.status.idle":"2025-02-07T16:45:55.417597Z","shell.execute_reply.started":"2025-02-07T16:44:29.937604Z","shell.execute_reply":"2025-02-07T16:45:55.416816Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_model(model3, train_loader, val_loader)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T16:45:55.418800Z","iopub.execute_input":"2025-02-07T16:45:55.419105Z","iopub.status.idle":"2025-02-07T18:34:51.964534Z","shell.execute_reply.started":"2025-02-07T16:45:55.419083Z","shell.execute_reply":"2025-02-07T18:34:51.963553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Testing Model 3 (ConvNeXt Large)...\")\nacc3, f1_3 = test_model(model3, val_loader)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T18:36:39.429152Z","iopub.execute_input":"2025-02-07T18:36:39.429474Z","iopub.status.idle":"2025-02-07T18:38:17.237232Z","shell.execute_reply.started":"2025-02-07T18:36:39.429450Z","shell.execute_reply":"2025-02-07T18:38:17.236345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Starting weighted fusion prediction...\")\n\ndef weighted_fusion(models, weights, dataloader):\n    models = [m.eval() for m in models]\n    total_preds = []\n    total_labels = []\n    \n    with torch.no_grad():\n        for images, labels in dataloader:\n            images = images.cuda()\n            outputs = [m(images) for m in models]\n            fused_output = sum(w * o for w, o in zip(weights, outputs)) / sum(weights)\n            preds = torch.argmax(fused_output, dim=1).cpu().numpy()\n            total_preds.extend(preds)\n            total_labels.extend(labels.numpy())\n    \n    accuracy = accuracy_score(total_labels, total_preds)\n    f1 = f1_score(total_labels, total_preds, average='weighted')\n    print(f\"Fusion Accuracy: {accuracy:.4f}, F1-score: {f1:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T18:38:30.162147Z","iopub.execute_input":"2025-02-07T18:38:30.162426Z","iopub.status.idle":"2025-02-07T18:38:30.169003Z","shell.execute_reply.started":"2025-02-07T18:38:30.162403Z","shell.execute_reply":"2025-02-07T18:38:30.168114Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"weights = [0.5, 0.0, 0.5]  # Adjusted based on validation performance\nweighted_fusion([model1, model2, model3], weights, val_loader)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T18:58:22.946535Z","iopub.execute_input":"2025-02-07T18:58:22.946854Z","iopub.status.idle":"2025-02-07T19:00:33.333020Z","shell.execute_reply.started":"2025-02-07T18:58:22.946810Z","shell.execute_reply":"2025-02-07T19:00:33.331921Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\n\n# Save individual model states and fusion weights\ntorch.save({\n    'model1_state_dict': model1.state_dict(),\n    'model2_state_dict': model2.state_dict(),\n    'model3_state_dict': model3.state_dict(),\n    'fusion_weights': weights\n}, \"/kaggle/working/weighted_fusion_model.pth\")\n\nprint(\"Weighted fusion model and weights saved successfully at /kaggle/working/weighted_fusion_model.pth\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T19:18:16.066403Z","iopub.execute_input":"2025-02-07T19:18:16.066717Z","iopub.status.idle":"2025-02-07T19:18:18.427728Z","shell.execute_reply.started":"2025-02-07T19:18:16.066693Z","shell.execute_reply":"2025-02-07T19:18:18.427017Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load the saved model and weights\ncheckpoint = torch.load(\"/kaggle/working/weighted_fusion_model.pth\")\n\n# Restore model states\nmodel1.load_state_dict(checkpoint['model1_state_dict'])\nmodel2.load_state_dict(checkpoint['model2_state_dict'])\nmodel3.load_state_dict(checkpoint['model3_state_dict'])\n\n# Restore fusion weights\nweights = checkpoint['fusion_weights']\n\nprint(\"Weighted fusion model and weights loaded successfully!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T19:18:38.945408Z","iopub.execute_input":"2025-02-07T19:18:38.945690Z","iopub.status.idle":"2025-02-07T19:18:40.530897Z","shell.execute_reply.started":"2025-02-07T19:18:38.945668Z","shell.execute_reply":"2025-02-07T19:18:40.530166Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save all three models\nmodel1_path = \"/kaggle/working/model1.pth\"\nmodel2_path = \"/kaggle/working/model2.pth\"\nmodel3_path = \"/kaggle/working/model3.pth\"\n\ntorch.save(model1.state_dict(), model1_path)\ntorch.save(model2.state_dict(), model2_path)\ntorch.save(model3.state_dict(), model3_path)\n\n# Download models to local computer\nfrom IPython.display import FileLink\n\ndisplay(FileLink(model1_path))\ndisplay(FileLink(model2_path))\ndisplay(FileLink(model3_path))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-07T19:19:00.957984Z","iopub.execute_input":"2025-02-07T19:19:00.958267Z","iopub.status.idle":"2025-02-07T19:19:03.590584Z","shell.execute_reply.started":"2025-02-07T19:19:00.958246Z","shell.execute_reply":"2025-02-07T19:19:03.589915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}