{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader, Dataset, random_split\nfrom torchvision import transforms\nfrom torchvision.models import vit_b_16\nimport pandas as pd\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nimport os","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:08:38.241371Z","iopub.execute_input":"2024-10-27T09:08:38.242320Z","iopub.status.idle":"2024-10-27T09:08:43.528831Z","shell.execute_reply.started":"2024-10-27T09:08:38.242275Z","shell.execute_reply":"2024-10-27T09:08:43.527989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define paths and parameters\ntrain_csv_path = '/kaggle/input/aptos2019-blindness-detection/train.csv'\ntest_csv_path = '/kaggle/input/aptos2019-blindness-detection/test.csv'\ntrain_images_dir = '/kaggle/input/aptos2019-blindness-detection/train_images'  # Directory containing train images\ntest_images_dir = '/kaggle/input/aptos2019-blindness-detection/test_images'  # Directory containing test images\noutput_csv_path = '/kaggle/working/predictions.csv'\nmodel_save_dir = '/kaggle/working/models'  # Directory to save model checkpoints\nbatch_size = 32\nnum_epochs = 15\ninitial_learning_rate = 0.001\nlr_decay_factor = 0.1  # Factor by which the learning rate will decrease","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:08:46.640509Z","iopub.execute_input":"2024-10-27T09:08:46.641082Z","iopub.status.idle":"2024-10-27T09:08:46.646535Z","shell.execute_reply.started":"2024-10-27T09:08:46.641039Z","shell.execute_reply":"2024-10-27T09:08:46.645553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the directory to save model checkpoints if it doesn't exist\nif not os.path.exists(model_save_dir):\n    os.makedirs(model_save_dir)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:08:51.678523Z","iopub.execute_input":"2024-10-27T09:08:51.679008Z","iopub.status.idle":"2024-10-27T09:08:51.684524Z","shell.execute_reply.started":"2024-10-27T09:08:51.678948Z","shell.execute_reply":"2024-10-27T09:08:51.683428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load train.csv and test.csv\ntrain_df = pd.read_csv(train_csv_path)\ntest_df = pd.read_csv(test_csv_path)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:09:05.553886Z","iopub.execute_input":"2024-10-27T09:09:05.554263Z","iopub.status.idle":"2024-10-27T09:09:05.578298Z","shell.execute_reply.started":"2024-10-27T09:09:05.554226Z","shell.execute_reply":"2024-10-27T09:09:05.577515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Custom Dataset class\nclass DiabeticRetinopathyDataset(Dataset):\n    def __init__(self, df, images_dir, transform=None, train=True):\n        self.df = df\n        self.images_dir = images_dir\n        self.transform = transform\n        self.train = train\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        img_name = self.df.iloc[idx, 0]\n        img_path = os.path.join(self.images_dir, f\"{img_name}.png\")\n        image = Image.open(img_path).convert(\"L\")  # Convert to grayscale\n\n        if self.transform:\n            image = self.transform(image)\n\n        if self.train:\n            label = self.df.iloc[idx, 1]\n            return image, label\n        else:\n            return image, img_name","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:09:10.657164Z","iopub.execute_input":"2024-10-27T09:09:10.658046Z","iopub.status.idle":"2024-10-27T09:09:10.665695Z","shell.execute_reply.started":"2024-10-27T09:09:10.658003Z","shell.execute_reply":"2024-10-27T09:09:10.664480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Image transformations\ntransform = transforms.Compose([\n    transforms.Grayscale(num_output_channels=3),  # ViT expects 3-channel input\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])\n])","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:09:14.911155Z","iopub.execute_input":"2024-10-27T09:09:14.911506Z","iopub.status.idle":"2024-10-27T09:09:14.916999Z","shell.execute_reply.started":"2024-10-27T09:09:14.911472Z","shell.execute_reply":"2024-10-27T09:09:14.916018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the full train dataset\nfull_dataset = DiabeticRetinopathyDataset(train_df, train_images_dir, transform=transform)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:09:16.478445Z","iopub.execute_input":"2024-10-27T09:09:16.479139Z","iopub.status.idle":"2024-10-27T09:09:16.483225Z","shell.execute_reply.started":"2024-10-27T09:09:16.479096Z","shell.execute_reply":"2024-10-27T09:09:16.482331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split the train dataset into train, validation, and test sets\ntrain_size = int(0.65 * len(full_dataset))\nval_size = int(0.25 * len(full_dataset))\ntest_size = len(full_dataset) - train_size - val_size\ntrain_dataset, val_dataset, test_dataset = random_split(full_dataset, [train_size, val_size, test_size])","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:09:19.220470Z","iopub.execute_input":"2024-10-27T09:09:19.221228Z","iopub.status.idle":"2024-10-27T09:09:19.247319Z","shell.execute_reply.started":"2024-10-27T09:09:19.221184Z","shell.execute_reply":"2024-10-27T09:09:19.246362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create DataLoaders\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:09:30.518441Z","iopub.execute_input":"2024-10-27T09:09:30.518829Z","iopub.status.idle":"2024-10-27T09:09:30.524257Z","shell.execute_reply.started":"2024-10-27T09:09:30.518791Z","shell.execute_reply":"2024-10-27T09:09:30.523324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load pretrained Vision Transformer model\nmodel = vit_b_16(pretrained=True)\nmodel.heads = nn.Sequential(\n    nn.Linear(model.heads.head.in_features, 512),\n    nn.ReLU(),\n    nn.Linear(512, 5)\n)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:09:35.411997Z","iopub.execute_input":"2024-10-27T09:09:35.412879Z","iopub.status.idle":"2024-10-27T09:09:38.524592Z","shell.execute_reply.started":"2024-10-27T09:09:35.412837Z","shell.execute_reply":"2024-10-27T09:09:38.523777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda')\nmodel = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:09:48.829436Z","iopub.execute_input":"2024-10-27T09:09:48.830253Z","iopub.status.idle":"2024-10-27T09:09:49.313274Z","shell.execute_reply.started":"2024-10-27T09:09:48.830212Z","shell.execute_reply":"2024-10-27T09:09:49.312232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loss and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=initial_learning_rate)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:09:51.705884Z","iopub.execute_input":"2024-10-27T09:09:51.706496Z","iopub.status.idle":"2024-10-27T09:09:51.712065Z","shell.execute_reply.started":"2024-10-27T09:09:51.706456Z","shell.execute_reply":"2024-10-27T09:09:51.711107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Learning rate scheduler\nscheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=lr_decay_factor)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:09:54.119191Z","iopub.execute_input":"2024-10-27T09:09:54.119571Z","iopub.status.idle":"2024-10-27T09:09:54.124139Z","shell.execute_reply.started":"2024-10-27T09:09:54.119533Z","shell.execute_reply":"2024-10-27T09:09:54.123194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training function\ndef train_model(model, train_loader, val_loader, criterion, optimizer, scheduler, num_epochs):\n    best_val_loss = float(\"inf\")\n\n    for epoch in range(num_epochs):\n        # Training phase\n        model.train()\n        running_loss = 0.0\n        for images, labels in train_loader:\n            images, labels = images.to(device), labels.to(device)\n\n            optimizer.zero_grad()\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n\n            running_loss += loss.item()\n\n        avg_train_loss = running_loss / len(train_loader)\n        print(f\"Epoch [{epoch + 1}/{num_epochs}], Train Loss: {avg_train_loss:.5f}\")\n\n        # Validation phase\n        model.eval()\n        val_loss = 0.0\n        with torch.no_grad():\n            for images, labels in val_loader:\n                images, labels = images.to(device), labels.to(device)\n                outputs = model(images)\n                loss = criterion(outputs, labels)\n                val_loss += loss.item()\n\n        avg_val_loss = val_loss / len(val_loader)\n        print(f\"Epoch [{epoch + 1}/{num_epochs}], Val Loss: {avg_val_loss:.5f}\")\n\n        # Learning rate adjustment\n        scheduler.step()\n\n        # Model checkpointing\n        if avg_val_loss < best_val_loss:\n            best_val_loss = avg_val_loss\n            best_model_path = os.path.join(model_save_dir, f\"best_model_epoch_{epoch + 1}.pth\")\n            torch.save(model.state_dict(), best_model_path)\n            print(f\"Saved best model of epoch {epoch + 1} with validation loss: {best_val_loss:.5f}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:10:46.274988Z","iopub.execute_input":"2024-10-27T09:10:46.275618Z","iopub.status.idle":"2024-10-27T09:10:46.285723Z","shell.execute_reply.started":"2024-10-27T09:10:46.275576Z","shell.execute_reply":"2024-10-27T09:10:46.284697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model\ntrain_model(model, train_loader, val_loader, criterion, optimizer, scheduler, num_epochs)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T09:10:49.769081Z","iopub.execute_input":"2024-10-27T09:10:49.769799Z","iopub.status.idle":"2024-10-27T11:12:49.477723Z","shell.execute_reply.started":"2024-10-27T09:10:49.769757Z","shell.execute_reply":"2024-10-27T11:12:49.476714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prediction on test.csv images\ntest_transform = transforms.Compose([\n    transforms.Grayscale(num_output_channels=3),\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])\n])","metadata":{"execution":{"iopub.status.busy":"2024-10-27T11:15:49.060100Z","iopub.execute_input":"2024-10-27T11:15:49.060705Z","iopub.status.idle":"2024-10-27T11:15:49.066358Z","shell.execute_reply.started":"2024-10-27T11:15:49.060654Z","shell.execute_reply":"2024-10-27T11:15:49.065272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = DiabeticRetinopathyDataset(test_df, test_images_dir, transform=test_transform, train=False)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T11:15:49.255772Z","iopub.execute_input":"2024-10-27T11:15:49.257954Z","iopub.status.idle":"2024-10-27T11:15:49.263065Z","shell.execute_reply.started":"2024-10-27T11:15:49.257901Z","shell.execute_reply":"2024-10-27T11:15:49.262095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda')\nmodel = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-10-27T11:15:51.010364Z","iopub.execute_input":"2024-10-27T11:15:51.011084Z","iopub.status.idle":"2024-10-27T11:15:51.020160Z","shell.execute_reply.started":"2024-10-27T11:15:51.011043Z","shell.execute_reply":"2024-10-27T11:15:51.019260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_path = '/kaggle/working/models/best_model_epoch_9.pth'\nmodel.load_state_dict(torch.load(model_path, map_location=device))\nmodel.eval()","metadata":{"execution":{"iopub.status.busy":"2024-10-27T11:15:51.636926Z","iopub.execute_input":"2024-10-27T11:15:51.637735Z","iopub.status.idle":"2024-10-27T11:15:51.978033Z","shell.execute_reply.started":"2024-10-27T11:15:51.637683Z","shell.execute_reply":"2024-10-27T11:15:51.977165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = []\nwith torch.no_grad():\n    for images, img_names in test_loader:\n        images = images.to(device)\n        outputs = model(images)\n        _, preds = torch.max(outputs, 1)\n        predictions.extend(zip(img_names, preds.cpu().numpy()))","metadata":{"execution":{"iopub.status.busy":"2024-10-27T11:16:09.143740Z","iopub.execute_input":"2024-10-27T11:16:09.144111Z","iopub.status.idle":"2024-10-27T11:18:12.542690Z","shell.execute_reply.started":"2024-10-27T11:16:09.144075Z","shell.execute_reply":"2024-10-27T11:18:12.541666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save predictions to CSV\npredictions_df = pd.DataFrame(predictions, columns=['id_code', 'diagnosis'])\npredictions_df.to_csv(output_csv_path, index=False)\nprint(f\"Predictions saved to {output_csv_path}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-27T11:18:12.544478Z","iopub.execute_input":"2024-10-27T11:18:12.544853Z","iopub.status.idle":"2024-10-27T11:18:12.567160Z","shell.execute_reply.started":"2024-10-27T11:18:12.544815Z","shell.execute_reply":"2024-10-27T11:18:12.566149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}