{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":848739,"sourceType":"datasetVersion","datasetId":251095},{"sourceId":1399787,"sourceType":"datasetVersion","datasetId":252150}],"dockerImageVersionId":29186,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install efficientnet_pytorch","metadata":{"execution":{"iopub.status.busy":"2024-05-14T14:54:26.589167Z","iopub.execute_input":"2024-05-14T14:54:26.58957Z","iopub.status.idle":"2024-05-14T14:54:35.300485Z","shell.execute_reply.started":"2024-05-14T14:54:26.5895Z","shell.execute_reply":"2024-05-14T14:54:35.29974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nfrom os.path import isfile\nimport torch.nn.init as init\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport pandas as pd \nimport os\nfrom PIL import Image, ImageFilter\nprint(os.listdir(\"../input\"))\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom torch.optim import Adam, SGD, RMSprop\nimport time\nfrom torch.autograd import Variable\nimport torch.functional as F\nfrom tqdm import tqdm\nfrom sklearn import metrics\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\nimport urllib\nimport pickle\nimport cv2\nimport torch.nn.functional as F\nfrom torchvision import models\nimport seaborn as sns\nimport random\nimport sys\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix, roc_curve, auc","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","scrolled":true,"execution":{"iopub.status.busy":"2024-05-14T14:54:35.30252Z","iopub.execute_input":"2024-05-14T14:54:35.302772Z","iopub.status.idle":"2024-05-14T14:54:37.946518Z","shell.execute_reply.started":"2024-05-14T14:54:35.30273Z","shell.execute_reply":"2024-05-14T14:54:37.945855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# package_path = '../input/efficientnet/efficientnet-pytorch/EfficientNet-PyTorch/'\n# sys.path.append(package_path)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T14:54:37.94798Z","iopub.execute_input":"2024-05-14T14:54:37.948217Z","iopub.status.idle":"2024-05-14T14:54:37.951412Z","shell.execute_reply.started":"2024-05-14T14:54:37.948175Z","shell.execute_reply":"2024-05-14T14:54:37.950682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","metadata":{"execution":{"iopub.status.busy":"2024-05-14T14:54:37.952858Z","iopub.execute_input":"2024-05-14T14:54:37.953084Z","iopub.status.idle":"2024-05-14T14:54:37.962059Z","shell.execute_reply.started":"2024-05-14T14:54:37.953045Z","shell.execute_reply":"2024-05-14T14:54:37.961244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_classes = 5\nseed_everything(1234)\nlr          = 1e-3\nIMG_SIZE    = 256","metadata":{"execution":{"iopub.status.busy":"2024-05-14T14:54:37.965972Z","iopub.execute_input":"2024-05-14T14:54:37.966198Z","iopub.status.idle":"2024-05-14T14:54:37.974517Z","shell.execute_reply.started":"2024-05-14T14:54:37.96616Z","shell.execute_reply":"2024-05-14T14:54:37.973831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train      = '../input/aptos2019-blindness-detection/train_images/'\ntest       = '../input/aptos2019-blindness-detection/test_images/'\ntrain_csv  = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n\ntrain_df, val_df = train_test_split(train_csv, test_size=0.2, random_state=2018, stratify=train_csv.diagnosis) # Stratify is used to ensure that both the training and validation sets have a similar distribution of the 'diagnosis' column. This helps maintain the same class distribution in both sets.\ntrain_df.reset_index(drop=True, inplace=True) # After splitting, this line resets the index of the train_df DataFrame. The drop=True argument means that the old index will be removed, and inplace=True modifies the DataFrame in place.\nval_df.reset_index(drop=True, inplace=True) \ntrain_df.head()","metadata":{"_uuid":"766f44c87272f67d632e519dce11cf54a3382696","execution":{"iopub.status.busy":"2024-05-14T14:54:37.977116Z","iopub.execute_input":"2024-05-14T14:54:37.97736Z","iopub.status.idle":"2024-05-14T14:54:38.020723Z","shell.execute_reply.started":"2024-05-14T14:54:37.977313Z","shell.execute_reply":"2024-05-14T14:54:38.019887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize ResNet-101\nfrom torchvision.models import resnet101\nmodel_resnet = resnet101(pretrained=True)\nnum_ftrs_resnet = model_resnet.fc.in_features\nmodel_resnet.fc = nn.Linear(num_ftrs_resnet, num_classes)\nmodel_resnet = model_resnet.cuda()\n\n# Initialize Inception-v3\nfrom torchvision.models import inception_v3\nmodel_inception = inception_v3(pretrained=True, aux_logits=True)\nnum_ftrs_inception = model_inception.fc.in_features\nmodel_inception.fc = nn.Linear(num_ftrs_inception, num_classes)\n# Handle auxiliary logits for Inception-v3\nnum_ftrs_aux = model_inception.AuxLogits.fc.in_features\nmodel_inception.AuxLogits.fc = nn.Linear(num_ftrs_aux, num_classes)\nmodel_inception = model_inception.cuda()\n\n# Initialize EfficientNet\nfrom efficientnet_pytorch import EfficientNet\nmodel_efficientnet = EfficientNet.from_pretrained('efficientnet-b0')\nnum_ftrs_efficientnet = model_efficientnet._fc.in_features\nmodel_efficientnet._fc = nn.Linear(num_ftrs_efficientnet, num_classes)\nmodel_efficientnet = model_efficientnet.cuda()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T14:54:38.021908Z","iopub.execute_input":"2024-05-14T14:54:38.022164Z","iopub.status.idle":"2024-05-14T14:54:49.156429Z","shell.execute_reply.started":"2024-05-14T14:54:38.022119Z","shell.execute_reply":"2024-05-14T14:54:49.155653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This function takes a filename p as input and expands it to a full file path.\ndef expand_path(p):\n    p = str(p)\n    if isfile(train + p + \".png\"):\n        return train + (p + \".png\")\n    if isfile(train_2015 + p + '.png'):\n        return train_2015 + (p + \".png\")\n    if isfile(test + p + \".png\"):\n        return test + (p + \".png\")\n    return p\n\n# This function is used to display a grid of images.\ndef p_show(imgs, label_name=None, per_row=3):\n    n = len(imgs)\n    rows = (n + per_row - 1)//per_row\n    cols = min(per_row, n)\n    fig, axes = plt.subplots(rows,cols, figsize=(15,15))\n    for ax in axes.flatten(): ax.axis('off')\n    for i,(p, ax) in enumerate(zip(imgs, axes.flatten())): \n        img = Image.open(expand_path(p))\n        ax.imshow(img)\n        ax.set_title(train_df[train_df.id_code == p].diagnosis.values)","metadata":{"_uuid":"64d7e44b053ac654c681e77b04de74ba32020fbd","execution":{"iopub.status.busy":"2024-05-14T14:54:49.157531Z","iopub.execute_input":"2024-05-14T14:54:49.157826Z","iopub.status.idle":"2024-05-14T14:54:49.168863Z","shell.execute_reply.started":"2024-05-14T14:54:49.157776Z","shell.execute_reply":"2024-05-14T14:54:49.16805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs = []\nfor p in train_df.id_code:\n    imgs.append(p)\n    if len(imgs) == 16: break\np_show(imgs)","metadata":{"_uuid":"b4739904397dd22d058c36769034b91964dcb9fe","execution":{"iopub.status.busy":"2024-05-14T14:54:49.170158Z","iopub.execute_input":"2024-05-14T14:54:49.170433Z","iopub.status.idle":"2024-05-14T14:54:55.724223Z","shell.execute_reply.started":"2024-05-14T14:54:49.170381Z","shell.execute_reply":"2024-05-14T14:54:55.723368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#The Code from: https://www.kaggle.com/ratthachat/aptos-updated-albumentation-meets-grad-cam\n# In summary, both functions aim to crop images by removing regions with pixel intensities below a specified threshold (tolerance). This can be useful for eliminating unwanted dark or uninformative areas in images, depending on the application.\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img","metadata":{"execution":{"iopub.status.busy":"2024-05-14T14:54:55.725738Z","iopub.execute_input":"2024-05-14T14:54:55.726017Z","iopub.status.idle":"2024-05-14T14:54:55.737304Z","shell.execute_reply.started":"2024-05-14T14:54:55.725964Z","shell.execute_reply":"2024-05-14T14:54:55.736438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport cv2\nfrom torchvision.transforms import ToPILImage, ToTensor, Normalize, Compose\n\nclass MyDataset(Dataset):\n    \n    def __init__(self, dataframe, transform=None, model_name='resnet'):\n        self.df = dataframe\n        self.transform = transform\n        self.model_name = model_name\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def expand_path(self, p):\n        # Assuming 'train', 'test', and 'train_2015' are defined and accessible\n        p = str(p)\n        if os.path.isfile(os.path.join(train, p + \".png\")):\n            return os.path.join(train, p + \".png\")\n        if os.path.isfile(os.path.join(train_2015, p + '.png')):\n            return os.path.join(train_2015, p + \".png\")\n        if os.path.isfile(os.path.join(test, p + \".png\")):\n            return os.path.join(test, p + \".png\")\n        return p\n\n    def __getitem__(self, idx):\n        label = self.df.diagnosis.values[idx]\n        # Ensure label is a long type if not already\n        label = torch.tensor(label, dtype=torch.long)\n        \n        p = self.df.id_code.values[idx]\n        p_path = self.expand_path(p)\n        image = cv2.imread(p_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = crop_image_from_gray(image)\n\n        # Resize image based on the model\n        if self.model_name == 'inception':\n            image = cv2.resize(image, (299, 299))\n        else:\n            image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n\n        image = cv2.addWeighted(image, 4, cv2.GaussianBlur(image, (0,0), 30), -4, 128)\n        image = ToPILImage()(image)\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        return image, label","metadata":{"_uuid":"21908baa8df4e398b0d49a5146ce544504637c5a","execution":{"iopub.status.busy":"2024-05-14T14:54:55.738413Z","iopub.execute_input":"2024-05-14T14:54:55.738702Z","iopub.status.idle":"2024-05-14T14:54:55.756682Z","shell.execute_reply.started":"2024-05-14T14:54:55.738596Z","shell.execute_reply":"2024-05-14T14:54:55.755721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Common transform for all models\ntrain_transform = transforms.Compose([\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation((-120, 120)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\nresnet_trainset = MyDataset(train_df, transform=train_transform, model_name='resnet')\nresnet_train_loader = torch.utils.data.DataLoader(resnet_trainset, batch_size=32, shuffle=True, num_workers=4)\nresnet_valset = MyDataset(val_df, transform=train_transform, model_name='resnet')\nresnet_val_loader = torch.utils.data.DataLoader(resnet_valset, batch_size=32, shuffle=False, num_workers=4)\n\ninception_trainset = MyDataset(train_df, transform=train_transform, model_name='inception')\ninception_train_loader = torch.utils.data.DataLoader(inception_trainset, batch_size=32, shuffle=True, num_workers=4)\ninception_valset = MyDataset(val_df, transform=train_transform, model_name='inception')\ninception_val_loader = torch.utils.data.DataLoader(inception_valset, batch_size=32, shuffle=False, num_workers=4)","metadata":{"_uuid":"f590638fd07b9aefe2210a39612ac77e0689c0c1","execution":{"iopub.status.busy":"2024-05-14T14:54:55.757948Z","iopub.execute_input":"2024-05-14T14:54:55.758213Z","iopub.status.idle":"2024-05-14T14:54:55.771556Z","shell.execute_reply.started":"2024-05-14T14:54:55.758171Z","shell.execute_reply":"2024-05-14T14:54:55.770961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\n\noptimizer_resnet = torch.optim.Adam(model_resnet.parameters(), lr=lr, weight_decay=1e-5)\nscheduler_resnet = torch.optim.lr_scheduler.StepLR(optimizer_resnet, step_size=5, gamma=0.1)\n\noptimizer_efficientnet = torch.optim.Adam(model_efficientnet.parameters(), lr=lr, weight_decay=1e-5)\nscheduler_efficientnet = torch.optim.lr_scheduler.StepLR(optimizer_efficientnet, step_size=5, gamma=0.1)\n\noptimizer_inception = torch.optim.Adam(model_inception.parameters(), lr=lr, weight_decay=1e-5)\nscheduler_inception = torch.optim.lr_scheduler.StepLR(optimizer_inception, step_size=5, gamma=0.1)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T14:54:55.772705Z","iopub.execute_input":"2024-05-14T14:54:55.772967Z","iopub.status.idle":"2024-05-14T14:54:55.790712Z","shell.execute_reply.started":"2024-05-14T14:54:55.77292Z","shell.execute_reply":"2024-05-14T14:54:55.790154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, optimizer, train_loader, criterion, is_inception=False):\n    model.train()\n    avg_loss = 0.\n    correct = 0\n    total = 0\n\n    for idx, (imgs, labels) in enumerate(train_loader):\n        imgs_train, labels_train = imgs.cuda(), torch.squeeze(labels.cuda())\n        optimizer.zero_grad()\n\n        if is_inception:\n            output_train, aux_output_train = model(imgs_train)\n            loss1 = criterion(output_train, labels_train)\n            loss2 = criterion(aux_output_train, labels_train)\n            loss = loss1 + 0.4 * loss2\n        else:\n            output_train = model(imgs_train)\n            loss = criterion(output_train, labels_train)\n\n        _, predicted = torch.max(output_train.data, 1)\n        total += labels_train.size(0)\n        correct += (predicted == labels_train).sum().item()\n\n        loss.backward()\n        optimizer.step()\n        avg_loss += loss.item() / len(train_loader)\n\n    avg_accuracy = correct / total\n    return avg_loss, avg_accuracy\n\ndef test_model(model, val_loader, criterion, is_inception=False):\n    model.eval()\n    total_loss = 0\n    total_correct = 0\n    with torch.no_grad():\n        for imgs, labels in val_loader:\n            imgs, labels = imgs.cuda(), labels.cuda()\n            if is_inception and model.training:\n                # Handle Inception model's auxiliary output in training mode\n                output, aux_output = model(imgs)\n                loss1 = criterion(output, labels)\n                loss2 = criterion(aux_output, labels)\n                loss = loss1 + 0.4*loss2\n            else:\n                output = model(imgs)\n                loss = criterion(output, labels)\n            total_loss += loss.item()\n            _, preds = torch.max(output, 1)\n            total_correct += torch.sum(preds == labels.data)\n    avg_loss = total_loss / len(val_loader.dataset)\n    accuracy = total_correct.double() / len(val_loader.dataset)\n    return avg_loss, accuracy","metadata":{"_uuid":"c338feda0eee741964b4c3d736c30b1e0a7e3ace","execution":{"iopub.status.busy":"2024-05-14T14:54:55.792168Z","iopub.execute_input":"2024-05-14T14:54:55.792449Z","iopub.status.idle":"2024-05-14T14:54:55.810325Z","shell.execute_reply.started":"2024-05-14T14:54:55.79239Z","shell.execute_reply":"2024-05-14T14:54:55.809594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_epochs = 30\n\n# Initialize empty lists to store data\ntrain_losses_resnet = []\nval_losses_resnet = []\ntrain_accuracies_resnet = []\nval_accuracies_resnet = []\n\ntrain_losses_efficientnet = []\nval_losses_efficientnet = []\ntrain_accuracies_efficientnet = []\nval_accuracies_efficientnet = []\n\ntrain_losses_inception = []\nval_losses_inception = []\ntrain_accuracies_inception = []\nval_accuracies_inception = []\n\nfor model, optimizer, scheduler, train_loader, val_loader in [\n    (model_resnet, optimizer_resnet, scheduler_resnet, resnet_train_loader, resnet_val_loader), \n    (model_efficientnet, optimizer_efficientnet, scheduler_efficientnet, resnet_train_loader, resnet_val_loader), \n    (model_inception, optimizer_inception, scheduler_inception, inception_train_loader, inception_val_loader),\n]:\n    best_avg_loss = 100.0\n    is_inception = (model.__class__.__name__ == \"Inception3\")\n\n    for epoch in range(n_epochs):  \n        print(f'Training {model.__class__.__name__}, Epoch: {epoch+1}/{n_epochs}')\n        avg_loss, avg_accuracy = train_model(model, optimizer, train_loader, criterion, is_inception)\n        avg_val_loss, val_accuracy = test_model(model, val_loader, criterion, is_inception)\n        scheduler.step()\n\n        # Append losses and accuracies to the respective lists\n        if model == model_resnet:\n            train_losses_resnet.append(avg_loss)\n            val_losses_resnet.append(avg_val_loss)\n            train_accuracies_resnet.append(avg_accuracy)\n            val_accuracies_resnet.append(val_accuracy)\n        elif model == model_efficientnet:\n            train_losses_efficientnet.append(avg_loss)\n            val_losses_efficientnet.append(avg_val_loss)\n            train_accuracies_efficientnet.append(avg_accuracy)\n            val_accuracies_efficientnet.append(val_accuracy)\n        elif model == model_inception:\n            train_losses_inception.append(avg_loss)\n            val_losses_inception.append(avg_val_loss)\n            train_accuracies_inception.append(avg_accuracy)\n            val_accuracies_inception.append(val_accuracy)\n\n        # Save model if validation loss improved\n        if avg_val_loss < best_avg_loss:\n            best_avg_loss = avg_val_loss\n            torch.save(model.state_dict(), f'weight_best_{model.__class__.__name__}.pt')","metadata":{"_uuid":"3562bd2ec1b0650519ca196bfc0e60eb139ca180","execution":{"iopub.status.busy":"2024-05-14T14:54:55.81182Z","iopub.execute_input":"2024-05-14T14:54:55.812055Z","iopub.status.idle":"2024-05-14T15:15:59.361788Z","shell.execute_reply.started":"2024-05-14T14:54:55.812015Z","shell.execute_reply":"2024-05-14T15:15:59.360565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load the Best Model Weights","metadata":{}},{"cell_type":"code","source":"# Make sure models are in evaluation mode\nmodel_resnet.eval()\nmodel_efficientnet.eval()\nmodel_inception.eval()\n\n# Load the saved best weights\nmodel_resnet.load_state_dict(torch.load('weight_best_ResNet.pt'))\nmodel_efficientnet.load_state_dict(torch.load('weight_best_EfficientNet.pt'))\nmodel_inception.load_state_dict(torch.load('weight_best_Inception3.pt'))","metadata":{"execution":{"iopub.status.busy":"2024-05-14T15:15:59.363896Z","iopub.execute_input":"2024-05-14T15:15:59.364254Z","iopub.status.idle":"2024-05-14T15:15:59.708506Z","shell.execute_reply.started":"2024-05-14T15:15:59.364189Z","shell.execute_reply":"2024-05-14T15:15:59.707772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluate Each Model","metadata":{}},{"cell_type":"code","source":"def evaluate_model(model, val_loader):\n    model.eval()\n    correct = 0\n    total = 0\n    all_labels = []\n    all_predictions = []\n    with torch.no_grad():\n        for images, labels in val_loader:\n            images, labels = images.cuda(), labels.cuda()\n            outputs = model(images)\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n            all_labels.extend(labels.cpu().numpy())\n            all_predictions.extend(predicted.cpu().numpy())\n\n    accuracy = 100 * correct / total\n    precision = precision_score(all_labels, all_predictions, average='weighted')\n    recall = recall_score(all_labels, all_predictions, average='weighted')\n    f1 = f1_score(all_labels, all_predictions, average='weighted')\n\n    return accuracy, precision, recall, f1","metadata":{"execution":{"iopub.status.busy":"2024-05-14T15:15:59.71025Z","iopub.execute_input":"2024-05-14T15:15:59.710586Z","iopub.status.idle":"2024-05-14T15:15:59.720931Z","shell.execute_reply.started":"2024-05-14T15:15:59.710528Z","shell.execute_reply":"2024-05-14T15:15:59.720231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# At the end of training for each model, call the evaluate_model function\naccuracy_resnet, precision_resnet, recall_resnet, f1_score_resnet = evaluate_model(model_resnet, resnet_val_loader)\naccuracy_efficientnet, precision_efficientnet, recall_efficientnet, f1_score_efficientnet = evaluate_model(model_efficientnet, resnet_val_loader)\naccuracy_inception, precision_inception, recall_inception, f1_score_inception = evaluate_model(model_inception, inception_val_loader)\n\nprint(f'ResNet-101 Accuracy: {accuracy_resnet:.2f}%')\nprint(f'ResNet-101 Precision: {precision_resnet:.2f}')\nprint(f'ResNet-101 Recall: {recall_resnet:.2f}')\nprint(f'ResNet-101 F1-Score: {f1_score_resnet:.2f}')\n\nprint(f'EfficientNet Accuracy: {accuracy_efficientnet:.2f}%')\nprint(f'EfficientNet Precision: {precision_efficientnet:.2f}')\nprint(f'EfficientNet Recall: {recall_efficientnet:.2f}')\nprint(f'EfficientNet F1-Score: {f1_score_efficientnet:.2f}')\n\nprint(f'Inception-v3 Accuracy: {accuracy_inception:.2f}%')\nprint(f'Inception-v3 Precision: {precision_inception:.2f}')\nprint(f'Inception-v3 Recall: {recall_inception:.2f}')\nprint(f'Inception-v3 F1-Score: {f1_score_inception:.2f}')","metadata":{"execution":{"iopub.status.busy":"2024-05-14T15:15:59.722189Z","iopub.execute_input":"2024-05-14T15:15:59.722476Z","iopub.status.idle":"2024-05-14T15:20:10.962969Z","shell.execute_reply.started":"2024-05-14T15:15:59.722426Z","shell.execute_reply":"2024-05-14T15:20:10.96206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import VotingClassifier\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n\n# Define a function to get predictions from a PyTorch model\ndef get_predictions(model, data_loader):\n    model.eval()\n    predictions = []\n    true_labels = []\n    with torch.no_grad():\n        for inputs, labels in data_loader:\n            inputs = inputs.cuda()\n            labels = labels.cuda()\n            outputs = model(inputs)\n            _, preds = torch.max(outputs, 1)\n            predictions.extend(preds.cpu().numpy())\n            true_labels.extend(labels.cpu().numpy())\n    return predictions, true_labels\n\n# Get predictions for each model\npredictions_resnet, labels = get_predictions(model_resnet, resnet_val_loader)\npredictions_inception, _ = get_predictions(model_inception, inception_val_loader)\npredictions_efficientnet, _ = get_predictions(model_efficientnet,resnet_val_loader )\n\n# Create an ensemble model\nensemble_predictions = []\nfor pred_resnet, pred_inception, pred_efficientnet in zip(predictions_resnet, predictions_inception, predictions_efficientnet):\n    if pred_resnet == pred_inception == pred_efficientnet:  \n        ensemble_predictions.append(pred_inception)\n    # Perform majority voting\n    else:\n        ensemble_predictions.append(max(set([pred_resnet, pred_inception, pred_efficientnet]), key = [pred_resnet, pred_inception, pred_efficientnet].count))\n\n# Calculate evaluation metrics for the ensemble\naccuracy_ensemble = accuracy_score(labels, ensemble_predictions) * 100.0\nprecision_ensemble = precision_score(labels, ensemble_predictions, average='macro')\nrecall_ensemble = recall_score(labels, ensemble_predictions, average='macro')\nf1_score_ensemble = f1_score(labels, ensemble_predictions, average='macro')\n\n# Save the ensemble model if the accuracy improves\nbest_accuracy = max(accuracy_resnet, accuracy_inception, accuracy_efficientnet)\nprint(f'Initialized Best Accuracy: {best_accuracy:.2f}')\n\n# Print results\nprint(f'Ensemble Model Accuracy: {accuracy_ensemble:.2f}%')\nprint(f'Ensemble Model Precision: {precision_ensemble:.2f}')\nprint(f'Ensemble Model Recall: {recall_ensemble:.2f}')\nprint(f'Ensemble Model F1-Score: {f1_score_ensemble:.2f}')\n","metadata":{"execution":{"iopub.status.busy":"2024-05-14T15:20:10.964819Z","iopub.execute_input":"2024-05-14T15:20:10.965108Z","iopub.status.idle":"2024-05-14T15:24:23.523131Z","shell.execute_reply.started":"2024-05-14T15:20:10.965054Z","shell.execute_reply":"2024-05-14T15:24:23.52208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Create subplots for loss and accuracy\nplt.figure(figsize=(12, 4))\n\n# Plot ResNet results\nplt.subplot(1, 2, 1)\nplt.plot(range(1, n_epochs + 1), val_losses_resnet, label='ResNet', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_inception, label='Inception', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_efficientnet, label='EfficientNet', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Validation Loss')\nplt.title('Validation Loss Curves')\nplt.legend()\n\nplt.subplot(1, 2, 2)\nplt.plot(range(1, n_epochs + 1), val_accuracies_resnet, label='ResNet', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_inception, label='Inception', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_efficientnet, label='EfficientNet', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Validation Accuracy')\nplt.title('Validation Accuracy Curves')\nplt.legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T15:24:23.524943Z","iopub.execute_input":"2024-05-14T15:24:23.525311Z","iopub.status.idle":"2024-05-14T15:24:24.214496Z","shell.execute_reply.started":"2024-05-14T15:24:23.525249Z","shell.execute_reply":"2024-05-14T15:24:24.213226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Create subplots for loss and accuracy\nplt.figure(figsize=(16, 8))\n\n# Plot ResNet results\nplt.subplot(2, 3, 1)\nplt.plot(range(1, n_epochs + 1), train_losses_resnet, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_resnet, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('ResNet Loss Curves')\nplt.legend()\n\nplt.subplot(2, 3, 2)\nplt.plot(range(1, n_epochs + 1), train_accuracies_resnet, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_resnet, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('ResNet Accuracy Curves')\nplt.legend()\n\n# Plot Inception results\nplt.subplot(2, 3, 3)\nplt.plot(range(1, n_epochs + 1), train_losses_inception, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_inception, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Inception Loss Curves')\nplt.legend()\n\nplt.subplot(2, 3, 4)\nplt.plot(range(1, n_epochs + 1), train_accuracies_inception, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_inception, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('Inception Accuracy Curves')\nplt.legend()\n\n# Plot EfficientNet results\nplt.subplot(2, 3, 5)\nplt.plot(range(1, n_epochs + 1), train_losses_efficientnet, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_efficientnet, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('EfficientNet Loss Curves')\nplt.legend()\n\nplt.subplot(2, 3, 6)\nplt.plot(range(1, n_epochs + 1), train_accuracies_efficientnet, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_efficientnet, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('EfficientNet Accuracy Curves')\nplt.legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T15:24:24.220786Z","iopub.execute_input":"2024-05-14T15:24:24.223954Z","iopub.status.idle":"2024-05-14T15:24:26.010459Z","shell.execute_reply.started":"2024-05-14T15:24:24.221153Z","shell.execute_reply":"2024-05-14T15:24:26.009646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a bar chart for final validation accuracy\nmodels = ['ResNet', 'Inception', 'EfficientNet']\nfinal_accuracies = [val_accuracies_resnet[-1], val_accuracies_inception[-1], val_accuracies_efficientnet[-1]]\n\nplt.figure(figsize=(8, 5))\nplt.bar(models, final_accuracies)\nplt.xlabel('Model')\nplt.ylabel('Final Validation Accuracy')\nplt.title('Comparison of Final Validation Accuracy')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T15:24:26.011798Z","iopub.execute_input":"2024-05-14T15:24:26.012062Z","iopub.status.idle":"2024-05-14T15:24:26.266515Z","shell.execute_reply.started":"2024-05-14T15:24:26.012013Z","shell.execute_reply":"2024-05-14T15:24:26.265082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport itertools\n\n# After evaluating each model\n# Example for ResNet model\ndef plot_roc_curve_and_confusion_matrix(model, val_loader):\n    model.eval()\n    all_labels = []\n    all_probs = []\n    with torch.no_grad():\n        for images, labels in val_loader:\n            images, labels = images.cuda(), labels.cuda()\n            outputs = model(images)\n            probabilities = torch.softmax(outputs, dim=1).cpu().numpy()\n            all_labels.extend(labels.cpu().numpy())\n            all_probs.extend(probabilities)\n\n    # Calculate ROC curve for each class\n    fpr = dict()\n    tpr = dict()\n    roc_auc = dict()\n    for i in range(num_classes):  # Replace num_classes with the actual number of classes\n        fpr[i], tpr[i], _ = roc_curve(np.array(all_labels) == i, np.array(all_probs)[:, i])\n        roc_auc[i] = auc(fpr[i], tpr[i])\n\n    # Plot ROC curves\n    plt.figure(figsize=(8, 6))\n    for i in range(num_classes):\n        plt.plot(fpr[i], tpr[i], lw=2, label=f'Class {i} (AUC = {roc_auc[i]:0.2f})')\n    plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='black')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver Operating Characteristic (ROC) Curve')\n    plt.legend(loc='best')\n\n    # Plot confusion matrix\n    plt.figure(figsize=(8, 6))\n    cm = confusion_matrix(all_labels, np.argmax(all_probs, axis=1))\n    plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)\n    plt.title('Confusion Matrix')\n    plt.colorbar()\n    tick_marks = np.arange(num_classes)  # Replace num_classes with the actual number of classes\n    plt.xticks(tick_marks, [f'Class {i}' for i in range(num_classes)], rotation=45)\n    plt.yticks(tick_marks, [f'Class {i}' for i in range(num_classes)])\n    plt.xlabel('Predicted')\n    plt.ylabel('True')\n\n    # Annotate boxes with counts\n    fmt = 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n\n    plt.show()\n\n# Example usage for ResNet model\nplot_roc_curve_and_confusion_matrix(model_resnet, resnet_val_loader)\nplot_roc_curve_and_confusion_matrix(model_efficientnet, resnet_val_loader)\nplot_roc_curve_and_confusion_matrix(model_inception, inception_val_loader)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T15:24:26.268659Z","iopub.execute_input":"2024-05-14T15:24:26.269119Z","iopub.status.idle":"2024-05-14T15:28:40.994541Z","shell.execute_reply.started":"2024-05-14T15:24:26.269005Z","shell.execute_reply":"2024-05-14T15:28:40.993341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\ndef visualize_predictions_percentage(model, val_loader):\n    model.eval()\n    all_predictions = []\n    with torch.no_grad():\n        for images, _ in val_loader:\n            images = images.cuda()\n            outputs = model(images)\n            _, predicted = torch.max(outputs, 1)\n            all_predictions.extend(predicted.cpu().numpy())\n\n    # Calculate percentage distribution of predictions\n    unique_labels, counts = np.unique(all_predictions, return_counts=True)\n    percentages = (counts / len(all_predictions)) * 100\n\n    # Create a pie chart\n    plt.figure(figsize=(8, 6))\n    plt.pie(percentages, labels=unique_labels, autopct='%1.1f%%', startangle=140)\n    plt.title(f\"Percentage of Predictions by Class for {model}\")\n    plt.show()\n\n# Example usage for ResNet model\nvisualize_predictions_percentage(model_resnet, resnet_val_loader)\nvisualize_predictions_percentage(model_efficientnet, resnet_val_loader)\nvisualize_predictions_percentage(model_inception, inception_val_loader)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-14T15:28:40.997142Z","iopub.execute_input":"2024-05-14T15:28:40.99792Z","iopub.status.idle":"2024-05-14T15:32:58.745295Z","shell.execute_reply.started":"2024-05-14T15:28:40.997841Z","shell.execute_reply":"2024-05-14T15:32:58.744288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-05-14T15:32:58.747076Z","iopub.execute_input":"2024-05-14T15:32:58.747357Z","iopub.status.idle":"2024-05-14T15:32:58.76269Z","shell.execute_reply.started":"2024-05-14T15:32:58.747303Z","shell.execute_reply":"2024-05-14T15:32:58.761853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T15:32:58.764307Z","iopub.execute_input":"2024-05-14T15:32:58.76472Z","iopub.status.idle":"2024-05-14T15:32:58.775194Z","shell.execute_reply.started":"2024-05-14T15:32:58.764657Z","shell.execute_reply":"2024-05-14T15:32:58.774297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self, dataframe, transform=None, model_name='resnet'):\n        self.df = dataframe\n        self.root_dir = test\n        self.transform = transform\n        self.model_name = model_name\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        img_id = self.df.iloc[idx, 0]  # Assuming the first column is the image id\n        img_path = os.path.join(self.root_dir, f\"{img_id}.png\")\n        image = cv2.imread(img_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = crop_image_from_gray(image)  # Assuming you have this function defined\n\n        # Resize image based on the model\n        if self.model_name == 'inception':\n            image = cv2.resize(image, (299, 299))\n        else:\n            image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))  # Assuming IMG_SIZE is defined\n\n        image = cv2.addWeighted(image, 4, cv2.GaussianBlur(image, (0, 0), 30), -4, 128)\n        image = ToPILImage()(image)\n\n        if self.transform:\n            image = self.transform(image)\n\n        return image, img_id\n\ntest_transform = transforms.Compose([\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation((-120, 120)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\n# Define test datasets and loaders for each model\nresnet_testset = MyDataset(test_df, transform=test_transform, model_name='resnet')\nresnet_test_loader = torch.utils.data.DataLoader(resnet_testset, batch_size=32, shuffle=False, num_workers=4)\n\ninception_testset = MyDataset(test_df, transform=test_transform, model_name='inception')\ninception_test_loader = torch.utils.data.DataLoader(inception_testset, batch_size=32, shuffle=False, num_workers=4)\n\nefficientnet_testset = MyDataset(test_df, transform=test_transform, model_name='efficientnet')\nefficientnet_test_loader = torch.utils.data.DataLoader(efficientnet_testset, batch_size=32, shuffle=False, num_workers=4)\n\n# Define a function to get predictions from a PyTorch model\ndef get_predictions(model, data_loader):\n    model.eval()\n    predictions = []\n    with torch.no_grad():\n        for inputs, _ in data_loader:\n            inputs = inputs.cuda()\n            outputs = model(inputs)\n            _, preds = torch.max(outputs, 1)\n            predictions.extend(preds.cpu().numpy())\n    return predictions\n\n# Get predictions for each model on the test set\npredictions_resnet = get_predictions(model_resnet, resnet_test_loader)\npredictions_inception = get_predictions(model_inception, inception_test_loader)\npredictions_efficientnet = get_predictions(model_efficientnet, efficientnet_test_loader)\n\n# Ensemble predictions using Voting Classifier\nensemble_predictions = []\nfor pred_resnet, pred_inception, pred_efficientnet in zip(predictions_resnet, predictions_inception, predictions_efficientnet):\n    if pred_resnet == pred_inception == pred_efficientnet:  \n        ensemble_predictions.append(pred_inception)\n    else:\n        ensemble_predictions.append(max(set([pred_resnet, pred_inception, pred_efficientnet]), key=[pred_resnet, pred_inception, pred_efficientnet].count))\n\n# Convert predictions to DataFrame with appropriate column name\npredictions_df_resnet = pd.DataFrame(predictions_resnet, columns=['label'])\npredictions_df_inception = pd.DataFrame(predictions_inception, columns=['label'])\npredictions_df_efficientnet = pd.DataFrame(predictions_efficientnet, columns=['label'])\npredictions_df_ensemble = pd.DataFrame(ensemble_predictions, columns=['label'])\n\n# Save predictions to CSV files\npredictions_df_resnet.to_csv(\"predictions_resnet.csv\", index=False)\npredictions_df_inception.to_csv(\"predictions_inception.csv\", index=False)\npredictions_df_efficientnet.to_csv(\"predictions_efficientnet.csv\", index=False)\npredictions_df_ensemble.to_csv(\"predictions_ensemble.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T15:58:45.350403Z","iopub.execute_input":"2024-05-14T15:58:45.350776Z","iopub.status.idle":"2024-05-14T16:06:24.47477Z","shell.execute_reply.started":"2024-05-14T15:58:45.350714Z","shell.execute_reply":"2024-05-14T16:06:24.473812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_df_resnet","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:12:39.95533Z","iopub.execute_input":"2024-05-14T16:12:39.95571Z","iopub.status.idle":"2024-05-14T16:12:39.987121Z","shell.execute_reply.started":"2024-05-14T16:12:39.955656Z","shell.execute_reply":"2024-05-14T16:12:39.985996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_df_ensemble","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:13:07.002079Z","iopub.execute_input":"2024-05-14T16:13:07.002475Z","iopub.status.idle":"2024-05-14T16:13:07.017486Z","shell.execute_reply.started":"2024-05-14T16:13:07.002411Z","shell.execute_reply":"2024-05-14T16:13:07.016534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}