{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":418427,"sourceType":"datasetVersion","datasetId":187021},{"sourceId":527603,"sourceType":"datasetVersion","datasetId":250877},{"sourceId":848739,"sourceType":"datasetVersion","datasetId":251095},{"sourceId":1399787,"sourceType":"datasetVersion","datasetId":252150},{"sourceId":7521612,"sourceType":"datasetVersion","datasetId":4381613}],"dockerImageVersionId":29188,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install efficientnet_pytorch","metadata":{"execution":{"iopub.status.busy":"2024-04-23T12:30:50.490586Z","iopub.execute_input":"2024-04-23T12:30:50.490870Z","iopub.status.idle":"2024-04-23T12:30:59.283056Z","shell.execute_reply.started":"2024-04-23T12:30:50.490812Z","shell.execute_reply":"2024-04-23T12:30:59.282222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nfrom os.path import isfile\nimport torch.nn.init as init\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport pandas as pd \nimport os\nfrom PIL import Image, ImageFilter\nprint(os.listdir(\"../input\"))\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom torch.optim import Adam, SGD, RMSprop\nimport time\nfrom torch.autograd import Variable\nimport torch.functional as F\nfrom tqdm import tqdm\nfrom sklearn import metrics\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\nimport urllib\nimport pickle\nimport cv2\nimport torch.nn.functional as F\nfrom torchvision import models\nimport seaborn as sns\nimport random\nimport sys\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix, roc_curve, auc","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","scrolled":true,"execution":{"iopub.status.busy":"2024-04-23T12:30:59.285332Z","iopub.execute_input":"2024-04-23T12:30:59.285595Z","iopub.status.idle":"2024-04-23T12:31:01.995836Z","shell.execute_reply.started":"2024-04-23T12:30:59.285544Z","shell.execute_reply":"2024-04-23T12:31:01.995148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# package_path = '../input/efficientnet/efficientnet-pytorch/EfficientNet-PyTorch/'\n# sys.path.append(package_path)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T12:31:01.997346Z","iopub.execute_input":"2024-04-23T12:31:01.997589Z","iopub.status.idle":"2024-04-23T12:31:02.000962Z","shell.execute_reply.started":"2024-04-23T12:31:01.997549Z","shell.execute_reply":"2024-04-23T12:31:02.000077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","metadata":{"execution":{"iopub.status.busy":"2024-04-23T12:31:02.002094Z","iopub.execute_input":"2024-04-23T12:31:02.002329Z","iopub.status.idle":"2024-04-23T12:31:02.012501Z","shell.execute_reply.started":"2024-04-23T12:31:02.002289Z","shell.execute_reply":"2024-04-23T12:31:02.011779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_classes = 5\nseed_everything(1234)\nlr          = 1e-3\nIMG_SIZE    = 256","metadata":{"execution":{"iopub.status.busy":"2024-04-23T12:31:02.016496Z","iopub.execute_input":"2024-04-23T12:31:02.016770Z","iopub.status.idle":"2024-04-23T12:31:02.026065Z","shell.execute_reply.started":"2024-04-23T12:31:02.016716Z","shell.execute_reply":"2024-04-23T12:31:02.025362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train      = '../input/aptos2019-blindness-detection/train_images/'\ntest       = '../input/aptos2019-blindness-detection/test_images/'\ntrain_csv  = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n\ntrain_df, val_df = train_test_split(train_csv, test_size=0.1, random_state=2018, stratify=train_csv.diagnosis) # Stratify is used to ensure that both the training and validation sets have a similar distribution of the 'diagnosis' column. This helps maintain the same class distribution in both sets.\ntrain_df.reset_index(drop=True, inplace=True) # After splitting, this line resets the index of the train_df DataFrame. The drop=True argument means that the old index will be removed, and inplace=True modifies the DataFrame in place.\nval_df.reset_index(drop=True, inplace=True) \ntrain_df.head()","metadata":{"_uuid":"766f44c87272f67d632e519dce11cf54a3382696","execution":{"iopub.status.busy":"2024-04-23T12:31:02.028793Z","iopub.execute_input":"2024-04-23T12:31:02.029056Z","iopub.status.idle":"2024-04-23T12:31:02.070088Z","shell.execute_reply.started":"2024-04-23T12:31:02.028996Z","shell.execute_reply":"2024-04-23T12:31:02.069329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize ResNet-101\nfrom torchvision.models import resnet101\nmodel_resnet = resnet101(pretrained=True)\nnum_ftrs_resnet = model_resnet.fc.in_features\nmodel_resnet.fc = nn.Linear(num_ftrs_resnet, num_classes)\nmodel_resnet = model_resnet.cuda()\n\n# Initialize Inception-v3\nfrom torchvision.models import inception_v3\nmodel_inception = inception_v3(pretrained=True, aux_logits=True)\nnum_ftrs_inception = model_inception.fc.in_features\nmodel_inception.fc = nn.Linear(num_ftrs_inception, num_classes)\n# Handle auxiliary logits for Inception-v3\nnum_ftrs_aux = model_inception.AuxLogits.fc.in_features\nmodel_inception.AuxLogits.fc = nn.Linear(num_ftrs_aux, num_classes)\nmodel_inception = model_inception.cuda()\n\n# Initialize EfficientNet\nfrom efficientnet_pytorch import EfficientNet\nmodel_efficientnet = EfficientNet.from_pretrained('efficientnet-b0')\nnum_ftrs_efficientnet = model_efficientnet._fc.in_features\nmodel_efficientnet._fc = nn.Linear(num_ftrs_efficientnet, num_classes)\nmodel_efficientnet = model_efficientnet.cuda()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T12:31:02.071349Z","iopub.execute_input":"2024-04-23T12:31:02.071637Z","iopub.status.idle":"2024-04-23T12:31:16.374315Z","shell.execute_reply.started":"2024-04-23T12:31:02.071568Z","shell.execute_reply":"2024-04-23T12:31:16.373392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This function takes a filename p as input and expands it to a full file path.\ndef expand_path(p):\n    p = str(p)\n    if isfile(train + p + \".png\"):\n        return train + (p + \".png\")\n    if isfile(train_2015 + p + '.png'):\n        return train_2015 + (p + \".png\")\n    if isfile(test + p + \".png\"):\n        return test + (p + \".png\")\n    return p\n\n# This function is used to display a grid of images.\ndef p_show(imgs, label_name=None, per_row=3):\n    n = len(imgs)\n    rows = (n + per_row - 1)//per_row\n    cols = min(per_row, n)\n    fig, axes = plt.subplots(rows,cols, figsize=(15,15))\n    for ax in axes.flatten(): ax.axis('off')\n    for i,(p, ax) in enumerate(zip(imgs, axes.flatten())): \n        img = Image.open(expand_path(p))\n        ax.imshow(img)\n        ax.set_title(train_df[train_df.id_code == p].diagnosis.values)","metadata":{"_uuid":"64d7e44b053ac654c681e77b04de74ba32020fbd","execution":{"iopub.status.busy":"2024-04-23T12:31:16.375748Z","iopub.execute_input":"2024-04-23T12:31:16.376089Z","iopub.status.idle":"2024-04-23T12:31:16.386590Z","shell.execute_reply.started":"2024-04-23T12:31:16.376022Z","shell.execute_reply":"2024-04-23T12:31:16.385616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs = []\nfor p in train_df.id_code:\n    imgs.append(p)\n    if len(imgs) == 16: break\np_show(imgs)","metadata":{"_uuid":"b4739904397dd22d058c36769034b91964dcb9fe","execution":{"iopub.status.busy":"2024-04-23T12:31:16.388123Z","iopub.execute_input":"2024-04-23T12:31:16.388436Z","iopub.status.idle":"2024-04-23T12:31:21.950726Z","shell.execute_reply.started":"2024-04-23T12:31:16.388378Z","shell.execute_reply":"2024-04-23T12:31:21.949978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#The Code from: https://www.kaggle.com/ratthachat/aptos-updated-albumentation-meets-grad-cam\n# In summary, both functions aim to crop images by removing regions with pixel intensities below a specified threshold (tolerance). This can be useful for eliminating unwanted dark or uninformative areas in images, depending on the application.\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img","metadata":{"execution":{"iopub.status.busy":"2024-04-23T12:31:21.952271Z","iopub.execute_input":"2024-04-23T12:31:21.952536Z","iopub.status.idle":"2024-04-23T12:31:21.964302Z","shell.execute_reply.started":"2024-04-23T12:31:21.952484Z","shell.execute_reply":"2024-04-23T12:31:21.963212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport cv2\nfrom torchvision.transforms import ToPILImage, ToTensor, Normalize, Compose\n\nclass MyDataset(Dataset):\n    \n    def __init__(self, dataframe, transform=None, model_name='resnet'):\n        self.df = dataframe\n        self.transform = transform\n        self.model_name = model_name\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def expand_path(self, p):\n        # Assuming 'train', 'test', and 'train_2015' are defined and accessible\n        p = str(p)\n        if os.path.isfile(os.path.join(train, p + \".png\")):\n            return os.path.join(train, p + \".png\")\n        if os.path.isfile(os.path.join(train_2015, p + '.png')):\n            return os.path.join(train_2015, p + \".png\")\n        if os.path.isfile(os.path.join(test, p + \".png\")):\n            return os.path.join(test, p + \".png\")\n        return p\n\n    def __getitem__(self, idx):\n        label = self.df.diagnosis.values[idx]\n        # Ensure label is a long type if not already\n        label = torch.tensor(label, dtype=torch.long)\n        \n        p = self.df.id_code.values[idx]\n        p_path = self.expand_path(p)\n        image = cv2.imread(p_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = crop_image_from_gray(image)\n\n        # Resize image based on the model\n        if self.model_name == 'inception':\n            image = cv2.resize(image, (299, 299))\n        else:\n            image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n\n        image = cv2.addWeighted(image, 4, cv2.GaussianBlur(image, (0,0), 30), -4, 128)\n        image = ToPILImage()(image)\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        return image, label","metadata":{"_uuid":"21908baa8df4e398b0d49a5146ce544504637c5a","execution":{"iopub.status.busy":"2024-04-23T12:31:21.965864Z","iopub.execute_input":"2024-04-23T12:31:21.966134Z","iopub.status.idle":"2024-04-23T12:31:21.983944Z","shell.execute_reply.started":"2024-04-23T12:31:21.966083Z","shell.execute_reply":"2024-04-23T12:31:21.983157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Common transform for all models\ntrain_transform = transforms.Compose([\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation((-120, 120)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\nresnet_trainset = MyDataset(train_df, transform=train_transform, model_name='resnet')\nresnet_train_loader = torch.utils.data.DataLoader(resnet_trainset, batch_size=32, shuffle=True, num_workers=4)\nresnet_valset = MyDataset(val_df, transform=train_transform, model_name='resnet')\nresnet_val_loader = torch.utils.data.DataLoader(resnet_valset, batch_size=32, shuffle=False, num_workers=4)\n\ninception_trainset = MyDataset(train_df, transform=train_transform, model_name='inception')\ninception_train_loader = torch.utils.data.DataLoader(inception_trainset, batch_size=32, shuffle=True, num_workers=4)\ninception_valset = MyDataset(val_df, transform=train_transform, model_name='inception')\ninception_val_loader = torch.utils.data.DataLoader(inception_valset, batch_size=32, shuffle=False, num_workers=4)","metadata":{"_uuid":"f590638fd07b9aefe2210a39612ac77e0689c0c1","execution":{"iopub.status.busy":"2024-04-23T12:31:21.985186Z","iopub.execute_input":"2024-04-23T12:31:21.985434Z","iopub.status.idle":"2024-04-23T12:31:21.998048Z","shell.execute_reply.started":"2024-04-23T12:31:21.985387Z","shell.execute_reply":"2024-04-23T12:31:21.997250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\n\noptimizer_resnet = torch.optim.Adam(model_resnet.parameters(), lr=lr, weight_decay=1e-5)\nscheduler_resnet = torch.optim.lr_scheduler.StepLR(optimizer_resnet, step_size=5, gamma=0.1)\n\noptimizer_efficientnet = torch.optim.Adam(model_efficientnet.parameters(), lr=lr, weight_decay=1e-5)\nscheduler_efficientnet = torch.optim.lr_scheduler.StepLR(optimizer_efficientnet, step_size=5, gamma=0.1)\n\noptimizer_inception = torch.optim.Adam(model_inception.parameters(), lr=lr, weight_decay=1e-5)\nscheduler_inception = torch.optim.lr_scheduler.StepLR(optimizer_inception, step_size=5, gamma=0.1)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T12:31:21.999525Z","iopub.execute_input":"2024-04-23T12:31:21.999778Z","iopub.status.idle":"2024-04-23T12:31:22.017891Z","shell.execute_reply.started":"2024-04-23T12:31:21.999728Z","shell.execute_reply":"2024-04-23T12:31:22.017334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, optimizer, train_loader, criterion, is_inception=False):\n    model.train()\n    avg_loss = 0.\n    correct = 0\n    total = 0\n\n    for idx, (imgs, labels) in enumerate(train_loader):\n        imgs_train, labels_train = imgs.cuda(), torch.squeeze(labels.cuda())\n        optimizer.zero_grad()\n\n        if is_inception:\n            output_train, aux_output_train = model(imgs_train)\n            loss1 = criterion(output_train, labels_train)\n            loss2 = criterion(aux_output_train, labels_train)\n            loss = loss1 + 0.4 * loss2\n        else:\n            output_train = model(imgs_train)\n            loss = criterion(output_train, labels_train)\n\n        _, predicted = torch.max(output_train.data, 1)\n        total += labels_train.size(0)\n        correct += (predicted == labels_train).sum().item()\n\n        loss.backward()\n        optimizer.step()\n        avg_loss += loss.item() / len(train_loader)\n\n    avg_accuracy = correct / total\n    return avg_loss, avg_accuracy\n\ndef test_model(model, val_loader, criterion, is_inception=False):\n    model.eval()\n    total_loss = 0\n    total_correct = 0\n    with torch.no_grad():\n        for imgs, labels in val_loader:\n            imgs, labels = imgs.cuda(), labels.cuda()\n            if is_inception and model.training:\n                # Handle Inception model's auxiliary output in training mode\n                output, aux_output = model(imgs)\n                loss1 = criterion(output, labels)\n                loss2 = criterion(aux_output, labels)\n                loss = loss1 + 0.4*loss2\n            else:\n                output = model(imgs)\n                loss = criterion(output, labels)\n            total_loss += loss.item()\n            _, preds = torch.max(output, 1)\n            total_correct += torch.sum(preds == labels.data)\n    avg_loss = total_loss / len(val_loader.dataset)\n    accuracy = total_correct.double() / len(val_loader.dataset)\n    return avg_loss, accuracy","metadata":{"_uuid":"c338feda0eee741964b4c3d736c30b1e0a7e3ace","execution":{"iopub.status.busy":"2024-04-23T12:31:22.019223Z","iopub.execute_input":"2024-04-23T12:31:22.019533Z","iopub.status.idle":"2024-04-23T12:31:22.036929Z","shell.execute_reply.started":"2024-04-23T12:31:22.019470Z","shell.execute_reply":"2024-04-23T12:31:22.036140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_epochs = 15\n\n# Initialize empty lists to store data\ntrain_losses_resnet = []\nval_losses_resnet = []\ntrain_accuracies_resnet = []\nval_accuracies_resnet = []\n\ntrain_losses_efficientnet = []\nval_losses_efficientnet = []\ntrain_accuracies_efficientnet = []\nval_accuracies_efficientnet = []\n\ntrain_losses_inception = []\nval_losses_inception = []\ntrain_accuracies_inception = []\nval_accuracies_inception = []\n\nfor model, optimizer, scheduler, train_loader, val_loader in [\n    (model_resnet, optimizer_resnet, scheduler_resnet, resnet_train_loader, resnet_val_loader), \n    (model_efficientnet, optimizer_efficientnet, scheduler_efficientnet, resnet_train_loader, resnet_val_loader), \n    (model_inception, optimizer_inception, scheduler_inception, inception_train_loader, inception_val_loader),\n]:\n    best_avg_loss = 100.0\n    is_inception = (model.__class__.__name__ == \"Inception3\")\n\n    for epoch in range(n_epochs):  \n        print(f'Training {model.__class__.__name__}, Epoch: {epoch+1}/{n_epochs}')\n        avg_loss, avg_accuracy = train_model(model, optimizer, train_loader, criterion, is_inception)\n        avg_val_loss, val_accuracy = test_model(model, val_loader, criterion, is_inception)\n        scheduler.step()\n\n        # Append losses and accuracies to the respective lists\n        if model == model_resnet:\n            train_losses_resnet.append(avg_loss)\n            val_losses_resnet.append(avg_val_loss)\n            train_accuracies_resnet.append(avg_accuracy)\n            val_accuracies_resnet.append(val_accuracy)\n        elif model == model_efficientnet:\n            train_losses_efficientnet.append(avg_loss)\n            val_losses_efficientnet.append(avg_val_loss)\n            train_accuracies_efficientnet.append(avg_accuracy)\n            val_accuracies_efficientnet.append(val_accuracy)\n        elif model == model_inception:\n            train_losses_inception.append(avg_loss)\n            val_losses_inception.append(avg_val_loss)\n            train_accuracies_inception.append(avg_accuracy)\n            val_accuracies_inception.append(val_accuracy)\n\n        # Save model if validation loss improved\n        if avg_val_loss < best_avg_loss:\n            best_avg_loss = avg_val_loss\n            torch.save(model.state_dict(), f'weight_best_{model.__class__.__name__}.pt')","metadata":{"_uuid":"3562bd2ec1b0650519ca196bfc0e60eb139ca180","execution":{"iopub.status.busy":"2024-04-23T12:31:22.038199Z","iopub.execute_input":"2024-04-23T12:31:22.038418Z","iopub.status.idle":"2024-04-23T14:36:53.024163Z","shell.execute_reply.started":"2024-04-23T12:31:22.038380Z","shell.execute_reply":"2024-04-23T14:36:53.023063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load the Best Model Weights","metadata":{}},{"cell_type":"code","source":"# Make sure models are in evaluation mode\nmodel_resnet.eval()\nmodel_efficientnet.eval()\nmodel_inception.eval()\n\n# Load the saved best weights\nmodel_resnet.load_state_dict(torch.load('weight_best_ResNet.pt'))\nmodel_efficientnet.load_state_dict(torch.load('weight_best_EfficientNet.pt'))\nmodel_inception.load_state_dict(torch.load('weight_best_Inception3.pt'))","metadata":{"execution":{"iopub.status.busy":"2024-04-23T14:36:53.026003Z","iopub.execute_input":"2024-04-23T14:36:53.026314Z","iopub.status.idle":"2024-04-23T14:36:53.399217Z","shell.execute_reply.started":"2024-04-23T14:36:53.026258Z","shell.execute_reply":"2024-04-23T14:36:53.398307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluate Each Model","metadata":{}},{"cell_type":"code","source":"def evaluate_model(model, val_loader):\n    model.eval()\n    correct = 0\n    total = 0\n    all_labels = []\n    all_predictions = []\n    with torch.no_grad():\n        for images, labels in val_loader:\n            images, labels = images.cuda(), labels.cuda()\n            outputs = model(images)\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n            all_labels.extend(labels.cpu().numpy())\n            all_predictions.extend(predicted.cpu().numpy())\n\n    accuracy = 100 * correct / total\n    precision = precision_score(all_labels, all_predictions, average='weighted')\n    recall = recall_score(all_labels, all_predictions, average='weighted')\n    f1 = f1_score(all_labels, all_predictions, average='weighted')\n\n    return accuracy, precision, recall, f1","metadata":{"execution":{"iopub.status.busy":"2024-04-23T14:36:53.400490Z","iopub.execute_input":"2024-04-23T14:36:53.400754Z","iopub.status.idle":"2024-04-23T14:36:53.410860Z","shell.execute_reply.started":"2024-04-23T14:36:53.400704Z","shell.execute_reply":"2024-04-23T14:36:53.410156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# At the end of training for each model, call the evaluate_model function\naccuracy_resnet, precision_resnet, recall_resnet, f1_score_resnet = evaluate_model(model_resnet, resnet_val_loader)\naccuracy_efficientnet, precision_efficientnet, recall_efficientnet, f1_score_efficientnet = evaluate_model(model_efficientnet, resnet_val_loader)\naccuracy_inception, precision_inception, recall_inception, f1_score_inception = evaluate_model(model_inception, inception_val_loader)\n\nprint(f'ResNet-101 Accuracy: {accuracy_resnet:.2f}%')\nprint(f'ResNet-101 Precision: {precision_resnet:.2f}')\nprint(f'ResNet-101 Recall: {recall_resnet:.2f}')\nprint(f'ResNet-101 F1-Score: {f1_score_resnet:.2f}')\n\nprint(f'EfficientNet Accuracy: {accuracy_efficientnet:.2f}%')\nprint(f'EfficientNet Precision: {precision_efficientnet:.2f}')\nprint(f'EfficientNet Recall: {recall_efficientnet:.2f}')\nprint(f'EfficientNet F1-Score: {f1_score_efficientnet:.2f}')\n\nprint(f'Inception-v3 Accuracy: {accuracy_inception:.2f}%')\nprint(f'Inception-v3 Precision: {precision_inception:.2f}')\nprint(f'Inception-v3 Recall: {recall_inception:.2f}')\nprint(f'Inception-v3 F1-Score: {f1_score_inception:.2f}')","metadata":{"execution":{"iopub.status.busy":"2024-04-23T14:36:53.412215Z","iopub.execute_input":"2024-04-23T14:36:53.412480Z","iopub.status.idle":"2024-04-23T14:38:59.333148Z","shell.execute_reply.started":"2024-04-23T14:36:53.412426Z","shell.execute_reply":"2024-04-23T14:38:59.332260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import VotingClassifier\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n\n# Define a function to get predictions from a PyTorch model\ndef get_predictions(model, data_loader):\n    model.eval()\n    predictions = []\n    true_labels = []\n    with torch.no_grad():\n        for inputs, labels in data_loader:\n            inputs = inputs.cuda()\n            labels = labels.cuda()\n            outputs = model(inputs)\n            _, preds = torch.max(outputs, 1)\n            predictions.extend(preds.cpu().numpy())\n            true_labels.extend(labels.cpu().numpy())\n    return predictions, true_labels\n\n# Get predictions for each model\npredictions_resnet, labels = get_predictions(model_resnet, resnet_val_loader)\npredictions_inception, _ = get_predictions(model_inception, inception_val_loader)\npredictions_efficientnet, _ = get_predictions(model_efficientnet,resnet_val_loader )\n\n# Create an ensemble model\nensemble_predictions = []\nfor pred_resnet, pred_inception, pred_efficientnet in zip(predictions_resnet, predictions_inception, predictions_efficientnet):\n    # Perform majority voting\n    ensemble_predictions.append(max(set([pred_resnet, pred_inception, pred_efficientnet]), key = [pred_resnet, pred_inception, pred_efficientnet].count))\n\n# Calculate evaluation metrics for the ensemble\naccuracy_ensemble = accuracy_score(labels, ensemble_predictions)\nprecision_ensemble = precision_score(labels, ensemble_predictions, average='macro')\nrecall_ensemble = recall_score(labels, ensemble_predictions, average='macro')\nf1_score_ensemble = f1_score(labels, ensemble_predictions, average='macro')\n\n# Save the ensemble model if the accuracy improves\nbest_accuracy = max(accuracy_resnet, accuracy_inception, accuracy_efficientnet)\nprint(f'Initialized Best Accuracy: {best_accuracy:.2f}')\n\nif accuracy_ensemble > best_accuracy:\n    best_accuracy = accuracy_ensemble\n    torch.save(ensemble_model.state_dict(), 'ensemble_model_best.pt')\n\n# Print results\nprint(f'Ensemble Model Accuracy: {accuracy_ensemble:.2f}%')\nprint(f'Ensemble Model Precision: {precision_ensemble:.2f}')\nprint(f'Ensemble Model Recall: {recall_ensemble:.2f}')\nprint(f'Ensemble Model F1-Score: {f1_score_ensemble:.2f}')\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T14:55:01.627725Z","iopub.execute_input":"2024-04-23T14:55:01.628130Z","iopub.status.idle":"2024-04-23T14:57:07.477211Z","shell.execute_reply.started":"2024-04-23T14:55:01.628068Z","shell.execute_reply":"2024-04-23T14:57:07.476029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Create subplots for loss and accuracy\nplt.figure(figsize=(12, 4))\n\n# Plot ResNet results\nplt.subplot(1, 2, 1)\nplt.plot(range(1, n_epochs + 1), val_losses_resnet, label='ResNet', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_inception, label='Inception', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_efficientnet, label='EfficientNet', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Validation Loss')\nplt.title('Validation Loss Curves')\nplt.legend()\n\nplt.subplot(1, 2, 2)\nplt.plot(range(1, n_epochs + 1), val_accuracies_resnet, label='ResNet', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_inception, label='Inception', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_efficientnet, label='EfficientNet', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Validation Accuracy')\nplt.title('Validation Accuracy Curves')\nplt.legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T14:41:05.956872Z","iopub.status.idle":"2024-04-23T14:41:05.957392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Create subplots for loss and accuracy\nplt.figure(figsize=(16, 8))\n\n# Plot ResNet results\nplt.subplot(2, 3, 1)\nplt.plot(range(1, n_epochs + 1), train_losses_resnet, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_resnet, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('ResNet Loss Curves')\nplt.legend()\n\nplt.subplot(2, 3, 2)\nplt.plot(range(1, n_epochs + 1), train_accuracies_resnet, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_resnet, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('ResNet Accuracy Curves')\nplt.legend()\n\n# Plot Inception results\nplt.subplot(2, 3, 3)\nplt.plot(range(1, n_epochs + 1), train_losses_inception, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_inception, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Inception Loss Curves')\nplt.legend()\n\nplt.subplot(2, 3, 4)\nplt.plot(range(1, n_epochs + 1), train_accuracies_inception, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_inception, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('Inception Accuracy Curves')\nplt.legend()\n\n# Plot EfficientNet results\nplt.subplot(2, 3, 5)\nplt.plot(range(1, n_epochs + 1), train_losses_efficientnet, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_efficientnet, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('EfficientNet Loss Curves')\nplt.legend()\n\nplt.subplot(2, 3, 6)\nplt.plot(range(1, n_epochs + 1), train_accuracies_efficientnet, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_efficientnet, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('EfficientNet Accuracy Curves')\nplt.legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T14:41:05.958846Z","iopub.status.idle":"2024-04-23T14:41:05.959338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a bar chart for final validation accuracy\nmodels = ['ResNet', 'Inception', 'EfficientNet']\nfinal_accuracies = [val_accuracies_resnet[-1], val_accuracies_inception[-1], val_accuracies_efficientnet[-1]]\n\nplt.figure(figsize=(8, 5))\nplt.bar(models, final_accuracies)\nplt.xlabel('Model')\nplt.ylabel('Final Validation Accuracy')\nplt.title('Comparison of Final Validation Accuracy')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T14:41:05.960527Z","iopub.status.idle":"2024-04-23T14:41:05.960989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport itertools\n\n# After evaluating each model\n# Example for ResNet model\ndef plot_roc_curve_and_confusion_matrix(model, val_loader):\n    model.eval()\n    all_labels = []\n    all_probs = []\n    with torch.no_grad():\n        for images, labels in val_loader:\n            images, labels = images.cuda(), labels.cuda()\n            outputs = model(images)\n            probabilities = torch.softmax(outputs, dim=1).cpu().numpy()\n            all_labels.extend(labels.cpu().numpy())\n            all_probs.extend(probabilities)\n\n    # Calculate ROC curve for each class\n    fpr = dict()\n    tpr = dict()\n    roc_auc = dict()\n    for i in range(num_classes):  # Replace num_classes with the actual number of classes\n        fpr[i], tpr[i], _ = roc_curve(np.array(all_labels) == i, np.array(all_probs)[:, i])\n        roc_auc[i] = auc(fpr[i], tpr[i])\n\n    # Plot ROC curves\n    plt.figure(figsize=(8, 6))\n    for i in range(num_classes):\n        plt.plot(fpr[i], tpr[i], lw=2, label=f'Class {i} (AUC = {roc_auc[i]:0.2f})')\n    plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='black')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver Operating Characteristic (ROC) Curve')\n    plt.legend(loc='best')\n\n    # Plot confusion matrix\n    plt.figure(figsize=(8, 6))\n    cm = confusion_matrix(all_labels, np.argmax(all_probs, axis=1))\n    plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)\n    plt.title('Confusion Matrix')\n    plt.colorbar()\n    tick_marks = np.arange(num_classes)  # Replace num_classes with the actual number of classes\n    plt.xticks(tick_marks, [f'Class {i}' for i in range(num_classes)], rotation=45)\n    plt.yticks(tick_marks, [f'Class {i}' for i in range(num_classes)])\n    plt.xlabel('Predicted')\n    plt.ylabel('True')\n\n    # Annotate boxes with counts\n    fmt = 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n\n    plt.show()\n\n# Example usage for ResNet model\nplot_roc_curve_and_confusion_matrix(model_resnet, resnet_val_loader)\nplot_roc_curve_and_confusion_matrix(model_efficientnet, resnet_val_loader)\nplot_roc_curve_and_confusion_matrix(model_inception, inception_val_loader)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T14:41:05.962267Z","iopub.status.idle":"2024-04-23T14:41:05.962883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\ndef visualize_predictions_percentage(model, val_loader):\n    model.eval()\n    all_predictions = []\n    with torch.no_grad():\n        for images, _ in val_loader:\n            images = images.cuda()\n            outputs = model(images)\n            _, predicted = torch.max(outputs, 1)\n            all_predictions.extend(predicted.cpu().numpy())\n\n    # Calculate percentage distribution of predictions\n    unique_labels, counts = np.unique(all_predictions, return_counts=True)\n    percentages = (counts / len(all_predictions)) * 100\n\n    # Create a pie chart\n    plt.figure(figsize=(8, 6))\n    plt.pie(percentages, labels=unique_labels, autopct='%1.1f%%', startangle=140)\n    plt.title(f\"Percentage of Predictions by Class for {model}\")\n    plt.show()\n\n# Example usage for ResNet model\nvisualize_predictions_percentage(model_resnet, resnet_val_loader)\nvisualize_predictions_percentage(model_efficientnet, resnet_val_loader)\nvisualize_predictions_percentage(model_inception, inception_val_loader)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T14:41:05.964135Z","iopub.status.idle":"2024-04-23T14:41:05.964771Z"},"trusted":true},"execution_count":null,"outputs":[]}]}