{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":14774,"databundleVersionId":875431,"sourceType":"competition"},{"sourceId":418427,"sourceType":"datasetVersion","datasetId":187021},{"sourceId":527603,"sourceType":"datasetVersion","datasetId":250877},{"sourceId":848739,"sourceType":"datasetVersion","datasetId":251095},{"sourceId":1399787,"sourceType":"datasetVersion","datasetId":252150},{"sourceId":7521612,"sourceType":"datasetVersion","datasetId":4381613}],"dockerImageVersionId":29186,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"version2-3. Fix Nvidia apex installation Error\n\nInference: https://www.kaggle.com/chanhu/eye-inference-num-class-1-ver3","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet_pytorch","metadata":{"execution":{"iopub.status.busy":"2024-01-31T15:10:39.456335Z","iopub.execute_input":"2024-01-31T15:10:39.456683Z","iopub.status.idle":"2024-01-31T15:10:45.452663Z","shell.execute_reply.started":"2024-01-31T15:10:39.456628Z","shell.execute_reply":"2024-01-31T15:10:45.451629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nfrom os.path import isfile\nimport torch.nn.init as init\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport pandas as pd \nimport os\nfrom PIL import Image, ImageFilter\nprint(os.listdir(\"../input\"))\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nfrom torch.optim import Adam, SGD, RMSprop\nimport time\nfrom torch.autograd import Variable\nimport torch.functional as F\nfrom tqdm import tqdm\nfrom sklearn import metrics\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\nimport urllib\nimport pickle\nimport cv2\nimport torch.nn.functional as F\nfrom torchvision import models\nimport seaborn as sns\nimport random\nimport sys","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","scrolled":true,"execution":{"iopub.status.busy":"2024-01-31T15:10:45.455472Z","iopub.execute_input":"2024-01-31T15:10:45.455863Z","iopub.status.idle":"2024-01-31T15:10:45.466779Z","shell.execute_reply.started":"2024-01-31T15:10:45.455803Z","shell.execute_reply":"2024-01-31T15:10:45.465834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# package_path = '../input/efficientnet/efficientnet-pytorch/EfficientNet-PyTorch/'\n# sys.path.append(package_path)","metadata":{"execution":{"iopub.status.busy":"2024-01-31T15:10:45.46812Z","iopub.execute_input":"2024-01-31T15:10:45.46841Z","iopub.status.idle":"2024-01-31T15:10:45.477655Z","shell.execute_reply.started":"2024-01-31T15:10:45.468345Z","shell.execute_reply":"2024-01-31T15:10:45.477072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","metadata":{"execution":{"iopub.status.busy":"2024-01-31T15:10:45.478854Z","iopub.execute_input":"2024-01-31T15:10:45.479174Z","iopub.status.idle":"2024-01-31T15:10:45.488296Z","shell.execute_reply.started":"2024-01-31T15:10:45.479089Z","shell.execute_reply":"2024-01-31T15:10:45.48763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_classes = 1\nseed_everything(1234)\nlr          = 1e-3\nIMG_SIZE    = 256","metadata":{"execution":{"iopub.status.busy":"2024-01-31T15:10:45.49175Z","iopub.execute_input":"2024-01-31T15:10:45.49211Z","iopub.status.idle":"2024-01-31T15:10:45.498247Z","shell.execute_reply.started":"2024-01-31T15:10:45.49205Z","shell.execute_reply":"2024-01-31T15:10:45.497632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train      = '../input/aptos2019-blindness-detection/train_images/'\ntest       = '../input/aptos2019-blindness-detection/test_images/'\ntrain_csv  = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n\ntrain_df, val_df = train_test_split(train_csv, test_size=0.1, random_state=2018, stratify=train_csv.diagnosis)\ntrain_df.reset_index(drop=True, inplace=True)\nval_df.reset_index(drop=True, inplace=True)\ntrain_df.head()","metadata":{"_uuid":"766f44c87272f67d632e519dce11cf54a3382696","execution":{"iopub.status.busy":"2024-01-31T15:10:45.500243Z","iopub.execute_input":"2024-01-31T15:10:45.500729Z","iopub.status.idle":"2024-01-31T15:10:45.52922Z","shell.execute_reply.started":"2024-01-31T15:10:45.500545Z","shell.execute_reply":"2024-01-31T15:10:45.528459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize ResNet-101\nfrom torchvision.models import resnet101\nmodel_resnet = resnet101(pretrained=True)\nnum_ftrs_resnet = model_resnet.fc.in_features\nmodel_resnet.fc = nn.Linear(num_ftrs_resnet, num_classes)\nmodel_resnet = model_resnet.cuda()\n\n# Initialize Inception-v3\nfrom torchvision.models import inception_v3\nmodel_inception = inception_v3(pretrained=True, aux_logits=True)\nnum_ftrs_inception = model_inception.fc.in_features\nmodel_inception.fc = nn.Linear(num_ftrs_inception, num_classes)\n# Handle auxiliary logits for Inception-v3\nnum_ftrs_aux = model_inception.AuxLogits.fc.in_features\nmodel_inception.AuxLogits.fc = nn.Linear(num_ftrs_aux, num_classes)\nmodel_inception = model_inception.cuda()\n\n# Initialize EfficientNet\nfrom efficientnet_pytorch import EfficientNet\nmodel_efficientnet = EfficientNet.from_pretrained('efficientnet-b0')\nnum_ftrs_efficientnet = model_efficientnet._fc.in_features\nmodel_efficientnet._fc = nn.Linear(num_ftrs_efficientnet, num_classes)\nmodel_efficientnet = model_efficientnet.cuda()","metadata":{"execution":{"iopub.status.busy":"2024-01-31T15:10:45.530766Z","iopub.execute_input":"2024-01-31T15:10:45.531112Z","iopub.status.idle":"2024-01-31T15:10:49.458469Z","shell.execute_reply.started":"2024-01-31T15:10:45.531053Z","shell.execute_reply":"2024-01-31T15:10:49.457472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def expand_path(p):\n    p = str(p)\n    if isfile(train + p + \".png\"):\n        return train + (p + \".png\")\n    if isfile(train_2015 + p + '.png'):\n        return train_2015 + (p + \".png\")\n    if isfile(test + p + \".png\"):\n        return test + (p + \".png\")\n    return p\n\ndef p_show(imgs, label_name=None, per_row=3):\n    n = len(imgs)\n    rows = (n + per_row - 1)//per_row\n    cols = min(per_row, n)\n    fig, axes = plt.subplots(rows,cols, figsize=(15,15))\n    for ax in axes.flatten(): ax.axis('off')\n    for i,(p, ax) in enumerate(zip(imgs, axes.flatten())): \n        img = Image.open(expand_path(p))\n        ax.imshow(img)\n        ax.set_title(train_df[train_df.id_code == p].diagnosis.values)","metadata":{"_uuid":"64d7e44b053ac654c681e77b04de74ba32020fbd","execution":{"iopub.status.busy":"2024-01-31T15:10:49.459998Z","iopub.execute_input":"2024-01-31T15:10:49.460353Z","iopub.status.idle":"2024-01-31T15:10:49.471511Z","shell.execute_reply.started":"2024-01-31T15:10:49.460283Z","shell.execute_reply":"2024-01-31T15:10:49.470632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs = []\nfor p in train_df.id_code:\n    imgs.append(p)\n    if len(imgs) == 16: break\np_show(imgs)","metadata":{"_uuid":"b4739904397dd22d058c36769034b91964dcb9fe","execution":{"iopub.status.busy":"2024-01-31T15:10:49.472801Z","iopub.execute_input":"2024-01-31T15:10:49.473098Z","iopub.status.idle":"2024-01-31T15:10:54.058903Z","shell.execute_reply.started":"2024-01-31T15:10:49.473048Z","shell.execute_reply":"2024-01-31T15:10:54.057982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#The Code from: https://www.kaggle.com/ratthachat/aptos-updated-albumentation-meets-grad-cam\n\ndef crop_image1(img,tol=7):\n    # img is image data\n    # tol  is tolerance\n        \n    mask = img>tol\n    return img[np.ix_(mask.any(1),mask.any(0))]\n\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n    #         print(img1.shape,img2.shape,img3.shape)\n            img = np.stack([img1,img2,img3],axis=-1)\n    #         print(img.shape)\n        return img","metadata":{"execution":{"iopub.status.busy":"2024-01-31T15:10:54.060124Z","iopub.execute_input":"2024-01-31T15:10:54.060358Z","iopub.status.idle":"2024-01-31T15:10:54.073482Z","shell.execute_reply.started":"2024-01-31T15:10:54.06032Z","shell.execute_reply":"2024-01-31T15:10:54.072553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyDataset(Dataset):\n    \n    def __init__(self, dataframe, transform=None, model_name='resnet'):\n        self.df = dataframe\n        self.transform = transform\n        self.model_name = model_name\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def expand_path(self, p):\n        # Assuming 'train', 'test', and 'train_2015' are defined and accessible\n        p = str(p)\n        if os.path.isfile(os.path.join(train, p + \".png\")):\n            return os.path.join(train, p + \".png\")\n        if os.path.isfile(os.path.join(train_2015, p + '.png')):\n            return os.path.join(train_2015, p + \".png\")\n        if os.path.isfile(os.path.join(test, p + \".png\")):\n            return os.path.join(test, p + \".png\")\n        return p\n\n    def __getitem__(self, idx):\n        label = self.df.diagnosis.values[idx]\n        label = np.expand_dims(label, -1)\n        \n        p = self.df.id_code.values[idx]\n        p_path = self.expand_path(p)\n        image = cv2.imread(p_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = crop_image_from_gray(image)\n\n        # Resize image based on the model\n        if self.model_name == 'inception':\n            image = cv2.resize(image, (299, 299))\n        else:\n            image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n\n        image = cv2.addWeighted(image, 4, cv2.GaussianBlur(image, (0,0), 30), -4, 128)\n        image = transforms.ToPILImage()(image)\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        return image, label\n","metadata":{"_uuid":"21908baa8df4e398b0d49a5146ce544504637c5a","execution":{"iopub.status.busy":"2024-01-31T15:10:54.075026Z","iopub.execute_input":"2024-01-31T15:10:54.075354Z","iopub.status.idle":"2024-01-31T15:10:54.092382Z","shell.execute_reply.started":"2024-01-31T15:10:54.075296Z","shell.execute_reply":"2024-01-31T15:10:54.091519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Common transform for all models\ntrain_transform = transforms.Compose([\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation((-120, 120)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\n# For ResNet and EfficientNet\ntrainset_resnet = MyDataset(train_df, transform=train_transform, model_name='resnet')\ntrain_loader_resnet = torch.utils.data.DataLoader(trainset_resnet, batch_size=32, shuffle=True, num_workers=4)\nvalset_resnet = MyDataset(val_df, transform=train_transform, model_name='resnet')\nval_loader_resnet = torch.utils.data.DataLoader(valset_resnet, batch_size=32, shuffle=False, num_workers=4)\n\n# For Inception\ntrainset_inception = MyDataset(train_df, transform=train_transform, model_name='inception')\ntrain_loader_inception = torch.utils.data.DataLoader(trainset_inception, batch_size=32, shuffle=True, num_workers=4)\nvalset_inception = MyDataset(val_df, transform=train_transform, model_name='inception')\nval_loader_inception = torch.utils.data.DataLoader(valset_inception, batch_size=32, shuffle=False, num_workers=4)","metadata":{"_uuid":"f590638fd07b9aefe2210a39612ac77e0689c0c1","execution":{"iopub.status.busy":"2024-01-31T15:10:54.093951Z","iopub.execute_input":"2024-01-31T15:10:54.094467Z","iopub.status.idle":"2024-01-31T15:10:54.106975Z","shell.execute_reply.started":"2024-01-31T15:10:54.094248Z","shell.execute_reply":"2024-01-31T15:10:54.106257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.MSELoss()\n\noptimizer_resnet = torch.optim.Adam(model_resnet.parameters(), lr=lr, weight_decay=1e-5)\nscheduler_resnet = torch.optim.lr_scheduler.StepLR(optimizer_resnet, step_size=5, gamma=0.1)\n\noptimizer_efficientnet = torch.optim.Adam(model_efficientnet.parameters(), lr=lr, weight_decay=1e-5)\nscheduler_efficientnet = torch.optim.lr_scheduler.StepLR(optimizer_efficientnet, step_size=5, gamma=0.1)\n\noptimizer_inception = torch.optim.Adam(model_inception.parameters(), lr=lr, weight_decay=1e-5)\nscheduler_inception = torch.optim.lr_scheduler.StepLR(optimizer_inception, step_size=5, gamma=0.1)","metadata":{"execution":{"iopub.status.busy":"2024-01-31T15:10:54.108199Z","iopub.execute_input":"2024-01-31T15:10:54.108431Z","iopub.status.idle":"2024-01-31T15:10:54.125144Z","shell.execute_reply.started":"2024-01-31T15:10:54.108387Z","shell.execute_reply":"2024-01-31T15:10:54.124578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, optimizer, train_loader, criterion, is_inception=False):\n    model.train()\n    avg_loss = 0.\n    correct = 0\n    total = 0\n\n    for idx, (imgs, labels) in enumerate(train_loader):\n        imgs_train, labels_train = imgs.cuda(), labels.cuda()\n        optimizer.zero_grad()\n\n        if is_inception:\n            output_train, aux_output_train = model(imgs_train)\n            loss1 = criterion(output_train, labels_train.float())\n            loss2 = criterion(aux_output_train, labels_train.float())\n            loss = loss1 + 0.4 * loss2\n        else:\n            output_train = model(imgs_train)\n            loss = criterion(output_train, labels_train.float())\n\n        _, predicted = torch.max(output_train.data, 1)\n        total += labels_train.size(0)\n        correct += (predicted == labels_train).sum().item()\n\n        loss.backward()\n        optimizer.step()\n        avg_loss += loss.item() / len(train_loader)\n\n    avg_accuracy = 100 * correct / total\n    return avg_loss, avg_accuracy\n\n\ndef test_model(model, val_loader, criterion, is_inception=False):\n    avg_val_loss = 0.\n    correct = 0\n    total = 0\n    model.eval()\n    with torch.no_grad():\n        for idx, (imgs, labels) in enumerate(val_loader):\n            imgs_valid, labels_valid = imgs.cuda(), labels.cuda()\n\n            # Handle Inception v3 model output\n            if is_inception:\n                outputs = model(imgs_valid)\n                if type(outputs) == tuple:  # When model outputs both main and auxiliary outputs\n                    output_test = outputs[0]  # Use only the main output\n                else:  # In evaluation mode, only main output is returned\n                    output_test = outputs\n            else:\n                output_test = model(imgs_valid)\n\n            avg_val_loss += criterion(output_test, labels_valid.float()).item() / len(val_loader)\n            _, predicted = torch.max(output_test.data, 1)\n            total += labels_valid.size(0)\n            correct += (predicted == labels_valid).sum().item()\n\n    accuracy = 100 * correct / total\n    return avg_val_loss, accuracy","metadata":{"_uuid":"c338feda0eee741964b4c3d736c30b1e0a7e3ace","execution":{"iopub.status.busy":"2024-01-31T15:10:54.126154Z","iopub.execute_input":"2024-01-31T15:10:54.12643Z","iopub.status.idle":"2024-01-31T15:10:54.141166Z","shell.execute_reply.started":"2024-01-31T15:10:54.12637Z","shell.execute_reply":"2024-01-31T15:10:54.140368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EarlyStopping:\n    def __init__(self, patience=7, min_delta=0):\n        self.patience = patience\n        self.min_delta = min_delta\n        self.counter = 0\n        self.best_loss = None\n        self.early_stop = False\n\n    def __call__(self, val_loss):\n        if self.best_loss is None:\n            self.best_loss = val_loss\n        elif val_loss > self.best_loss - self.min_delta:\n            self.counter += 1\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_loss = val_loss\n            self.counter = 0","metadata":{"execution":{"iopub.status.busy":"2024-01-31T15:10:54.142482Z","iopub.execute_input":"2024-01-31T15:10:54.142778Z","iopub.status.idle":"2024-01-31T15:10:54.1534Z","shell.execute_reply.started":"2024-01-31T15:10:54.142723Z","shell.execute_reply":"2024-01-31T15:10:54.152745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_epochs = 1\n\n# Initialize empty lists to store data\ntrain_losses_resnet = []\nval_losses_resnet = []\ntrain_accuracies_resnet = []\nval_accuracies_resnet = []\n\ntrain_losses_efficientnet = []\nval_losses_efficientnet = []\ntrain_accuracies_efficientnet = []\nval_accuracies_efficientnet = []\n\ntrain_losses_inception = []\nval_losses_inception = []\ntrain_accuracies_inception = []\nval_accuracies_inception = []\n\nfor model, optimizer, scheduler, train_loader, val_loader in [\n    (model_resnet, optimizer_resnet, scheduler_resnet, train_loader_resnet, val_loader_resnet), \n    (model_efficientnet, optimizer_efficientnet, scheduler_efficientnet, train_loader_resnet, val_loader_resnet), \n    (model_inception, optimizer_inception, scheduler_inception, train_loader_inception, val_loader_inception),\n]:\n    early_stopper = EarlyStopping(patience=5, min_delta=0.001)\n    best_avg_loss = 100.0\n    is_inception = (model.__class__.__name__ == \"Inception3\")\n\n    for epoch in range(n_epochs):  \n        print(f'Training {model.__class__.__name__}, Epoch: {epoch+1}/{n_epochs}')\n        avg_loss, avg_accuracy = train_model(model, optimizer, train_loader, criterion, is_inception)\n        avg_val_loss, val_accuracy = test_model(model, val_loader, criterion, is_inception)\n        scheduler.step()\n\n        # Append losses and accuracies to the respective lists\n        if model == model_resnet:\n            train_losses_resnet.append(avg_loss)\n            val_losses_resnet.append(avg_val_loss)\n            train_accuracies_resnet.append(avg_accuracy)\n            val_accuracies_resnet.append(val_accuracy)\n        elif model == model_efficientnet:\n            train_losses_efficientnet.append(avg_loss)\n            val_losses_efficientnet.append(avg_val_loss)\n            train_accuracies_efficientnet.append(avg_accuracy)\n            val_accuracies_efficientnet.append(val_accuracy)\n        elif model == model_inception:\n            train_losses_inception.append(avg_loss)\n            val_losses_inception.append(avg_val_loss)\n            train_accuracies_inception.append(avg_accuracy)\n            val_accuracies_inception.append(val_accuracy)\n\n        # Save model if validation loss improved\n        if avg_val_loss < best_avg_loss:\n            best_avg_loss = avg_val_loss\n            torch.save(model.state_dict(), f'weight_best_{model.__class__.__name__}.pt')\n        else:\n            early_stopper(avg_val_loss)\n            if early_stopper.early_stop:\n                print(f\"Early stopping for {model.__class__.__name__} at epoch {epoch+1}\")\n                break","metadata":{"_uuid":"3562bd2ec1b0650519ca196bfc0e60eb139ca180","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load the Best Model Weights","metadata":{}},{"cell_type":"code","source":"# Make sure models are in evaluation mode\nmodel_resnet.eval()\nmodel_efficientnet.eval()\nmodel_inception.eval()\n\n# Load the saved best weights\nmodel_resnet.load_state_dict(torch.load('weight_best_ResNet.pt'))\nmodel_efficientnet.load_state_dict(torch.load('weight_best_EfficientNet.pt'))\nmodel_inception.load_state_dict(torch.load('weight_best_Inception3.pt'))","metadata":{"execution":{"iopub.status.busy":"2024-01-31T14:59:07.985081Z","iopub.status.idle":"2024-01-31T14:59:07.985689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluate Each Model","metadata":{}},{"cell_type":"code","source":"def evaluate_model(model, val_loader):\n    model.eval()\n    correct = 0\n    total = 0\n    all_labels = []\n    all_predictions = []\n    with torch.no_grad():\n        for images, labels in val_loader:\n            images, labels = images.cuda(), labels.cuda()\n            outputs = model(images)\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n            all_labels.extend(labels.cpu().numpy())\n            all_predictions.extend(predicted.cpu().numpy())\n\n    accuracy = 100 * correct / total\n    precision = precision_score(all_labels, all_predictions, average='weighted')\n    recall = recall_score(all_labels, all_predictions, average='weighted')\n    f1 = f1_score(all_labels, all_predictions, average='weighted')\n\n    return accuracy, precision, recall, f1","metadata":{"execution":{"iopub.status.busy":"2024-01-31T14:59:07.987582Z","iopub.status.idle":"2024-01-31T14:59:07.988154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# At the end of training for each model, call the evaluate_model function\naccuracy_resnet, precision_resnet, recall_resnet, f1_score_resnet = evaluate_model(model_resnet, val_loader)\naccuracy_efficientnet, precision_efficientnet, recall_efficientnet, f1_score_efficientnet = evaluate_model(model_efficientnet, val_loader)\naccuracy_inception, precision_inception, recall_inception, f1_score_inception = evaluate_model(model_inception, val_loader)\n\nprint(f'ResNet-101 Accuracy: {accuracy_resnet:.2f}%')\nprint(f'ResNet-101 Precision: {precision_resnet:.2f}')\nprint(f'ResNet-101 Recall: {recall_resnet:.2f}')\nprint(f'ResNet-101 F1-Score: {f1_score_resnet:.2f}')\n\nprint(f'EfficientNet Accuracy: {accuracy_efficientnet:.2f}%')\nprint(f'EfficientNet Precision: {precision_efficientnet:.2f}')\nprint(f'EfficientNet Recall: {recall_efficientnet:.2f}')\nprint(f'EfficientNet F1-Score: {f1_score_efficientnet:.2f}')\n\nprint(f'Inception-v3 Accuracy: {accuracy_inception:.2f}%')\nprint(f'Inception-v3 Precision: {precision_inception:.2f}')\nprint(f'Inception-v3 Recall: {recall_inception:.2f}')\nprint(f'Inception-v3 F1-Score: {f1_score_inception:.2f}')","metadata":{"execution":{"iopub.status.busy":"2024-01-31T14:59:07.989819Z","iopub.status.idle":"2024-01-31T14:59:07.990383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Create subplots for loss and accuracy\nplt.figure(figsize=(12, 4))\n\n# Plot ResNet results\nplt.subplot(1, 2, 1)\nplt.plot(range(1, n_epochs + 1), val_losses_resnet, label='ResNet', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_inception, label='Inception', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_efficientnet, label='EfficientNet', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Validation Loss')\nplt.title('Validation Loss Curves')\nplt.legend()\n\nplt.subplot(1, 2, 2)\nplt.plot(range(1, n_epochs + 1), val_accuracies_resnet, label='ResNet', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_inception, label='Inception', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_efficientnet, label='EfficientNet', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Validation Accuracy')\nplt.title('Validation Accuracy Curves')\nplt.legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-31T14:59:07.991706Z","iopub.status.idle":"2024-01-31T14:59:07.992359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Create subplots for loss and accuracy\nplt.figure(figsize=(16, 8))\n\n# Plot ResNet results\nplt.subplot(2, 3, 1)\nplt.plot(range(1, n_epochs + 1), train_losses_resnet, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_resnet, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('ResNet Loss Curves')\nplt.legend()\n\nplt.subplot(2, 3, 2)\nplt.plot(range(1, n_epochs + 1), train_accuracies_resnet, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_resnet, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('ResNet Accuracy Curves')\nplt.legend()\n\n# Plot Inception results\nplt.subplot(2, 3, 3)\nplt.plot(range(1, n_epochs + 1), train_losses_inception, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_inception, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Inception Loss Curves')\nplt.legend()\n\nplt.subplot(2, 3, 4)\nplt.plot(range(1, n_epochs + 1), train_accuracies_inception, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_inception, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('Inception Accuracy Curves')\nplt.legend()\n\n# Plot EfficientNet results\nplt.subplot(2, 3, 5)\nplt.plot(range(1, n_epochs + 1), train_losses_efficientnet, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_losses_efficientnet, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('EfficientNet Loss Curves')\nplt.legend()\n\nplt.subplot(2, 3, 6)\nplt.plot(range(1, n_epochs + 1), train_accuracies_efficientnet, label='Training', marker='o')\nplt.plot(range(1, n_epochs + 1), val_accuracies_efficientnet, label='Validation', marker='o')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('EfficientNet Accuracy Curves')\nplt.legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-31T14:59:07.993906Z","iopub.status.idle":"2024-01-31T14:59:07.994505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a bar chart for final validation accuracy\nmodels = ['ResNet', 'Inception', 'EfficientNet']\nfinal_accuracies = [val_accuracies_resnet[-1], val_accuracies_inception[-1], val_accuracies_efficientnet[-1]]\n\nplt.figure(figsize=(8, 5))\nplt.bar(models, final_accuracies)\nplt.xlabel('Model')\nplt.ylabel('Final Validation Accuracy')\nplt.title('Comparison of Final Validation Accuracy')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-31T14:59:07.995896Z","iopub.status.idle":"2024-01-31T14:59:07.996456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch\n# from torch.utils.data import DataLoader\n# from torchvision import transforms\n# import pandas as pd\n# from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score\n\n# # Define the validation transformations\n# val_transform = transforms.Compose([\n#     transforms.RandomHorizontalFlip(),  # Assuming used during training\n#     transforms.RandomRotation((-120, 120)),  # Assuming used during training\n#     transforms.ToTensor(),\n#     transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n# ])\n\n# # Create the validation dataset and DataLoader\n# valset = MyDataset(val_df, transform=val_transform)\n# val_loader = DataLoader(valset, batch_size=32, shuffle=False, num_workers=4)\n\n# # Define evaluation function\n# def evaluate_model(model, val_loader):\n#     model.eval()\n#     all_labels = []\n#     all_predictions = []\n#     with torch.no_grad():\n#         for i, (images, labels) in enumerate(val_loader):\n#             if i >= 3:  # Limiting to first 3 batches for prediction print\n#                 break\n#             images, labels = images.cuda(), labels.cuda()\n#             outputs = model(images)\n#             _, predicted = torch.max(outputs.data, 1)\n#             print(f\"Batch {i+1} Predictions:\", predicted)  # Debug print for predictions\n#             all_labels.extend(labels.cpu().numpy())\n#             all_predictions.extend(predicted.cpu().numpy())\n\n#     accuracy = accuracy_score(all_labels, all_predictions)\n#     precision = precision_score(all_labels, all_predictions, average='weighted')\n#     recall = recall_score(all_labels, all_predictions, average='weighted')\n#     f1 = f1_score(all_labels, all_predictions, average='weighted')\n\n#     return accuracy, precision, recall, f1\n\n# # Load model weights\n# model_resnet.load_state_dict(torch.load('../input/model-weights/weight_best_ResNet.pt'))\n# model_efficientnet.load_state_dict(torch.load('../input/model-weights/weight_best_EfficientNet.pt'))\n# model_inception.load_state_dict(torch.load('../input/model-weights/weight_best_Inception3.pt'))\n\n# # Evaluate the models\n# # Note: The prediction output will be printed for the first few batches\n# accuracy_resnet, precision_resnet, recall_resnet, f1_score_resnet = evaluate_model(model_resnet, val_loader)\n# accuracy_efficientnet, precision_efficientnet, recall_efficientnet, f1_score_efficientnet = evaluate_model(model_efficientnet, val_loader)\n# accuracy_inception, precision_inception, recall_inception, f1_score_inception = evaluate_model(model_inception, val_loader)\n\n# # Print evaluation results\n# print(f'ResNet-101 Accuracy: {accuracy_resnet:.2f}%')\n# print(f'ResNet-101 Precision: {precision_resnet:.2f}')\n# print(f'ResNet-101 Recall: {recall_resnet:.2f}')\n# print(f'ResNet-101 F1-Score: {f1_score_resnet:.2f}')\n\n# print(f'EfficientNet Accuracy: {accuracy_efficientnet:.2f}%')\n# print(f'EfficientNet Precision: {precision_efficientnet:.2f}')\n# print(f'EfficientNet Recall: {recall_efficientnet:.2f}')\n# print(f'EfficientNet F1-Score: {f1_score_efficientnet:.2f}')\n\n# print(f'Inception-v3 Accuracy: {accuracy_inception:.2f}%')\n# print(f'Inception-v3 Precision: {precision_inception:.2f}')\n# print(f'Inception-v3 Recall: {recall_inception:.2f}')\n# print(f'Inception-v3 F1-Score: {f1_score_inception:.2f}')\n","metadata":{"execution":{"iopub.status.busy":"2024-01-31T15:29:57.706969Z","iopub.execute_input":"2024-01-31T15:29:57.707314Z","iopub.status.idle":"2024-01-31T15:31:24.57877Z","shell.execute_reply.started":"2024-01-31T15:29:57.707255Z","shell.execute_reply":"2024-01-31T15:31:24.577806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_predictions_and_labels(model, val_loader):\n#     model.eval()\n#     all_labels = []\n#     all_predictions = []\n#     with torch.no_grad():\n#         for images, labels in val_loader:\n#             images, labels = images.cuda(), labels.cuda()\n#             outputs = model(images)\n#             _, predicted = torch.max(outputs.data, 1)\n#             # Ensure labels and predictions are flattened\n#             all_labels.extend(labels.cpu().view(-1).numpy())\n#             all_predictions.extend(predicted.cpu().view(-1).numpy())\n#     return all_predictions, all_labels\n\n# predicted_labels_resnet, actual_labels_resnet = get_predictions_and_labels(model_resnet, val_loader)\n# predicted_labels_inception, actual_labels_inception = get_predictions_and_labels(model_inception, val_loader)\n# predicted_labels_efficientnet, actual_labels_efficientnet = get_predictions_and_labels(model_efficientnet, val_loader)\n\n# # Repeat for other models as needed\n# import numpy as np\n# from collections import Counter\n\n# def analyze_predictions(predicted, actual):\n#     predicted_count = Counter(predicted)\n#     actual_count = Counter(actual)\n    \n#     total_predictions = len(predicted)\n#     print(\"Predicted Distribution (%):\")\n#     for label, count in predicted_count.items():\n#         print(f\"Label {label}: {count / total_predictions * 100:.2f}%\")\n\n#     print(\"\\nActual Distribution (%):\")\n#     for label, count in actual_count.items():\n#         print(f\"Label {label}: {count / total_predictions * 100:.2f}%\")\n\n# analyze_predictions(predicted_labels_resnet, actual_labels_resnet)\n# analyze_predictions(predicted_labels_inception, actual_labels_inception)\n# analyze_predictions(predicted_labels_efficientnet, actual_labels_efficientnet)\n# # Repeat for other models as needed\n","metadata":{"execution":{"iopub.status.busy":"2024-01-31T15:44:03.094097Z","iopub.execute_input":"2024-01-31T15:44:03.094405Z","iopub.status.idle":"2024-01-31T15:46:09.073574Z","shell.execute_reply.started":"2024-01-31T15:44:03.094364Z","shell.execute_reply":"2024-01-31T15:46:09.072657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}