{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"},{"sourceId":1253590,"sourceType":"datasetVersion","datasetId":720563},{"sourceId":7515484,"sourceType":"datasetVersion","datasetId":4377612}],"dockerImageVersionId":30648,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\ninput_directory = '/kaggle/input'\n\nfor dirpath, dirnames, filenames in os.walk(input_directory):\n    for dirname in dirnames:\n        print(dirname)\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-30T18:48:27.868783Z","iopub.execute_input":"2024-01-30T18:48:27.869701Z","iopub.status.idle":"2024-01-30T18:49:57.088394Z","shell.execute_reply.started":"2024-01-30T18:48:27.869662Z","shell.execute_reply":"2024-01-30T18:49:57.087204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport os\nfrom collections import OrderedDict\nimport torch\nfrom torch import nn\nfrom torch import optim\nimport torch.nn.functional as F\nfrom torchvision import datasets, transforms, models\n\n#import custom_models\n\n#python packages\nfrom PIL import Image\nfrom tqdm import tqdm\n#from tqdm.notebook import tqdm\nimport gc\nimport datetime\nimport copy\nimport matplotlib.pyplot as plt\nimport time\nfrom skimage import io\n#torch\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, random_split\n#torchvision\nimport torchvision\nfrom torchvision import datasets, models, transforms\nprint(\"PyTorch Version: \",torch.__version__)\nprint(\"Torchvision Version: \",torchvision.__version__)\n\nimport random\nimport cv2\nimport warnings\nimport seaborn as sns\nimport ssl\nssl._create_default_https_context = ssl._create_unverified_context\n","metadata":{"execution":{"iopub.status.busy":"2024-01-30T18:49:57.090444Z","iopub.execute_input":"2024-01-30T18:49:57.091474Z","iopub.status.idle":"2024-01-30T18:50:04.270806Z","shell.execute_reply.started":"2024-01-30T18:49:57.091419Z","shell.execute_reply":"2024-01-30T18:50:04.269937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.models as models","metadata":{"execution":{"iopub.status.busy":"2024-01-30T18:50:04.27207Z","iopub.execute_input":"2024-01-30T18:50:04.272521Z","iopub.status.idle":"2024-01-30T18:50:04.277015Z","shell.execute_reply.started":"2024-01-30T18:50:04.272494Z","shell.execute_reply":"2024-01-30T18:50:04.276089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\nwandb.init(project='Skin Melanoma Detection Project', save_code=True,)","metadata":{"execution":{"iopub.status.busy":"2024-01-30T18:50:52.265862Z","iopub.execute_input":"2024-01-30T18:50:52.266847Z","iopub.status.idle":"2024-01-30T18:51:36.617866Z","shell.execute_reply.started":"2024-01-30T18:50:52.266815Z","shell.execute_reply":"2024-01-30T18:51:36.616914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AdvancedHairAugmentation:\n    \"\"\"\n    Impose an image of a hair to the target image\n\n    Args:\n        hairs (int): maximum number of hairs to impose\n        hairs_folder (str): path to the folder with hairs images\n    \"\"\"\n\n    def __init__(self, hairs: int = 5, hairs_folder: str = \"\"):\n        self.hairs = hairs\n        self.hairs_folder = hairs_folder\n\n    def __call__(self, img_path):\n        \"\"\"\n        Args:\n            img (PIL Image): Image to draw hairs on.\n\n        Returns:\n            PIL Image: Image with drawn hairs.\n        \"\"\"\n        img = cv2.imread(img_path)\n        n_hairs = random.randint(0, self.hairs)\n        \n        if not n_hairs:\n            return img\n        \n        height, width, _ = img.shape  # target image width and height\n        hair_images = [im for im in os.listdir(self.hairs_folder) if 'png' in im]\n        \n        for _ in range(n_hairs):\n            hair = cv2.imread(os.path.join(self.hairs_folder, random.choice(hair_images)))\n            hair = cv2.flip(hair, random.choice([-1, 0, 1]))\n            hair = cv2.rotate(hair, random.choice([0, 1, 2]))\n\n            h_height, h_width, _ = hair.shape  # hair image width and height\n            roi_ho = random.randint(0, img.shape[0] - hair.shape[0])\n            roi_wo = random.randint(0, img.shape[1] - hair.shape[1])\n            roi = img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]\n\n            # Creating a mask and inverse mask\n            img2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)\n            ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)\n            mask_inv = cv2.bitwise_not(mask)\n\n            # Now black-out the area of hair in ROI\n            img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n\n            # Take only region of hair from hair image.\n            hair_fg = cv2.bitwise_and(hair, hair, mask=mask)\n\n            # Put hair in ROI and modify the target image\n            dst = cv2.add(img_bg, hair_fg)\n\n            img[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst\n                \n        return img\n\n    def __repr__(self):\n        return f'{self.__class__.__name__}(hairs={self.hairs}, hairs_folder=\"{self.hairs_folder}\")'","metadata":{"execution":{"iopub.status.busy":"2024-01-30T18:51:48.647899Z","iopub.execute_input":"2024-01-30T18:51:48.648821Z","iopub.status.idle":"2024-01-30T18:51:49.236809Z","shell.execute_reply.started":"2024-01-30T18:51:48.648789Z","shell.execute_reply":"2024-01-30T18:51:49.235731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader, ConcatDataset\nfrom PIL import Image\nimport torchvision\nclass MultimodalDataset(Dataset):\n    \"\"\"\n    Custom dataset definition\n    \"\"\"\n    def __init__(self, csv_path, img_path, mode='train', transform=None):\n        \"\"\"\n        \"\"\"\n        self.df = pd.read_csv(csv_path)\n        self.img_path = img_path\n        self.mode= mode\n        self.transform = transform\n        \n            \n    def __getitem__(self, index):\n        \"\"\"\n        \"\"\"\n        img_name = self.df.iloc[index][\"image_name\"] + '.jpg'\n        img_path = os.path.join(self.img_path, img_name)\n        image = Image.open(img_path)\n        \n        dtype = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor # ???\n        \n        if self.mode == 'train':\n            if self.df.iloc[index][\"augmented\"]==1:\n                image = AdvancedHairAugmentation(hairs_folder=\"/kaggle/input/melanoma-hairs\")(img_path)\n                image = Image.fromarray(image, 'RGB')\n            else:  \n                image = image.convert(\"RGB\")\n                \n            image = np.asarray(image)\n            if self.transform is not None:\n                image = self.transform(image)\n            labels = self.df.iloc[index][\"target\"]\n            return image, labels\n            \n        elif self.mode == 'val':\n            image = np.asarray(image)\n            if self.transform is not None:\n                image = self.transform(image)\n            labels = self.df.iloc[index][\"target\"]\n            return image, labels\n        \n        else: #when self.mode=='test'\n            image = np.asarray(image)\n            if self.transform is not None:\n                image = self.transform(image)\n            return image, self.df.iloc[index][\"image_name\"]\n\n    def __len__(self):\n        return len(self.df)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-30T18:51:50.839137Z","iopub.execute_input":"2024-01-30T18:51:50.83982Z","iopub.status.idle":"2024-01-30T18:51:51.444734Z","shell.execute_reply.started":"2024-01-30T18:51:50.839789Z","shell.execute_reply":"2024-01-30T18:51:51.44382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataloaders(input_size, batch_size, augment=False, shuffle = True):\n    # How to transform the image when you are loading them.\n    # you'll likely want to mess with the transforms on the training set.\n    \n    # For now, we resize/crop the image to the correct input size for our network,\n    # then convert it to a [C,H,W] tensor, then normalize it to values with a given mean/stdev. These normalization constants\n    # are derived from aggregating lots of data and happen to produce better results.\n    data_transforms = {\n        'train': transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.Resize(input_size),\n            transforms.CenterCrop(input_size),\n            #Add extra transformations for data augmentation\n            transforms.RandomApply([\n                transforms.RandomChoice([\n                    transforms.RandomAffine(degrees=20),\n                    transforms.RandomAffine(degrees=0,scale=(0.1, 0.15)),\n                    transforms.RandomAffine(degrees=0,translate=(0.2,0.2)),\n                    #transforms.RandomAffine(degrees=0,shear=0.15),\n                    transforms.RandomHorizontalFlip(p=1.0)\n                ] if augment else [transforms.RandomAffine(degrees=0)])#else do nothing\n            ], p=0.5),\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.225])\n        ]),\n        'val': transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.Resize(input_size),\n            transforms.CenterCrop(input_size),\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.225])\n        ]),\n        'test': transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.Resize(input_size),\n            transforms.CenterCrop(input_size),\n            transforms.ToTensor(),\n            transforms.Normalize([0.5], [0.225])\n        ])\n    }\n    \n    data_subsets = {x: MultimodalDataset(csv_path=\"/kaggle/input/melanomacsv/\"+x+\".csv\", \n                                         img_path = image_path_dict[x],\n                                         mode = x,\n                                         transform=data_transforms[x]) for x in data_transforms.keys()}\n    # Create training and validation dataloaders\n    # Never shuffle the test set\n    dataloaders_dict = {x: DataLoader(data_subsets[x], batch_size=batch_size, shuffle=False if x != 'train' else shuffle, num_workers=4) for x in data_transforms.keys()}\n    return dataloaders_dict","metadata":{"execution":{"iopub.status.busy":"2024-01-30T18:51:57.879463Z","iopub.execute_input":"2024-01-30T18:51:57.880482Z","iopub.status.idle":"2024-01-30T18:51:58.493728Z","shell.execute_reply.started":"2024-01-30T18:51:57.880399Z","shell.execute_reply":"2024-01-30T18:51:58.49277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path_dict = {'train': \"../input/siim-isic-melanoma-classification/jpeg/train\",\n                  'val': \"../input/siim-isic-melanoma-classification/jpeg/train\" ,\n                  'test': \"../input/siim-isic-melanoma-classification/jpeg/test\"}","metadata":{"execution":{"iopub.status.busy":"2024-01-30T18:52:02.129535Z","iopub.execute_input":"2024-01-30T18:52:02.13019Z","iopub.status.idle":"2024-01-30T18:52:02.71454Z","shell.execute_reply.started":"2024-01-30T18:52:02.130157Z","shell.execute_reply":"2024-01-30T18:52:02.711079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataloaders = get_dataloaders(input_size=224, batch_size=64,augment=False, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-30T18:52:08.058828Z","iopub.execute_input":"2024-01-30T18:52:08.0597Z","iopub.status.idle":"2024-01-30T18:52:08.64234Z","shell.execute_reply.started":"2024-01-30T18:52:08.059666Z","shell.execute_reply":"2024-01-30T18:52:08.641424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = dataloaders['train']\nval_loader = dataloaders['val']\ntest_loader = dataloaders['test']","metadata":{"execution":{"iopub.status.busy":"2024-01-30T18:52:10.837144Z","iopub.execute_input":"2024-01-30T18:52:10.837757Z","iopub.status.idle":"2024-01-30T18:52:14.71501Z","shell.execute_reply.started":"2024-01-30T18:52:10.837728Z","shell.execute_reply":"2024-01-30T18:52:14.714081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader","metadata":{"execution":{"iopub.status.busy":"2024-01-30T18:52:20.057093Z","iopub.execute_input":"2024-01-30T18:52:20.057798Z","iopub.status.idle":"2024-01-30T18:52:20.645955Z","shell.execute_reply.started":"2024-01-30T18:52:20.057769Z","shell.execute_reply":"2024-01-30T18:52:20.645021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_loader","metadata":{"execution":{"iopub.status.busy":"2024-01-30T18:52:23.237523Z","iopub.execute_input":"2024-01-30T18:52:23.238454Z","iopub.status.idle":"2024-01-30T18:52:23.930819Z","shell.execute_reply.started":"2024-01-30T18:52:23.23842Z","shell.execute_reply":"2024-01-30T18:52:23.929909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_loader","metadata":{"execution":{"iopub.status.busy":"2024-01-30T18:52:26.145878Z","iopub.execute_input":"2024-01-30T18:52:26.146757Z","iopub.status.idle":"2024-01-30T18:52:26.810848Z","shell.execute_reply.started":"2024-01-30T18:52:26.146723Z","shell.execute_reply":"2024-01-30T18:52:26.81002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torchvision import models\nfrom collections import OrderedDict\n\n# Load pre-trained ResNet18\nmodel = models.resnet18(pretrained=True)\n\n# Freeze all parameters\nfor param in model.parameters():\n    param.requires_grad = False\n\n# Modify the classifier\nfc = nn.Sequential(\n    nn.Linear(512, 100),\n    nn.ReLU(),\n    nn.Linear(100, 2),\n    nn.LogSoftmax(dim=1)\n)\n\n# Replace the classifier in the model\nmodel.fc = fc\n\n# Define the device (cuda or cpu)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# Move the model to the GPU\nmodel.to(device)\n\n# Print the modified ResNet18 model\nprint(model)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-30T18:53:07.207052Z","iopub.execute_input":"2024-01-30T18:53:07.207381Z","iopub.status.idle":"2024-01-30T18:53:08.9092Z","shell.execute_reply.started":"2024-01-30T18:53:07.207357Z","shell.execute_reply":"2024-01-30T18:53:08.908245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Define hyperparameters\nnum_epochs = 10\nbatch_size = 64\nlearning_rate = 1e-7\n\n# Define loss function and optimizer\n#criterion = nn.NLLLoss()\n#optimizer = optim.Adam(model.parameters(), lr=learning_rate)\noptimizer = optim.SGD(model.parameters(), lr=1e-7)  # Initial learning rate for LR finder\ncriterion = torch.nn.CrossEntropyLoss()\n\n# Learning rate finder code\nlr_finder = LRFinder(model, optimizer, criterion, device=\"cuda\")\nlr_finder.range_test(train_loader, end_lr=10, num_iter=100)\nlr_finder.plot_lr_find()\n\n\ndef train_model(model, train_loader, criterion, optimizer, device):\n    model.train()\n    total_loss = 0.0\n    correct = 0\n    total_samples = 0\n\n    for inputs, labels in train_loader:\n        inputs, labels = inputs.to(device), labels.to(device)\n\n        optimizer.zero_grad()\n\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n\n        loss.backward()\n        optimizer.step()\n\n        total_loss += loss.item()\n\n        _, predicted = torch.max(outputs.data, 1)\n        total_samples += labels.size(0)\n        correct += (predicted == labels).sum().item()\n\n    loss_avg = total_loss / len(train_loader)\n    accuracy = correct / total_samples\n\n    # Log metrics to WandB\n    wandb.log({'Train Loss': loss_avg, 'Train Accuracy': accuracy})\n\n    return loss_avg, accuracy\n\ndef test_model(model, test_loader, criterion, device, phase):\n    model.eval()\n    total_loss = 0.0\n    correct = 0\n    total_samples = 0\n\n    with torch.no_grad():\n        for inputs, labels in test_loader:\n            inputs, labels = inputs.to(device), labels.to(device)\n\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n\n            total_loss += loss.item()\n\n            _, predicted = torch.max(outputs.data, 1)\n            total_samples += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n    loss_avg = total_loss / len(test_loader)\n    accuracy = correct / total_samples\n\n    # Log metrics to WandB\n    wandb.log({f'{phase} Loss': loss_avg, f'{phase} Accuracy': accuracy})\n\n    return loss_avg, accuracy\n\n# Training loop with WandB logging\nfor epoch in range(num_epochs):\n    train_loss, train_accuracy = train_model(model, dataloaders['train'], criterion, optimizer, device)\n    print(f\"Epoch [{epoch+1}/{num_epochs}], Training Loss: {train_loss:.4f}, Training Accuracy: {train_accuracy:.4f}\")\n\n    # Save checkpoint after each epoch\n    checkpoint = {\n        'epoch': epoch + 1,\n        'model_state_dict': model.state_dict(),\n        'optimizer_state_dict': optimizer.state_dict(),\n        'train_loss': train_loss,\n        'train_accuracy': train_accuracy,\n    }\n    torch.save(checkpoint, f'checkpoint_epoch_{epoch+1}.pt')\n\n    # Validation\n    val_loss, val_accuracy = test_model(model, dataloaders['val'], criterion, device, 'Validation')\n    print(f\"Validation Loss: {val_loss:.4f}, Validation Accuracy: {val_accuracy:.4f}\")\n\n# Testing loop\n#test_loss, test_accuracy = test_model(model, dataloaders['test'], criterion, device, 'Test')\n#print(f\"Test Loss: {test_loss:.4f}, Test Accuracy: {test_accuracy:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-01-30T18:53:14.934644Z","iopub.execute_input":"2024-01-30T18:53:14.935427Z","iopub.status.idle":"2024-01-30T19:37:25.844245Z","shell.execute_reply.started":"2024-01-30T18:53:14.935394Z","shell.execute_reply":"2024-01-30T19:37:25.842705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_model(model, test_loader, criterion, device, phase):\n    model.eval()\n    total_loss = 0.0\n    correct = 0\n    total_samples = 0\n\n    with torch.no_grad():\n        for batch_idx, (inputs, labels) in enumerate(test_loader):\n            inputs, labels = inputs.to(device), labels.to(device)\n\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n\n            total_loss += loss.item()\n\n            _, predicted = torch.max(outputs.data, 1)\n            total_samples += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n        # Calculate average loss and accuracy\n        loss_avg = total_loss / len(test_loader)\n        accuracy = correct / total_samples\n\n        print(f'{phase} Set - Loss: {loss_avg:.4f}, Accuracy: {accuracy:.4f}')\n\n    # Log metrics to WandB\n    wandb.log({f'{phase} Loss': loss_avg, f'{phase} Accuracy': accuracy})\n\n    return loss_avg, accuracy\n","metadata":{"execution":{"iopub.status.busy":"2024-01-30T19:39:55.328471Z","iopub.execute_input":"2024-01-30T19:39:55.32925Z","iopub.status.idle":"2024-01-30T19:39:56.038922Z","shell.execute_reply.started":"2024-01-30T19:39:55.329214Z","shell.execute_reply":"2024-01-30T19:39:56.038004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, classification_report\n\ndef evaluate_model(model, dataloader, criterion, device):\n    model.eval()\n    all_labels = []\n    all_predictions = []\n\n    with torch.no_grad():\n        for inputs, labels in dataloader:\n            inputs, labels = inputs.to(device), labels.to(device)\n\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n\n            _, predicted = torch.max(outputs.data, 1)\n\n            all_labels.extend(labels.cpu().numpy())\n            all_predictions.extend(predicted.cpu().numpy())\n\n    # Calculate confusion matrix\n    cm = confusion_matrix(all_labels, all_predictions)\n\n    # Calculate classification report\n    report = classification_report(all_labels, all_predictions, target_names=['Class 0', 'Class 1'])\n\n    return cm, report\n","metadata":{"execution":{"iopub.status.busy":"2024-01-30T19:50:44.214578Z","iopub.execute_input":"2024-01-30T19:50:44.215466Z","iopub.status.idle":"2024-01-30T19:50:44.886192Z","shell.execute_reply.started":"2024-01-30T19:50:44.21543Z","shell.execute_reply":"2024-01-30T19:50:44.885292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate the model on the validation set\nval_confusion_matrix, val_classification_report = evaluate_model(model, dataloaders['val'], criterion, device)\n\n# Evaluate the model on the test set\n#test_confusion_matrix, test_classification_report = evaluate_model(model, dataloaders['test'], criterion, device)\n\n# Print the results\nprint(\"Validation Set Metrics:\")\nprint(\"Confusion Matrix:\")\nprint(val_confusion_matrix)\nprint(\"\\nClassification Report:\")\nprint(val_classification_report)\n\n#print(\"\\nTest Set Metrics:\")\n#print(\"Confusion Matrix:\")\n#print(test_confusion_matrix)\n#print(\"\\nClassification Report:\")\n#print(test_classification_report)","metadata":{"execution":{"iopub.status.busy":"2024-01-30T19:58:30.153166Z","iopub.execute_input":"2024-01-30T19:58:30.153596Z","iopub.status.idle":"2024-01-30T19:59:23.788929Z","shell.execute_reply.started":"2024-01-30T19:58:30.153561Z","shell.execute_reply":"2024-01-30T19:59:23.787937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}