{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Setup**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport matplotlib.pyplot as plt \nimport os \nimport cv2 \nimport PIL\n\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score\n\nimport torch\nfrom torch import nn \nfrom torch import optim as O\nfrom torch.nn import Module\nfrom torch.nn import functional as f\nfrom torch.nn.modules.activation import Sigmoid\nfrom torch.nn.modules.upsampling import UpsamplingBilinear2d\nfrom torch.nn.modules.conv import Conv2d\nfrom torch.optim.lr_scheduler import LinearLR\nfrom torch.utils.data import Dataset \nfrom torch.utils.data import DataLoader\n\n\nimport torchvision\nfrom torchvision import utils as Vision_utils\nfrom torchvision import models\nfrom torchvision.transforms import functional as TF\n\nfrom glob import glob\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom tqdm import tqdm ","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:58:57.807272Z","iopub.execute_input":"2022-07-01T12:58:57.808373Z","iopub.status.idle":"2022-07-01T12:59:02.629168Z","shell.execute_reply.started":"2022-07-01T12:58:57.808234Z","shell.execute_reply":"2022-07-01T12:59:02.627689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Loading the Data and Exploratory Data Analysis**\n\nI have loaded the dataset, and did general data analysis, like how many unique images, classes, subjects etc.","metadata":{}},{"cell_type":"code","source":"## I am reading the csv file using a pandas dataframe adn then adding a columnn name file_path containing the image\n## path for the corresponding entry in the dataset\n\narr = os.listdir(\"/kaggle/input\")\ndata_dir = os.path.join(\"/kaggle/input\", arr[0])\nlist_dir = os.listdir(data_dir)\npath_for_train_dataset = os.path.join(data_dir, list_dir[1])\ndataframe = pd.read_csv(path_for_train_dataset)\ndataframe.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:02.631824Z","iopub.execute_input":"2022-07-01T12:59:02.632747Z","iopub.status.idle":"2022-07-01T12:59:02.706321Z","shell.execute_reply.started":"2022-07-01T12:59:02.632692Z","shell.execute_reply":"2022-07-01T12:59:02.704856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del path_for_train_dataset, list_dir","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:02.708454Z","iopub.execute_input":"2022-07-01T12:59:02.708909Z","iopub.status.idle":"2022-07-01T12:59:02.718588Z","shell.execute_reply.started":"2022-07-01T12:59:02.708867Z","shell.execute_reply":"2022-07-01T12:59:02.716847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## train file list\ntrain_file = glob(os.path.join(data_dir, \"imgs/train/*/*.jpg\"))\n\n## Now we have a list of images of training dataset, let us add a path to the training pandas dataframe \ndataframe[\"file_path\"] = dataframe.apply(lambda x : os.path.join(data_dir, 'imgs/train', x.classname, x.img), axis = 1)\ndataframe[\"labels\"] = dataframe[\"classname\"].map(lambda x : int(x[1]))\ndataframe.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:02.724145Z","iopub.execute_input":"2022-07-01T12:59:02.725999Z","iopub.status.idle":"2022-07-01T12:59:08.640151Z","shell.execute_reply.started":"2022-07-01T12:59:02.725955Z","shell.execute_reply":"2022-07-01T12:59:08.638852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_dataset = len(dataframe)\nprint(\"The number of Images in the dataset :\", num_dataset)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:08.642386Z","iopub.execute_input":"2022-07-01T12:59:08.642875Z","iopub.status.idle":"2022-07-01T12:59:08.651052Z","shell.execute_reply.started":"2022-07-01T12:59:08.642829Z","shell.execute_reply":"2022-07-01T12:59:08.649635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Now we have a Dataframe of the dataset. Let us use it to visualize the dataset and plotting relative graphs\n## Some raw visualisation led me to find that there are diffent photos of a single person.\n## That is let us find the subject \narr = dataframe[\"subject\"].value_counts()\nplt.bar(arr.index, arr.values)\nplt.grid()\nplt.xticks(rotation = 90)\nplt.xlabel(\"Subjects\")\nplt.ylabel(\"Number of images\")\nplt.show()\n\nprint(\"Number of Different Subject :\", len(arr.index))\nprint(\"Average Number of Images per subject :\", arr.values.sum()//len(arr.index))","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:08.652678Z","iopub.execute_input":"2022-07-01T12:59:08.654125Z","iopub.status.idle":"2022-07-01T12:59:09.052629Z","shell.execute_reply.started":"2022-07-01T12:59:08.654079Z","shell.execute_reply":"2022-07-01T12:59:09.051021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## So this survey has been done on 26 differnet subject \n## Each subject has nearly 862 images each\n## Let us also make a visualisation about the labels on the dataset.\narr = dataframe[\"labels\"].value_counts()\nplt.bar(arr.index, arr.values)\nplt.grid()\nplt.xticks(rotation = 90)\nplt.xlabel(\"Labels\")\nplt.ylabel(\"Number of images\")\nplt.show()\n\nprint(\"Number of labels :\", len(arr.index))\nprint(\"Average Number of Images per Label :\", arr.values.sum()/len(arr.index))","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:09.054386Z","iopub.execute_input":"2022-07-01T12:59:09.056599Z","iopub.status.idle":"2022-07-01T12:59:09.279160Z","shell.execute_reply.started":"2022-07-01T12:59:09.056549Z","shell.execute_reply":"2022-07-01T12:59:09.277692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Shuffling the Dataframe and splitting into train, test and validation dataset\ndataframe = dataframe.sample(frac = 1)\ndataframe.reset_index(inplace = True)\ndataframe.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:09.282109Z","iopub.execute_input":"2022-07-01T12:59:09.283095Z","iopub.status.idle":"2022-07-01T12:59:09.308218Z","shell.execute_reply.started":"2022-07-01T12:59:09.283050Z","shell.execute_reply":"2022-07-01T12:59:09.306851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = OneHotEncoder(sparse = False)\nlabels = np.array(dataframe[\"labels\"])\nlabels ","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:09.310003Z","iopub.execute_input":"2022-07-01T12:59:09.310445Z","iopub.status.idle":"2022-07-01T12:59:09.321906Z","shell.execute_reply.started":"2022-07-01T12:59:09.310383Z","shell.execute_reply":"2022-07-01T12:59:09.320370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = encoder.fit_transform(labels.reshape((22424, 1)))\nlabels.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:09.329039Z","iopub.execute_input":"2022-07-01T12:59:09.330228Z","iopub.status.idle":"2022-07-01T12:59:09.344277Z","shell.execute_reply.started":"2022-07-01T12:59:09.330172Z","shell.execute_reply":"2022-07-01T12:59:09.342772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = labels[0 : 16000]\nval = labels[16000 : 20000]\ntest = labels[20000 : ]","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:09.346303Z","iopub.execute_input":"2022-07-01T12:59:09.346756Z","iopub.status.idle":"2022-07-01T12:59:09.354401Z","shell.execute_reply.started":"2022-07-01T12:59:09.346714Z","shell.execute_reply":"2022-07-01T12:59:09.352876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.shape, val.shape, test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:09.356812Z","iopub.execute_input":"2022-07-01T12:59:09.357340Z","iopub.status.idle":"2022-07-01T12:59:09.368569Z","shell.execute_reply.started":"2022-07-01T12:59:09.357298Z","shell.execute_reply":"2022-07-01T12:59:09.366873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = dataframe[0 : 16000]\nval_df = dataframe[16000 : 20000]\ntest_df = dataframe[20000 : ]\n\ndel dataframe, labels ","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:09.371332Z","iopub.execute_input":"2022-07-01T12:59:09.372224Z","iopub.status.idle":"2022-07-01T12:59:09.381210Z","shell.execute_reply.started":"2022-07-01T12:59:09.372159Z","shell.execute_reply":"2022-07-01T12:59:09.379622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Building the DataPipeline**","metadata":{}},{"cell_type":"code","source":"class DriverDataset(Dataset) :\n    \n    def __init__(self, df, labels, transform = None) :\n        super(DriverDataset, self).__init__()\n        self.img_dir = df[\"file_path\"]\n        self.labels = labels\n        self.transform = transform \n        \n    def __len__(self) :\n        return(len(self.labels))\n    \n    def __getitem__(self, index) :\n        img_pth = self.img_dir[index]\n        \n        image_ = cv2.imread(img_pth)\n        image_ = cv2.cvtColor(image_, cv2.COLOR_BGR2RGB)\n        image_ = np.array(image_, dtype = np.float32)\n        \n        label = self.labels[index]\n        \n        if self.transform is not None :\n            augmentations = self.transform(image = image_)\n            image_ = augmentations[\"image\"]\n            \n        return image_, label","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:09.382961Z","iopub.execute_input":"2022-07-01T12:59:09.383753Z","iopub.status.idle":"2022-07-01T12:59:09.396692Z","shell.execute_reply.started":"2022-07-01T12:59:09.383706Z","shell.execute_reply":"2022-07-01T12:59:09.395016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_loaders(train_df, train_label, val_df, val_label, batch_size, train_transform, val_transform, num_workers = 2, pin_memory = True) :\n    \n    trainDS = DriverDataset(train_df, train_label, train_transform)\n    valDS = DriverDataset(val_df, val_label, val_transform)\n    \n    train_loader = DataLoader(trainDS, batch_size, num_workers = num_workers, pin_memory = pin_memory, shuffle = False)\n    val_loader = DataLoader(valDS, batch_size, num_workers = num_workers, pin_memory = pin_memory, shuffle = False)\n    \n    return train_loader, val_loader","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:09.398636Z","iopub.execute_input":"2022-07-01T12:59:09.399260Z","iopub.status.idle":"2022-07-01T12:59:09.413139Z","shell.execute_reply.started":"2022-07-01T12:59:09.399220Z","shell.execute_reply":"2022-07-01T12:59:09.411849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Other Utility Functions**","metadata":{}},{"cell_type":"code","source":"def save_checkpoint(state, filename = \"my_checkpoint.pth.tar\") :\n    print(\"=> Saving Checkpoint\")\n    torch.save(state, filename)\n\ndef load_checkpoint(checkpoint, model) :\n    print(\"=> Loading Checkpoint\")\n    model.load_state_dict(checkpoint[\"state_dict\"])\n    \ndef eval_model(model):\n    correct = 0.0\n    total = 0.0\n    with torch.no_grad():\n        for i, data in enumerate(test_loader, 0):\n            images, labels = data\n            images = images.to(device)\n            labels = labels.to(device)\n            \n            outputs = model_ft(images)\n            _, predicted = torch.max(outputs.data, 1)\n            \n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n    test_acc = 100.0 * correct / total\n    print('Accuracy of the network on the test images: %d %%' % (test_acc))\n    return test_acc\n            \n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:09.414981Z","iopub.execute_input":"2022-07-01T12:59:09.416040Z","iopub.status.idle":"2022-07-01T12:59:09.429145Z","shell.execute_reply.started":"2022-07-01T12:59:09.415969Z","shell.execute_reply":"2022-07-01T12:59:09.427829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Preparing the HyperParameters**","metadata":{}},{"cell_type":"code","source":"learning_rate = 3e-4\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nbatch_size = 16\nnum_epochs = 30\nImage_height = 256\nImage_width = 256\npin_memory = True \nload_model = False\nnum_workers = 2","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:09.432176Z","iopub.execute_input":"2022-07-01T12:59:09.433113Z","iopub.status.idle":"2022-07-01T12:59:09.520836Z","shell.execute_reply.started":"2022-07-01T12:59:09.433054Z","shell.execute_reply":"2022-07-01T12:59:09.519380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Building the Model**","metadata":{}},{"cell_type":"code","source":"## I am going to use the ResNeXt-101 pretrained on imagenet\n\nmodel = models.resnet18(pretrained = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:09.523073Z","iopub.execute_input":"2022-07-01T12:59:09.524196Z","iopub.status.idle":"2022-07-01T12:59:09.838424Z","shell.execute_reply.started":"2022-07-01T12:59:09.524150Z","shell.execute_reply":"2022-07-01T12:59:09.836968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:09.841150Z","iopub.execute_input":"2022-07-01T12:59:09.841959Z","iopub.status.idle":"2022-07-01T12:59:09.854522Z","shell.execute_reply.started":"2022-07-01T12:59:09.841912Z","shell.execute_reply":"2022-07-01T12:59:09.852731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fc = nn.Sequential(nn.Linear(512, 10),\n                                nn.Sigmoid())\nmodel = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:09.856769Z","iopub.execute_input":"2022-07-01T12:59:09.858142Z","iopub.status.idle":"2022-07-01T12:59:13.434225Z","shell.execute_reply.started":"2022-07-01T12:59:09.858095Z","shell.execute_reply":"2022-07-01T12:59:13.432830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Image Augmentation using Albumentations\n\ntrain_transform = A.Compose(\n    [\n        A.Resize(Image_height, Image_width),\n        A.Rotate(limit = 35, p = 1.0),\n        A.HorizontalFlip(p = 0.5),\n        A.VerticalFlip(p = 0.1),\n        A.Normalize(\n            mean = [0.0, 0.0, 0.0],\n            std = [1.0, 1.0, 1.0],\n            max_pixel_value = 255.0\n        ),\n        ToTensorV2(),  \n    ]\n)\n\nval_transforms = A.Compose(\n    [\n        A.Resize(Image_height, Image_width), \n        A.Normalize(\n            mean = [0.0, 0.0, 0.0],\n            std = [1.0, 1.0, 1.0],\n            max_pixel_value = 255.0\n        ),\n        ToTensorV2(),\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:13.436183Z","iopub.execute_input":"2022-07-01T12:59:13.438025Z","iopub.status.idle":"2022-07-01T12:59:13.448493Z","shell.execute_reply.started":"2022-07-01T12:59:13.437965Z","shell.execute_reply":"2022-07-01T12:59:13.447048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Defining the train Function\n\ndef train_fn(loader, model, optimizer, loss_fn, scaler) :\n    loop = tqdm(loader)\n\n    for batch_idx, (data, targets) in enumerate(loop) :\n        data = data.to(device)\n        targets = targets.to(device)\n\n        ## Forward\n        with torch.cuda.amp.autocast() :\n            predictions = model(data)\n            loss = loss_fn(predictions, targets)\n\n        ## Backwards \n        optimizer.zero_grad()\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n\n        ## Update the tqdm loader\n        loop.set_postfix(loss = loss.item())","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:13.450347Z","iopub.execute_input":"2022-07-01T12:59:13.451089Z","iopub.status.idle":"2022-07-01T12:59:13.464942Z","shell.execute_reply.started":"2022-07-01T12:59:13.451043Z","shell.execute_reply":"2022-07-01T12:59:13.463553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = O.Adam(model.parameters(), lr = learning_rate)\nscheduler = LinearLR(optimizer, 0.33, 1)\ntrain_loader, val_loader = get_loaders(\n            train_df, train, val_df, val, batch_size, train_transform, val_transforms,\n            num_workers, pin_memory \n        )\nloss_fn = nn.BCEWithLogitsLoss()\n\ndef Training(load_model) :\n    if load_model :\n        load_checkpoint(torch.load(\"my_checkpoint.pth.tar\"), model)\n\n    #check_accuracy(val_loader, model, device)\n    scaler = torch.cuda.amp.GradScaler()\n\n    for epoch in range(num_epochs) :\n        train_fn(train_loader, model, optimizer, loss_fn, scaler)\n        scheduler.step()\n\n        ## Saving the Model\n        checkpoint = {\n            \"state_dict\" : model.state_dict(), \n            \"optimizer\" : optimizer.state_dict()   \n        }\n\n        save_checkpoint(checkpoint)\n\n        ## Checking the Accuracy \n        #check_accuracy(self.val_loader, self.model, self.device)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:13.466447Z","iopub.execute_input":"2022-07-01T12:59:13.467373Z","iopub.status.idle":"2022-07-01T12:59:13.483174Z","shell.execute_reply.started":"2022-07-01T12:59:13.467339Z","shell.execute_reply":"2022-07-01T12:59:13.481840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Training(False)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T12:59:13.484996Z","iopub.execute_input":"2022-07-01T12:59:13.486388Z","iopub.status.idle":"2022-07-01T14:30:38.931488Z","shell.execute_reply.started":"2022-07-01T12:59:13.486356Z","shell.execute_reply":"2022-07-01T14:30:38.929809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = list()\nwith torch.no_grad() :\n    for img_pth in test_df[\"file_path\"] :\n        image_ = cv2.imread(img_pth)\n        image_ = cv2.cvtColor(image_, cv2.COLOR_BGR2RGB)\n        image_ = cv2.resize(image_, (224, 224))\n        image_ = np.array(image_, dtype = np.float32)\n        image_ = torch.tensor(image_).to(device)\n        image_ = torch.reshape(image_, (1, 3, 224, 224))\n        y = np.array(model(image_).squeeze().cpu())\n        y_pred.append(y)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-01T14:30:38.933876Z","iopub.execute_input":"2022-07-01T14:30:38.934748Z","iopub.status.idle":"2022-07-01T14:31:31.936553Z","shell.execute_reply.started":"2022-07-01T14:30:38.934683Z","shell.execute_reply":"2022-07-01T14:31:31.935239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = np.array(y_pred)\ny_pred.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-01T14:31:31.938733Z","iopub.execute_input":"2022-07-01T14:31:31.939162Z","iopub.status.idle":"2022-07-01T14:31:31.952959Z","shell.execute_reply.started":"2022-07-01T14:31:31.939119Z","shell.execute_reply":"2022-07-01T14:31:31.951501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-01T14:31:31.955047Z","iopub.execute_input":"2022-07-01T14:31:31.955750Z","iopub.status.idle":"2022-07-01T14:31:31.968015Z","shell.execute_reply.started":"2022-07-01T14:31:31.955705Z","shell.execute_reply":"2022-07-01T14:31:31.966672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-01T14:31:31.975732Z","iopub.execute_input":"2022-07-01T14:31:31.976082Z","iopub.status.idle":"2022-07-01T14:31:31.986713Z","shell.execute_reply.started":"2022-07-01T14:31:31.976054Z","shell.execute_reply":"2022-07-01T14:31:31.985250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = np.argmax(y_pred, 1)\ny_true = np.argmax(test, 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T14:31:31.988553Z","iopub.execute_input":"2022-07-01T14:31:31.989344Z","iopub.status.idle":"2022-07-01T14:31:31.997846Z","shell.execute_reply.started":"2022-07-01T14:31:31.989302Z","shell.execute_reply":"2022-07-01T14:31:31.996484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(y_true, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T14:31:32.001002Z","iopub.execute_input":"2022-07-01T14:31:32.001364Z","iopub.status.idle":"2022-07-01T14:31:32.013877Z","shell.execute_reply.started":"2022-07-01T14:31:32.001321Z","shell.execute_reply":"2022-07-01T14:31:32.012007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_matrix(y_true, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T14:31:32.015526Z","iopub.execute_input":"2022-07-01T14:31:32.016377Z","iopub.status.idle":"2022-07-01T14:31:32.030887Z","shell.execute_reply.started":"2022-07-01T14:31:32.016334Z","shell.execute_reply":"2022-07-01T14:31:32.029570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}