{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# install packages\n!pip install vit-pytorch\n!pip install linformer\n","metadata":{"execution":{"iopub.status.busy":"2022-06-01T10:00:01.140427Z","iopub.execute_input":"2022-06-01T10:00:01.140968Z","iopub.status.idle":"2022-06-01T10:00:12.430886Z","shell.execute_reply.started":"2022-06-01T10:00:01.140859Z","shell.execute_reply":"2022-06-01T10:00:12.429778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\n\nimport pandas as pd\nimport numpy as np\nfrom random import shuffle\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\n\n\nimport pandas as pd\nimport numpy as np\nfrom random import shuffle\nfrom sklearn.metrics import precision_recall_curve, roc_curve\nimport matplotlib.pyplot as plt\nimport wandb\n\n# from vit_pytorch.efficient import ViT as ViT_eff\n# from vit_pytorch import ViT\n# from linformer import Linformer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-06T05:27:23.846531Z","iopub.execute_input":"2022-06-06T05:27:23.847016Z","iopub.status.idle":"2022-06-06T05:27:27.928772Z","shell.execute_reply.started":"2022-06-06T05:27:23.846930Z","shell.execute_reply":"2022-06-06T05:27:27.927532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset\n\nCustom dataset class for Aptos 2019","metadata":{}},{"cell_type":"markdown","source":"\n### Data utils","metadata":{}},{"cell_type":"code","source":"class_list = ['No DR', 'DR']\ndef binarize(x):\n    if x != 0:\n        return 1\n    else:\n        return x\n    \ndef balance(df, target='binary_target'):\n    # We can balance the dataset\n    df_0 = df[df[target] == 0]\n    df_1 = df[df[target] == 1].sample(len(df_0), random_state=101)\n    df_data = pd.concat([df_0, df_1], axis=0).reset_index(drop=True)\n    df_data = shuffle(df_data)\n    \n    return df_data","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:27:27.931358Z","iopub.execute_input":"2022-06-06T05:27:27.932257Z","iopub.status.idle":"2022-06-06T05:27:27.940841Z","shell.execute_reply.started":"2022-06-06T05:27:27.932220Z","shell.execute_reply":"2022-06-06T05:27:27.939321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_validate_test_split(df, train_percent=.6, validate_percent=.2, \n                              seed=None):\n    \n\n    np.random.seed(seed)\n    perm = np.random.permutation(df.index)\n    m = len(df.index)\n    train_end = int(train_percent * m)\n    validate_end = int(validate_percent * m) + train_end\n    train = df.iloc[perm[:train_end]]\n    validate = df.iloc[perm[train_end:validate_end]]\n    test = df.iloc[perm[validate_end:]]\n\n    return train, validate, test\n\n\ndef get_splits(csv_path,  target_col='diagnosis', balance=False):\n    df = pd.read_csv(csv_path)\n    # TODO: conditional statmenent if to binarize the labels\n    \n    df['binary_target'] = df[target_col].apply(binarize)\n    \n    if balance:\n        df = balance(df)\n    \n    # Split the entire dataframe into train, val and test splits\n    df_data = df.drop(target_col, axis=1).reset_index(drop=True)\n    train_df, validate_df, test_df = train_validate_test_split(df_data)\n\n    # reset_index for all\n    train_df = train_df.reset_index(drop=True)\n    validate_df = validate_df.reset_index(drop=True)\n    test_df = test_df.reset_index(drop=True)\n\n    return train_df, validate_df, test_df","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:27:27.942636Z","iopub.execute_input":"2022-06-06T05:27:27.943817Z","iopub.status.idle":"2022-06-06T05:27:27.958093Z","shell.execute_reply.started":"2022-06-06T05:27:27.943773Z","shell.execute_reply":"2022-06-06T05:27:27.956785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AptosDataset(Dataset):\n    def __init__(self, df_data, data_dir, transform):\n        super().__init__()\n        self.df = df_data.values\n        self.data_dir = data_dir\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_name, label = self.df[index]\n        img_path = os.path.join(self.data_dir, img_name+'.png')\n        image = cv2.imread(img_path)\n        image = self.transform(image)\n            \n        return image, label","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:27:27.961371Z","iopub.execute_input":"2022-06-06T05:27:27.962058Z","iopub.status.idle":"2022-06-06T05:27:27.972617Z","shell.execute_reply.started":"2022-06-06T05:27:27.962013Z","shell.execute_reply":"2022-06-06T05:27:27.971129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DataLoader","metadata":{}},{"cell_type":"code","source":"## Data Module\ndef collate_fn(examples):\n    pixel_values = torch.stack([example[0] for example in examples])\n    labels = torch.tensor([example[1] for example in examples])\n    return {\"pixel_values\": pixel_values, \"labels\": labels}","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:27:27.975009Z","iopub.execute_input":"2022-06-06T05:27:27.975576Z","iopub.status.idle":"2022-06-06T05:27:27.988294Z","shell.execute_reply.started":"2022-06-06T05:27:27.975531Z","shell.execute_reply":"2022-06-06T05:27:27.986954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AptosDataModule():\n    def __init__(self, batch_size, train_df, valid_df, test_df, _train_transforms, _val_transforms, data_dir: str = './'):\n        super().__init__()\n        self.data_dir = data_dir\n        self.batch_size = batch_size\n        self.train_df = train_df\n        self.valid_df = valid_df\n        self.test_df = test_df\n        self._train_transforms = _train_transforms\n        self._val_transforms = _val_transforms\n\n        self.train_set = AptosDataset(self.train_df, self.data_dir, transform=self._train_transforms)\n        self.validate_set = AptosDataset(self.valid_df, self.data_dir, transform=self._val_transforms)\n        self.test_set = AptosDataset(self.test_df, self.data_dir, transform=self._val_transforms)\n\n    def train_dataloader(self):\n        return DataLoader(self.train_set, batch_size=self.batch_size,\n                          shuffle=True, collate_fn=collate_fn, num_workers=2)\n\n    def val_dataloader(self):\n        return DataLoader(self.validate_set, batch_size=self.batch_size, \n                          shuffle=False, collate_fn=collate_fn,num_workers=2)\n    \n    def test_dataloader(self):\n        return DataLoader(self.test_set, batch_size=self.batch_size,\n                          shuffle=False, collate_fn=collate_fn, num_workers=2)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:27:27.990032Z","iopub.execute_input":"2022-06-06T05:27:27.991307Z","iopub.status.idle":"2022-06-06T05:27:28.005029Z","shell.execute_reply.started":"2022-06-06T05:27:27.991275Z","shell.execute_reply":"2022-06-06T05:27:28.003979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AptosDataModule():\n    def __init__(self, batch_size, train_df, valid_df, test_df, _train_transforms, _val_transforms, data_dir: str = './'):\n        super().__init__()\n        self.data_dir = data_dir\n        self.batch_size = batch_size\n        self.train_df = train_df\n        self.valid_df = valid_df\n        self.test_df = test_df\n        self._train_transforms = _train_transforms\n        self._val_transforms = _val_transforms\n\n        self.train_set = AptosDataset(self.train_df, self.data_dir, transform=self._train_transforms)\n        self.validate_set = AptosDataset(self.valid_df, self.data_dir, transform=self._val_transforms)\n        self.test_set = AptosDataset(self.test_df, self.data_dir, transform=self._val_transforms)\n\n    def train_dataloader(self):\n        return DataLoader(self.train_set, batch_size=self.batch_size,\n                          shuffle=True, collate_fn=collate_fn, num_workers=2)\n\n    def val_dataloader(self):\n        return DataLoader(self.validate_set, batch_size=self.batch_size, \n                          shuffle=False, collate_fn=collate_fn,num_workers=2)\n    \n    def test_dataloader(self):\n        return DataLoader(self.test_set, batch_size=self.batch_size,\n                          shuffle=False, collate_fn=collate_fn, num_workers=2)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:27:28.006507Z","iopub.execute_input":"2022-06-06T05:27:28.007850Z","iopub.status.idle":"2022-06-06T05:27:28.022804Z","shell.execute_reply.started":"2022-06-06T05:27:28.007750Z","shell.execute_reply":"2022-06-06T05:27:28.021716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import ViTFeatureExtractor","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:27:28.026087Z","iopub.execute_input":"2022-06-06T05:27:28.026753Z","iopub.status.idle":"2022-06-06T05:27:33.375376Z","shell.execute_reply.started":"2022-06-06T05:27:28.026673Z","shell.execute_reply":"2022-06-06T05:27:33.374306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchvision.transforms import (CenterCrop, \n                                    Compose, \n                                    Normalize, \n                                    RandomHorizontalFlip,\n                                    RandomResizedCrop, \n                                    Resize,\n                                    ToPILImage,\n                                    ToTensor,)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:27:33.378066Z","iopub.execute_input":"2022-06-06T05:27:33.378521Z","iopub.status.idle":"2022-06-06T05:27:33.644992Z","shell.execute_reply.started":"2022-06-06T05:27:33.378478Z","shell.execute_reply":"2022-06-06T05:27:33.644000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform(pre_train_model):\n\n    feature_extractor = ViTFeatureExtractor.from_pretrained(pre_train_model)\n\n\n    normalize = Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)\n    _train_transforms = Compose(\n            [\n                ToPILImage(mode='RGB'),\n                RandomResizedCrop(feature_extractor.size),\n                RandomHorizontalFlip(),\n                ToTensor(),\n                normalize,\n            ]\n        )\n\n    _val_transforms = Compose(\n            [\n                ToPILImage(mode='RGB'),\n                Resize(feature_extractor.size),\n                CenterCrop(feature_extractor.size),\n                ToTensor(),\n                normalize,\n            ]\n        )\n\n    return _train_transforms, _val_transforms","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:27:33.646496Z","iopub.execute_input":"2022-06-06T05:27:33.646936Z","iopub.status.idle":"2022-06-06T05:27:33.656097Z","shell.execute_reply.started":"2022-06-06T05:27:33.646890Z","shell.execute_reply":"2022-06-06T05:27:33.654909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_train_model = \"google/vit-base-patch16-224-in21k\" #\"google/vit-large-patch16-224-in21k\"","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:27:33.659096Z","iopub.execute_input":"2022-06-06T05:27:33.659680Z","iopub.status.idle":"2022-06-06T05:27:33.668677Z","shell.execute_reply.started":"2022-06-06T05:27:33.659633Z","shell.execute_reply":"2022-06-06T05:27:33.667627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_train_transforms, _val_transforms = transform(pre_train_model)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:27:33.669873Z","iopub.execute_input":"2022-06-06T05:27:33.671810Z","iopub.status.idle":"2022-06-06T05:27:34.625811Z","shell.execute_reply.started":"2022-06-06T05:27:33.671754Z","shell.execute_reply":"2022-06-06T05:27:34.624723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nbatch_size = 16\nData_Path = '../input/aptos2019-blindness-detection'\ntrain_dir = Data_Path+ \"/train_images\"\ncsv_file = Data_Path + \"/train.csv\"\ntarget_col = 'diagnosis'\ntarget_names = ['No DR', 'DR']\nnum_epochs = 1\nlearning_rate=0.005\nweight_decay=0.0002\n\ntrain_df, validate_df, test_df  = get_splits(csv_path=csv_file, target_col=target_col)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:27:37.526738Z","iopub.execute_input":"2022-06-06T05:27:37.527258Z","iopub.status.idle":"2022-06-06T05:27:37.575195Z","shell.execute_reply.started":"2022-06-06T05:27:37.527202Z","shell.execute_reply":"2022-06-06T05:27:37.574171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_module = AptosDataModule(batch_size,\n                            train_df,\n                            validate_df, \n                            test_df,\n                            _train_transforms,\n                            _val_transforms,\n                            train_dir\n                            )","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:27:37.896428Z","iopub.execute_input":"2022-06-06T05:27:37.896848Z","iopub.status.idle":"2022-06-06T05:27:37.904935Z","shell.execute_reply.started":"2022-06-06T05:27:37.896816Z","shell.execute_reply":"2022-06-06T05:27:37.901731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_loader = data_module.train_dataloader()\nval_data_loader = data_module.val_dataloader()\ntest_data_loader = data_module.test_dataloader()","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:27:39.306273Z","iopub.execute_input":"2022-06-06T05:27:39.306704Z","iopub.status.idle":"2022-06-06T05:27:39.312472Z","shell.execute_reply.started":"2022-06-06T05:27:39.306670Z","shell.execute_reply":"2022-06-06T05:27:39.311347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pretrain_ ViT Model","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\nfrom transformers import ViTModel\nfrom transformers import ViTConfig\nfrom transformers import PretrainedConfig\nfrom transformers import PreTrainedModel","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:54:10.336418Z","iopub.execute_input":"2022-06-06T05:54:10.337084Z","iopub.status.idle":"2022-06-06T05:54:10.342941Z","shell.execute_reply.started":"2022-06-06T05:54:10.337049Z","shell.execute_reply":"2022-06-06T05:54:10.341703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-06-06T05:46:57.315690Z","iopub.execute_input":"2022-06-06T05:46:57.316074Z","iopub.status.idle":"2022-06-06T05:46:57.387122Z","shell.execute_reply.started":"2022-06-06T05:46:57.316042Z","shell.execute_reply":"2022-06-06T05:46:57.386010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vit = ViTModel.from_pretrained(pre_train_model,return_dict=True)\nconfig = ViTConfig().from_pretrained(pre_train_model,\n                                    attention_probs_dropout_prob = 0.2)\nprint(config)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T06:01:09.136105Z","iopub.execute_input":"2022-06-06T06:01:09.136590Z","iopub.status.idle":"2022-06-06T06:01:11.859402Z","shell.execute_reply.started":"2022-06-06T06:01:09.136559Z","shell.execute_reply":"2022-06-06T06:01:11.858167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save_pretrained\nconfig.save_pretrained(save_directory=\"./\")","metadata":{"execution":{"iopub.status.busy":"2022-06-06T06:02:51.162434Z","iopub.execute_input":"2022-06-06T06:02:51.163028Z","iopub.status.idle":"2022-06-06T06:02:51.176607Z","shell.execute_reply.started":"2022-06-06T06:02:51.162981Z","shell.execute_reply":"2022-06-06T06:02:51.175266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ViTModuleImageClassification(PreTrainedModel):\n    def __init__(self, pre_train_model,drop_out, num_classes, config):\n        super(ViTModuleImageClassification, self).__init__(config)\n        self.vit = ViTModel.from_pretrained(pre_train_model,return_dict=True) # made changes in the function not to return dict\n        self.body = nn.Sequential(*list(self.vit.children())[:-2]) # return the model without the last two layers\n        self.dropout = nn.Dropout(drop_out)\n        self.num_classes = num_classes\n        self.classifier = nn.Linear(self.vit.config.hidden_size, self.num_classes)\n    \n\n    def forward(self, pixel_values):\n        outputs = self.body(pixel_values) # main model\n        output = self.dropout(outputs.last_hidden_state[:,0])\n        att_weights = outputs.attentions\n        logits = self.classifier(output)\n        \n        return logits","metadata":{"execution":{"iopub.status.busy":"2022-06-06T06:15:31.095920Z","iopub.execute_input":"2022-06-06T06:15:31.096326Z","iopub.status.idle":"2022-06-06T06:15:31.106435Z","shell.execute_reply.started":"2022-06-06T06:15:31.096295Z","shell.execute_reply":"2022-06-06T06:15:31.105341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_CLASSES = 2\nmodel = ViTModuleImageClassification(pre_train_model,config=config, drop_out=0.1, num_classes=NUM_CLASSES)\nmodel = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T06:15:51.706880Z","iopub.execute_input":"2022-06-06T06:15:51.707371Z","iopub.status.idle":"2022-06-06T06:15:59.332696Z","shell.execute_reply.started":"2022-06-06T06:15:51.707339Z","shell.execute_reply":"2022-06-06T06:15:59.331664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-06-06T06:19:03.071527Z","iopub.execute_input":"2022-06-06T06:19:03.071958Z","iopub.status.idle":"2022-06-06T06:19:03.683656Z","shell.execute_reply.started":"2022-06-06T06:19:03.071929Z","shell.execute_reply":"2022-06-06T06:19:03.682588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train and evaluation FUnction","metadata":{}},{"cell_type":"code","source":"def train(model, criterion, device, train_data_loader, optimizer, epoch):\n    model.train()\n    loss_x = 0\n    correct_train = 0\n\n    for i, (batch) in enumerate(train_data_loader):\n        pixel_values = batch['pixel_values'].to(device)\n        labels = batch['labels'].to(device)\n\n        optimizer.zero_grad()\n        logits = model(pixel_values)\n        _,predictions = torch.max(logits, 1)\n\n        loss = criterion(logits, labels)\n        loss_x += loss.item()\n        correct_train += (predictions == labels).sum().item()\n\n        loss.backward()\n        optimizer.step()\n\n    # scheduler.step()\n    accuracy = 100*float(correct_train)/len(train_data_loader.dataset)\n    loss_ = loss_x / len(train_data_loader.dataset)\n    print(\"Training accuracy for epoch {} is {}\".format(epoch, accuracy))\n    print(\"Training Loss for epoch {} is {}\".format(epoch, loss_))\n\ndef evaluate(model, criterion, device, val_data_loader, epoch):\n\n    model.eval()\n    loss_x = 0\n    correct_test = 0\n\n    with torch.no_grad():\n        for i, (batch) in enumerate(val_data_loader):\n\n            pixel_values = batch['pixel_values'].to(device)\n            labels = batch['labels'].to(device)\n\n            logits = model(pixel_values)\n            _,predictions = torch.max(logits, 1)\n\n            loss = criterion(logits, labels)\n\n            loss_x += loss.item()\n            \n            correct_test += (predictions == labels).sum().item()\n\n    acc = 100*float(correct_test)/len(val_data_loader.dataset)\n    loss_ = loss_x / len(val_data_loader.dataset)\n\n    print(\"Val accuracy for epoch{} is {}\".format(epoch,acc))\n    print(\"Val Loss for epoch {} is {}\".format(epoch, loss_))\n","metadata":{"execution":{"iopub.status.busy":"2022-06-06T06:19:20.656373Z","iopub.execute_input":"2022-06-06T06:19:20.656946Z","iopub.status.idle":"2022-06-06T06:19:20.683218Z","shell.execute_reply.started":"2022-06-06T06:19:20.656905Z","shell.execute_reply":"2022-06-06T06:19:20.682276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test(test_data_loader, model):\n    predictions, targets = [], []\n\n    with torch.no_grad():\n        model.eval()\n\n        for i, (batch) in enumerate(test_data_loader):\n            # evaluate the model on the test set\n            pixel_values = batch['pixel_values'].to(device)\n            labels = batch['labels'].to(device)\n\n            logits = model(pixel_values)  # [bs, n_class]\n            _,pred = torch.max(logits, 1)  # [bs]\n\n\n            targets.extend(labels.cpu().numpy())\n            predictions.extend(pred.cpu().numpy())\n\n    return predictions, targets\n","metadata":{"execution":{"iopub.status.busy":"2022-06-06T06:19:21.335553Z","iopub.execute_input":"2022-06-06T06:19:21.335974Z","iopub.status.idle":"2022-06-06T06:19:21.344327Z","shell.execute_reply.started":"2022-06-06T06:19:21.335927Z","shell.execute_reply":"2022-06-06T06:19:21.343084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn","metadata":{"execution":{"iopub.status.busy":"2022-06-06T06:19:22.035791Z","iopub.execute_input":"2022-06-06T06:19:22.036668Z","iopub.status.idle":"2022-06-06T06:19:22.043057Z","shell.execute_reply.started":"2022-06-06T06:19:22.036635Z","shell.execute_reply":"2022-06-06T06:19:22.041911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n# scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[5, 10, 15], gamma=0.1, verbose=True)\ncriterion = nn.CrossEntropyLoss().to(device)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-06T06:19:22.746168Z","iopub.execute_input":"2022-06-06T06:19:22.746627Z","iopub.status.idle":"2022-06-06T06:19:22.754602Z","shell.execute_reply.started":"2022-06-06T06:19:22.746595Z","shell.execute_reply":"2022-06-06T06:19:22.753486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(num_epochs):\n    train(model, criterion, device, train_data_loader, optimizer, epoch)\n    evaluate(model, criterion, device, val_data_loader, epoch)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-06T06:19:23.735684Z","iopub.execute_input":"2022-06-06T06:19:23.736285Z","iopub.status.idle":"2022-06-06T06:24:37.896359Z","shell.execute_reply.started":"2022-06-06T06:19:23.736252Z","shell.execute_reply":"2022-06-06T06:24:37.894005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_pretrained(save_directory='./')","metadata":{"execution":{"iopub.status.busy":"2022-06-06T06:30:58.917322Z","iopub.execute_input":"2022-06-06T06:30:58.918006Z","iopub.status.idle":"2022-06-06T06:30:58.924573Z","shell.execute_reply.started":"2022-06-06T06:30:58.917971Z","shell.execute_reply":"2022-06-06T06:30:58.923502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import ViTForImageClassification","metadata":{"execution":{"iopub.status.busy":"2022-06-06T06:28:23.807634Z","iopub.execute_input":"2022-06-06T06:28:23.808068Z","iopub.status.idle":"2022-06-06T06:28:23.813717Z","shell.execute_reply.started":"2022-06-06T06:28:23.808037Z","shell.execute_reply":"2022-06-06T06:28:23.812483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load ViT\nvit = ViTModel.from_pretrained(\"./config.json\").to(\n    device\n)\nvit.eval()","metadata":{"execution":{"iopub.status.busy":"2022-06-06T06:30:21.772273Z","iopub.execute_input":"2022-06-06T06:30:21.772910Z","iopub.status.idle":"2022-06-06T06:30:21.807828Z","shell.execute_reply.started":"2022-06-06T06:30:21.772877Z","shell.execute_reply":"2022-06-06T06:30:21.806330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Vision transformer Explainability**\n\n[Reference Link](https://github.com/jacobgil/vit-explain)","metadata":{}},{"cell_type":"code","source":"# for name, module in model.named_modules():\n#     print(name)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\natt_name = 'attention.output'\nattentions = []\nattention_gradients = []\ndef get_attention(module, input, output):\n    attentions.append(output.cpu())\n    \n    \ndef get_attention_gradient(module, grad_input, grad_output):\n    attention_gradients.append(grad_input[0].cpu())\n    \nfor name, module in model.named_modules():\n    if att_name in name:\n        module.register_forward_hook(get_attention)\n        module.register_backward_hook(get_attention_gradient)\n        \n        \n","metadata":{"execution":{"iopub.status.busy":"2022-06-04T10:45:15.805406Z","iopub.execute_input":"2022-06-04T10:45:15.805767Z","iopub.status.idle":"2022-06-04T10:45:15.812918Z","shell.execute_reply.started":"2022-06-04T10:45:15.805735Z","shell.execute_reply":"2022-06-04T10:45:15.811957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"attentions","metadata":{"execution":{"iopub.status.busy":"2022-06-04T10:45:29.373821Z","iopub.execute_input":"2022-06-04T10:45:29.374399Z","iopub.status.idle":"2022-06-04T10:45:29.385731Z","shell.execute_reply.started":"2022-06-04T10:45:29.374361Z","shell.execute_reply":"2022-06-04T10:45:29.384685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom PIL import Image\nimport numpy\nimport sys\nfrom torchvision import transforms\nimport numpy as np\nimport cv2\n\ndef grad_rollout(attentions, gradients, discard_ratio):\n    result = torch.eye(attentions[0].size(-1))\n    with torch.no_grad():\n        for attention, grad in zip(attentions, gradients):                \n            weights = grad\n            attention_heads_fused = (attention*weights).mean(axis=1)\n            attention_heads_fused[attention_heads_fused < 0] = 0\n\n            # Drop the lowest attentions, but\n            # don't drop the class token\n            flat = attention_heads_fused.view(attention_heads_fused.size(0), -1)\n            _, indices = flat.topk(int(flat.size(-1)*discard_ratio), -1, False)\n            #indices = indices[indices != 0]\n            flat[0, indices] = 0\n\n            I = torch.eye(attention_heads_fused.size(-1))\n            a = (attention_heads_fused + 1.0*I)/2\n            a = a / a.sum(dim=-1)\n            result = torch.matmul(a, result)\n    \n    # Look at the total attention between the class token,\n    # and the image patches\n    mask = result[0, 0 , 1 :]\n    # In case of 224x224 image, this brings us from 196 to 14\n    width = int(mask.size(-1)**0.5)\n    mask = mask.reshape(width, width).numpy()\n    mask = mask / np.max(mask)\n    return mask    \n\nclass VITAttentionGradRollout:\n    def __init__(self, model, attention_layer_name='vit.encoder.layer.11.output.dense',\n        discard_ratio=0.9):\n        self.model = model\n        self.discard_ratio = discard_ratio\n        for name, module in self.model.named_modules():\n            if attention_layer_name in name:\n                module.register_forward_hook(self.get_attention)\n                module.register_backward_hook(self.get_attention_gradient)\n\n        self.attentions = []\n        self.attention_gradients = []\n\n    def get_attention(self, module, input, output):\n        self.attentions.append(output.cpu())\n\n    def get_attention_gradient(self, module, grad_input, grad_output):\n        self.attention_gradients.append(grad_input[0].cpu())\n\n    def __call__(self, input_tensor, category_index):\n        self.model.zero_grad()\n        output = self.model(input_tensor)\n        output = output['logits']\n        category_mask = torch.zeros(output.size())\n        category_mask[:, category_index] = 1\n        loss = (output*category_mask).sum()\n        loss.backward()\n\n        return grad_rollout(self.attentions, self.attention_gradients,\n            self.discard_ratio)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom PIL import Image\nimport numpy\nimport sys\nfrom torchvision import transforms\nimport numpy as np\nimport cv2\n\ndef rollout(attentions, discard_ratio, head_fusion):\n    result = torch.eye(attentions[0].size(-1))\n    with torch.no_grad():\n        for attention in attentions:\n            if head_fusion == \"mean\":\n                attention_heads_fused = attention.mean(axis=1)\n            elif head_fusion == \"max\":\n                attention_heads_fused = attention.max(axis=1)[0]\n            elif head_fusion == \"min\":\n                attention_heads_fused = attention.min(axis=1)[0]\n            else:\n                raise \"Attention head fusion type Not supported\"\n\n            # Drop the lowest attentions, but\n            # don't drop the class token\n            flat = attention_heads_fused.view(attention_heads_fused.size(0), -1)\n            _, indices = flat.topk(int(flat.size(-1)*discard_ratio), -1, False)\n            indices = indices[indices != 0]\n            flat[0, indices] = 0\n\n            I = torch.eye(attention_heads_fused.size(-1))\n            a = (attention_heads_fused + 1.0*I)/2\n            a = a / a.sum(dim=-1)\n\n            result = torch.matmul(a, result)\n    \n    # Look at the total attention between the class token,\n    # and the image patches\n    mask = result[0, 0 , 1 :]\n    # In case of 224x224 image, this brings us from 196 to 14\n    width = int(mask.size(-1)**0.5)\n    mask = mask.reshape(width, width).numpy()\n    mask = mask / np.max(mask)\n    return mask    \n\nclass VITAttentionRollout:\n    def __init__(self, model, attention_layer_name='attn_drop', head_fusion=\"mean\",\n        discard_ratio=0.9):\n        self.model = model\n        self.head_fusion = head_fusion\n        self.discard_ratio = discard_ratio\n        for name, module in self.model.named_modules():\n            if attention_layer_name in name:\n                module.register_forward_hook(self.get_attention)\n\n        self.attentions = []\n\n    def get_attention(self, module, input, output):\n        self.attentions.append(output.cpu())\n\n    def __call__(self, input_tensor):\n        self.attentions = []\n        with torch.no_grad():\n            output = self.model(input_tensor)\n\n        return rollout(self.attentions, self.discard_ratio, self.head_fusion)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\nx = x.to('cpu')\n\ngrad_rollout = VITAttentionGradRollout(model, discard_ratio=0.9)\nmask = grad_rollout(x, category_index=0)\n\natt_name = 'vit.embeddings.patch_embeddings'#vit.encoder.layer.11.output.dense'\nattentions = []\n\n\nfor name, module in model.named_modules():\n    print(name)\n    \ndef get_attention(module, input, output):\n    attentions.append(output.cpu())\nfor name, module in model.named_modules():\n    if att_name in name:\n        module.register_forward_hook(get_attention)\n        \nattentions","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## trying from scratch","metadata":{}},{"cell_type":"code","source":"# from torch.autograd import Variable\n# import torch\n\n# \"\"\"I cannot load the attention weights with this.\"\"\"\n# mod = ViTModule(pre_train_model, drop_out=0.1, num_classes=2)\n# mod = mod.to(device)\n\n# x=Variable(torch.FloatTensor(1, 3, 224,224))\n# x = x.to(device)\n\n# reuslts = mod(x)\n# # reuslts['logits'].size()\n# # reuslts['att_weights']\n\n# from torch.autograd import Variable\n# import torch\n\n# \"\"\"I cannot load the attention weights with this.\"\"\"\n# model = ViTModule(pre_train_model, drop_out=0.1, num_classes=2)\n# model = model.to(device)\n\n# x=Variable(torch.FloatTensor(16, 3, 224,224))\n# x = x.to(device)\n\n# logits, att_weights = model(x, output_attentions=True)\n# att_weights\n\n\n# \"\"\" From using ViTForImageCalssification\n#     we can load the atention weights\"\"\"\n# from transformers import ViTForImageClassification\n\n# vit = ViTForImageClassification.from_pretrained(\"google/vit-base-patch16-224\").to(\n#     device\n# )\n\n# result = vit(x, output_attentions=True)\n# attention_probs = torch.stack(result[1]).squeeze(1)\n# attention_probs.size()\n# result[0].size()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vit = ViTModel.from_pretrained(pre_train_model,output_attentions=true, return_dict=True) # made changes in the function not to return dict\nbody = nn.Sequential(*list(vit.children())[:-2]) # return the model without the last two layers\ndropout = nn.Dropout(drop_out)\nclassifier = nn.Linear(self.vit.config.hidden_size, self.num_classes)\n    \n\n    def forward(self, pixel_values, output_attentions=False):\n        outputs = self.body(pixel_values) # main model\n        output = self.dropout(outputs.last_hidden_state[:,0])\n        att_weights = outputs.attentions\n        logits = self.classifier(output)\n        if output_attentions:\n            return {'logits': logits,\n                   'att_weights':att_weights}\n        \n        return logits","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reuslts['att_weights'].size()","metadata":{"execution":{"iopub.status.busy":"2022-06-04T08:48:50.560485Z","iopub.execute_input":"2022-06-04T08:48:50.561289Z","iopub.status.idle":"2022-06-04T08:48:50.591148Z","shell.execute_reply.started":"2022-06-04T08:48:50.561256Z","shell.execute_reply":"2022-06-04T08:48:50.589553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_pretrained(\"weights_hf_test.pt\")","metadata":{"execution":{"iopub.status.busy":"2022-06-04T07:42:01.475112Z","iopub.execute_input":"2022-06-04T07:42:01.475519Z","iopub.status.idle":"2022-06-04T07:42:01.513813Z","shell.execute_reply.started":"2022-06-04T07:42:01.475482Z","shell.execute_reply":"2022-06-04T07:42:01.512596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save model\n\npath = \"weights_test.pt\"\nprint(\"Model saved at {}\".format(path))\ntorch.save(model, path)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-06-04T08:08:15.925219Z","iopub.execute_input":"2022-06-04T08:08:15.926153Z","iopub.status.idle":"2022-06-04T08:08:16.505471Z","shell.execute_reply.started":"2022-06-04T08:08:15.926101Z","shell.execute_reply":"2022-06-04T08:08:16.504591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=load_checkpoint('./weights_test.pt', model)\nmodel.save_pretrained(\"my path\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vit = ViTForImageClassification.from_pretrained('')","metadata":{"execution":{"iopub.status.busy":"2022-06-04T08:09:16.935536Z","iopub.execute_input":"2022-06-04T08:09:16.936335Z","iopub.status.idle":"2022-06-04T08:09:17.452683Z","shell.execute_reply.started":"2022-06-04T08:09:16.936301Z","shell.execute_reply":"2022-06-04T08:09:17.451512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch = next(iter(test_data_loader))\nx = batch['pixel_values']\ny = batch['labels']\nprint(x.shape, y.shape)","metadata":{"execution":{"iopub.status.busy":"2022-06-02T12:40:09.162359Z","iopub.execute_input":"2022-06-02T12:40:09.162889Z","iopub.status.idle":"2022-06-02T12:40:16.779196Z","shell.execute_reply.started":"2022-06-02T12:40:09.162847Z","shell.execute_reply":"2022-06-02T12:40:16.777385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import ViTForImageClassification","metadata":{"execution":{"iopub.status.busy":"2022-06-02T12:44:43.486176Z","iopub.execute_input":"2022-06-02T12:44:43.487346Z","iopub.status.idle":"2022-06-02T12:44:43.495048Z","shell.execute_reply.started":"2022-06-02T12:44:43.487303Z","shell.execute_reply":"2022-06-02T12:44:43.493783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trained_model = torch.load('./weights_test.pt')","metadata":{"execution":{"iopub.status.busy":"2022-06-02T12:40:16.781783Z","iopub.execute_input":"2022-06-02T12:40:16.782255Z","iopub.status.idle":"2022-06-02T12:40:17.627221Z","shell.execute_reply.started":"2022-06-02T12:40:16.78219Z","shell.execute_reply":"2022-06-02T12:40:17.626388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"module= ViTModule(pre_train_model, drop_out=0.1, num_classes=2)\n\nmodel.load_state_dict(torch.load('./weights_test.pt'))","metadata":{"execution":{"iopub.status.busy":"2022-06-02T12:50:15.291601Z","iopub.execute_input":"2022-06-02T12:50:15.291981Z","iopub.status.idle":"2022-06-02T12:50:24.502159Z","shell.execute_reply.started":"2022-06-02T12:50:15.291948Z","shell.execute_reply":"2022-06-02T12:50:24.501125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vit = ViTForImageClassification.from_pretrained(\"google/vit-large-patch16-224-in21k\",\n                                               state_dict=trained_model).to(device)\nvit.eval()","metadata":{"execution":{"iopub.status.busy":"2022-06-02T12:46:51.571807Z","iopub.execute_input":"2022-06-02T12:46:51.572174Z","iopub.status.idle":"2022-06-02T12:47:04.754991Z","shell.execute_reply.started":"2022-06-02T12:46:51.572145Z","shell.execute_reply":"2022-06-02T12:47:04.754067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trained_model.eval()\nresult = trained_model(x.to(device), output_attentions=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-02T12:41:22.250056Z","iopub.execute_input":"2022-06-02T12:41:22.250753Z","iopub.status.idle":"2022-06-02T12:41:22.297004Z","shell.execute_reply.started":"2022-06-02T12:41:22.250714Z","shell.execute_reply":"2022-06-02T12:41:22.296162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"att = result.last_hidden_state","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_id = result[0].argmax()","metadata":{"execution":{"iopub.status.busy":"2022-06-02T12:42:15.37012Z","iopub.execute_input":"2022-06-02T12:42:15.370903Z","iopub.status.idle":"2022-06-02T12:42:15.377602Z","shell.execute_reply.started":"2022-06-02T12:42:15.370864Z","shell.execute_reply":"2022-06-02T12:42:15.374918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_id","metadata":{"execution":{"iopub.status.busy":"2022-06-02T12:42:23.029569Z","iopub.execute_input":"2022-06-02T12:42:23.02996Z","iopub.status.idle":"2022-06-02T12:42:23.037821Z","shell.execute_reply.started":"2022-06-02T12:42:23.02993Z","shell.execute_reply":"2022-06-02T12:42:23.03692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nattention_probs = torch.stack(result[1]).squeeze(1)\n\n# Average the attention at each layer over all heads\nattention_probs = torch.mean(attention_probs, dim=1)\nresidual = torch.eye(attention_probs.size(-1)).to(device)\nattention_probs = 0.5 * attention_probs + 0.5 * residual","metadata":{"execution":{"iopub.status.busy":"2022-06-02T12:41:30.510122Z","iopub.execute_input":"2022-06-02T12:41:30.511001Z","iopub.status.idle":"2022-06-02T12:41:30.567086Z","shell.execute_reply.started":"2022-06-02T12:41:30.510968Z","shell.execute_reply":"2022-06-02T12:41:30.566066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Testing Model","metadata":{}},{"cell_type":"code","source":"# Test Model Phase\npredictions, targets = test(test_data_loader, model)","metadata":{"execution":{"iopub.status.busy":"2022-06-02T06:26:04.192744Z","iopub.execute_input":"2022-06-02T06:26:04.193258Z","iopub.status.idle":"2022-06-02T06:27:29.207979Z","shell.execute_reply.started":"2022-06-02T06:26:04.193189Z","shell.execute_reply":"2022-06-02T06:27:29.206445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score","metadata":{"execution":{"iopub.status.busy":"2022-06-02T06:27:29.2111Z","iopub.execute_input":"2022-06-02T06:27:29.211625Z","iopub.status.idle":"2022-06-02T06:27:29.217244Z","shell.execute_reply.started":"2022-06-02T06:27:29.211563Z","shell.execute_reply":"2022-06-02T06:27:29.216204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy = accuracy_score(targets, predictions)","metadata":{"execution":{"iopub.status.busy":"2022-06-02T06:27:29.218927Z","iopub.execute_input":"2022-06-02T06:27:29.219646Z","iopub.status.idle":"2022-06-02T06:27:29.23327Z","shell.execute_reply.started":"2022-06-02T06:27:29.2196Z","shell.execute_reply":"2022-06-02T06:27:29.232257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy","metadata":{"execution":{"iopub.status.busy":"2022-06-02T06:27:29.23458Z","iopub.execute_input":"2022-06-02T06:27:29.235042Z","iopub.status.idle":"2022-06-02T06:27:29.255836Z","shell.execute_reply.started":"2022-06-02T06:27:29.235002Z","shell.execute_reply":"2022-06-02T06:27:29.254744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report","metadata":{"execution":{"iopub.status.busy":"2022-06-02T06:27:29.257853Z","iopub.execute_input":"2022-06-02T06:27:29.258978Z","iopub.status.idle":"2022-06-02T06:27:29.265396Z","shell.execute_reply.started":"2022-06-02T06:27:29.258931Z","shell.execute_reply":"2022-06-02T06:27:29.2642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# confusion matrix\nprint('This is the  classification report:...')\nprint(classification_report(targets, predictions, digits=3, target_names=target_names), '\\n')\n\n","metadata":{"execution":{"iopub.status.busy":"2022-06-02T06:27:29.268561Z","iopub.execute_input":"2022-06-02T06:27:29.269326Z","iopub.status.idle":"2022-06-02T06:27:29.28746Z","shell.execute_reply.started":"2022-06-02T06:27:29.26925Z","shell.execute_reply":"2022-06-02T06:27:29.286417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import precision_recall_curve","metadata":{"execution":{"iopub.status.busy":"2022-06-02T06:27:29.288882Z","iopub.execute_input":"2022-06-02T06:27:29.290041Z","iopub.status.idle":"2022-06-02T06:27:29.296165Z","shell.execute_reply.started":"2022-06-02T06:27:29.289994Z","shell.execute_reply":"2022-06-02T06:27:29.294436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# wandb.sklearn.plot_confusion_matrix(targets, predictions, target_names)\n\n# Log plot of ROC\n\n# precision recall curve\n#calculate precision and recall\nprecision, recall, thresholds = precision_recall_curve(targets, predictions)\n\n#create precision recall curve\nfig, ax = plt.subplots()\nax.plot(recall, precision, color='purple')\n#add axis labels to plot\nax.set_title('Precision-Recall Curve')\nax.set_ylabel('Precision')\nax.set_xlabel('Recall')","metadata":{"execution":{"iopub.status.busy":"2022-06-02T06:29:27.190139Z","iopub.execute_input":"2022-06-02T06:29:27.191312Z","iopub.status.idle":"2022-06-02T06:29:27.46586Z","shell.execute_reply.started":"2022-06-02T06:29:27.19128Z","shell.execute_reply":"2022-06-02T06:29:27.464533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import PrecisionRecallDisplay","metadata":{"execution":{"iopub.status.busy":"2022-06-02T06:29:54.470429Z","iopub.execute_input":"2022-06-02T06:29:54.470884Z","iopub.status.idle":"2022-06-02T06:29:54.476224Z","shell.execute_reply.started":"2022-06-02T06:29:54.470854Z","shell.execute_reply":"2022-06-02T06:29:54.475094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display = PrecisionRecallDisplay.from_predictions(targets, predictions, name=\"LinearSVC\")\n_ = display.ax_.set_title(\"2-class Precision-Recall curve\")","metadata":{"execution":{"iopub.status.busy":"2022-06-02T06:30:52.800681Z","iopub.execute_input":"2022-06-02T06:30:52.801082Z","iopub.status.idle":"2022-06-02T06:30:53.026758Z","shell.execute_reply.started":"2022-06-02T06:30:52.801053Z","shell.execute_reply":"2022-06-02T06:30:53.025678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fpr, tpr, _ = roc_curve(targets, predictions)\nplt.plot(fpr, tpr, lw=2)\n\nplt.xlabel(\"false positive rate\")\nplt.ylabel(\"true positive rate\")\nplt.legend(loc=\"best\")\nplt.title(\"ROC curve\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-02T06:31:07.361847Z","iopub.execute_input":"2022-06-02T06:31:07.362227Z","iopub.status.idle":"2022-06-02T06:31:07.583013Z","shell.execute_reply.started":"2022-06-02T06:31:07.362197Z","shell.execute_reply":"2022-06-02T06:31:07.581716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi\n","metadata":{"execution":{"iopub.status.busy":"2022-06-02T06:32:06.223336Z","iopub.execute_input":"2022-06-02T06:32:06.223801Z","iopub.status.idle":"2022-06-02T06:32:07.100852Z","shell.execute_reply.started":"2022-06-02T06:32:06.223754Z","shell.execute_reply":"2022-06-02T06:32:07.099512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}