{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Import Libraries**","metadata":{}},{"cell_type":"code","source":"! conda install -c conda-forge gdcm -y;","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:34:17.343375Z","iopub.execute_input":"2021-09-18T17:34:17.347200Z","iopub.status.idle":"2021-09-18T17:35:17.678187Z","shell.execute_reply.started":"2021-09-18T17:34:17.347069Z","shell.execute_reply":"2021-09-18T17:35:17.677010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"../input/timmeffnetv2\")\n\nimport platform\nimport numpy as np\nimport pandas as pd\nimport os\nfrom tqdm.notebook import tqdm\nimport cv2\nimport pydicom\nimport glob\nimport gc\nfrom math import ceil\nimport matplotlib.pyplot as plt\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\n\nimport albumentations as A\nimport seaborn as sn\n\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-18T17:35:17.680860Z","iopub.execute_input":"2021-09-18T17:35:17.681314Z","iopub.status.idle":"2021-09-18T17:35:24.338059Z","shell.execute_reply.started":"2021-09-18T17:35:17.681255Z","shell.execute_reply":"2021-09-18T17:35:24.336934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Load Metadata**","metadata":{}},{"cell_type":"code","source":"train_image = pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")\ntrain_study = pd.read_csv(\"../input/siim-covid19-detection/train_study_level.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:35:24.339632Z","iopub.execute_input":"2021-09-18T17:35:24.340002Z","iopub.status.idle":"2021-09-18T17:35:24.422734Z","shell.execute_reply.started":"2021-09-18T17:35:24.339956Z","shell.execute_reply":"2021-09-18T17:35:24.421938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DIR = \"../input/siim-covid19-detection/train/\"\ntrain_study['StudyInstanceUID'] = train_study['id'].apply(lambda x: x.replace('_study', ''))\ntrain = train_image.merge(train_study, on='StudyInstanceUID')\n\n# Make a path folder\npaths = []\nfor instance_id in tqdm(train['StudyInstanceUID']):\n    paths.append(glob.glob(os.path.join(TRAIN_DIR, instance_id +\"/*/*\"))[0])\n\ntrain['path'] = paths\n\ntrain = train.drop(['id_x', 'id_y'], axis=1)\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:35:24.425848Z","iopub.execute_input":"2021-09-18T17:35:24.426198Z","iopub.status.idle":"2021-09-18T17:35:59.182400Z","shell.execute_reply.started":"2021-09-18T17:35:24.426157Z","shell.execute_reply":"2021-09-18T17:35:59.180981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Exploratory Analysis**","metadata":{}},{"cell_type":"code","source":"one_hot_encode_target = train.iloc[:, 3:7]\nlabels = one_hot_encode_target.columns\ncounts_classes = one_hot_encode_target.sum(axis = 0)\nperc_counts = 100 * counts_classes / counts_classes.sum()","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:35:59.184995Z","iopub.execute_input":"2021-09-18T17:35:59.185386Z","iopub.status.idle":"2021-09-18T17:35:59.211666Z","shell.execute_reply.started":"2021-09-18T17:35:59.185344Z","shell.execute_reply":"2021-09-18T17:35:59.210692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (16, 10))\n\n#define Seaborn color palette to use\ncolors = sn.color_palette('pastel')[0:5]\n\n#create pie chart\n_ = plt.pie(perc_counts, labels = labels, colors = colors, autopct='%.0f%%')","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:35:59.214183Z","iopub.execute_input":"2021-09-18T17:35:59.214642Z","iopub.status.idle":"2021-09-18T17:35:59.411298Z","shell.execute_reply.started":"2021-09-18T17:35:59.214595Z","shell.execute_reply":"2021-09-18T17:35:59.409930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Load DIACOM Dataset**","metadata":{}},{"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:35:59.413500Z","iopub.execute_input":"2021-09-18T17:35:59.414035Z","iopub.status.idle":"2021-09-18T17:35:59.423351Z","shell.execute_reply.started":"2021-09-18T17:35:59.413988Z","shell.execute_reply":"2021-09-18T17:35:59.421958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    image_size = (224, 224)\n    train_bs = 32\n    valid_bs = 16\n    num_workers = 8\n    num_finetuning_epochs = 10\n    num_total_epochs = 25\n    scaler = GradScaler()","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:37:57.902821Z","iopub.execute_input":"2021-09-18T17:37:57.903223Z","iopub.status.idle":"2021-09-18T17:37:57.912144Z","shell.execute_reply.started":"2021-09-18T17:37:57.903191Z","shell.execute_reply":"2021-09-18T17:37:57.911095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,5,figsize = (25,10))\n\nfor i, path in enumerate(train['path'][0:5]):\n    image = dicom2array(path)\n    image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)\n    image = cv2.resize(image, (400, 400))\n    ax[i].imshow(image)","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:35:59.475084Z","iopub.execute_input":"2021-09-18T17:35:59.475437Z","iopub.status.idle":"2021-09-18T17:36:03.885713Z","shell.execute_reply.started":"2021-09-18T17:35:59.475399Z","shell.execute_reply":"2021-09-18T17:36:03.884660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SIIMData(Dataset):\n    def __init__(self, df, is_train = True, augments = None, img_size = Config.image_size):\n        super().__init__()\n        self.df = df\n        self.is_train = is_train\n        self.augments = augments\n        self.img_size = img_size\n        \n    def __getitem__(self, idx):\n        image_id = self.df['StudyInstanceUID'].values[idx] \n        image_path = self.df['path'].values[idx]\n        image = dicom2array(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)\n        image = cv2.resize(image, Config.image_size)\n        \n        # Augments must be albumentations\n        if self.augments:\n            image = self.augments(image=image)['image']\n        else:\n            image = torch.tensor(image, dtype=torch.float)\n        \n        if self.is_train:\n            label = self.df[self.df['StudyInstanceUID'] == image_id].values.tolist()[0][3:7]\n            return image, torch.tensor(label)\n        \n        return image\n    \n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:36:03.890610Z","iopub.execute_input":"2021-09-18T17:36:03.891307Z","iopub.status.idle":"2021-09-18T17:36:03.905717Z","shell.execute_reply.started":"2021-09-18T17:36:03.891263Z","shell.execute_reply":"2021-09-18T17:36:03.904579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = A.Compose({\n        A.HorizontalFlip(p=0.5),\n        A.Rotate(limit=(-40, 40)),\n        A.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n        })\n\nvalid_test_transform = A.Compose({\n        A.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n        })","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:36:03.907698Z","iopub.execute_input":"2021-09-18T17:36:03.908157Z","iopub.status.idle":"2021-09-18T17:36:03.923244Z","shell.execute_reply.started":"2021-09-18T17:36:03.908115Z","shell.execute_reply":"2021-09-18T17:36:03.922003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Splitting the Dataset** ","metadata":{}},{"cell_type":"code","source":"# train[\"class\"] = train.iloc[:, 3:7].to_numpy().argmax(axis = 1)\n\n# train = train[:150]\n# np.unique(train[\"class\"].to_numpy(), return_counts = True)","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:36:03.925242Z","iopub.execute_input":"2021-09-18T17:36:03.925667Z","iopub.status.idle":"2021-09-18T17:36:03.934212Z","shell.execute_reply.started":"2021-09-18T17:36:03.925626Z","shell.execute_reply":"2021-09-18T17:36:03.933222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"class\"] = train.iloc[:, 3:7].to_numpy().argmax(axis = 1)\n\ntrain, valid = train_test_split(train, stratify = train[\"class\"], shuffle = True, test_size=0.3)\n\nvalid, test = train_test_split(valid, stratify = valid[\"class\"], shuffle = True, test_size=0.5)","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:36:03.936262Z","iopub.execute_input":"2021-09-18T17:36:03.936777Z","iopub.status.idle":"2021-09-18T17:36:03.962961Z","shell.execute_reply.started":"2021-09-18T17:36:03.936735Z","shell.execute_reply":"2021-09-18T17:36:03.961763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = SIIMData(train, augments = train_transform)\nvalid_data = SIIMData(valid, augments = valid_test_transform)\ntest_data = SIIMData(test, augments = valid_test_transform)","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:36:03.964792Z","iopub.execute_input":"2021-09-18T17:36:03.965194Z","iopub.status.idle":"2021-09-18T17:36:03.972019Z","shell.execute_reply.started":"2021-09-18T17:36:03.965151Z","shell.execute_reply":"2021-09-18T17:36:03.970577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, class_sample_count = np.unique(train[\"class\"].to_numpy(), return_counts = True)\n#Balance the Training Dataset\n\nweight = 1. / class_sample_count\nsamples_weight = np.array([weight[t] for t in train[\"class\"]])\n\nsamples_weight = torch.from_numpy(samples_weight)\nsamples_weight = samples_weight.double()\nsampler = WeightedRandomSampler(samples_weight, len(samples_weight))","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:36:03.974278Z","iopub.execute_input":"2021-09-18T17:36:03.975101Z","iopub.status.idle":"2021-09-18T17:36:03.997633Z","shell.execute_reply.started":"2021-09-18T17:36:03.975057Z","shell.execute_reply":"2021-09-18T17:36:03.996684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(dataset = train_data, batch_size = Config.train_bs, num_workers = Config.num_workers, sampler = sampler)\nvalid_loader = DataLoader(dataset = valid_data, batch_size = Config.valid_bs, num_workers = Config.num_workers)\ntest_loader = DataLoader(dataset = test_data, batch_size = Config.valid_bs, num_workers = Config.num_workers)","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:36:04.001108Z","iopub.execute_input":"2021-09-18T17:36:04.001364Z","iopub.status.idle":"2021-09-18T17:36:04.008267Z","shell.execute_reply.started":"2021-09-18T17:36:04.001338Z","shell.execute_reply":"2021-09-18T17:36:04.007043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **DenseNet201**","metadata":{}},{"cell_type":"code","source":"# There is a bug in pytorch when downloading .pt through wget :(\n# ! wget -c https://github.com/PedroRASB/COVID-19-Twice-Transfer-DNNs/blob/master/NetC.pt","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:36:04.009953Z","iopub.execute_input":"2021-09-18T17:36:04.011181Z","iopub.status.idle":"2021-09-18T17:36:04.017937Z","shell.execute_reply.started":"2021-09-18T17:36:04.011136Z","shell.execute_reply":"2021-09-18T17:36:04.016837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DenseNet201(nn.Module):\n\n    def __init__(self, num_classes = 4):\n        super(DenseNet201, self).__init__()\n        self.model = torch.load(\"../input/covidnet/NetC.pt\", map_location=torch.device('cpu'))\n        \n        #Freezing the loaded model\n        for param in self.model.parameters():\n            param.requires_grad = False\n            \n        #The output layer is unfreeze\n        self.model.classifier = nn.Linear(self.model.classifier.in_features, num_classes)\n        \n    def unfreeze_all_layers(self):\n        for param in self.model.parameters():\n            param.requires_grad = True\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-09-18T17:36:04.019914Z","iopub.execute_input":"2021-09-18T17:36:04.020983Z","iopub.status.idle":"2021-09-18T17:36:04.032349Z","shell.execute_reply.started":"2021-09-18T17:36:04.020939Z","shell.execute_reply":"2021-09-18T17:36:04.031284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Training the model**","metadata":{}},{"cell_type":"code","source":"class Trainer:\n    def __init__(self, model, train_loader, valid_loader, optimizer, criterion, device):\n        \"\"\"\n        Constructor for Trainer class\n        \"\"\"\n        self.model = model\n        self.train = train_loader\n        self.valid = valid_loader\n        self.optim = optimizer\n        self.criterion = criterion\n        self.device = device\n        self.scaler = GradScaler()\n    \n    def train_one_epoch(self):\n        \"\"\"\n        Runs one epoch of training, backpropagation and optimization\n        \"\"\"\n        self.model.train()\n        \n        running_loss = 0\n        running_acc = 0\n        \n        for xtrain, ytrain in self.train:\n            xtrain = xtrain.to(self.device).float()\n            ytrain = ytrain.to(self.device).float()\n            xtrain = xtrain.permute(0, 3, 1, 2)\n            \n            with autocast():\n                # Get predictions\n                z = self.model(xtrain)\n\n                # Training\n                train_loss = self.criterion(z, ytrain)\n                self.scaler.scale(train_loss).backward()\n                \n                self.scaler.step(self.optim)\n                self.scaler.update()\n                self.optim.zero_grad(set_to_none=True)\n\n                # For averaging and reporting later\n                running_loss += train_loss\n\n                # Convert the predictions and corresponding labels to right form\n                train_preds = torch.argmax(z, 1).detach().cpu().numpy()\n                train_labels = ytrain.detach().cpu().numpy().argmax(axis = 1)\n                \n                running_acc += ( train_preds == train_labels ).sum()                \n        \n        # Now average the running loss over all batches and return\n        running_loss /= train.shape[0]\n        running_acc  /= train.shape[0]\n        print(f\"Training Loss: {running_loss:.4f}\")\n        print(f\"Training Accuracy: {running_acc:.4f}\\n\")\n        \n        # Free up memory\n        del train_preds, train_labels, xtrain, ytrain, z\n        gc.collect()\n        torch.cuda.empty_cache()\n        \n        return (running_loss, running_acc)\n\n    def valid_one_epoch(self):\n        \"\"\"\n        Runs one epoch of prediction\n        \"\"\"        \n        model.eval()\n                \n        with torch.no_grad():\n            \n            running_loss = 0\n            running_acc = 0\n            \n            for xval, yval in self.valid:\n                xval = xval.to(self.device).float()\n                yval = yval.to(self.device).float()\n                xval = xval.permute(0, 3, 1, 2)\n                \n                val_z = self.model(xval)\n                \n                val_loss = self.criterion(val_z, yval)\n                \n                running_loss += val_loss.item()\n                \n                val_preds = torch.argmax(val_z, 1).detach().cpu().numpy()\n                val_labels = yval.detach().cpu().numpy().argmax(axis = 1)\n                \n                running_acc += ( val_preds == val_labels ).sum()\n            \n            # Get the final loss\n            running_loss /= valid.shape[0]\n            running_acc  /= valid.shape[0]\n            \n            print(f\"Validation Loss: {running_loss:.4f}\")\n            print(f\"Validation Accuracy: {running_acc:.4f}\")\n            \n            # Free up memory\n            del val_labels, val_preds, xval, yval, val_z\n            gc.collect()\n            torch.cuda.empty_cache()\n            \n        return (running_loss, running_acc)","metadata":{"execution":{"iopub.status.busy":"2021-09-18T18:18:44.583019Z","iopub.execute_input":"2021-09-18T18:18:44.583366Z","iopub.status.idle":"2021-09-18T18:18:44.607683Z","shell.execute_reply.started":"2021-09-18T18:18:44.583323Z","shell.execute_reply":"2021-09-18T18:18:44.606588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Code\nprint(f\"[INFO] Training on {train.shape[0]} samples and validation on {valid.shape[0]} samples\")\n\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n\nmodel = DenseNet201().to(device)\n    \n# optim = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.001)\n\noptim = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.01)\n\ncriterion = nn.BCEWithLogitsLoss()\n\ntrainer = Trainer(\n    model=model,\n    train_loader=train_loader,\n    valid_loader=valid_loader,\n    optimizer=optim,\n    criterion=criterion,\n    device=device,\n)\n\ntrain_losses = []\nvalid_losses = []\n\ntrain_acc = []\nvalid_acc = []\n\nbest_acc = 0\n\nfor epoch in range(Config.num_finetuning_epochs):\n    print(f\"{'-'*25} EPOCH: {epoch+1}/{Config.num_total_epochs} {'-'*25}\")\n\n    # Run one training epoch\n    current_train_loss, current_train_acc = trainer.train_one_epoch()\n    train_losses.append(current_train_loss)\n    train_acc.append(current_train_acc)\n\n    # Run one validation epoch\n    current_val_loss, current_val_acc = trainer.valid_one_epoch()\n    valid_losses.append(current_val_loss)\n    valid_acc.append(current_val_acc)\n    \n    if(current_val_acc > best_acc):\n        best_acc = current_val_acc\n        torch.save(trainer.model, 'best-model.pt')\n\n    # Empty CUDA cache\n    torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2021-09-18T18:18:49.417400Z","iopub.execute_input":"2021-09-18T18:18:49.417690Z","iopub.status.idle":"2021-09-18T20:34:12.462792Z","shell.execute_reply.started":"2021-09-18T18:18:49.417661Z","shell.execute_reply":"2021-09-18T20:34:12.460354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Evaluate the model**","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nfig, ax = plt.subplots(1,2,figsize = (25,10))\n\nax[0].plot(train_losses, label='Training loss')\nax[0].plot(valid_losses, label='Validation loss')\nax[0].set_xlabel(\"Epochs\")\nax[0].set_ylabel(\"Loss\")\nax[0].legend(frameon=False)\n\nax[1].plot(train_acc, label='Training accuracy')\nax[1].plot(valid_acc, label='Validation accuracy')\nax[1].set_xlabel(\"Epochs\")\nax[1].set_ylabel(\"Accuracy\")\nax[1].legend(frameon=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-18T20:34:12.467664Z","iopub.execute_input":"2021-09-18T20:34:12.467972Z","iopub.status.idle":"2021-09-18T20:34:13.669804Z","shell.execute_reply.started":"2021-09-18T20:34:12.467884Z","shell.execute_reply":"2021-09-18T20:34:13.668861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Finetuning**","metadata":{}},{"cell_type":"code","source":"# Training Code -- Finetuning\nprint(\"\\nFinetuning\\n\")\n\ntrainer.model.unfreeze_all_layers()\ntrainer.optimizer = torch.optim.SGD(trainer.model.parameters(), lr=0.0001, momentum=0.9, weight_decay=0.01)\n\nfor epoch in range(Config.num_finetuning_epochs, Config.num_total_epochs):\n    print(f\"{'-'*25} EPOCH: {epoch+1}/{Config.num_total_epochs} {'-'*25}\")\n\n    # Run one training epoch\n    current_train_loss, current_train_acc = trainer.train_one_epoch()\n    train_losses.append(current_train_loss)\n    train_acc.append(current_train_acc)\n\n    # Run one validation epoch\n    current_val_loss, current_val_acc = trainer.valid_one_epoch()\n    valid_losses.append(current_val_loss)\n    valid_acc.append(current_val_acc)\n    \n    if(current_val_acc > best_acc):\n        best_acc = current_val_acc\n        torch.save(trainer.model, 'best-model.pt')\n\n    # Empty CUDA cache\n    torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2021-09-18T20:34:13.671509Z","iopub.execute_input":"2021-09-18T20:34:13.671951Z","iopub.status.idle":"2021-09-19T00:06:18.480731Z","shell.execute_reply.started":"2021-09-18T20:34:13.671907Z","shell.execute_reply":"2021-09-19T00:06:18.478409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Evaluate the model**","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nfig, ax = plt.subplots(1,2,figsize = (25,10))\n\nax[0].plot(train_losses, label='Training loss')\nax[0].plot(valid_losses, label='Validation loss')\nax[0].set_xlabel(\"Epochs\")\nax[0].set_ylabel(\"Loss\")\nax[0].legend(frameon=False)\n\nax[1].plot(train_acc, label='Training accuracy')\nax[1].plot(valid_acc, label='Validation accuracy')\nax[1].set_xlabel(\"Epochs\")\nax[1].set_ylabel(\"Accuracy\")\nax[1].legend(frameon=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-19T00:06:18.485695Z","iopub.execute_input":"2021-09-19T00:06:18.486612Z","iopub.status.idle":"2021-09-19T00:06:19.338298Z","shell.execute_reply.started":"2021-09-19T00:06:18.486569Z","shell.execute_reply":"2021-09-19T00:06:19.337217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test-the-model\nmodel.eval()  # it-disables-dropout\n\nconfusion_matrix = np.zeros((4,4))\n\nwith torch.no_grad():\n\n    for images, labels in test_loader:\n        \n        images = images.to(device).float()\n        labels = labels.to(device).float()\n        images = images.permute(0, 3, 1, 2)\n        \n        outputs = model(images)\n        test_preds = torch.argmax(outputs, 1).detach().cpu().numpy()\n        test_labels = labels.detach().cpu().numpy().argmax(axis = 1)\n            \n        for i in range(test_preds.shape[0]):\n            confusion_matrix[test_preds[i], test_labels[i]] += 1\n\n# Save \n#torch.save(model.state_dict(), 'model.ckpt')","metadata":{"execution":{"iopub.status.busy":"2021-09-19T00:06:19.339752Z","iopub.execute_input":"2021-09-19T00:06:19.341145Z","iopub.status.idle":"2021-09-19T00:08:58.437323Z","shell.execute_reply.started":"2021-09-19T00:06:19.341093Z","shell.execute_reply":"2021-09-19T00:08:58.435956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_matrix","metadata":{"execution":{"iopub.status.busy":"2021-09-19T00:08:58.440658Z","iopub.execute_input":"2021-09-19T00:08:58.440993Z","iopub.status.idle":"2021-09-19T00:08:58.451283Z","shell.execute_reply.started":"2021-09-19T00:08:58.440943Z","shell.execute_reply":"2021-09-19T00:08:58.449897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sn\n\nlabels = [\"Negative\\nfor\\nPneumonia\", \"Typical\\nAppearance\", \"Indeterminate\\nAppearance\", \"Atypical\\nAppearance\"]\n\ndf_cm = pd.DataFrame(confusion_matrix, labels, labels)\nplt.figure(figsize=(10,7))\nsn.set(font_scale=1.4) # for label size\nax = sn.heatmap(df_cm, annot=True, annot_kws={\"size\": 16}) # font size\n\nax.set_xlabel(\"True Labels\")\nax.set_ylabel(\"Predicted Labels\")\nax.xaxis.set_label_position('top')\nplt.tick_params(axis='both', which='major', labelsize=10, labelbottom = False, bottom=False, top = False, labeltop=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-19T00:08:58.453334Z","iopub.execute_input":"2021-09-19T00:08:58.453792Z","iopub.status.idle":"2021-09-19T00:08:59.173523Z","shell.execute_reply.started":"2021-09-19T00:08:58.453751Z","shell.execute_reply":"2021-09-19T00:08:59.172612Z"},"trusted":true},"execution_count":null,"outputs":[]}]}