{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Import Libraries**","metadata":{}},{"cell_type":"code","source":"! conda install -c conda-forge gdcm -y;","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:06.637633Z","iopub.execute_input":"2021-09-20T09:46:06.637905Z","iopub.status.idle":"2021-09-20T09:46:35.848645Z","shell.execute_reply.started":"2021-09-20T09:46:06.637877Z","shell.execute_reply":"2021-09-20T09:46:35.847833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"../input/timmeffnetv2\")\n\nimport platform\nimport numpy as np\nimport pandas as pd\nimport os\nfrom tqdm.notebook import tqdm\nimport cv2\nimport pydicom\nimport glob\nimport gc\nfrom math import ceil\nimport matplotlib.pyplot as plt\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\n\nimport albumentations as A\nimport seaborn as sn\n\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-20T09:46:35.851913Z","iopub.execute_input":"2021-09-20T09:46:35.852193Z","iopub.status.idle":"2021-09-20T09:46:35.862132Z","shell.execute_reply.started":"2021-09-20T09:46:35.852155Z","shell.execute_reply":"2021-09-20T09:46:35.861321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Load Metadata**","metadata":{}},{"cell_type":"code","source":"train_image = pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")\ntrain_study = pd.read_csv(\"../input/siim-covid19-detection/train_study_level.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:35.864987Z","iopub.execute_input":"2021-09-20T09:46:35.865277Z","iopub.status.idle":"2021-09-20T09:46:35.904174Z","shell.execute_reply.started":"2021-09-20T09:46:35.865252Z","shell.execute_reply":"2021-09-20T09:46:35.903539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DIR = \"../input/siim-covid19-detection/train/\"\ntrain_study['StudyInstanceUID'] = train_study['id'].apply(lambda x: x.replace('_study', ''))\ntrain = train_image.merge(train_study, on='StudyInstanceUID')\n\n# Make a path folder\npaths = []\nfor instance_id in tqdm(train['StudyInstanceUID']):\n    paths.append(glob.glob(os.path.join(TRAIN_DIR, instance_id +\"/*/*\"))[0])\n\ntrain['path'] = paths\n\ntrain = train.drop(['id_x', 'id_y'], axis=1)\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:35.905824Z","iopub.execute_input":"2021-09-20T09:46:35.906014Z","iopub.status.idle":"2021-09-20T09:46:40.614414Z","shell.execute_reply.started":"2021-09-20T09:46:35.905992Z","shell.execute_reply":"2021-09-20T09:46:40.613796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Exploratory Analysis**","metadata":{}},{"cell_type":"code","source":"one_hot_encode_target = train.iloc[:, 3:7]\nlabels = one_hot_encode_target.columns\ncounts_classes = one_hot_encode_target.sum(axis = 0)\nperc_counts = 100 * counts_classes / counts_classes.sum()","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:40.616992Z","iopub.execute_input":"2021-09-20T09:46:40.617202Z","iopub.status.idle":"2021-09-20T09:46:40.625699Z","shell.execute_reply.started":"2021-09-20T09:46:40.617177Z","shell.execute_reply":"2021-09-20T09:46:40.625034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (16, 10))\n\n#define Seaborn color palette to use\ncolors = sn.color_palette('pastel')[0:5]\n\n#create pie chart\n_ = plt.pie(perc_counts, labels = labels, colors = colors, autopct='%.0f%%')","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:40.626860Z","iopub.execute_input":"2021-09-20T09:46:40.627130Z","iopub.status.idle":"2021-09-20T09:46:40.756058Z","shell.execute_reply.started":"2021-09-20T09:46:40.627097Z","shell.execute_reply":"2021-09-20T09:46:40.755411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Load DIACOM Dataset**","metadata":{}},{"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:40.757041Z","iopub.execute_input":"2021-09-20T09:46:40.757301Z","iopub.status.idle":"2021-09-20T09:46:40.765880Z","shell.execute_reply.started":"2021-09-20T09:46:40.757265Z","shell.execute_reply":"2021-09-20T09:46:40.764731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    image_size = (224, 224)\n    train_bs = 32\n    valid_bs = 16\n    num_workers = 8\n    num_finetuning_epochs = 5\n    num_total_epochs = 15\n#     num_finetuning_epochs = 3\n#     num_total_epochs = 5\n    scaler = GradScaler()","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:40.767025Z","iopub.execute_input":"2021-09-20T09:46:40.768115Z","iopub.status.idle":"2021-09-20T09:46:40.774804Z","shell.execute_reply.started":"2021-09-20T09:46:40.768078Z","shell.execute_reply":"2021-09-20T09:46:40.774049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,5,figsize = (25,10))\n\nfor i, path in enumerate(train['path'][0:5]):\n    image = dicom2array(path)\n    image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)\n    image = cv2.resize(image, (400, 400))\n    ax[i].imshow(image)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:40.775918Z","iopub.execute_input":"2021-09-20T09:46:40.776196Z","iopub.status.idle":"2021-09-20T09:46:42.240401Z","shell.execute_reply.started":"2021-09-20T09:46:40.776160Z","shell.execute_reply":"2021-09-20T09:46:42.239727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SIIMData(Dataset):\n    def __init__(self, df, is_train = True, augments = None, img_size = Config.image_size):\n        super().__init__()\n        self.df = df\n        self.is_train = is_train\n        self.augments = augments\n        self.img_size = img_size\n        \n    def __getitem__(self, idx):\n        image_id = self.df['StudyInstanceUID'].values[idx] \n        image_path = self.df['path'].values[idx]\n        image = dicom2array(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)\n        image = cv2.resize(image, Config.image_size)\n        \n        # Augments must be albumentations\n        if self.augments:\n            image = self.augments(image=image)['image']\n        else:\n            image = torch.tensor(image, dtype=torch.float)\n        \n        if self.is_train:\n            label = self.df[self.df['StudyInstanceUID'] == image_id].values.tolist()[0][8]\n            return image, torch.tensor(label, dtype = torch.long)\n        \n        return image\n    \n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:42.243296Z","iopub.execute_input":"2021-09-20T09:46:42.243712Z","iopub.status.idle":"2021-09-20T09:46:42.253439Z","shell.execute_reply.started":"2021-09-20T09:46:42.243670Z","shell.execute_reply":"2021-09-20T09:46:42.252681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = A.Compose({\n        A.HorizontalFlip(p=0.5),\n        A.Rotate(limit=(-40, 40)),\n        A.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n        })\n\nvalid_test_transform = A.Compose({\n        A.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n        })","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:42.254738Z","iopub.execute_input":"2021-09-20T09:46:42.255010Z","iopub.status.idle":"2021-09-20T09:46:42.267291Z","shell.execute_reply.started":"2021-09-20T09:46:42.254977Z","shell.execute_reply":"2021-09-20T09:46:42.266552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Splitting the Dataset** ","metadata":{}},{"cell_type":"code","source":"#train = train[:200]\n#np.unique(train[\"class\"].to_numpy(), return_counts = True)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:42.269276Z","iopub.execute_input":"2021-09-20T09:46:42.269905Z","iopub.status.idle":"2021-09-20T09:46:42.274977Z","shell.execute_reply.started":"2021-09-20T09:46:42.269867Z","shell.execute_reply":"2021-09-20T09:46:42.274302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"class\"] = train.iloc[:, 3:7].to_numpy().argmax(axis = 1)\n\ntrain, valid = train_test_split(train, stratify = train[\"class\"], shuffle = True, test_size=0.3)\n\nvalid, test = train_test_split(valid, stratify = valid[\"class\"], shuffle = True, test_size=0.5)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:42.276267Z","iopub.execute_input":"2021-09-20T09:46:42.276576Z","iopub.status.idle":"2021-09-20T09:46:42.295987Z","shell.execute_reply.started":"2021-09-20T09:46:42.276542Z","shell.execute_reply":"2021-09-20T09:46:42.295244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = SIIMData(train, augments = train_transform)\nvalid_data = SIIMData(valid, augments = valid_test_transform)\ntest_data = SIIMData(test, augments = valid_test_transform)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:42.298812Z","iopub.execute_input":"2021-09-20T09:46:42.299012Z","iopub.status.idle":"2021-09-20T09:46:42.304264Z","shell.execute_reply.started":"2021-09-20T09:46:42.298989Z","shell.execute_reply":"2021-09-20T09:46:42.303371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, class_sample_count = np.unique(train[\"class\"].to_numpy(), return_counts = True)\n#Balance the Training Dataset\n\nweight = 1. / class_sample_count\nsamples_weight = np.array([weight[t] for t in train[\"class\"]])\n\nsamples_weight = torch.from_numpy(samples_weight)\nsamples_weight = samples_weight.double()\nsampler = WeightedRandomSampler(samples_weight, len(samples_weight), replacement = True)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:42.305558Z","iopub.execute_input":"2021-09-20T09:46:42.306092Z","iopub.status.idle":"2021-09-20T09:46:42.316872Z","shell.execute_reply.started":"2021-09-20T09:46:42.306053Z","shell.execute_reply":"2021-09-20T09:46:42.316024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(dataset = train_data, batch_size = Config.train_bs, num_workers = Config.num_workers, sampler = sampler)\nvalid_loader = DataLoader(dataset = valid_data, batch_size = Config.valid_bs, num_workers = Config.num_workers)\ntest_loader = DataLoader(dataset = test_data, batch_size = Config.valid_bs, num_workers = Config.num_workers)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:42.318247Z","iopub.execute_input":"2021-09-20T09:46:42.318630Z","iopub.status.idle":"2021-09-20T09:46:42.328944Z","shell.execute_reply.started":"2021-09-20T09:46:42.318591Z","shell.execute_reply":"2021-09-20T09:46:42.328082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **DenseNet201**","metadata":{}},{"cell_type":"code","source":"# There is a bug in pytorch when downloading .pt through wget :(\n# ! wget -c https://github.com/PedroRASB/COVID-19-Twice-Transfer-DNNs/blob/master/NetC.pt","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:42.330139Z","iopub.execute_input":"2021-09-20T09:46:42.330631Z","iopub.status.idle":"2021-09-20T09:46:42.337098Z","shell.execute_reply.started":"2021-09-20T09:46:42.330592Z","shell.execute_reply":"2021-09-20T09:46:42.336319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DenseNet201(nn.Module):\n\n    def __init__(self, num_classes = 4):\n        super(DenseNet201, self).__init__()\n        self.model = torch.load(\"../input/covidnet/NetC.pt\", map_location=torch.device('cuda:0'))\n        \n        #Freezing the loaded model\n        for param in self.model.parameters():\n            param.requires_grad = False\n            \n        #The output layer is unfreeze\n        self.model.classifier = nn.Linear(self.model.classifier.in_features, num_classes)\n        \n    def unfreeze_all_layers(self):\n        for param in self.model.parameters():\n            param.requires_grad = True\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:42.338578Z","iopub.execute_input":"2021-09-20T09:46:42.338877Z","iopub.status.idle":"2021-09-20T09:46:42.347507Z","shell.execute_reply.started":"2021-09-20T09:46:42.338840Z","shell.execute_reply":"2021-09-20T09:46:42.346530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Training the model**","metadata":{}},{"cell_type":"code","source":"class Trainer:\n    def __init__(self, model, train_loader, valid_loader, optimizer, criterion, device):\n        \"\"\"\n        Constructor for Trainer class\n        \"\"\"\n        self.model = model\n        self.train = train_loader\n        self.valid = valid_loader\n        self.optim = optimizer\n        self.criterion = criterion\n        self.device = device\n        self.scaler = GradScaler()\n        \n    def multi_acc(self, y_pred, y_test):\n        y_pred_softmax = torch.log_softmax(y_pred, dim = 1)\n        _, y_pred_tags = torch.max(y_pred_softmax, dim = 1)    \n\n        correct_pred = (y_pred_tags == y_test).float()\n        return correct_pred.sum() \n    \n    def train_one_epoch(self):\n        \"\"\"\n        Runs one epoch of training, backpropagation and optimization\n        \"\"\"\n        self.model.train()\n        \n        running_loss = 0\n        running_acc = 0\n        \n        for xtrain, ytrain in self.train:\n            xtrain = xtrain.to(self.device).float()\n            ytrain = ytrain.to(self.device).long()\n            xtrain = xtrain.permute(0, 3, 1, 2)\n            \n            with autocast():\n                # Get predictions\n                z = self.model(xtrain)\n\n                # Training\n                train_loss = self.criterion(z, ytrain)\n                self.scaler.scale(train_loss).backward()\n                \n                self.scaler.step(self.optim)\n                self.scaler.update()\n                self.optim.zero_grad(set_to_none=True)\n\n                # For averaging and reporting later\n                running_loss += train_loss.item()\n                running_acc += self.multi_acc(z, ytrain).item()             \n        \n        # Now average the running loss over all batches and return\n        running_loss /= len(self.train)\n        running_acc  /= train.shape[0]\n        print(f\"Training Loss: {running_loss:.4f}\")\n        print(f\"Training Accuracy: {running_acc:.4f}\\n\")\n        \n        # Free up memory\n        del train_loss, xtrain, ytrain, z\n        gc.collect()\n        torch.cuda.empty_cache()\n        \n        return (running_loss, running_acc)\n\n    def valid_one_epoch(self):\n        \"\"\"\n        Runs one epoch of prediction\n        \"\"\"        \n        model.eval()\n                \n        with torch.no_grad():\n            \n            running_loss = 0\n            running_acc = 0\n            \n            for xval, yval in self.valid:\n                xval = xval.to(self.device).float()\n                yval = yval.to(self.device).long()\n                xval = xval.permute(0, 3, 1, 2)\n                \n                val_z = self.model(xval)\n                \n                val_loss = self.criterion(val_z, yval)\n                \n                running_loss += val_loss.item()                \n                running_acc += self.multi_acc(val_z, yval).item()\n            \n            # Get the final loss\n            running_loss /= len(self.valid)\n            running_acc  /= valid.shape[0]\n            \n            print(f\"Validation Loss: {running_loss:.4f}\")\n            print(f\"Validation Accuracy: {running_acc:.4f}\")\n            \n            # Free up memory\n            del val_loss, xval, yval, val_z\n            gc.collect()\n            torch.cuda.empty_cache()\n            \n        return (running_loss, running_acc)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:42.349119Z","iopub.execute_input":"2021-09-20T09:46:42.349575Z","iopub.status.idle":"2021-09-20T09:46:42.369297Z","shell.execute_reply.started":"2021-09-20T09:46:42.349537Z","shell.execute_reply":"2021-09-20T09:46:42.368601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Code\nprint(f\"[INFO] Training on {train.shape[0]} samples and validation on {valid.shape[0]} samples\")\n\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n\nmodel = DenseNet201().to(device)\n    \n# optim = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.001)\n\noptim = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.01)\n\ncriterion = nn.CrossEntropyLoss()\n\ntrainer = Trainer(\n    model=model,\n    train_loader=train_loader,\n    valid_loader=valid_loader,\n    optimizer=optim,\n    criterion=criterion,\n    device=device,\n)\n\ntrain_losses = []\nvalid_losses = []\n\ntrain_acc = []\nvalid_acc = []\n\nbest_acc = 0\n\nfor epoch in range(Config.num_finetuning_epochs):\n    print(f\"{'-'*25} EPOCH: {epoch+1}/{Config.num_total_epochs} {'-'*25}\")\n\n    # Run one training epoch\n    current_train_loss, current_train_acc = trainer.train_one_epoch()\n    train_losses.append(current_train_loss)\n    train_acc.append(current_train_acc)\n\n    # Run one validation epoch\n    current_val_loss, current_val_acc = trainer.valid_one_epoch()\n    valid_losses.append(current_val_loss)\n    valid_acc.append(current_val_acc)\n    \n    if(current_val_acc > best_acc):\n        best_acc = current_val_acc\n        torch.save(trainer.model, 'best-model.pt')\n\n    # Empty CUDA cache\n    torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:46:42.370993Z","iopub.execute_input":"2021-09-20T09:46:42.371194Z","iopub.status.idle":"2021-09-20T10:43:13.218614Z","shell.execute_reply.started":"2021-09-20T09:46:42.371170Z","shell.execute_reply":"2021-09-20T10:43:13.216561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Evaluate the model**","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nfig, ax = plt.subplots(1,2,figsize = (25,10))\n\nax[0].plot(train_losses, label='Training loss')\nax[0].plot(valid_losses, label='Validation loss')\nax[0].set_xlabel(\"Epochs\")\nax[0].set_ylabel(\"Loss\")\nax[0].legend(frameon=False)\n\nax[1].plot(train_acc, label='Training accuracy')\nax[1].plot(valid_acc, label='Validation accuracy')\nax[1].set_xlabel(\"Epochs\")\nax[1].set_ylabel(\"Accuracy\")\nax[1].legend(frameon=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T10:43:13.227570Z","iopub.execute_input":"2021-09-20T10:43:13.229509Z","iopub.status.idle":"2021-09-20T10:43:14.252557Z","shell.execute_reply.started":"2021-09-20T10:43:13.229455Z","shell.execute_reply":"2021-09-20T10:43:14.251846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Finetuning**","metadata":{}},{"cell_type":"code","source":"# Training Code -- Finetuning\nprint(\"\\nFinetuning\\n\")\n\ntrainer.model.unfreeze_all_layers()\ntrainer.optimizer = torch.optim.SGD(trainer.model.parameters(), lr=0.0001, momentum=0.9, weight_decay=0.01)\n\nfor epoch in range(Config.num_finetuning_epochs, Config.num_total_epochs):\n    print(f\"{'-'*25} EPOCH: {epoch+1}/{Config.num_total_epochs} {'-'*25}\")\n\n    # Run one training epoch\n    current_train_loss, current_train_acc = trainer.train_one_epoch()\n    train_losses.append(current_train_loss)\n    train_acc.append(current_train_acc)\n\n    # Run one validation epoch\n    current_val_loss, current_val_acc = trainer.valid_one_epoch()\n    valid_losses.append(current_val_loss)\n    valid_acc.append(current_val_acc)\n    \n    if(current_val_acc > best_acc):\n        best_acc = current_val_acc\n        torch.save(trainer.model, 'best-model.pt')\n\n    # Empty CUDA cache\n    torch.cuda.empty_cache()\n\ntorch.save(trainer.model, 'last-model.pt')","metadata":{"execution":{"iopub.status.busy":"2021-09-20T10:43:14.253928Z","iopub.execute_input":"2021-09-20T10:43:14.254194Z","iopub.status.idle":"2021-09-20T12:36:56.292699Z","shell.execute_reply.started":"2021-09-20T10:43:14.254159Z","shell.execute_reply":"2021-09-20T12:36:56.290840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Evaluate the model**","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nfig, ax = plt.subplots(1,2,figsize = (25,10))\n\nax[0].plot(train_losses, label='Training loss')\nax[0].plot(valid_losses, label='Validation loss')\nax[0].set_xlabel(\"Epochs\")\nax[0].set_ylabel(\"Loss\")\nax[0].legend(frameon=False)\n\nax[1].plot(train_acc, label='Training accuracy')\nax[1].plot(valid_acc, label='Validation accuracy')\nax[1].set_xlabel(\"Epochs\")\nax[1].set_ylabel(\"Accuracy\")\nax[1].legend(frameon=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T12:36:56.296124Z","iopub.execute_input":"2021-09-20T12:36:56.296684Z","iopub.status.idle":"2021-09-20T12:36:57.030967Z","shell.execute_reply.started":"2021-09-20T12:36:56.296643Z","shell.execute_reply":"2021-09-20T12:36:57.030297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test-the-model\ntrainer.model.eval()  # it-disables-dropout\n\nconfusion_matrix = np.zeros((4,4))\n\nground_truth_test = []\npredicted_test = []\n\nwith torch.no_grad():\n\n    for images, labels in test_loader:\n        \n        images = images.to(device).float()\n        test_labels = labels.cpu().long().numpy()\n        images = images.permute(0, 3, 1, 2)\n        \n        outputs = trainer.model(images)\n        \n        y_pred_softmax = torch.log_softmax(outputs, dim = 1)\n        _, y_pred_tags = torch.max(y_pred_softmax, dim = 1)\n        test_preds = y_pred_tags.cpu().numpy()\n                                                    \n        for i in range(test_preds.shape[0]):\n            confusion_matrix[test_preds[i], test_labels[i]] += 1\n            ground_truth_test.append(test_labels[i])\n            predicted_test.append(test_preds[i])\n\n# Save \n#torch.save(model.state_dict(), 'model.ckpt')","metadata":{"execution":{"iopub.status.busy":"2021-09-20T13:01:56.290523Z","iopub.execute_input":"2021-09-20T13:01:56.291035Z","iopub.status.idle":"2021-09-20T13:04:07.786118Z","shell.execute_reply.started":"2021-09-20T13:01:56.290995Z","shell.execute_reply":"2021-09-20T13:04:07.784934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_matrix","metadata":{"execution":{"iopub.status.busy":"2021-09-20T12:39:10.770788Z","iopub.execute_input":"2021-09-20T12:39:10.771071Z","iopub.status.idle":"2021-09-20T12:39:10.780521Z","shell.execute_reply.started":"2021-09-20T12:39:10.771035Z","shell.execute_reply":"2021-09-20T12:39:10.779679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sn\n\nlabels = [\"Negative\\nfor\\nPneumonia\", \"Typical\\nAppearance\", \"Indeterminate\\nAppearance\", \"Atypical\\nAppearance\"]\n\ndf_cm = pd.DataFrame(confusion_matrix, labels, labels)\nplt.figure(figsize=(10,7))\nsn.set(font_scale=1.4) # for label size\nax = sn.heatmap(df_cm, annot=True, annot_kws={\"size\": 16}) # font size\n\nax.set_xlabel(\"True Labels\")\nax.set_ylabel(\"Predicted Labels\")\nax.xaxis.set_label_position('top')\nplt.tick_params(axis='both', which='major', labelsize=10, labelbottom = False, bottom=False, top = False, labeltop=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T12:39:10.782402Z","iopub.execute_input":"2021-09-20T12:39:10.782683Z","iopub.status.idle":"2021-09-20T12:39:11.344551Z","shell.execute_reply.started":"2021-09-20T12:39:10.782650Z","shell.execute_reply":"2021-09-20T12:39:11.343833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\n\ntarget_names = [\"Negative for Pneumonia\", \"Typical Appearance\", \"Indeterminate Appearance\", \"Atypical Appearance\"]\nprint(classification_report(ground_truth_test, predicted_test, target_names=target_names))","metadata":{"execution":{"iopub.status.busy":"2021-09-20T13:04:16.595400Z","iopub.execute_input":"2021-09-20T13:04:16.596058Z","iopub.status.idle":"2021-09-20T13:04:16.611086Z","shell.execute_reply.started":"2021-09-20T13:04:16.596017Z","shell.execute_reply":"2021-09-20T13:04:16.610193Z"},"trusted":true},"execution_count":null,"outputs":[]}]}