{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:43:22.583285Z","iopub.execute_input":"2022-05-04T11:43:22.583563Z","iopub.status.idle":"2022-05-04T11:43:34.625710Z","shell.execute_reply.started":"2022-05-04T11:43:22.583534Z","shell.execute_reply":"2022-05-04T11:43:34.625167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import related libraries","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nimport pydicom\nimport numpy as np\nimport cv2\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:43:34.627042Z","iopub.execute_input":"2022-05-04T11:43:34.627334Z","iopub.status.idle":"2022-05-04T11:43:34.634731Z","shell.execute_reply.started":"2022-05-04T11:43:34.627299Z","shell.execute_reply":"2022-05-04T11:43:34.634179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels_path = \"../input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:43:34.635694Z","iopub.execute_input":"2022-05-04T11:43:34.636058Z","iopub.status.idle":"2022-05-04T11:43:34.641326Z","shell.execute_reply.started":"2022-05-04T11:43:34.636024Z","shell.execute_reply":"2022-05-04T11:43:34.640758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = pd.read_csv(train_labels_path)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:43:34.643162Z","iopub.execute_input":"2022-05-04T11:43:34.643474Z","iopub.status.idle":"2022-05-04T11:43:34.746739Z","shell.execute_reply.started":"2022-05-04T11:43:34.643442Z","shell.execute_reply":"2022-05-04T11:43:34.746225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:43:34.748626Z","iopub.execute_input":"2022-05-04T11:43:34.748955Z","iopub.status.idle":"2022-05-04T11:43:34.773891Z","shell.execute_reply.started":"2022-05-04T11:43:34.748920Z","shell.execute_reply":"2022-05-04T11:43:34.772868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:43:34.777781Z","iopub.execute_input":"2022-05-04T11:43:34.778171Z","iopub.status.idle":"2022-05-04T11:43:34.787526Z","shell.execute_reply.started":"2022-05-04T11:43:34.778134Z","shell.execute_reply":"2022-05-04T11:43:34.786763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.drop_duplicates(\"patientId\")","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:43:34.788897Z","iopub.execute_input":"2022-05-04T11:43:34.789356Z","iopub.status.idle":"2022-05-04T11:43:34.832419Z","shell.execute_reply.started":"2022-05-04T11:43:34.789262Z","shell.execute_reply":"2022-05-04T11:43:34.831790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = labels.drop_duplicates(\"patientId\")\nlabels.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:43:34.833500Z","iopub.execute_input":"2022-05-04T11:43:34.833860Z","iopub.status.idle":"2022-05-04T11:43:34.853104Z","shell.execute_reply.started":"2022-05-04T11:43:34.833816Z","shell.execute_reply":"2022-05-04T11:43:34.851649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_PATH = Path(\"../input/rsna-pneumonia-detection-challenge/stage_2_train_images\")\nSAVE_PATH = Path(\"/root/Processed\")","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:43:34.854215Z","iopub.execute_input":"2022-05-04T11:43:34.854565Z","iopub.status.idle":"2022-05-04T11:43:34.858683Z","shell.execute_reply.started":"2022-05-04T11:43:34.854534Z","shell.execute_reply":"2022-05-04T11:43:34.857805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 3, figsize=(9, 9))\nc = 0\n\nfor i in range(3):\n    for j in range(3):\n        patient_id = labels.patientId.iloc[c]\n        dcm_path = ROOT_PATH/patient_id\n        dcm_path = dcm_path.with_suffix(\".dcm\")\n        dcm = pydicom.read_file(dcm_path).pixel_array\n        \n        label = labels[\"Target\"].iloc[c]\n        \n        axis[i][j].imshow(dcm, cmap=\"bone\")\n        axis[i][j].set_title(label)\n        c+=1","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:43:34.862728Z","iopub.execute_input":"2022-05-04T11:43:34.863267Z","iopub.status.idle":"2022-05-04T11:43:37.170325Z","shell.execute_reply.started":"2022-05-04T11:43:34.863232Z","shell.execute_reply":"2022-05-04T11:43:37.169706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dicom Reading & Effective storage\n\nX-ray --> DICOM format \n\nNormalizasyon içi her bir görüntüdeki piksellerin ortalam ve standart sapması hesaplandı\n\nArdından oluşturulan görüntüler ikili etiketlerine göre iki ayrı klasörde saklanır:\n* 0:  zatürre belirtisi gösteren X-Ray görüntüleri\n* 1:  zatürre belirtisi göstermeyen X-Ray görüntüleri","metadata":{}},{"cell_type":"code","source":"sums, sums_squared = 0, 0\n\nfor c, patient_id in enumerate(tqdm(labels.patientId)):\n    patient_id = labels.patientId.iloc[c]\n    dcm_path = ROOT_PATH/patient_id\n    dcm_path = dcm_path.with_suffix(\".dcm\")\n    \n    # Read the dicom file with pydicom and standardize the array\n    dcm = pydicom.read_file(dcm_path).pixel_array / 255\n    \n    # Let's use a shape of 224x224\n    # In order to use less space when storing the image we convert it to float16\n    dcm_array = cv2.resize(dcm, (224,224)).astype(np.float16)\n    \n    # Retrieve the corresponding label\n    label = labels.Target.iloc[c]\n    \n    # 4/5 train split, 1/5 val split\n    train_or_val = \"train\" if c < 24000 else \"val\"\n    \n    current_save_path = SAVE_PATH/train_or_val/str(label)  # Define save path and create if necessary\n    current_save_path.mkdir(parents=True, exist_ok=True)\n    np.save(current_save_path/patient_id, dcm_array) # Save the array in the corresponding directory\n    \n    ### updating the sums and sums_squared just for training data\n    normalizer = 224*224  # Normalize sum of image\n    if train_or_val == \"train\":    # Only use train data to compute dataset statistics\n        sums += np.sum(dcm_array) / normalizer\n        sums_squared += (dcm_array**2).sum() / normalizer","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:43:37.171721Z","iopub.execute_input":"2022-05-04T11:43:37.172189Z","iopub.status.idle":"2022-05-04T11:49:41.492311Z","shell.execute_reply.started":"2022-05-04T11:43:37.172154Z","shell.execute_reply":"2022-05-04T11:49:41.491594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean = sums / 24000\nstd = np.sqrt((sums_squared / 24000) - (mean**2))","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:41.493567Z","iopub.execute_input":"2022-05-04T11:49:41.493964Z","iopub.status.idle":"2022-05-04T11:49:41.498329Z","shell.execute_reply.started":"2022-05-04T11:49:41.493927Z","shell.execute_reply":"2022-05-04T11:49:41.497698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we will use mean and std later to normalize the dataset \nmean, std","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:41.501156Z","iopub.execute_input":"2022-05-04T11:49:41.501596Z","iopub.status.idle":"2022-05-04T11:49:41.628730Z","shell.execute_reply.started":"2022-05-04T11:49:41.501556Z","shell.execute_reply":"2022-05-04T11:49:41.627960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training","metadata":{}},{"cell_type":"code","source":"import torch\nimport torchvision\nfrom torchvision import transforms\nimport torchmetrics\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom pytorch_lightning.loggers import TensorBoardLogger\nfrom tqdm.notebook import tqdm\nimport numpy as np\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:41.630288Z","iopub.execute_input":"2022-05-04T11:49:41.630650Z","iopub.status.idle":"2022-05-04T11:49:41.638339Z","shell.execute_reply.started":"2022-05-04T11:49:41.630615Z","shell.execute_reply":"2022-05-04T11:49:41.637653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_file(path):\n    return np.load(path).astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:41.639146Z","iopub.execute_input":"2022-05-04T11:49:41.639379Z","iopub.status.idle":"2022-05-04T11:49:41.647448Z","shell.execute_reply.started":"2022-05-04T11:49:41.639306Z","shell.execute_reply":"2022-05-04T11:49:41.646622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tranforms = transforms.Compose([\n    transforms.ToTensor(),                     # Convert numpy array to tensor\n    transforms.Normalize(0.49, 0.248),         # Use calculated mean and std before\n    transforms.RandomAffine(degrees=(-5,5), translate=(0,0.05), scale=(0.9,1.1)),   # Data Augmentation\n    transforms.RandomResizedCrop((224,224), scale=(0.35,1))    #RandomResizedCrops which applies a random crop of the \n])                                                            #image and resizes it to the original image size (224x224)\n\n### validation data is not augmented\nval_transform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(0.49,0.248)\n])","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:41.648605Z","iopub.execute_input":"2022-05-04T11:49:41.648785Z","iopub.status.idle":"2022-05-04T11:49:41.658671Z","shell.execute_reply.started":"2022-05-04T11:49:41.648763Z","shell.execute_reply":"2022-05-04T11:49:41.657940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = torchvision.datasets.DatasetFolder(\"/root/Processed/train\", loader=load_file,\n                                                extensions=\"npy\", transform=train_tranforms)\nval_data = torchvision.datasets.DatasetFolder(\"/root/Processed/val\", loader=load_file,\n                                             extensions=\"npy\", transform=val_transform)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:41.659898Z","iopub.execute_input":"2022-05-04T11:49:41.660145Z","iopub.status.idle":"2022-05-04T11:49:41.913994Z","shell.execute_reply.started":"2022-05-04T11:49:41.660112Z","shell.execute_reply":"2022-05-04T11:49:41.913281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's see some augmented train images\nfig, axis = plt.subplots(2,2, figsize=(9,9))\n\nfor i in range(2):\n    for j in range(2):\n        random_index = np.random.randint(0,24000)\n        x_ray, label = train_data[random_index]\n        axis[i][j].imshow(x_ray[0], cmap=\"bone\")\n        axis[i][j].set_title(label)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:41.915271Z","iopub.execute_input":"2022-05-04T11:49:41.915604Z","iopub.status.idle":"2022-05-04T11:49:42.581216Z","shell.execute_reply.started":"2022-05-04T11:49:41.915568Z","shell.execute_reply":"2022-05-04T11:49:42.579435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 128\nnum_workers = 2\n\ntrain_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, num_workers=num_workers, shuffle=True)\n# shuffle=True --> Imagine a case that at first fit 1000 X-ray images that do not show signs of pneumonia to model  or vice versa\n\nval_loader = torch.utils.data.DataLoader(val_data, batch_size=batch_size, num_workers=num_workers, shuffle=False)\n# shuffle=False --> don't want to shuffle data in order to get a deterministic evaluation.","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:42.582625Z","iopub.execute_input":"2022-05-04T11:49:42.583061Z","iopub.status.idle":"2022-05-04T11:49:42.588764Z","shell.execute_reply.started":"2022-05-04T11:49:42.583027Z","shell.execute_reply":"2022-05-04T11:49:42.588133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"There are {len(train_data)} train images and {len(val_data)} val images\")","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:42.590272Z","iopub.execute_input":"2022-05-04T11:49:42.590730Z","iopub.status.idle":"2022-05-04T11:49:42.600501Z","shell.execute_reply.started":"2022-05-04T11:49:42.590695Z","shell.execute_reply":"2022-05-04T11:49:42.599778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distribution of the  train labels\nnp.unique(train_data.targets, return_counts=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:42.601749Z","iopub.execute_input":"2022-05-04T11:49:42.602259Z","iopub.status.idle":"2022-05-04T11:49:42.614565Z","shell.execute_reply.started":"2022-05-04T11:49:42.602201Z","shell.execute_reply":"2022-05-04T11:49:42.613739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distribution of the  validation labels\nnp.unique(val_data.targets, return_counts=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:42.615953Z","iopub.execute_input":"2022-05-04T11:49:42.616335Z","iopub.status.idle":"2022-05-04T11:49:42.625764Z","shell.execute_reply.started":"2022-05-04T11:49:42.616298Z","shell.execute_reply":"2022-05-04T11:49:42.624878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torchvision.models.resnet18()","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:42.627067Z","iopub.execute_input":"2022-05-04T11:49:42.627461Z","iopub.status.idle":"2022-05-04T11:49:42.782115Z","shell.execute_reply.started":"2022-05-04T11:49:42.627370Z","shell.execute_reply":"2022-05-04T11:49:42.781272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PneumoniaModel(pl.LightningModule):\n    \n    def __init__(self):\n        \n        super().__init__()\n        \n        self.model = torchvision.models.resnet18()  # don't need to create a model from scratch\n        \n        # change conv1 from 3 to 1 input channels\n        self.model.conv1 = torch.nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n        \n        # change out_feature of the last fully connected layer (called fc in resnet18) from 1000 to 1\n        self.model.fc = torch.nn.Linear(in_features=512, out_features=1, bias=True)\n        \n        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=1e-4)\n        self.loss_fn = torch.nn.BCEWithLogitsLoss(pos_weight=torch.tensor([3]))\n        \n        # simple accuracy computation\n        self.train_acc = torchmetrics.Accuracy()\n        self.val_acc = torchmetrics.Accuracy()\n    \n    # prediction \n    def forward(self, data):\n        pred = self.model(data)\n        return pred\n    \n    def training_step(self, batch, batch_idx):\n        x_ray, label = batch\n        label = label.float()   # Convert label to float (just needed for loss computation)\n        pred = self(x_ray)[:,0]   # Prediction: Make sure prediction and label have same shape\n        loss = self.loss_fn(pred, label)   # Compute the loss\n        \n        # Log loss and batch accuracy\n        self.log(\"Train Loss\", loss)\n        self.log(\"Step Train ACC\", self.train_acc(torch.sigmoid(pred), label.int()))\n        \n        return loss\n    \n    def training_epoch_end(self, outs):\n        # After one epoch compute the whole train_data accuracy\n        self.log(\"Train ACC\", self.train_acc.compute())\n        \n    def validation_step(self, batch, batch_idx):\n        # Same steps as in the training_step\n        x_ray, label = batch\n        label = label.float()   # Convert label to float (just needed for loss computation)\n        pred = self(x_ray)[:,0]   # Prediction: Make sure prediction and label have same shape\n        loss = self.loss_fn(pred, label)   # Compute the loss\n        \n        # Log validation metrics\n        self.log(\"Validation Loss\", loss)\n        self.log(\"Step Validation ACC\", self.val_acc(torch.sigmoid(pred), label.int()))\n    \n    def validation_epoch_end(self, outs):\n        self.log(\"Validation ACC\", self.val_acc.compute())\n        \n    def configure_optimizers(self):\n        \n        return [self.optimizer]","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:42.783645Z","iopub.execute_input":"2022-05-04T11:49:42.783918Z","iopub.status.idle":"2022-05-04T11:49:42.797174Z","shell.execute_reply.started":"2022-05-04T11:49:42.783884Z","shell.execute_reply":"2022-05-04T11:49:42.796426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = PneumoniaModel()","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:42.798516Z","iopub.execute_input":"2022-05-04T11:49:42.798957Z","iopub.status.idle":"2022-05-04T11:49:42.957843Z","shell.execute_reply.started":"2022-05-04T11:49:42.798923Z","shell.execute_reply":"2022-05-04T11:49:42.957050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the checkpoint callback (validation accuracy en yüksek olan 10 modeli kaydediyorum.)\ncheckpoint_callback = ModelCheckpoint(monitor=\"Validation ACC\", save_top_k=10, mode=\"max\")","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:42.959302Z","iopub.execute_input":"2022-05-04T11:49:42.959564Z","iopub.status.idle":"2022-05-04T11:49:42.963961Z","shell.execute_reply.started":"2022-05-04T11:49:42.959530Z","shell.execute_reply":"2022-05-04T11:49:42.963248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpus = 1\ntrainer = pl.Trainer(gpus=gpus, logger= TensorBoardLogger(save_dir=\"./logs\"), log_every_n_steps=1,\n                     callbacks=checkpoint_callback, max_epochs=10)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:42.965331Z","iopub.execute_input":"2022-05-04T11:49:42.965761Z","iopub.status.idle":"2022-05-04T11:49:42.976874Z","shell.execute_reply.started":"2022-05-04T11:49:42.965725Z","shell.execute_reply":"2022-05-04T11:49:42.976194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.fit(model, train_loader, val_loader)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:49:42.981151Z","iopub.execute_input":"2022-05-04T11:49:42.981358Z","iopub.status.idle":"2022-05-04T12:01:01.872394Z","shell.execute_reply.started":"2022-05-04T11:49:42.981336Z","shell.execute_reply":"2022-05-04T12:01:01.871609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluation","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = PneumoniaModel.load_from_checkpoint(\"../input/weights/weights/weights_1.ckpt\") # without weighted loss\nmodel.eval()\nmodel.to(device);","metadata":{"execution":{"iopub.status.busy":"2022-05-04T12:01:01.875768Z","iopub.execute_input":"2022-05-04T12:01:01.876255Z","iopub.status.idle":"2022-05-04T12:01:03.733086Z","shell.execute_reply.started":"2022-05-04T12:01:01.876224Z","shell.execute_reply":"2022-05-04T12:01:03.732299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loss = sum(loss(h(x[i]), y[i])) / M","metadata":{}},{"cell_type":"code","source":"#Compute prediction on the complete validation set and store predictions and labels\npreds = []\nlabels = []\nwith torch.no_grad():\n    for data, label in tqdm(val_data):\n        data = data.to(device).float().unsqueeze(0)\n        pred = torch.sigmoid(model(data)[0].cpu())\n        preds.append(pred)\n        labels.append(label)\npreds = torch.tensor(preds)\nlabels = torch.tensor(labels).int()","metadata":{"execution":{"iopub.status.busy":"2022-05-04T12:01:03.734533Z","iopub.execute_input":"2022-05-04T12:01:03.734837Z","iopub.status.idle":"2022-05-04T12:01:15.174379Z","shell.execute_reply.started":"2022-05-04T12:01:03.734783Z","shell.execute_reply":"2022-05-04T12:01:15.173564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = torchmetrics.Accuracy()(preds, labels)\nprecision = torchmetrics.Precision()(preds, labels)\nrecall = torchmetrics.Recall()(preds, labels)\n#auc = torchmetrics.AUC()\ncm = torchmetrics.ConfusionMatrix(num_classes=2)(preds, labels)\ncm_threshed = torchmetrics.ConfusionMatrix(num_classes=2)(preds, labels)\n\nprint(f\"Val Accuracy: {acc}\")\nprint(f\"Val Precision: {precision}\")\nprint(f\"Val Recall: {recall}\")\nprint(f\"Confusion Matrix:\\n {cm}\")\n\n\n# high recall means that the model rarely misses a case of pneumonia. However, the low precision means that\n# images without pneumonia are also classified as pneumonic. \n\n# confusion matrix show us 1936 images were correctly classified into class \"0\", in contrast 334 images were\n# correctly classified as pneumonic. 271 pneumonias missed. however, 143 images were classfied incorrectly\n# classified into class \"1\". \n\n# FN ratio is high. so that the recall score is low","metadata":{"execution":{"iopub.status.busy":"2022-05-04T12:01:15.176136Z","iopub.execute_input":"2022-05-04T12:01:15.176472Z","iopub.status.idle":"2022-05-04T12:01:15.194051Z","shell.execute_reply.started":"2022-05-04T12:01:15.176434Z","shell.execute_reply":"2022-05-04T12:01:15.193331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# recall --> TP / (TP + FN)\n334/(334+271)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-05-04T12:01:15.195018Z","iopub.execute_input":"2022-05-04T12:01:15.196962Z","iopub.status.idle":"2022-05-04T12:01:15.203487Z","shell.execute_reply.started":"2022-05-04T12:01:15.196925Z","shell.execute_reply":"2022-05-04T12:01:15.202736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = PneumoniaModel.load_from_checkpoint(\"../input/weights/weights/weights_3.ckpt\") # with weighted loss\nmodel.eval()\nmodel.to(device);","metadata":{"execution":{"iopub.status.busy":"2022-05-04T12:01:15.204418Z","iopub.execute_input":"2022-05-04T12:01:15.205949Z","iopub.status.idle":"2022-05-04T12:01:17.188920Z","shell.execute_reply.started":"2022-05-04T12:01:15.205912Z","shell.execute_reply":"2022-05-04T12:01:17.187954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Weighted Loss = sum(loss(h(x[i]), y[i]) * w[i] ) / sum(w[i])","metadata":{}},{"cell_type":"code","source":"preds = []\nlabels = []\nwith torch.no_grad():\n    for data, label in tqdm(val_data):\n        data = data.to(device).float().unsqueeze(0)\n        pred = torch.sigmoid(model(data)[0].cpu())\n        preds.append(pred)\n        labels.append(label)\npreds = torch.tensor(preds)\nlabels = torch.tensor(labels).int()","metadata":{"execution":{"iopub.status.busy":"2022-05-04T12:01:17.190486Z","iopub.execute_input":"2022-05-04T12:01:17.190844Z","iopub.status.idle":"2022-05-04T12:01:28.365163Z","shell.execute_reply.started":"2022-05-04T12:01:17.190780Z","shell.execute_reply":"2022-05-04T12:01:28.364471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = torchmetrics.Accuracy()(preds, labels)\nprecision = torchmetrics.Precision()(preds, labels)\nrecall = torchmetrics.Recall()(preds, labels)\nauc = torchmetrics.AUC(reorder=True)(preds, labels)\nroc = torchmetrics.ROC()(preds, labels)\ncm = torchmetrics.ConfusionMatrix(num_classes=2)(preds, labels)\ncm_threshed = torchmetrics.ConfusionMatrix(num_classes=2, threshold=0.25)(preds, labels)\n\nprint(f\"Val Accuracy: {acc}\")\nprint(f\"Val Precision: {precision}\")\nprint(f\"Val Recall: {recall}\")\nprint(f\"Val AUC: {auc}\")\nprint(f\"Val ROC: {roc}\")\nprint(f\"Confusion Matrix:\\n {cm}\")\n\n# confusion matrix show us 1537 images were correctly classified into class \"0\", in contrast 517 images were\n# correctly classified as pneumonic. 88 pneumonias missed. however, 542 images were classfied incorrectly\n# classified into class \"1\". \n\n# Whether it's such a result can be considered good. \n# Low FN ratio and high recall score","metadata":{"execution":{"iopub.status.busy":"2022-05-04T12:24:22.481209Z","iopub.execute_input":"2022-05-04T12:24:22.481493Z","iopub.status.idle":"2022-05-04T12:24:22.504611Z","shell.execute_reply.started":"2022-05-04T12:24:22.481463Z","shell.execute_reply":"2022-05-04T12:24:22.503784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"517 / (88 + 517)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:07.378365Z","iopub.execute_input":"2022-05-04T11:37:07.379086Z","iopub.status.idle":"2022-05-04T11:37:07.384670Z","shell.execute_reply.started":"2022-05-04T11:37:07.379049Z","shell.execute_reply":"2022-05-04T11:37:07.383878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axis = plt.subplots(3, 3, figsize=(9, 9))\n\nfor i in range(3):\n    for j in range(3):\n        rnd_idx = np.random.randint(0, len(preds))\n        axis[i][j].imshow(val_data[rnd_idx][0][0], cmap=\"bone\")\n        axis[i][j].set_title(f\"Pred:{int(preds[rnd_idx] > 0.5)}, Label:{labels[rnd_idx]}\")\n        axis[i][j].axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:07.385940Z","iopub.execute_input":"2022-05-04T11:37:07.386759Z","iopub.status.idle":"2022-05-04T11:37:08.011320Z","shell.execute_reply.started":"2022-05-04T11:37:07.386722Z","shell.execute_reply":"2022-05-04T11:37:08.010711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Interpretability","metadata":{}},{"cell_type":"code","source":"%matplotlib notebook\nimport torch\nimport torchvision\nfrom torchvision import transforms\nimport pytorch_lightning as pl\nimport numpy as np\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:08.012689Z","iopub.execute_input":"2022-05-04T11:37:08.013118Z","iopub.status.idle":"2022-05-04T11:37:08.023734Z","shell.execute_reply.started":"2022-05-04T11:37:08.013085Z","shell.execute_reply":"2022-05-04T11:37:08.023095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_file(path):\n    return np.load(path).astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:08.025180Z","iopub.execute_input":"2022-05-04T11:37:08.025383Z","iopub.status.idle":"2022-05-04T11:37:08.032453Z","shell.execute_reply.started":"2022-05-04T11:37:08.025359Z","shell.execute_reply":"2022-05-04T11:37:08.031757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_transforms = transforms.Compose([\n                                transforms.ToTensor(),\n                                transforms.Normalize(0.49, 0.248),\n])\n\nval_dataset = torchvision.datasets.DatasetFolder(\"/root/Processed/val/\", loader=load_file, extensions=\"npy\", transform=val_transforms)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:08.033946Z","iopub.execute_input":"2022-05-04T11:37:08.034484Z","iopub.status.idle":"2022-05-04T11:37:08.056733Z","shell.execute_reply.started":"2022-05-04T11:37:08.034447Z","shell.execute_reply":"2022-05-04T11:37:08.056129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# resneti tekrar inceleyelim\ntemp_model = torchvision.models.resnet18()\ntemp_model;","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:08.057916Z","iopub.execute_input":"2022-05-04T11:37:08.058357Z","iopub.status.idle":"2022-05-04T11:37:08.211478Z","shell.execute_reply.started":"2022-05-04T11:37:08.058323Z","shell.execute_reply":"2022-05-04T11:37:08.210725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_model.children()","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:08.212690Z","iopub.execute_input":"2022-05-04T11:37:08.212963Z","iopub.status.idle":"2022-05-04T11:37:08.218843Z","shell.execute_reply.started":"2022-05-04T11:37:08.212928Z","shell.execute_reply":"2022-05-04T11:37:08.218113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(temp_model.children());","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:08.219925Z","iopub.execute_input":"2022-05-04T11:37:08.220570Z","iopub.status.idle":"2022-05-04T11:37:08.227835Z","shell.execute_reply.started":"2022-05-04T11:37:08.220533Z","shell.execute_reply":"2022-05-04T11:37:08.226751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(temp_model.children())[:-2]  # get all layers up to avgpool","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:08.229291Z","iopub.execute_input":"2022-05-04T11:37:08.230069Z","iopub.status.idle":"2022-05-04T11:37:08.238100Z","shell.execute_reply.started":"2022-05-04T11:37:08.230032Z","shell.execute_reply":"2022-05-04T11:37:08.237317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.nn.Sequential(*list(temp_model.children())[:-2])","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:08.239513Z","iopub.execute_input":"2022-05-04T11:37:08.239987Z","iopub.status.idle":"2022-05-04T11:37:08.250172Z","shell.execute_reply.started":"2022-05-04T11:37:08.239951Z","shell.execute_reply":"2022-05-04T11:37:08.249403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PneumoniaModel(pl.LightningModule):\n    \n    def __init__(self):\n        super().__init__()\n        \n        self.model = torchvision.models.resnet18()\n        \n        # Change conv1 from 3 to 1 input channels\n        self.model.conv1 = torch.nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n        \n        # Change out_feature of the last fully connected layer (called fc in resnet18) from 1000 to 1\n        self.model.fc = torch.nn.Linear(in_features=512, out_features=1)\n        \n        # Extract the feature map\n        self.feature_map = torch.nn.Sequential(*list(self.model.children())[:-2])\n        \n    def forward(self, data):\n        \n        # Compute feature map\n        feature_map = self.feature_map(data)\n        \n        # Use Adaptive Average Pooling as in the original model\n        avg_pool_output = torch.nn.functional.adaptive_avg_pool2d(input=feature_map, output_size=(1,1))\n        \n        # Flatten the output into a 512 element vector\n        avg_output_flattened = torch.flatten(avg_pool_output)\n        \n        # Compute prediction\n        pred = self.model.fc(avg_output_flattened)\n        return pred, feature_map","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:08.251413Z","iopub.execute_input":"2022-05-04T11:37:08.251938Z","iopub.status.idle":"2022-05-04T11:37:08.262257Z","shell.execute_reply.started":"2022-05-04T11:37:08.251901Z","shell.execute_reply":"2022-05-04T11:37:08.261472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = PneumoniaModel.load_from_checkpoint(\"../input/weights/weights/weights_3.ckpt\", strict=False)\nmodel.eval();","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:08.263615Z","iopub.execute_input":"2022-05-04T11:37:08.264079Z","iopub.status.idle":"2022-05-04T11:37:08.549193Z","shell.execute_reply.started":"2022-05-04T11:37:08.264044Z","shell.execute_reply":"2022-05-04T11:37:08.548420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cam(model, img):\n    with torch.no_grad():\n        pred, features = model(img.unsqueeze(0))\n    features = features.reshape(512,49)  # (512, 7, 7)\n    weight_params = list(model.model.fc.parameters())[0]\n    weight = weight_params[0].detach()\n    \n    cam = torch.matmul(weight, features)  # The key idea of Cam is to multiply the output of the last convolutional layer\n                                          # with the weights of the subsequent fully connected layer.\n    \n    cam_img = cam.reshape(7,7).cpu()\n    return cam_img, torch.sigmoid(pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:08.550760Z","iopub.execute_input":"2022-05-04T11:37:08.551522Z","iopub.status.idle":"2022-05-04T11:37:08.558195Z","shell.execute_reply.started":"2022-05-04T11:37:08.551483Z","shell.execute_reply":"2022-05-04T11:37:08.557395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize(img, cam, pred):\n    img = img[0]\n    cam = transforms.functional.resize(cam.unsqueeze(0), (224,224))[0]\n    \n    fig, axis = plt.subplots(1,2)\n    axis[0].imshow(img, cmap=\"bone\")\n    axis[1].imshow(img, cmap=\"bone\")\n    axis[1].imshow(cam, alpha=0.5, cmap=\"jet\")\n    plt.title(pred>0.5)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:08.559245Z","iopub.execute_input":"2022-05-04T11:37:08.559497Z","iopub.status.idle":"2022-05-04T11:37:08.568049Z","shell.execute_reply.started":"2022-05-04T11:37:08.559460Z","shell.execute_reply":"2022-05-04T11:37:08.567173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Demo","metadata":{}},{"cell_type":"code","source":"img = val_dataset[-32][0]   # Select a subject\nactivation_map, pred = cam(model, img)  # Compute the Class activation map given the subject","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:08.569643Z","iopub.execute_input":"2022-05-04T11:37:08.570501Z","iopub.status.idle":"2022-05-04T11:37:08.725467Z","shell.execute_reply.started":"2022-05-04T11:37:08.570463Z","shell.execute_reply":"2022-05-04T11:37:08.724664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\nvisualize(img, activation_map, pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:08.727715Z","iopub.execute_input":"2022-05-04T11:37:08.728291Z","iopub.status.idle":"2022-05-04T11:37:09.082350Z","shell.execute_reply.started":"2022-05-04T11:37:08.728236Z","shell.execute_reply":"2022-05-04T11:37:09.081585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### The heatmap focuses on the area which shows signs of pneumonia","metadata":{"execution":{"iopub.status.busy":"2022-05-04T11:37:09.083672Z","iopub.execute_input":"2022-05-04T11:37:09.083996Z","iopub.status.idle":"2022-05-04T11:37:09.088117Z","shell.execute_reply.started":"2022-05-04T11:37:09.083960Z","shell.execute_reply":"2022-05-04T11:37:09.087169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}