{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nsys.path.append('../input/monai-v070')","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:45:35.234421Z","iopub.execute_input":"2021-12-24T09:45:35.235026Z","iopub.status.idle":"2021-12-24T09:45:35.239519Z","shell.execute_reply.started":"2021-12-24T09:45:35.234983Z","shell.execute_reply":"2021-12-24T09:45:35.238409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport glob\nimport pydicom\nimport random\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom tqdm import tqdm\nimport seaborn as sns\nimport time\nimport datetime\nfrom dataclasses import dataclass, field\nfrom sklearn.model_selection import train_test_split, KFold, StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import f1_score\nimport matplotlib.pyplot as plt\nfrom copy import deepcopy\nfrom monai.data import CacheDataset, DataLoader\nfrom monai.transforms import *\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(42)\n\nclass AverageMeter:\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n        self.history = []\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\nclass History:\n    def __init__(self):\n        self.history = []\n    \n    def add(self, val):\n        self.history.append(val)","metadata":{"execution":{"iopub.status.busy":"2021-12-24T11:14:28.582123Z","iopub.execute_input":"2021-12-24T11:14:28.582614Z","iopub.status.idle":"2021-12-24T11:14:28.594751Z","shell.execute_reply.started":"2021-12-24T11:14:28.582578Z","shell.execute_reply":"2021-12-24T11:14:28.593802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = '../input/rsna-miccai-brain-tumor-radiogenomic-classification/'\nMRI_TYPES = [\"FLAIR\", \"T1w\", \"T2w\", \"T1wCE\"]","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:45:35.258856Z","iopub.execute_input":"2021-12-24T09:45:35.259316Z","iopub.status.idle":"2021-12-24T09:45:35.265205Z","shell.execute_reply.started":"2021-12-24T09:45:35.259279Z","shell.execute_reply":"2021-12-24T09:45:35.264348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BrainTumorDataset(CacheDataset):\n    def __init__(self, root_dir, patient_ids, mri_types, annotations, section, *args, **kwargs):\n        self.root_dir = root_dir\n        self.patient_ids = patient_ids\n        self.mri_types = mri_types\n        self.annotations = annotations\n        data = self.get_data()\n        if section is not None:\n            train_data, val_data = train_test_split(data, test_size=0.2, random_state=42)\n            data = train_data if section=='train' else val_data\n        super(BrainTumorDataset, self).__init__(data, *args, **kwargs)\n    \n    def get_data(self):\n        data = []\n        for patient_id in tqdm(self.patient_ids):\n            if self.annotations is not None:\n                label = self.annotations[self.annotations['BraTS21ID'] \n                                         == int(patient_id)]['MGMT_value'].item()\n            else:\n                label = 0 \n            for slice_path in self.get_patient_slice_paths(patient_id):\n                data.append({\n                    'image': slice_path,\n                    'label': label,\n                    'patient_id': patient_id\n                })\n        return data\n    \n    def get_patient_slice_paths(self, patient_id):\n        '''\n        Returns an array of all the images of a particular type for a particular patient ID\n        '''\n        assert(set(self.mri_types) <= set(MRI_TYPES))\n        patient_path = os.path.join(self.root_dir, str(patient_id).zfill(5))\n        patient_slice_paths = []\n        for mri_type in self.mri_types:\n            paths = sorted(\n                glob.glob(os.path.join(patient_path, mri_type, \"*.dcm\")), \n                key=lambda x: int(x[:-4].split(\"-\")[-1]),\n            )\n\n            num_images = len(paths)\n            start = int(num_images * 0.25)\n            end = int(num_images * 0.75)\n\n            interval = 3\n            if num_images < 10: \n                interval = 1\n            patient_slice_paths.extend(paths[start:end:interval])\n        return patient_slice_paths\n    \nclass LoadDicomd(MapTransform):\n    def __init__(self, img_size, *args, **kwargs):\n        self.img_size = img_size\n        super(LoadDicomd, self).__init__(*args, **kwargs)\n    \n    def __call__(self, data):\n        d = dict(data)\n        for key in self.keys:\n            d[key] = self.load_dicom(d[key])\n        return d\n\n    def load_dicom(self, path):\n        dicom = pydicom.read_file(path)\n        data = dicom.pixel_array\n        if np.max(data) != 0:\n            data = data / np.max(data)\n        data = (data * 255).astype(np.uint8)\n        data = cv2.resize(data, (self.img_size, self.img_size)) / 255\n        return np.expand_dims(data, axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:45:35.266865Z","iopub.execute_input":"2021-12-24T09:45:35.267203Z","iopub.status.idle":"2021-12-24T09:45:35.285448Z","shell.execute_reply.started":"2021-12-24T09:45:35.267169Z","shell.execute_reply":"2021-12-24T09:45:35.284564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Simple2dCNN(nn.Module):\n    def __init__(self, \n                 input_channels=1, \n                 n_classes=2, \n                 img_size=32, \n                 conv1_filters=128,\n                 conv2_filters=64,\n                 dropout_prob=0.1,\n                 fc1_units=48):\n        super(Simple2dCNN, self).__init__()\n        \n        self.relu = nn.ReLU()\n        \n        self.conv1 = nn.Conv2d(input_channels, conv1_filters, 4)\n        self.maxpool1 = nn.MaxPool2d(2)\n        self.bn1 = nn.BatchNorm2d(conv1_filters)\n        \n        self.conv2 = nn.Conv2d(conv1_filters, conv2_filters, 2)\n        self.maxpool2 = nn.MaxPool2d(1)\n        self.bn2 = nn.BatchNorm2d(conv2_filters)\n        \n        self.dropout = nn.Dropout(dropout_prob)\n        last_feature_map_size = (img_size - 3) // 2 - 1\n        self.fc1 = nn.Linear(conv2_filters * last_feature_map_size**2, fc1_units)\n        self.fc2 = nn.Linear(fc1_units, n_classes)\n\n    def forward(self, x):\n        x = self.relu(self.conv1(x))\n        x = self.bn1(x)\n        x = self.maxpool1(x)\n        x = self.dropout(x)\n        \n        x = self.relu(self.conv2(x))\n        x = self.bn2(x)\n        x = self.maxpool2(x) \n        \n        x = self.dropout(x)\n        x = x.view(x.size(0), -1) \n        x = self.relu(self.fc1(x)) \n        x = self.fc2(x) \n        return x","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:45:35.288164Z","iopub.execute_input":"2021-12-24T09:45:35.288502Z","iopub.status.idle":"2021-12-24T09:45:35.300067Z","shell.execute_reply.started":"2021-12-24T09:45:35.288465Z","shell.execute_reply":"2021-12-24T09:45:35.299419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@dataclass\nclass Config:\n    train_dir: str = os.path.join(DATA_DIR, 'train')\n    test_dir: str = os.path.join(DATA_DIR, 'test')\n    annotation_path: str = os.path.join(DATA_DIR, 'train_labels.csv')\n    n_classes: int = 2\n    img_size: int = 32\n    n_workers: int = 4\n    early_stopping_rounds: int = 3\n    n_folds: int = 5\n        \n        \nclass Pipeline:\n    def __init__(self, config):\n        self.args = config\n        self.device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')\n        self.annotations = None\n        self.model = None\n        self.load_model()\n        self.preaugment_transform = [\n            LoadDicomd(keys=\"image\", img_size=self.args.img_size),\n        ]\n        self.augment_transform = [] \n        self.postaugment_transform = [\n            ToTensord(keys=\"image\", dtype=torch.float),\n            ToTensord(keys=\"label\", dtype=torch.int64),\n        ]\n        \n    def load_annotations(self):\n        self.annotations = pd.read_csv(self.args.annotation_path)\n        # exclude 3 cases\n        self.annotations = self.annotations[~self.annotations['BraTS21ID'].isin([109, 123, 709])]\n        self.annotations = self.annotations.reset_index(drop=True)\n        skf = StratifiedKFold(n_splits=self.args.n_folds, shuffle=True, random_state=42)\n        folds = skf.split(self.annotations['BraTS21ID'].values, self.annotations['MGMT_value'].values)\n        for i, (train_indices, val_indices) in enumerate(folds):\n            self.annotations.loc[val_indices, 'fold'] = i\n        self.annotations['fold'] = self.annotations['fold'].astype(int)\n    \n    def load_model(self, weights_path=None):\n        self.model = Simple2dCNN(input_channels=1, \n                                 n_classes=self.args.n_classes,\n                                 img_size=self.args.img_size).to(self.device)\n        if weights_path:\n            weights = torch.load(weights_path, map_location=self.device)\n            self.model.load_state_dict(weights)\n        return self.model\n        \n    def prepare_datasets(self, mri_types, fold, cache_rate):\n        \n        train_ids = self.annotations[self.annotations['fold']!=fold]['BraTS21ID'].values.tolist()\n        test_ids = self.annotations[self.annotations['fold']!=fold]['BraTS21ID'].values.tolist()\n        \n        train_transform = Compose(\n            self.preaugment_transform +\n            self.augment_transform +\n            self.postaugment_transform\n        )\n        val_transform = Compose(\n            self.preaugment_transform +\n            self.postaugment_transform\n        )\n        \n        \n        train_ds = BrainTumorDataset(root_dir=self.args.train_dir, \n                                     patient_ids=train_ids, \n                                     mri_types=mri_types,  \n                                     annotations=self.annotations,\n                                     transform=train_transform,\n                                     section='train',\n                                     cache_rate=cache_rate,\n                                     num_workers=self.args.n_workers)\n        val_ds = BrainTumorDataset(root_dir=self.args.train_dir, \n                                   patient_ids=train_ids, \n                                   mri_types=mri_types,  \n                                   annotations=self.annotations,\n                                   transform=val_transform,\n                                   section='val',\n                                   cache_rate=cache_rate,\n                                   num_workers=self.args.n_workers)\n        test_ds = BrainTumorDataset(root_dir=self.args.train_dir, \n                                           patient_ids=test_ids, \n                                           mri_types=mri_types, \n                                           annotations=self.annotations, \n                                           transform=val_transform,\n                                           section='val',\n                                           cache_rate=cache_rate,\n                                           num_workers=self.args.n_workers)\n        return train_ds, val_ds, test_ds\n    \n    def prepare_test_dataset(self, mri_types, cache_rate):\n        test_transform = Compose(\n            self.preaugment_transform +\n            self.postaugment_transform\n        )\n        test_ids = [int(patient_id) for patient_id in os.listdir(self.args.test_dir)]\n        test_ids = sorted(test_ids, key=lambda x: int(x))\n        test_ds = BrainTumorDataset(root_dir=self.args.test_dir, \n                                    patient_ids=test_ids, \n                                    mri_types=mri_types, \n                                    annotations=None, \n                                    transform=test_transform,\n                                    section=None,\n                                    cache_rate=cache_rate,\n                                    num_workers=self.args.n_workers)\n        return test_ds\n    \n    def train_epoch(self, loader, loss_function, optimizer, verbose):\n        self.model.train()\n        summary_loss = AverageMeter()\n        start = time.time()\n        n = len(loader)\n        patient_ids_all = []\n        probabilities_all = []\n        labels_all = []\n        for step, batch_data in enumerate(loader):\n            inputs, labels, patient_ids = (\n                batch_data[\"image\"].to(self.device), # (None, 1, 32, 32)\n                batch_data[\"label\"].to(self.device), # (None, )\n                batch_data[\"patient_id\"]\n            )\n            patient_ids_all.extend(patient_ids)\n            \n            labels_all.extend(labels.tolist())\n            batch_size = inputs.size(0)\n            \n            optimizer.zero_grad()\n            outputs = self.model(inputs) # (None, 2)\n            loss = loss_function(outputs, labels)\n            probabilities = F.softmax(outputs, dim=1)[:, 1].tolist()\n            probabilities_all.extend(probabilities)\n            loss.backward()\n            optimizer.step()\n            \n            summary_loss.update(loss.item(), batch_size)\n            if verbose:\n                print('Train step {}/{}, loss: {:.5f}'.format(step + 1, n, \n                                                              summary_loss.avg), end='\\r')\n        elapsed_time = str(datetime.timedelta(seconds=time.time() - start))\n        print('Train loss: {:.5f} - time: {}'.format(summary_loss.avg, elapsed_time))\n        result = {\n            'BraTS21ID': list(map(lambda x: x.item(), patient_ids_all)), \n            'probability': probabilities_all,\n            'label': labels_all\n        }\n        result = pd.DataFrame(result)\n        slice_auc = roc_auc_score(result['label'], result['probability'])\n        result = result.groupby(\"BraTS21ID\", as_index=False).mean()\n        patient_auc = roc_auc_score(result['label'], result['probability'])\n        print('Patient AUC: {:.5f} - Slice AUC: {:.5f}'.format(patient_auc, slice_auc))\n        return summary_loss.avg, patient_auc\n    \n    def evaluate_epoch(self, loader, loss_function, verbose):\n        self.model.eval()\n        summary_loss = AverageMeter()\n        start = time.time()\n        n = len(loader)\n        patient_ids_all = []\n        probabilities_all = []\n        labels_all = []\n        with torch.no_grad():\n            for step, batch_data in enumerate(loader):\n                inputs, labels, patient_ids = (\n                    batch_data[\"image\"].to(self.device),\n                    batch_data[\"label\"].to(self.device), \n                    batch_data[\"patient_id\"], \n                )\n                batch_size = inputs.size(0)\n                \n                outputs = self.model(inputs) \n                loss = loss_function(outputs, labels)\n                \n                probabilities = F.softmax(outputs, dim=1)[:, 1].tolist()\n                probabilities_all.extend(probabilities)\n                labels_all.extend(labels.tolist())\n                patient_ids_all.extend(patient_ids)\n                \n                summary_loss.update(loss.item(), batch_size)\n                if verbose:\n                    print('Val step {}/{}, loss: {:.5f}'.format(step + 1, n, \n                                                                summary_loss.avg), end='\\r')\n        elapsed_time = str(datetime.timedelta(seconds=time.time() - start))\n        #print('Val loss: {:.5f} - time: {}'.format(summary_loss.avg, elapsed_time))\n        result = {\n            'BraTS21ID': list(map(lambda x: x.item(), patient_ids_all)), \n            'probability': probabilities_all,\n            'label': labels_all\n        }\n        result = pd.DataFrame(result)\n        slice_auc = roc_auc_score(result['label'], result['probability'])\n        result = result.groupby(\"BraTS21ID\", as_index=False).mean()\n        patient_auc = roc_auc_score(result['label'], result['probability'])\n        #print('Patient AUC: {:.5f} - Slice AUC: {:.5f}'.format(patient_auc, slice_auc))\n        \n        return summary_loss.avg, patient_auc, result\n    \n    def infer_epoch(self, loader, verbose):\n        self.model.eval()\n        start = time.time()\n        n = len(loader)\n        patient_ids_all = []\n        probabilities_all = []\n        with torch.no_grad():\n            for step, batch_data in enumerate(loader):\n                inputs, patient_ids = (\n                    batch_data[\"image\"].to(self.device), \n                    batch_data[\"patient_id\"], \n                )\n                batch_size = inputs.size(0)\n                \n                outputs = self.model(inputs) \n                \n                probabilities = F.softmax(outputs, dim=1)[:, 1].tolist()\n                probabilities_all.extend(probabilities)\n                patient_ids_all.extend(patient_ids)\n                if verbose:\n                    print('Infer step {}/{}'.format(step + 1, n), end='\\r')\n        \n        result = {\n            'BraTS21ID': list(map(lambda x: x.item(), patient_ids_all)), \n            'probability': probabilities_all,\n        }\n        result = pd.DataFrame(result)\n        result = result.groupby(\"BraTS21ID\", as_index=False).mean()\n        \n        elapsed_time = str(datetime.timedelta(seconds=time.time() - start))\n        print('Elapsed time: {}'.format(elapsed_time))\n        \n        return result\n    \n    def fit(self, train_ds, val_ds, test_ds, batch_size, epochs, lr, model_name, verbose):\n        auc = History()\n        loss = History()\n        train_auc = History()\n        train_loss = History()\n        train_loader = DataLoader(train_ds, \n                                  batch_size=batch_size, \n                                  shuffle=True,\n                                  num_workers=self.args.n_workers)\n        val_loader = DataLoader(val_ds, \n                                batch_size=batch_size, \n                                shuffle=False,\n                                num_workers=self.args.n_workers)\n        test_loader = DataLoader(test_ds, \n                                        batch_size=batch_size, \n                                        shuffle=False,\n                                        num_workers=self.args.n_workers)\n        loss_function = nn.CrossEntropyLoss()\n        optimizer = torch.optim.Adam(self.model.parameters(), lr=lr)\n        \n        current_metric = -np.inf\n        current_loss = np.inf\n        current_epoch = 1\n        current_state_dict = None\n        save_path = '{}.pth'\n        for epoch in range(1, epochs + 1):\n            print('\\nEpoch {}/{}:'.format(epoch, epochs))\n            trai_loss, trai_auc = self.train_epoch(train_loader, loss_function, optimizer, verbose)\n            train_loss.add(trai_loss)\n            train_auc.add(trai_auc)\n            print(' Validation:')\n            val_loss, val_metric, _ = self.evaluate_epoch(val_loader, loss_function, verbose)\n            auc.add(val_metric)\n            loss.add(val_loss)\n            \n            \n#             if val_loss < current_loss:\n            if val_metric > current_metric:\n                print('Val AUC improved from {:.5f} to {:.5f}'.format(current_metric, val_metric))\n                current_metric = val_metric\n                current_loss = val_loss\n                current_epoch = epoch\n                current_state_dict = deepcopy(self.model.state_dict())\n                \n            elif (epoch - current_epoch) > self.args.early_stopping_rounds:\n                print('Early stopping. Best model is epoch {}'.format(current_epoch))\n                print('Val loss: {:.5f}, Val auc: {:.5f}'.format(current_loss, current_metric))\n                print('Saving model...')\n                torch.save(current_state_dict, \n                           save_path.format(model_name))\n                break\n            if epoch == epochs:\n                print('Finished training. Best model is epoch {}'.format(current_epoch))\n                print('Val loss: {:.5f}, Val auc: {:.5f}'.format(current_loss, current_metric))\n                print('Saving model...')\n                torch.save(current_state_dict, \n                           save_path.format(model_name))\n        return auc.history, loss.history, train_loss.history, train_auc.history\n                \n    def evaluate(self, test_ds, batch_size, verbose):\n        test_loader = DataLoader(test_ds, \n                                        batch_size=batch_size, \n                                        shuffle=False,\n                                        num_workers=self.args.n_workers)\n        loss_function = nn.CrossEntropyLoss()\n        #print(' Test:')\n        _, test_metric, test_result = self.evaluate_epoch(test_loader, \n                                                                        loss_function, \n                                                                        verbose)\n        return test_metric, test_result\n    \n    def predict(self, test_ds, batch_size, verbose):\n        test_loader = DataLoader(test_ds, \n                                 batch_size=batch_size, \n                                 shuffle=False,\n                                 num_workers=self.args.n_workers)\n        test_result = self.infer_epoch(test_loader, verbose)\n        return test_result","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:45:35.368856Z","iopub.execute_input":"2021-12-24T09:45:35.369113Z","iopub.status.idle":"2021-12-24T09:45:35.425421Z","shell.execute_reply.started":"2021-12-24T09:45:35.369085Z","shell.execute_reply":"2021-12-24T09:45:35.424635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_size = 128\nbatch_size = 32\nn_workers = 8\nearly_stopping_rounds = 3\nn_folds = 4\nepochs = 50\nlr = 1e-4","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:45:35.429046Z","iopub.execute_input":"2021-12-24T09:45:35.429249Z","iopub.status.idle":"2021-12-24T09:45:35.437649Z","shell.execute_reply.started":"2021-12-24T09:45:35.429227Z","shell.execute_reply":"2021-12-24T09:45:35.436992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"args = Config(img_size=img_size, \n              n_workers=n_workers, \n              early_stopping_rounds=early_stopping_rounds,\n              n_folds=n_folds)\npipeline = Pipeline(args)\nmodel = pipeline.load_model()\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:45:35.439096Z","iopub.execute_input":"2021-12-24T09:45:35.439358Z","iopub.status.idle":"2021-12-24T09:45:41.223031Z","shell.execute_reply.started":"2021-12-24T09:45:35.439325Z","shell.execute_reply":"2021-12-24T09:45:41.222266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipeline.load_annotations()\nauc, loss, t_loss, t_auc = [], [], [], []\nfor mri in MRI_TYPES:\n    for fold in range(n_folds):\n        print(f'### Train {mri} on fold {fold}: ###')\n        train_ds, val_ds, test_ds = pipeline.prepare_datasets(mri_types=[mri], \n                                                                     fold=fold,\n                                                                     cache_rate=1.0)\n        pipeline.load_model()\n        auc_, loss_, t_loss_, t_auc_ = pipeline.fit(train_ds, val_ds, test_ds,\n                     batch_size=batch_size, epochs=epochs, lr=lr, \n                     model_name=f'{\"_\".join(mri)}_fold{fold}',\n                     verbose=True)\n        auc.append(auc_)\n        t_auc.append(t_auc_)\n        loss.append(loss_)\n        t_loss.append(t_loss_)","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-12-24T09:45:41.22417Z","iopub.execute_input":"2021-12-24T09:45:41.224407Z","iopub.status.idle":"2021-12-24T10:40:38.647268Z","shell.execute_reply.started":"2021-12-24T09:45:41.224374Z","shell.execute_reply":"2021-12-24T10:40:38.645215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def roc(fpr, tpr, roc_auc, mri):\n    plt.figure()\n    lw = 2\n    plt.plot(\n        fpr,\n        tpr,\n        color=\"darkorange\",\n        lw=lw,\n        label=\"ROC curve (area = %0.2f)\" % roc_auc,\n    )\n    plt.plot([0, 1], [0, 1], color=\"navy\", lw=lw, linestyle=\"--\")\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel(\"False Positive Rate\")\n    plt.ylabel(\"True Positive Rate\")\n    plt.title(\"ROC Curve for {} test dataset\".format(mri))\n    plt.legend(loc=\"lower right\")\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(rc={'figure.figsize':(20,13)})\nplot = sns.lineplot(data = auc[0], label = 'val AUC').set_title(MRI_TYPES[0])\nplot = sns.lineplot(data = loss[0], label = 'val Loss')\nplot = sns.lineplot(data = t_auc[0], label = 'train AUC')\nplot = sns.lineplot(data = t_loss[0], label = 'train Loss')\na = plot.set(xlabel='Epochs', ylabel='Value')","metadata":{"execution":{"iopub.status.busy":"2021-12-24T10:40:38.652526Z","iopub.execute_input":"2021-12-24T10:40:38.652786Z","iopub.status.idle":"2021-12-24T10:40:39.305579Z","shell.execute_reply.started":"2021-12-24T10:40:38.652757Z","shell.execute_reply":"2021-12-24T10:40:39.304838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=4\nsns.set(rc={'figure.figsize':(20,13)})\nplot = sns.lineplot(data = auc[i], label = 'val AUC').set_title(MRI_TYPES[1])\nplot = sns.lineplot(data = loss[i], label = 'val Loss')\nplot = sns.lineplot(data = t_auc[i], label = 'train AUC')\nplot = sns.lineplot(data = t_loss[i], label = 'train Loss')\na = plot.set(xlabel='Epochs', ylabel='Value')","metadata":{"execution":{"iopub.status.busy":"2021-12-24T10:40:39.306802Z","iopub.execute_input":"2021-12-24T10:40:39.307688Z","iopub.status.idle":"2021-12-24T10:40:39.801212Z","shell.execute_reply.started":"2021-12-24T10:40:39.307644Z","shell.execute_reply":"2021-12-24T10:40:39.800472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=8\nsns.set(rc={'figure.figsize':(20,13)})\nplot = sns.lineplot(data = auc[i], label = 'val AUC').set_title(MRI_TYPES[2])\nplot = sns.lineplot(data = loss[i], label = 'val Loss')\nplot = sns.lineplot(data = t_auc[i], label = 'train AUC')\nplot = sns.lineplot(data = t_loss[i], label = 'train Loss')\na = plot.set(xlabel='Epochs', ylabel='Value')","metadata":{"execution":{"iopub.status.busy":"2021-12-24T10:40:39.802671Z","iopub.execute_input":"2021-12-24T10:40:39.80329Z","iopub.status.idle":"2021-12-24T10:40:40.236605Z","shell.execute_reply.started":"2021-12-24T10:40:39.80325Z","shell.execute_reply":"2021-12-24T10:40:40.235829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=12\nsns.set(rc={'figure.figsize':(20,13)})\nplot = sns.lineplot(data = auc[i], label = 'val AUC').set_title(MRI_TYPES[3])\nplot = sns.lineplot(data = loss[i], label = 'val Loss')\nplot = sns.lineplot(data = t_auc[i], label = 'train AUC')\nplot = sns.lineplot(data = t_loss[i], label = 'train Loss')\na = plot.set(xlabel='Epochs', ylabel='Value')","metadata":{"execution":{"iopub.status.busy":"2021-12-24T10:40:40.237996Z","iopub.execute_input":"2021-12-24T10:40:40.238386Z","iopub.status.idle":"2021-12-24T10:40:40.677191Z","shell.execute_reply.started":"2021-12-24T10:40:40.23835Z","shell.execute_reply":"2021-12-24T10:40:40.676472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(rc={'figure.figsize':(20,13)})\nplot = sns.lineplot(data = auc, label = 'AUC')\nplot = sns.lineplot(data = loss, label = 'Loss')\na = plot.set(xlabel='Epochs', ylabel='Value')","metadata":{"execution":{"iopub.status.busy":"2021-12-24T10:40:40.678489Z","iopub.execute_input":"2021-12-24T10:40:40.67969Z","iopub.status.idle":"2021-12-24T10:40:41.908178Z","shell.execute_reply.started":"2021-12-24T10:40:40.679651Z","shell.execute_reply":"2021-12-24T10:40:41.907523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mri_types = MRI_TYPES\nfor mri in mri_types:\n    metrics = []\n    results = []\n    find_weight = lambda x: [w for w in os.listdir() if x in w][0]\n    weights_paths = [f'{\"_\".join(mri)}_fold{fold}.pth' for fold in range(n_folds)]\n    weights_paths = [find_weight(x) for x in weights_paths]\n    for fold, weights_path in enumerate(weights_paths):\n        #print(f'### Evaluate {mri_types} on fold {fold}: ###')\n        _, _, test_ds = pipeline.prepare_datasets(mri_types=mri_types, \n                                                         fold=fold,\n                                                         cache_rate=0.0)\n        pipeline.load_model(weights_path)\n        val_metric, val_result = pipeline.evaluate(test_ds, batch_size=batch_size, verbose=False)\n        metrics.append(val_metric)\n        results.append(val_result)\n    results = pd.concat(results, ignore_index=True)\n    mean_auc = np.mean(metrics)\n    oof_auc = roc_auc_score(results['label'], results['probability'])\n    f1 = f1_score(results['label'].astype('float'), results['probability'].apply(lambda x: 0 if x <0.5 else 1),average='binary')\n    print('---')\n    print(f'{mri} test result:')\n    print(' Mean AUC: {:.5f}'.format(mean_auc))\n    print(' Mean F1_score: {:.5f}'.format(f1))\n    fpr, tpr, _ = roc_curve(results['label'], results['probability'])\n    roc(fpr, tpr, mean_auc, mri)\n    print('---')","metadata":{"execution":{"iopub.status.busy":"2021-12-24T11:44:17.174332Z","iopub.execute_input":"2021-12-24T11:44:17.174614Z","iopub.status.idle":"2021-12-24T11:55:06.995645Z","shell.execute_reply.started":"2021-12-24T11:44:17.174584Z","shell.execute_reply":"2021-12-24T11:55:06.994739Z"},"trusted":true},"execution_count":null,"outputs":[]}]}