{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Note**: network architecture and original data split strategy are based on [this noteboook](https://www.kaggle.com/mmellinger66/brain-tumor-basic-tensorflow-model).\n\nIn this notebook, I want to demonstrate the importance of splitting data properly, especially when using 2d cnn where each data point is a slice. If we split the data normally, the validation AUC can be deceptively high because similar slices of the same patient can appear in both train and validation set. Thus we should split the data based on patient id. Here are some noteable changes I made compared to the original notebook:\n1. Use split-by-patient strategy on top of their original split strategy (I called it holdout sets in this notebook)\n2. Reimplement using pytorch\n3. Prediction for a patient is the average of slices probabilities (instead of the average of slices 0-1 predictions in the original notebook)\n4. Track both patient AUC and slice AUC when training\n5. Calculate out-of-fold AUC and average AUC across all folds to get a more reliable result\n6. Remove the result rounding part because I didn't find it meaningful","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('../input/monai-v070')","metadata":{"execution":{"iopub.status.busy":"2021-10-11T17:34:00.542659Z","iopub.execute_input":"2021-10-11T17:34:00.545639Z","iopub.status.idle":"2021-10-11T17:34:00.636782Z","shell.execute_reply.started":"2021-10-11T17:34:00.545553Z","shell.execute_reply":"2021-10-11T17:34:00.635536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport glob\nimport pydicom\nimport random\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom tqdm import tqdm\nimport time\nimport datetime\nfrom dataclasses import dataclass, field\nfrom sklearn.model_selection import train_test_split, KFold, StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom copy import deepcopy\n\nfrom monai.data import CacheDataset, DataLoader\nfrom monai.transforms import *\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(42)\n\nclass AverageMeter:\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2021-10-11T17:34:00.638491Z","iopub.execute_input":"2021-10-11T17:34:00.638775Z","iopub.status.idle":"2021-10-11T17:34:09.446471Z","shell.execute_reply.started":"2021-10-11T17:34:00.638742Z","shell.execute_reply":"2021-10-11T17:34:09.445748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = '../input/rsna-miccai-brain-tumor-radiogenomic-classification/'\nMRI_TYPES = [\"FLAIR\", \"T1w\", \"T2w\", \"T1wCE\"]","metadata":{"execution":{"iopub.status.busy":"2021-10-11T17:34:09.447721Z","iopub.execute_input":"2021-10-11T17:34:09.447964Z","iopub.status.idle":"2021-10-11T17:34:09.452316Z","shell.execute_reply.started":"2021-10-11T17:34:09.447932Z","shell.execute_reply":"2021-10-11T17:34:09.451647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"class BrainTumorDataset(CacheDataset):\n    def __init__(self, root_dir, patient_ids, mri_types, annotations, section, *args, **kwargs):\n        self.root_dir = root_dir\n        self.patient_ids = patient_ids\n        self.mri_types = mri_types\n        self.annotations = annotations\n        data = self.get_data()\n        if section is not None:\n            train_data, val_data = train_test_split(data, test_size=0.2, random_state=42)\n            data = train_data if section=='train' else val_data\n        super(BrainTumorDataset, self).__init__(data, *args, **kwargs)\n    \n    def get_data(self):\n        data = []\n        for patient_id in tqdm(self.patient_ids):\n            if self.annotations is not None:\n                label = self.annotations[self.annotations['BraTS21ID'] \n                                         == int(patient_id)]['MGMT_value'].item()\n            else:\n                label = 0 # dummy value\n            for slice_path in self.get_patient_slice_paths(patient_id):\n                data.append({\n                    'image': slice_path,\n                    'label': label,\n                    'patient_id': patient_id\n                })\n        return data\n    \n    def get_patient_slice_paths(self, patient_id):\n        '''\n        Returns an array of all the images of a particular type for a particular patient ID\n        '''\n        assert(set(self.mri_types) <= set(MRI_TYPES))\n        patient_path = os.path.join(self.root_dir, str(patient_id).zfill(5))\n        patient_slice_paths = []\n        for mri_type in self.mri_types:\n            paths = sorted(\n                glob.glob(os.path.join(patient_path, mri_type, \"*.dcm\")), \n                key=lambda x: int(x[:-4].split(\"-\")[-1]),\n            )\n\n            num_images = len(paths)\n            start = int(num_images * 0.25)\n            end = int(num_images * 0.75)\n\n            interval = 3\n            if num_images < 10: \n                interval = 1\n            patient_slice_paths.extend(paths[start:end:interval])\n        return patient_slice_paths\n    \nclass LoadDicomd(MapTransform):\n    def __init__(self, img_size, *args, **kwargs):\n        self.img_size = img_size\n        super(LoadDicomd, self).__init__(*args, **kwargs)\n    \n    def __call__(self, data):\n        d = dict(data)\n        for key in self.keys:\n            d[key] = self.load_dicom(d[key])\n        return d\n\n    def load_dicom(self, path):\n        ''' \n        Reads a DICOM image, standardizes so that the pixel values are between 0 and 1, \n        then rescales to 0 and 255\n        '''\n        dicom = pydicom.read_file(path)\n        data = dicom.pixel_array\n        if np.max(data) != 0:\n            data = data / np.max(data)\n        data = (data * 255).astype(np.uint8)\n        data = cv2.resize(data, (self.img_size, self.img_size)) / 255\n        return np.expand_dims(data, axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-10-11T17:34:09.453758Z","iopub.execute_input":"2021-10-11T17:34:09.454182Z","iopub.status.idle":"2021-10-11T17:34:09.472291Z","shell.execute_reply.started":"2021-10-11T17:34:09.454149Z","shell.execute_reply":"2021-10-11T17:34:09.471593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"class Simple2dCNN(nn.Module):\n    def __init__(self, \n                 input_channels=1, \n                 n_classes=2, \n                 img_size=32, \n                 conv1_filters=128,\n                 conv2_filters=64,\n                 dropout_prob=0.1,\n                 fc1_units=48):\n        super(Simple2dCNN, self).__init__()\n        \n        self.relu = nn.ReLU()\n        \n        self.conv1 = nn.Conv2d(input_channels, conv1_filters, 4)\n        self.maxpool1 = nn.MaxPool2d(2)\n        \n        self.conv2 = nn.Conv2d(conv1_filters, conv2_filters, 2)\n        self.maxpool2 = nn.MaxPool2d(1)\n        \n        self.dropout = nn.Dropout(dropout_prob)\n        last_feature_map_size = (img_size - 3) // 2 - 1\n        self.fc1 = nn.Linear(conv2_filters * last_feature_map_size**2, fc1_units)\n        self.fc2 = nn.Linear(fc1_units, n_classes)\n\n    def forward(self, x):\n        # (None, 1, 32, 32)\n        x = self.relu(self.conv1(x)) # (None, 128, 29, 29)\n        x = self.maxpool1(x) # (None, 128, 14, 14)\n        \n        x = self.relu(self.conv2(x)) # (None, 64, 13, 13)\n        x = self.maxpool2(x) # (None, 64, 13, 13)\n        \n        x = self.dropout(x)\n        x = x.view(x.size(0), -1) # (None, 64 * 13 * 13)\n        x = self.relu(self.fc1(x)) # (None, 48)\n        x = self.fc2(x) # (None, 2)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-10-11T17:34:09.474695Z","iopub.execute_input":"2021-10-11T17:34:09.475009Z","iopub.status.idle":"2021-10-11T17:34:09.487489Z","shell.execute_reply.started":"2021-10-11T17:34:09.474974Z","shell.execute_reply":"2021-10-11T17:34:09.486809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pipeline","metadata":{}},{"cell_type":"code","source":"@dataclass\nclass Config:\n    train_dir: str = os.path.join(DATA_DIR, 'train')\n    test_dir: str = os.path.join(DATA_DIR, 'test')\n    annotation_path: str = os.path.join(DATA_DIR, 'train_labels.csv')\n    n_classes: int = 2\n    img_size: int = 32\n    n_workers: int = 4\n    early_stopping_rounds: int = 3\n    n_folds: int = 5\n        \n        \nclass Pipeline:\n    def __init__(self, config):\n        self.args = config\n        self.device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')\n        self.annotations = None\n        self.model = None\n        self.load_model()\n        # transforms\n        self.preaugment_transform = [\n            LoadDicomd(keys=\"image\", img_size=self.args.img_size),\n        ]\n        self.augment_transform = [] # todo: add some augmentations\n        self.postaugment_transform = [\n            ToTensord(keys=\"image\", dtype=torch.float),\n            ToTensord(keys=\"label\", dtype=torch.int64),\n        ]\n        \n    def load_annotations(self):\n        self.annotations = pd.read_csv(self.args.annotation_path)\n        # exclude 3 cases\n        self.annotations = self.annotations[~self.annotations['BraTS21ID'].isin([109, 123, 709])]\n        self.annotations = self.annotations.reset_index(drop=True)\n        skf = StratifiedKFold(n_splits=self.args.n_folds, shuffle=True, random_state=42)\n        # split by patient, stratify based on target value\n        folds = skf.split(self.annotations['BraTS21ID'].values, self.annotations['MGMT_value'].values)\n        for i, (train_indices, val_indices) in enumerate(folds):\n            self.annotations.loc[val_indices, 'fold'] = i\n        self.annotations['fold'] = self.annotations['fold'].astype(int)\n    \n    def load_model(self, weights_path=None):\n        self.model = Simple2dCNN(input_channels=1, \n                                 n_classes=self.args.n_classes,\n                                 img_size=self.args.img_size).to(self.device)\n        if weights_path:\n            weights = torch.load(weights_path, map_location=self.device)\n            self.model.load_state_dict(weights)\n        \n    def prepare_datasets(self, mri_types, fold, cache_rate):\n        \"\"\"\n        Data format:\n        {\n            'image': torch tensor (batch_size, 1, 32, 32),\n            'label': torch tensor (batch_size, )\n            'patient_id'\n        }\n        Output: torch tensor (batch_size, 2)\n        \"\"\"\n        train_transform = Compose(\n            self.preaugment_transform +\n            self.augment_transform +\n            self.postaugment_transform\n        )\n        val_transform = Compose(\n            self.preaugment_transform +\n            self.postaugment_transform\n        )\n        \n        train_ids = self.annotations[self.annotations['fold']!=fold]['BraTS21ID'].values.tolist()\n        val_holdout_ids = self.annotations[self.annotations['fold']==fold]['BraTS21ID'].values.tolist()\n        \n        train_ds = BrainTumorDataset(root_dir=self.args.train_dir, \n                                     patient_ids=train_ids, \n                                     mri_types=mri_types,  \n                                     annotations=self.annotations,\n                                     transform=train_transform,\n                                     section='train',\n                                     cache_rate=cache_rate,\n                                     num_workers=self.args.n_workers)\n        val_ds = BrainTumorDataset(root_dir=self.args.train_dir, \n                                   patient_ids=train_ids, \n                                   mri_types=mri_types,  \n                                   annotations=self.annotations,\n                                   transform=val_transform,\n                                   section='val',\n                                   cache_rate=cache_rate,\n                                   num_workers=self.args.n_workers)\n        val_holdout_ds = BrainTumorDataset(root_dir=self.args.train_dir, \n                                           patient_ids=val_holdout_ids, \n                                           mri_types=mri_types, \n                                           annotations=self.annotations, \n                                           transform=val_transform,\n                                           section=None,\n                                           cache_rate=cache_rate,\n                                           num_workers=self.args.n_workers)\n        return train_ds, val_ds, val_holdout_ds\n    \n    def prepare_test_dataset(self, mri_types, cache_rate):\n        test_transform = Compose(\n            self.preaugment_transform +\n            self.postaugment_transform\n        )\n        test_ids = [int(patient_id) for patient_id in os.listdir(self.args.test_dir)]\n        test_ids = sorted(test_ids, key=lambda x: int(x))\n        test_ds = BrainTumorDataset(root_dir=self.args.test_dir, \n                                    patient_ids=test_ids, \n                                    mri_types=mri_types, \n                                    annotations=None, \n                                    transform=test_transform,\n                                    section=None,\n                                    cache_rate=cache_rate,\n                                    num_workers=self.args.n_workers)\n        return test_ds\n    \n    def train_epoch(self, loader, loss_function, optimizer, verbose):\n        self.model.train()\n        summary_loss = AverageMeter()\n        start = time.time()\n        n = len(loader)\n        for step, batch_data in enumerate(loader):\n            inputs, labels = (\n                batch_data[\"image\"].to(self.device), # (None, 1, 32, 32)\n                batch_data[\"label\"].to(self.device), # (None, )\n            )\n            batch_size = inputs.size(0)\n            # back propagation\n            optimizer.zero_grad()\n            outputs = self.model(inputs) # (None, 2)\n            loss = loss_function(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            # update stats\n            summary_loss.update(loss.item(), batch_size)\n            if verbose:\n                print('Train step {}/{}, loss: {:.5f}'.format(step + 1, n, \n                                                              summary_loss.avg), end='\\r')\n        elapsed_time = str(datetime.timedelta(seconds=time.time() - start))\n        print('Train loss: {:.5f} - time: {}'.format(summary_loss.avg, elapsed_time))\n        return summary_loss.avg\n    \n    def evaluate_epoch(self, loader, loss_function, verbose):\n        self.model.eval()\n        summary_loss = AverageMeter()\n        start = time.time()\n        n = len(loader)\n        patient_ids_all = []\n        probabilities_all = []\n        labels_all = []\n        with torch.no_grad():\n            for step, batch_data in enumerate(loader):\n                inputs, labels, patient_ids = (\n                    batch_data[\"image\"].to(self.device), # (None, 1, 32, 32)\n                    batch_data[\"label\"].to(self.device), # (None, )\n                    batch_data[\"patient_id\"], # (None, )\n                )\n                batch_size = inputs.size(0)\n                # back propagation\n                outputs = self.model(inputs) # (None, 2)\n                loss = loss_function(outputs, labels)\n                # update stats\n                probabilities = F.softmax(outputs, dim=1)[:, 1].tolist()\n                probabilities_all.extend(probabilities)\n                labels_all.extend(labels.tolist())\n                patient_ids_all.extend(patient_ids)\n                \n                summary_loss.update(loss.item(), batch_size)\n                if verbose:\n                    print('Val step {}/{}, loss: {:.5f}'.format(step + 1, n, \n                                                                summary_loss.avg), end='\\r')\n        elapsed_time = str(datetime.timedelta(seconds=time.time() - start))\n        print('Val loss: {:.5f} - time: {}'.format(summary_loss.avg, elapsed_time))\n        result = {\n            'BraTS21ID': list(map(lambda x: x.item(), patient_ids_all)), \n            'probability': probabilities_all,\n            'label': labels_all\n        }\n        result = pd.DataFrame(result)\n        slice_auc = roc_auc_score(result['label'], result['probability'])\n        result = result.groupby(\"BraTS21ID\", as_index=False).mean()\n        patient_auc = roc_auc_score(result['label'], result['probability'])\n        print('Patient AUC: {:.5f} - Slice AUC: {:.5f}'.format(patient_auc, slice_auc))\n        \n        return summary_loss.avg, patient_auc, result\n    \n    def infer_epoch(self, loader, verbose):\n        self.model.eval()\n        start = time.time()\n        n = len(loader)\n        patient_ids_all = []\n        probabilities_all = []\n        with torch.no_grad():\n            for step, batch_data in enumerate(loader):\n                inputs, patient_ids = (\n                    batch_data[\"image\"].to(self.device), # (None, 1, 32, 32)\n                    batch_data[\"patient_id\"], # (None, )\n                )\n                batch_size = inputs.size(0)\n                # forward\n                outputs = self.model(inputs) # (None, 2)\n                # update stats\n                probabilities = F.softmax(outputs, dim=1)[:, 1].tolist()\n                probabilities_all.extend(probabilities)\n                patient_ids_all.extend(patient_ids)\n                if verbose:\n                    print('Infer step {}/{}'.format(step + 1, n), end='\\r')\n        \n        result = {\n            'BraTS21ID': list(map(lambda x: x.item(), patient_ids_all)), \n            'probability': probabilities_all,\n        }\n        result = pd.DataFrame(result)\n        result = result.groupby(\"BraTS21ID\", as_index=False).mean()\n        \n        elapsed_time = str(datetime.timedelta(seconds=time.time() - start))\n        print('Elapsed time: {}'.format(elapsed_time))\n        \n        return result\n    \n    def fit(self, train_ds, val_ds, val_holdout_ds, batch_size, epochs, lr, model_name, verbose):\n        train_loader = DataLoader(train_ds, \n                                  batch_size=batch_size, \n                                  shuffle=True,\n                                  num_workers=self.args.n_workers)\n        val_loader = DataLoader(val_ds, \n                                batch_size=batch_size, \n                                shuffle=False,\n                                num_workers=self.args.n_workers)\n        val_holdout_loader = DataLoader(val_holdout_ds, \n                                        batch_size=batch_size, \n                                        shuffle=False,\n                                        num_workers=self.args.n_workers)\n        loss_function = nn.CrossEntropyLoss()\n        optimizer = torch.optim.Adam(self.model.parameters(), lr=lr)\n        \n        current_metric = -np.inf\n        current_loss = np.inf\n        current_epoch = 1\n        current_state_dict = None\n        save_path = '{}_imgsize{}_valloss{:.3f}_valauc{:.3f}.pth'\n        for epoch in range(1, epochs + 1):\n            print('\\nEpoch {}/{}:'.format(epoch, epochs))\n            train_loss = self.train_epoch(train_loader, loss_function, optimizer, verbose)\n            print(' Validation:')\n            val_loss, val_metric, _ = self.evaluate_epoch(val_loader, loss_function, verbose)\n            print(' Hold out:')\n            val_holdout_loss, val_holdout_metric, _ = self.evaluate_epoch(val_holdout_loader, \n                                                                          loss_function, \n                                                                          verbose)\n            \n#             if val_loss < current_loss:\n            if val_metric > current_metric:\n                print('Val AUC improved from {:.5f} to {:.5f}'.format(current_metric, val_metric))\n                current_metric = val_metric\n                current_loss = val_loss\n                current_epoch = epoch\n                current_state_dict = deepcopy(self.model.state_dict())\n                \n            elif (epoch - current_epoch) > self.args.early_stopping_rounds:\n                print('Early stopping. Best model is epoch {}'.format(current_epoch))\n                print('Val loss: {:.5f}, Val auc: {:.5f}'.format(current_loss, current_metric))\n                print('Saving model...')\n                torch.save(current_state_dict, \n                           save_path.format(model_name,\n                                            self.args.img_size, \n                                            current_loss, \n                                            current_metric))\n                break\n            if epoch == epochs:\n                print('Finished training. Best model is epoch {}'.format(current_epoch))\n                print('Val loss: {:.5f}, Val auc: {:.5f}'.format(current_loss, current_metric))\n                print('Saving model...')\n                torch.save(current_state_dict, \n                           save_path.format(model_name,\n                                            self.args.img_size, \n                                            current_loss, \n                                            current_metric))\n                \n    def evaluate(self, val_holdout_ds, batch_size, verbose):\n        val_holdout_loader = DataLoader(val_holdout_ds, \n                                        batch_size=batch_size, \n                                        shuffle=False,\n                                        num_workers=self.args.n_workers)\n        loss_function = nn.CrossEntropyLoss()\n        print(' Hold out:')\n        _, val_holdout_metric, val_holdout_result = self.evaluate_epoch(val_holdout_loader, \n                                                                        loss_function, \n                                                                        verbose)\n        return val_holdout_metric, val_holdout_result\n    \n    def predict(self, test_ds, batch_size, verbose):\n        test_loader = DataLoader(test_ds, \n                                 batch_size=batch_size, \n                                 shuffle=False,\n                                 num_workers=self.args.n_workers)\n        test_result = self.infer_epoch(test_loader, verbose)\n        return test_result","metadata":{"execution":{"iopub.status.busy":"2021-10-11T17:34:09.489021Z","iopub.execute_input":"2021-10-11T17:34:09.489349Z","iopub.status.idle":"2021-10-11T17:34:09.544467Z","shell.execute_reply.started":"2021-10-11T17:34:09.489311Z","shell.execute_reply":"2021-10-11T17:34:09.543727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mri_types = ['T1wCE']\nimg_size = 32\nbatch_size = 32\nn_workers = 4\nearly_stopping_rounds = 3\nn_folds = 5\nepochs = 50\nlr = 1e-3","metadata":{"execution":{"iopub.status.busy":"2021-10-11T17:34:09.547198Z","iopub.execute_input":"2021-10-11T17:34:09.547405Z","iopub.status.idle":"2021-10-11T17:34:09.558381Z","shell.execute_reply.started":"2021-10-11T17:34:09.547383Z","shell.execute_reply":"2021-10-11T17:34:09.557704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"args = Config(img_size=img_size, \n              n_workers=n_workers, \n              early_stopping_rounds=early_stopping_rounds,\n              n_folds=n_folds)\npipeline = Pipeline(args)","metadata":{"execution":{"iopub.status.busy":"2021-10-11T17:34:09.559584Z","iopub.execute_input":"2021-10-11T17:34:09.560089Z","iopub.status.idle":"2021-10-11T17:34:15.206538Z","shell.execute_reply.started":"2021-10-11T17:34:09.560047Z","shell.execute_reply":"2021-10-11T17:34:15.205816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"pipeline.load_annotations()\nfor fold in range(n_folds):\n    print(f'### Train {mri_types} on fold {fold}: ###')\n    train_ds, val_ds, val_holdout_ds = pipeline.prepare_datasets(mri_types=mri_types, \n                                                                 fold=fold,\n                                                                 cache_rate=1.0)\n    pipeline.load_model()\n    pipeline.fit(train_ds, val_ds, val_holdout_ds,\n                 batch_size=batch_size, epochs=epochs, lr=lr, \n                 model_name=f'{\"_\".join(mri_types)}_fold{fold}',\n                 verbose=True)","metadata":{"execution":{"iopub.status.busy":"2021-10-11T17:34:15.209801Z","iopub.execute_input":"2021-10-11T17:34:15.210011Z","iopub.status.idle":"2021-10-11T17:54:15.594346Z","shell.execute_reply.started":"2021-10-11T17:34:15.209987Z","shell.execute_reply":"2021-10-11T17:54:15.593468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluate","metadata":{}},{"cell_type":"code","source":"metrics = []\nresults = []\nfind_weight = lambda x: [w for w in os.listdir() if x in w][0]\nweights_paths = [f'{\"_\".join(mri_types)}_fold{fold}' for fold in range(n_folds)]\nweights_paths = [find_weight(x) for x in weights_paths]\nfor fold, weights_path in enumerate(weights_paths):\n    print(f'### Evaluate {mri_types} on fold {fold}: ###')\n    _, _, val_holdout_ds = pipeline.prepare_datasets(mri_types=mri_types, \n                                                     fold=fold,\n                                                     cache_rate=0.0)\n    pipeline.load_model(weights_path)\n    val_metric, val_result = pipeline.evaluate(val_holdout_ds, batch_size=batch_size, verbose=True)\n    metrics.append(val_metric)\n    results.append(val_result)\nresults = pd.concat(results, ignore_index=True)\nmean_auc = np.mean(metrics)\noof_auc = roc_auc_score(results['label'], results['probability'])\nprint('---')\nprint(f'{mri_types} holdout result:')\nprint(' Mean AUC: {:.5f}'.format(mean_auc))\nprint(' Out-of-fold AUC: {:.5f}'.format(oof_auc))\nprint('---')","metadata":{"execution":{"iopub.status.busy":"2021-10-11T17:54:15.595844Z","iopub.execute_input":"2021-10-11T17:54:15.596745Z","iopub.status.idle":"2021-10-11T17:54:57.643279Z","shell.execute_reply.started":"2021-10-11T17:54:15.596700Z","shell.execute_reply":"2021-10-11T17:54:57.642296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, although the validation AUCs are very high (0.8x to 0.9x) for every fold, the holdout AUC is only around 0.5, which is not even better than random guess.","metadata":{}},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"markdown","source":"Final prediction for the test set is the average of the predictions of all 5 models.","metadata":{}},{"cell_type":"code","source":"test_results = []\nfor fold, weights_path in enumerate(weights_paths):\n    print(f'### Inference {mri_types} on fold {fold}: ###')\n    test_ds = pipeline.prepare_test_dataset(mri_types=mri_types, cache_rate=0.0)\n    pipeline.load_model(weights_path)\n    test_result = pipeline.predict(test_ds, batch_size=batch_size, verbose=True)\n    test_results.append(test_result)","metadata":{"execution":{"iopub.status.busy":"2021-10-11T17:54:57.647977Z","iopub.execute_input":"2021-10-11T17:54:57.648290Z","iopub.status.idle":"2021-10-11T17:55:26.272632Z","shell.execute_reply.started":"2021-10-11T17:54:57.648249Z","shell.execute_reply":"2021-10-11T17:55:26.270933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction = pd.concat([x.set_index('BraTS21ID') for x in test_results], axis=1).mean(axis=1)\nprediction = pd.DataFrame(prediction, columns=['MGMT_value']).reset_index()\nprediction.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-11T17:55:26.274582Z","iopub.execute_input":"2021-10-11T17:55:26.274909Z","iopub.status.idle":"2021-10-11T17:55:26.294816Z","shell.execute_reply.started":"2021-10-11T17:55:26.274872Z","shell.execute_reply":"2021-10-11T17:55:26.294047Z"},"trusted":true},"execution_count":null,"outputs":[]}]}