{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook, I will try to optimize CV score of the split-by-patient strategy. The public score might be lower than my [previous notebook](https://www.kaggle.com/nanguyen/brain-tumor-2d-cnn-pytorch-split-by-patient), but I hope to get a more reliable CV result.","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('../input/monai-v070')\nimport os\nimport cv2\nimport glob\nimport pydicom\nimport random\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom tqdm import tqdm\nimport time\nimport datetime\nfrom dataclasses import dataclass, field\nfrom sklearn.model_selection import train_test_split, KFold, StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom copy import deepcopy\nimport gc\n\nfrom monai.data import CacheDataset, DataLoader\nfrom monai.transforms import *\nfrom monai.networks.nets import *\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(42)\n\nclass AverageMeter:\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2021-10-14T03:02:58.605738Z","iopub.execute_input":"2021-10-14T03:02:58.606796Z","iopub.status.idle":"2021-10-14T03:03:07.929378Z","shell.execute_reply.started":"2021-10-14T03:02:58.606686Z","shell.execute_reply":"2021-10-14T03:03:07.928367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = '../input/rsna-miccai-brain-tumor-radiogenomic-classification/'\nMRI_TYPES = [\"FLAIR\", \"T1w\", \"T2w\", \"T1wCE\"]","metadata":{"execution":{"iopub.status.busy":"2021-10-14T03:03:07.932943Z","iopub.execute_input":"2021-10-14T03:03:07.933157Z","iopub.status.idle":"2021-10-14T03:03:07.939143Z","shell.execute_reply.started":"2021-10-14T03:03:07.933132Z","shell.execute_reply":"2021-10-14T03:03:07.938386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"class BrainTumorDataset(CacheDataset):\n    def __init__(self, root_dir, patient_ids, mri_types, annotations, *args, **kwargs):\n        self.root_dir = root_dir\n        self.patient_ids = patient_ids\n        self.mri_types = mri_types\n        self.annotations = annotations\n        data = self.get_data()\n        super(BrainTumorDataset, self).__init__(data, *args, **kwargs)\n    \n    def get_data(self):\n        data = []\n        for patient_id in tqdm(self.patient_ids):\n            if self.annotations is not None:\n                label = self.annotations[self.annotations['BraTS21ID'] \n                                         == int(patient_id)]['MGMT_value'].item()\n            else:\n                label = 0 # dummy value\n            for slice_path in self.get_patient_slice_paths(patient_id):\n                data.append({\n                    'image': slice_path,\n                    'label': label,\n                    'patient_id': patient_id\n                })\n        return data\n    \n    def get_patient_slice_paths(self, patient_id):\n        '''\n        Returns an array of all the images of a particular type for a particular patient ID\n        '''\n        assert(set(self.mri_types) <= set(MRI_TYPES))\n        patient_path = os.path.join(self.root_dir, str(patient_id).zfill(5))\n        patient_slice_paths = []\n        for mri_type in self.mri_types:\n            paths = sorted(\n                glob.glob(os.path.join(patient_path, mri_type, \"*.dcm\")), \n                key=lambda x: int(x[:-4].split(\"-\")[-1]),\n            )\n\n            num_images = len(paths)\n            start = int(num_images * 0.25)\n            end = int(num_images * 0.75)\n\n            interval = 3\n            if num_images < 10: \n                interval = 1\n            patient_slice_paths.extend(paths[start:end:interval])\n        return patient_slice_paths\n    \nclass LoadDicomd(MapTransform):\n    def __init__(self, img_size, *args, **kwargs):\n        self.img_size = img_size\n        super(LoadDicomd, self).__init__(*args, **kwargs)\n    \n    def __call__(self, data):\n        d = dict(data)\n        for key in self.keys:\n            d[key] = self.load_dicom(d[key])\n        return d\n\n    def load_dicom(self, path):\n        ''' \n        Reads a DICOM image, standardizes so that the pixel values are between 0 and 1, \n        then rescales to 0 and 255\n        '''\n        dicom = pydicom.read_file(path)\n        data = dicom.pixel_array\n        if np.max(data) != 0:\n            data = data / np.max(data)\n        data = (data * 255).astype(np.uint8)\n        data = cv2.resize(data, (self.img_size, self.img_size)) / 255\n        return np.expand_dims(data, axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-10-14T03:03:07.942280Z","iopub.execute_input":"2021-10-14T03:03:07.942474Z","iopub.status.idle":"2021-10-14T03:03:07.959522Z","shell.execute_reply.started":"2021-10-14T03:03:07.942451Z","shell.execute_reply":"2021-10-14T03:03:07.958654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"class Simple2dCNN(nn.Module):\n    def __init__(self, \n                 input_channels=1, \n                 n_classes=2, \n                 img_size=32, \n                 conv1_filters=128,\n                 conv2_filters=64,\n                 dropout_prob=0.1,\n                 fc1_units=48):\n        super(Simple2dCNN, self).__init__()\n        \n        self.relu = nn.ReLU()\n        \n        self.conv1 = nn.Conv2d(input_channels, conv1_filters, 4)\n        self.maxpool1 = nn.MaxPool2d(2)\n        \n        self.conv2 = nn.Conv2d(conv1_filters, conv2_filters, 2)\n        self.maxpool2 = nn.MaxPool2d(1)\n        \n        self.dropout = nn.Dropout(dropout_prob)\n        last_feature_map_size = (img_size - 3) // 2 - 1\n        self.fc1 = nn.Linear(conv2_filters * last_feature_map_size**2, fc1_units)\n        self.fc2 = nn.Linear(fc1_units, n_classes)\n\n    def forward(self, x):\n        # (None, 1, 32, 32)\n        x = self.relu(self.conv1(x)) # (None, 128, 29, 29)\n        x = self.maxpool1(x) # (None, 128, 14, 14)\n        \n        x = self.relu(self.conv2(x)) # (None, 64, 13, 13)\n        x = self.maxpool2(x) # (None, 64, 13, 13)\n        \n        x = self.dropout(x)\n        x = x.view(x.size(0), -1) # (None, 64 * 13 * 13)\n        x = self.relu(self.fc1(x)) # (None, 48)\n        x = self.fc2(x) # (None, 2)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-10-14T03:03:07.960911Z","iopub.execute_input":"2021-10-14T03:03:07.961437Z","iopub.status.idle":"2021-10-14T03:03:07.973378Z","shell.execute_reply.started":"2021-10-14T03:03:07.961400Z","shell.execute_reply":"2021-10-14T03:03:07.972629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pipeline","metadata":{}},{"cell_type":"code","source":"@dataclass\nclass Config:\n    train_dir: str = os.path.join(DATA_DIR, 'train')\n    test_dir: str = os.path.join(DATA_DIR, 'test')\n    annotation_path: str = os.path.join(DATA_DIR, 'train_labels.csv')\n    model_name: str = 'efficientnet-b0'\n    n_classes: int = 2\n    img_size: int = 224\n    n_workers: int = 4\n    early_stopping_rounds: int = 3\n    n_folds: int = 5\n        \n        \nclass Pipeline:\n    def __init__(self, config):\n        self.args = config\n        self.device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')\n        self.annotations = None\n        self.model = None\n        self.load_model()\n        # transforms\n        self.preaugment_transform = [\n            LoadDicomd(keys=\"image\", img_size=self.args.img_size),\n        ]\n        self.augment_transform = [\n            RandAffined(\n                keys=\"image\",\n                prob=1.0,\n                rotate_range=(np.pi/9, np.pi/9),\n                scale_range=(0.1, 0.1),\n                shear_range=(0.1, 0.1),\n            ),\n            RandScaleIntensityd(keys=\"image\", factors=0.3, prob=1.0),\n            RandShiftIntensityd(keys=\"image\", offsets=0.3, prob=1.0),\n        ]\n        self.postaugment_transform = [\n            ToTensord(keys=\"image\", dtype=torch.float),\n            ToTensord(keys=\"label\", dtype=torch.int64),\n        ]\n        \n    def load_annotations(self):\n        self.annotations = pd.read_csv(self.args.annotation_path)\n        # exclude 3 cases\n        self.annotations = self.annotations[~self.annotations['BraTS21ID'].isin([109, 123, 709])]\n        self.annotations = self.annotations.reset_index(drop=True)\n        skf = StratifiedKFold(n_splits=self.args.n_folds, shuffle=True, random_state=42)\n        # split by patient, stratify based on target value\n        folds = skf.split(self.annotations['BraTS21ID'].values, self.annotations['MGMT_value'].values)\n        for i, (train_indices, val_indices) in enumerate(folds):\n            self.annotations.loc[val_indices, 'fold'] = i\n        self.annotations['fold'] = self.annotations['fold'].astype(int)\n    \n    def load_model(self, weights_path=None):\n        self.model = resnet18(spatial_dims=2, \n                              n_input_channels=1, \n                              num_classes=self.args.n_classes).to(self.device)\n#         self.model = EfficientNetBN(model_name=self.args.model_name,\n#                                     pretrained=False,\n#                                     spatial_dims=2,\n#                                     in_channels=1,\n#                                     num_classes=self.args.n_classes).to(self.device)\n\n#         self.model = Simple2dCNN(input_channels=1, \n#                                  n_classes=self.args.n_classes, \n#                                  img_size=self.args.img_size, \n#                                  conv1_filters=128,\n#                                  conv2_filters=64,\n#                                  dropout_prob=0.1,\n#                                  fc1_units=48).to(self.device)\n        if weights_path:\n            weights = torch.load(weights_path, map_location=self.device)\n            self.model.load_state_dict(weights)\n            \n    def prepare_datasets(self, mri_types, fold, cache_rate):\n        \"\"\"\n        Data format:\n        {\n            'image': torch tensor (batch_size, 1, img_size, img_size),\n            'label': torch tensor (batch_size, )\n            'patient_id'\n        }\n        \"\"\"\n        train_transform = Compose(\n            self.preaugment_transform +\n            self.augment_transform +\n            self.postaugment_transform\n        )\n        val_transform = Compose(\n            self.preaugment_transform +\n            self.postaugment_transform\n        )\n        \n        train_ids = self.annotations[self.annotations['fold']!=fold]['BraTS21ID'].values.tolist()\n        val_ids = self.annotations[self.annotations['fold']==fold]['BraTS21ID'].values.tolist()\n        \n        train_ds = BrainTumorDataset(root_dir=self.args.train_dir, \n                                     patient_ids=train_ids, \n                                     mri_types=mri_types,  \n                                     annotations=self.annotations,\n                                     transform=train_transform,\n                                     cache_rate=cache_rate,\n                                     num_workers=self.args.n_workers)\n        val_ds = BrainTumorDataset(root_dir=self.args.train_dir, \n                                   patient_ids=val_ids, \n                                   mri_types=mri_types, \n                                   annotations=self.annotations,\n                                   transform=val_transform,\n                                   cache_rate=cache_rate,\n                                   num_workers=self.args.n_workers)\n        return train_ds, val_ds\n    \n    def prepare_test_dataset(self, mri_types, cache_rate):\n        test_transform = Compose(\n            self.preaugment_transform +\n            self.postaugment_transform\n        )\n        test_ids = [int(patient_id) for patient_id in os.listdir(self.args.test_dir)]\n        test_ids = sorted(test_ids, key=lambda x: int(x))\n        test_ds = BrainTumorDataset(root_dir=self.args.test_dir, \n                                    patient_ids=test_ids, \n                                    mri_types=mri_types, \n                                    annotations=None, \n                                    transform=test_transform,\n                                    cache_rate=cache_rate,\n                                    num_workers=self.args.n_workers)\n        return test_ds\n    \n    def train_epoch(self, loader, loss_function, optimizer, verbose):\n        self.model.train()\n        summary_loss = AverageMeter()\n        start = time.time()\n        n = len(loader)\n        for step, batch_data in enumerate(loader):\n            inputs, labels = (\n                batch_data[\"image\"].to(self.device), # (None, 1, img_size, img_size)\n                batch_data[\"label\"].to(self.device), # (None, )\n            )\n            batch_size = inputs.size(0)\n            # back propagation\n            optimizer.zero_grad()\n            outputs = self.model(inputs) # (None, 2)\n            loss = loss_function(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            # update stats\n            summary_loss.update(loss.item(), batch_size)\n            if verbose:\n                print('Train step {}/{}, loss: {:.5f}'.format(step + 1, n, \n                                                              summary_loss.avg), end='\\r')\n        elapsed_time = str(datetime.timedelta(seconds=time.time() - start))\n        print('Train loss: {:.5f} - time: {}'.format(summary_loss.avg, elapsed_time))\n        return summary_loss.avg\n    \n    def evaluate_epoch(self, loader, loss_function, verbose):\n        self.model.eval()\n        summary_loss = AverageMeter()\n        start = time.time()\n        n = len(loader)\n        patient_ids_all = []\n        probabilities_all = []\n        labels_all = []\n        with torch.no_grad():\n            for step, batch_data in enumerate(loader):\n                inputs, labels, patient_ids = (\n                    batch_data[\"image\"].to(self.device), # (None, 1, img_size, img_size)\n                    batch_data[\"label\"].to(self.device), # (None, )\n                    batch_data[\"patient_id\"], # (None, )\n                )\n                batch_size = inputs.size(0)\n                # back propagation\n                outputs = self.model(inputs) # (None, 2)\n                loss = loss_function(outputs, labels)\n                # update stats\n                probabilities = F.softmax(outputs, dim=1)[:, 1].tolist()\n                probabilities_all.extend(probabilities)\n                labels_all.extend(labels.tolist())\n                patient_ids_all.extend(patient_ids)\n                \n                summary_loss.update(loss.item(), batch_size)\n                if verbose:\n                    print('Val step {}/{}, loss: {:.5f}'.format(step + 1, n, \n                                                                summary_loss.avg), end='\\r')\n        elapsed_time = str(datetime.timedelta(seconds=time.time() - start))\n        print('Val loss: {:.5f} - time: {}'.format(summary_loss.avg, elapsed_time))\n        result = {\n            'BraTS21ID': list(map(lambda x: x.item(), patient_ids_all)), \n            'probability': probabilities_all,\n            'label': labels_all\n        }\n        result = pd.DataFrame(result)\n        slice_auc = roc_auc_score(result['label'], result['probability'])\n        result = result.groupby(\"BraTS21ID\", as_index=False).mean()\n        patient_auc = roc_auc_score(result['label'], result['probability'])\n        print('Patient AUC: {:.5f} - Slice AUC: {:.5f}'.format(patient_auc, slice_auc))\n        \n        return summary_loss.avg, patient_auc, result\n    \n    def infer_epoch(self, loader, verbose):\n        self.model.eval()\n        start = time.time()\n        n = len(loader)\n        patient_ids_all = []\n        probabilities_all = []\n        with torch.no_grad():\n            for step, batch_data in enumerate(loader):\n                inputs, patient_ids = (\n                    batch_data[\"image\"].to(self.device), # (None, 1, img_size, img_size)\n                    batch_data[\"patient_id\"], # (None, )\n                )\n                batch_size = inputs.size(0)\n                # forward\n                outputs = self.model(inputs) # (None, 2)\n                # update stats\n                probabilities = F.softmax(outputs, dim=1)[:, 1].tolist()\n                probabilities_all.extend(probabilities)\n                patient_ids_all.extend(patient_ids)\n                if verbose:\n                    print('Infer step {}/{}'.format(step + 1, n), end='\\r')\n        \n        result = {\n            'BraTS21ID': list(map(lambda x: x.item(), patient_ids_all)), \n            'probability': probabilities_all,\n        }\n        result = pd.DataFrame(result)\n        result = result.groupby(\"BraTS21ID\", as_index=False).mean()\n        \n        elapsed_time = str(datetime.timedelta(seconds=time.time() - start))\n        print('Elapsed time: {}'.format(elapsed_time))\n        \n        return result\n    \n    def fit(self, train_ds, val_ds, batch_size, epochs, lr, model_name, verbose):\n        train_loader = DataLoader(train_ds, \n                                  batch_size=batch_size, \n                                  shuffle=True,\n                                  num_workers=self.args.n_workers)\n        val_loader = DataLoader(val_ds, \n                                batch_size=batch_size, \n                                shuffle=False,\n                                num_workers=self.args.n_workers)\n        loss_function = nn.CrossEntropyLoss()\n        optimizer = torch.optim.Adam(self.model.parameters(), lr=lr)\n        \n        current_metric = -np.inf\n        current_loss = np.inf\n        current_epoch = 1\n        current_state_dict = None\n        save_path = '{}_imgsize{}_valloss{:.3f}_valauc{:.3f}.pth'\n        for epoch in range(1, epochs + 1):\n            print('\\nEpoch {}/{}:'.format(epoch, epochs))\n            train_loss = self.train_epoch(train_loader, loss_function, optimizer, verbose)\n            val_loss, val_metric, _ = self.evaluate_epoch(val_loader, loss_function, verbose)\n            \n            if val_loss < current_loss:\n                print('Val loss improved from {:.5f} to {:.5f}'.format(current_loss, val_loss))\n#             if val_metric > current_metric:\n#                 print('Val AUC improved from {:.5f} to {:.5f}'.format(current_metric, val_metric))\n                current_metric = val_metric\n                current_loss = val_loss\n                current_epoch = epoch\n                current_state_dict = deepcopy(self.model.state_dict())\n                \n            elif (epoch - current_epoch) > self.args.early_stopping_rounds:\n                print('Early stopping. Best model is epoch {}'.format(current_epoch))\n                print('Val loss: {:.5f}, Val auc: {:.5f}'.format(current_loss, current_metric))\n                print('Saving model...')\n                torch.save(current_state_dict, \n                           save_path.format(model_name,\n                                            self.args.img_size, \n                                            current_loss, \n                                            current_metric))\n                break\n            if epoch == epochs:\n                print('Finished training. Best model is epoch {}'.format(current_epoch))\n                print('Val loss: {:.5f}, Val auc: {:.5f}'.format(current_loss, current_metric))\n                print('Saving model...')\n                torch.save(current_state_dict, \n                           save_path.format(model_name,\n                                            self.args.img_size, \n                                            current_loss, \n                                            current_metric))\n                \n    def evaluate(self, val_ds, batch_size, verbose):\n        val_loader = DataLoader(val_ds, \n                                batch_size=batch_size, \n                                shuffle=False,\n                                num_workers=self.args.n_workers)\n        loss_function = nn.CrossEntropyLoss()\n        _, val_metric, val_result = self.evaluate_epoch(val_loader, loss_function, verbose)\n        return val_metric, val_result\n    \n    def predict(self, test_ds, batch_size, verbose):\n        test_loader = DataLoader(test_ds, \n                                 batch_size=batch_size, \n                                 shuffle=False,\n                                 num_workers=self.args.n_workers)\n        test_result = self.infer_epoch(test_loader, verbose)\n        return test_result","metadata":{"execution":{"iopub.status.busy":"2021-10-14T03:03:07.976463Z","iopub.execute_input":"2021-10-14T03:03:07.976917Z","iopub.status.idle":"2021-10-14T03:03:08.030439Z","shell.execute_reply.started":"2021-10-14T03:03:07.976882Z","shell.execute_reply":"2021-10-14T03:03:08.029747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mri_types = ['FLAIR']\nmodel_name = 'resnet18' # '2dCNN' \nimg_size = 224 # 32\nbatch_size = 128\nn_workers = 4\nearly_stopping_rounds = 5\nn_folds = 5\nepochs = 50\nlr = 1e-5 ","metadata":{"execution":{"iopub.status.busy":"2021-10-14T03:03:08.031512Z","iopub.execute_input":"2021-10-14T03:03:08.031791Z","iopub.status.idle":"2021-10-14T03:03:08.040165Z","shell.execute_reply.started":"2021-10-14T03:03:08.031758Z","shell.execute_reply":"2021-10-14T03:03:08.039479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"args = Config(model_name=model_name,\n              img_size=img_size, \n              n_workers=n_workers, \n              early_stopping_rounds=early_stopping_rounds,\n              n_folds=n_folds)\npipeline = Pipeline(args)","metadata":{"execution":{"iopub.status.busy":"2021-10-14T03:03:08.041607Z","iopub.execute_input":"2021-10-14T03:03:08.041874Z","iopub.status.idle":"2021-10-14T03:03:13.946177Z","shell.execute_reply.started":"2021-10-14T03:03:08.041829Z","shell.execute_reply":"2021-10-14T03:03:13.945275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"def train(pipeline, mri_type, n_folds, batch_size, epochs, lr, model_name):\n    for fold in range(n_folds):\n        print(f'### Train {mri_type} on fold {fold}: ###')\n        train_ds, val_ds = pipeline.prepare_datasets(mri_types=[mri_type], \n                                                     fold=fold,\n                                                     cache_rate=1.0)\n        pipeline.load_model()\n        pipeline.fit(train_ds, val_ds,\n                     batch_size=batch_size, epochs=epochs, lr=lr, \n                     model_name=f'{model_name}_{mri_type}_fold{fold}',\n                     verbose=True)\n        del train_ds\n        del val_ds\n        gc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-10-14T03:03:13.952100Z","iopub.execute_input":"2021-10-14T03:03:13.952727Z","iopub.status.idle":"2021-10-14T03:03:13.965001Z","shell.execute_reply.started":"2021-10-14T03:03:13.952697Z","shell.execute_reply":"2021-10-14T03:03:13.963103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipeline.load_annotations()\nfor mri_type in MRI_TYPES:\n    train(pipeline, mri_type, n_folds, batch_size, epochs, lr, model_name)","metadata":{"execution":{"iopub.status.busy":"2021-10-14T03:03:13.966769Z","iopub.execute_input":"2021-10-14T03:03:13.967095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluate","metadata":{}},{"cell_type":"code","source":"def evaluate(pipeline, mri_type, n_folds, batch_size, model_name):\n    metrics = []\n    results = []\n    find_weight = lambda x: [w for w in os.listdir() if x in w][0]\n    weights_paths = [f'{model_name}_{mri_type}_fold{fold}' for fold in range(n_folds)]\n    weights_paths = [find_weight(x) for x in weights_paths]\n    for fold, weights_path in enumerate(weights_paths):\n        print(f'### Evaluate {mri_type} on fold {fold}: ###')\n        _, val_ds = pipeline.prepare_datasets(mri_types=[mri_type], \n                                              fold=fold,\n                                              cache_rate=0.0)\n        pipeline.load_model(weights_path)\n        val_metric, val_result = pipeline.evaluate(val_ds, batch_size=batch_size, verbose=True)\n        metrics.append(val_metric)\n        results.append(val_result)\n    results = pd.concat(results, ignore_index=True)\n    mean_auc = np.mean(metrics)\n    oof_auc = roc_auc_score(results['label'], results['probability'])\n    print('---')\n    print(f'{mri_type} validation result:')\n    print(' Mean AUC: {:.5f}'.format(mean_auc))\n    print(' Out-of-fold AUC: {:.5f}'.format(oof_auc))\n    print('---')\n    return results, mean_auc, oof_auc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_predictions = dict()\nmean_aucs = dict()\noof_aucs = dict()\nfor mri_type in MRI_TYPES:\n    oof_prediction, mean_auc, oof_auc = evaluate(pipeline, mri_type, n_folds, batch_size, model_name)\n    oof_predictions[mri_type] = oof_prediction\n    mean_aucs[mri_type] = mean_auc\n    oof_aucs[mri_type] = oof_auc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Blend modalities","metadata":{}},{"cell_type":"markdown","source":"In this section, I will try to to find the blending weights of the 4 modalities to optimize the out-of-fold AUC of the ensemble.","metadata":{}},{"cell_type":"code","source":"label_df = pd.read_csv(os.path.join(DATA_DIR, 'train_labels.csv')).set_index('BraTS21ID')\noof_df = pd.concat([oof_predictions[mri_type][['BraTS21ID', 'probability']].rename(columns={'probability': mri_type}).set_index('BraTS21ID') \n                    for mri_type in MRI_TYPES] + [label_df], axis=1, join='inner').reset_index()\noof_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_best_weight(oof_df):\n    w_range = np.arange(0.0, 1.01, 0.01)\n    best_auc = -np.inf\n    best_w = None\n    for w_flair in tqdm(w_range):\n        for w_t1w in w_range:\n            for w_t2w in w_range:\n                if w_flair + w_t1w + w_t2w > 1:\n                    continue\n                w_t1wce = 1 - (w_flair + w_t1w + w_t2w)\n                pred = (w_flair * oof_df['FLAIR'] + w_t1w * oof_df['T1w'] \n                        + w_t2w * oof_df['T2w'] + w_t1wce * oof_df['T1wCE'])\n                auc = roc_auc_score(oof_df['MGMT_value'], pred)\n                if auc > best_auc:\n                    best_auc = auc\n                    best_w = [w_flair, w_t1w, w_t2w, w_t1wce]\n    return best_w, best_auc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_w, best_auc = find_best_weight(oof_df)\nprint(best_w)\nprint(best_auc)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_df['pred'] = (best_w[0] * oof_df['FLAIR'] + best_w[1] * oof_df['T1w'] \n                  + best_w[2] * oof_df['T2w'] + best_w[3] * oof_df['T1wCE'])\noof_df[['BraTS21ID', 'pred']].to_csv('oof_prediction.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"markdown","source":"* Prediction of each modality is the average of the predictions of 5 folds.\n* Final prediction for the test set is the ensemble of 4 modalities.","metadata":{}},{"cell_type":"code","source":"def inference(pipeline, mri_type, n_folds, batch_size, model_name):\n    test_results = []\n    find_weight = lambda x: [w for w in os.listdir() if x in w][0]\n    weights_paths = [f'{model_name}_{mri_type}_fold{fold}' for fold in range(n_folds)]\n    weights_paths = [find_weight(x) for x in weights_paths]\n    for fold, weights_path in enumerate(weights_paths):\n        print(f'### Inference {mri_type} on fold {fold}: ###')\n        test_ds = pipeline.prepare_test_dataset(mri_types=[mri_type], cache_rate=0.0)\n        pipeline.load_model(weights_path)\n        test_result = pipeline.predict(test_ds, batch_size=batch_size, verbose=True)\n        test_results.append(test_result)\n    prediction = pd.concat([x.set_index('BraTS21ID') for x in test_results], axis=1).mean(axis=1)\n    prediction = pd.DataFrame(prediction, columns=['probability']).reset_index()\n    return prediction","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predictions = dict()\nfor mri_type in MRI_TYPES:\n    test_prediction = inference(pipeline, mri_type, n_folds, batch_size, model_name)\n    test_predictions[mri_type] = test_prediction","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.concat([test_predictions[mri_type][['BraTS21ID', 'probability']].rename(columns={'probability': mri_type}).set_index('BraTS21ID') \n                     for mri_type in MRI_TYPES], axis=1, join='inner').reset_index()\ntest_df['MGMT_value'] = (best_w[0] * test_df['FLAIR'] + best_w[1] * test_df['T1w'] \n                         + best_w[2] * test_df['T2w'] + best_w[3] * test_df['T1wCE'])\ntest_df[['BraTS21ID', 'MGMT_value']].to_csv('submission.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}