{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"My notebook extend from [notebook](https://www.kaggle.com/agrica/btrc-pretrained-medicalnet-finetuning-public) and use dataset from [dataset](https://www.kaggle.com/c/rsna-miccai-brain-tumor-radiogenomic-classification/discussion/253000)\nMy work:\n- Use Monai frame (develop for medical image and includes many architectures like ViT, efficienet, .... 2D and 3D)\n- Use Monai for Augmentation ( easy transform data into 2D and 3D planes).\n- Apply SAM optimizer\n- More infomation [Monai](http://https://github.com/Project-MONAI/tutorials)\nI hope my work can be useful for this challenge.\nMany thanks [agrica](https://www.kaggle.com/agrica) and [jonathanbesomi](https://www.kaggle.com/jonathanbesomi) for your work that is good point for starting this challenge.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-24T12:55:55.25123Z","iopub.execute_input":"2021-08-24T12:55:55.251605Z","iopub.status.idle":"2021-08-24T12:55:55.255557Z","shell.execute_reply.started":"2021-08-24T12:55:55.251521Z","shell.execute_reply":"2021-08-24T12:55:55.254714Z"}}},{"cell_type":"code","source":"!pip install wandb --upgrade","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:35:57.856291Z","iopub.execute_input":"2021-09-18T19:35:57.856686Z","iopub.status.idle":"2021-09-18T19:36:09.100101Z","shell.execute_reply.started":"2021-09-18T19:35:57.856608Z","shell.execute_reply":"2021-09-18T19:36:09.099201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/Project-MONAI/MONAI.git\n%cd MONAI/\n!pip install -e '.[all]'","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:36:09.103716Z","iopub.execute_input":"2021-09-18T19:36:09.103991Z","iopub.status.idle":"2021-09-18T19:38:13.431648Z","shell.execute_reply.started":"2021-09-18T19:36:09.103961Z","shell.execute_reply":"2021-09-18T19:38:13.430712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install adamp","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:38:13.435407Z","iopub.execute_input":"2021-09-18T19:38:13.435680Z","iopub.status.idle":"2021-09-18T19:38:24.362198Z","shell.execute_reply.started":"2021-09-18T19:38:13.435650Z","shell.execute_reply":"2021-09-18T19:38:24.361243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import monai\nfrom monai.data import ImageDataset\nfrom monai.transforms import Zoom,AddChannel, Compose, CenterScaleCrop,RandRotate90, Resize, ScaleIntensity, \\\nEnsureType,BorderPad,Zoom,RandZoom,SpatialPad,RandCoarseDropout,\\\nFlip,CenterScaleCrop,RandGaussianNoise,RandShiftIntensity,AdjustContrast,\\\nRandAdjustContrast\nfrom monai.transforms import RandCoarseDropout\n# system\nimport sys\nimport os\nimport random\nimport time\n\n# data processing\nimport numpy as np\nimport pandas as pd\n\n# dl librarires\n## torch\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\n\n## transformers\nfrom transformers import get_cosine_schedule_with_warmup\nfrom transformers.optimization import AdamW\n\n## sklearn\nfrom sklearn import model_selection\nfrom sklearn.metrics import roc_auc_score\n\n# wandb for logging\nimport wandb\nfrom kaggle_secrets import UserSecretsClient\n\n# plotting\nimport cv2\nimport matplotlib.pyplot as plt\nfrom IPython import display\n\n# med\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nfrom adamp import AdamP","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:38:24.365837Z","iopub.execute_input":"2021-09-18T19:38:24.366112Z","iopub.status.idle":"2021-09-18T19:38:34.797506Z","shell.execute_reply.started":"2021-09-18T19:38:24.366083Z","shell.execute_reply":"2021-09-18T19:38:34.796485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    # system\n    seed=42\n    no_cuda=False\n    # model\n    use_pretrained = True\n    # cohort to use\n    cohort = 'FLAIR'\n    optimizer_name = 'AdamW'\n    SAM = True\n    img_size=256\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    img_size = (128,128,128)\n    roi_scale = (0.8,0.8,0.8)\n    dropout_rate  = 0.3\n    visualize = True\nclass TrainerConfig:\n    num_epochs = 100\n    batch_size = 8\n    gradient_accumulation_steps = 1\n    \n    # optimizer\n    lr = 3e-4\n    warm_up_ratio = 0.1\n    weight_decay = 0.0005\n\n    # log every log_steps to wandb\n    log_steps = 20\n\n    # environment\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:38:34.799134Z","iopub.execute_input":"2021-09-18T19:38:34.799507Z","iopub.status.idle":"2021-09-18T19:38:34.827189Z","shell.execute_reply.started":"2021-09-18T19:38:34.799467Z","shell.execute_reply":"2021-09-18T19:38:34.826275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n# create 5 stratified folds, optionally pass rng generator \ndef get_folds(df, rng=42):\n    # read df\n    if isinstance(df, str):\n        train_df = pd.read_csv(df)\n    elif isinstance(df, pd.DataFrame):\n        train_df = df\n    else:\n        print(f\"Didn't understand data type for df: {type(df)}\")\n\n    \n\n    # shuffle before split\n    train_df = train_df.sample(frac=1, random_state=rng).reset_index(drop=True)\n\n    # get stratified splits with sklearn\n    kf = model_selection.StratifiedKFold(n_splits=5)\n    # train_df.loc[:, 'fold'] = 0\n    for fold_idx, (_, indices_test) in enumerate(kf.split(X=train_df.BraTS21ID, y=train_df.MGMT_value)):\n        train_df.loc[indices_test, 'fold'] = fold_idx\n    return train_df","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:38:34.828664Z","iopub.execute_input":"2021-09-18T19:38:34.829078Z","iopub.status.idle":"2021-09-18T19:38:34.838712Z","shell.execute_reply.started":"2021-09-18T19:38:34.829039Z","shell.execute_reply":"2021-09-18T19:38:34.837692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# system setup\nseed_everything(CFG.seed)\n\n# setup some wandb params\n# login wandb\nuser_secrets = UserSecretsClient()\nwandb_api = user_secrets.get_secret(\"wandb_api\")\nwandb.login(key=wandb_api)\n\n# config for logging\nconfig_wandb = {\n    'learning_rate': TrainerConfig.lr,\n    'num_epochs': TrainerConfig.num_epochs,\n    'batch_size': TrainerConfig.batch_size,\n    'warm_up_ratio': TrainerConfig.warm_up_ratio,\n    'weight_decay': TrainerConfig.weight_decay,\n}\nwandb_group_name = \"exp\" + wandb.util.generate_id()","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:38:34.840691Z","iopub.execute_input":"2021-09-18T19:38:34.840940Z","iopub.status.idle":"2021-09-18T19:38:36.654606Z","shell.execute_reply.started":"2021-09-18T19:38:34.840916Z","shell.execute_reply":"2021-09-18T19:38:36.653255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BTRCDataset(torch.utils.data.Dataset):\n    def __init__(self, df, data_dir,  transform=None, cohort='FLAIR'):\n        self.df = df\n        self.data_dir = data_dir\n        self.cohort = cohort\n        self.transform = transform\n        \n    def __getitem__(self, idx):\n        # get sample info\n        sample_id, target = self.df.loc[idx].values\n        \n        # get sample path. combination of dir, padded id and cohort\n        sample_dir = os.path.join(self.data_dir, f'{sample_id:05d}', self.cohort)\n        sample_files = os.listdir(sample_dir)\n        \n        # take subset of available images if n_images > 64\n        if len(sample_files) > 64:\n            sample_files = np.random.choice(sample_files, size=64, replace=False)\n        \n        # sort samples\n        sample_files = sorted(sample_files, key=lambda x: int(x[6:-4]))\n        \n        # load images\n        imgs = [self.read_img(os.path.join(sample_dir, path)) for path in sample_files]\n        imgs = np.stack(imgs)\n        \n        # resample images if not enough samples are available\n        if len(sample_files) < 64:\n            indices = sorted(np.random.choice(len(sample_files), size=64, replace=True))\n            imgs = np.stack(imgs[indices])     \n        imgs = np.stack(imgs)\n        if self.transform:\n            imgs = self.transform(imgs)\n#             print(imgs.shape)\n        return imgs.type(torch.float32), torch.tensor(target, dtype=torch.float32)\n\n    def __len__(self):\n        return len(self.df)\n    \n    def read_img(self, path):\n        if path.endswith('dcm'):\n            img = self.read_dicom(path)\n        elif path.endswith('png'):\n            img = self.read_png(path)\n        else:\n            print('unknown file format')\n        img = cv2.resize(img, (CFG.img_size[0], CFG.img_size[0]))\n        return img\n    \n    @staticmethod\n    def read_png(path):\n        img = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n        return img\n    \n    @staticmethod\n    def read_dicom(path):\n        dicom = pydicom.read_file(path)\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n        if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n            data = np.amax(data) - data\n        data = data - np.min(data)\n        data = data / np.max(data)\n        data = (data * 255).astype(np.uint8)\n        return data","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:38:36.656169Z","iopub.execute_input":"2021-09-18T19:38:36.656521Z","iopub.status.idle":"2021-09-18T19:38:36.682740Z","shell.execute_reply.started":"2021-09-18T19:38:36.656486Z","shell.execute_reply":"2021-09-18T19:38:36.680207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\nclass SAM(torch.optim.Optimizer):\n    def __init__(self, params, base_optimizer, rho=0.05, **kwargs):\n        assert rho >= 0.0, f\"Invalid rho, should be non-negative: {rho}\"\n\n        defaults = dict(rho=rho, **kwargs)\n        super(SAM, self).__init__(params, defaults)\n\n        self.base_optimizer = base_optimizer(self.param_groups, **kwargs)\n        self.param_groups = self.base_optimizer.param_groups\n\n    @torch.no_grad()\n    def first_step(self, zero_grad=False):\n        grad_norm = self._grad_norm()\n        for group in self.param_groups:\n            scale = group[\"rho\"] / (grad_norm + 1e-12)\n\n            for p in group[\"params\"]:\n                if p.grad is None: continue\n                e_w = p.grad * scale.to(p)\n                p.add_(e_w)  # climb to the local maximum \"w + e(w)\"\n                self.state[p][\"e_w\"] = e_w\n\n        if zero_grad: self.zero_grad()\n\n    @torch.no_grad()\n    def second_step(self, zero_grad=False):\n        for group in self.param_groups:\n            for p in group[\"params\"]:\n                if p.grad is None: continue\n                p.sub_(self.state[p][\"e_w\"])  # get back to \"w\" from \"w + e(w)\"\n\n        self.base_optimizer.step()  # do the actual \"sharpness-aware\" update\n\n        if zero_grad: self.zero_grad()\n\n    @torch.no_grad()\n    def step(self, closure=None):\n        assert closure is not None, \"Sharpness Aware Minimization requires closure, but it was not provided\"\n        closure = torch.enable_grad()(closure)  # the closure should do a full forward-backward pass\n\n        self.first_step(zero_grad=True)\n        closure()\n        self.second_step()\n\n    def _grad_norm(self):\n        shared_device = self.param_groups[0][\"params\"][0].device  # put everything on the same device, in case of model parallelism\n        norm = torch.norm(\n                    torch.stack([\n                        p.grad.norm(p=2).to(shared_device)\n                        for group in self.param_groups for p in group[\"params\"]\n                        if p.grad is not None\n                    ]),\n                    p=2\n               )\n        return norm","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:38:36.686265Z","iopub.execute_input":"2021-09-18T19:38:36.686686Z","iopub.status.idle":"2021-09-18T19:38:36.982844Z","shell.execute_reply.started":"2021-09-18T19:38:36.686646Z","shell.execute_reply":"2021-09-18T19:38:36.981658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"\n    Computes and stores the average and current value\n    Copied from: https://github.com/pytorch/examples/blob/master/imagenet/main.py\n    \"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\nclass Trainer:\n    def __init__(self, cfg: type(TrainerConfig),\n                 model: torch.nn.Module,\n                 model_path: str,\n                 dataset_train: Dataset = None,\n                 dataset_val: Dataset = None,\n                 wandb_run: wandb.sdk.wandb_run.Run = None):\n        self.cfg = cfg\n        self.model = model\n        self.best_model = None\n        self.model_path = model_path\n        self.wandb_run = wandb_run\n\n        # datasets\n        self.dataset_train = dataset_train\n        self.dataset_eval = dataset_val\n\n        # dataloaders, train/eval is optional\n        kwargs_dataloader = {'batch_size': self.cfg.batch_size, 'num_workers': 2}\n        if self.dataset_eval is not None:\n            self.dataloader_train = DataLoader(self.dataset_train, shuffle=True, **kwargs_dataloader)\n        if self.dataset_eval is not None:\n            self.dataloader_eval = DataLoader(self.dataset_eval, shuffle=False, **kwargs_dataloader)\n\n        # setup loss\n        self.loss_fnc = torch.nn.BCEWithLogitsLoss()\n\n        # if train set is provided, setup training\n        if dataset_train is not None:\n            # init optimizer\n            if CFG.optimizer_name == 'AdamW':\n                self.optimizer = AdamW(self.model.parameters(), self.cfg.lr, weight_decay=self.cfg.weight_decay)            \n            elif CFG.optimizer_name == 'SGD':\n                self.optimizer = torch.optim.SGD(self.model.parameters(), self.cfg.lr, weight_decay=self.cfg.weight_decay, momentum=0.9)\n#             elif CFG.SAM:\n#                 self.optimizer = SAM(model.parameters(), eval(CFG.optimizer_name), lr=self.cfg.lr, weight_decay=self.cfg.weight_decay)\n                \n            # setup lr scheduler\n            _n_steps = cfg.num_epochs * len(self.dataloader_train)\n            self.scheduler = get_cosine_schedule_with_warmup(\n                optimizer=self.optimizer,\n                num_warmup_steps=_n_steps * cfg.warm_up_ratio,\n                num_training_steps=_n_steps,\n            )\n            # call optimizer once so we can properly init the lr in step_train()\n            self.optimizer.zero_grad()\n            self.optimizer.step()\n            \n        else:\n            self.optimizer = None\n            self.scheduler = None\n            \n        self.epoch = 0\n\n    def train(self):\n        print(f'Training model for {self.cfg.num_epochs} epochs.')\n        print('Epoch | train_loss | eval_loss | train_auc | val_auc')\n        train_log = pd.DataFrame(columns=['epoch', 'train_loss', 'eval_loss', 'train_auc', 'eval_auc', 'lr'])\n        \n        best_loss = 1e3\n\n        for epoch in range(self.cfg.num_epochs):\n            train_loss, train_auc = self.step_train()\n            eval_loss, eval_auc = self.step_eval(return_predictions=False)\n            \n            log_item = {\n                'epoch': epoch,\n                'step': epoch*len(self.dataloader_train),\n                'train_loss': train_loss,\n                'eval_loss': eval_loss,\n                'train_auc': train_auc,\n                'eval_auc': eval_auc,\n                'lr': self.optimizer.param_groups[0]['lr']\n            }\n            self.wandb_run.log(log_item)\n            train_log = train_log.append(log_item, ignore_index=True)\n            print(f\"{epoch: <6}|{train_loss: >12.3f}|{eval_loss: >11.3f}|{train_auc: >11.3f}|{eval_auc: >8.3f}\")\n      \n            # checkpointing\n            if eval_loss < best_loss:\n                torch.save(self.model, self.model_path)\n                best_loss = eval_loss\n            self.epoch += 1\n        best_epoch = train_log.eval_loss.idxmin()\n        print(\"Training done. Best model at epoch {} with eval_loss {:3.2f} and auc {:3.2f}\".format(\n            train_log.loc[best_epoch, 'epoch'],\n            train_log.loc[best_epoch, 'eval_loss'],\n            train_log.loc[best_epoch, 'eval_auc']\n        ))\n        return train_log\n\n    def step_train(self):\n        self.model.train()\n        \n        # setup logging\n        loss_agg = AverageMeter()\n        targets = []\n        predictions = []\n        \n        # train one epoch\n        for batch_idx, (x, y) in enumerate(self.dataloader_train):\n            \n            x = x.to(self.cfg.device)\n            y = y.to(self.cfg.device)\n            # forward pass  \n            logits = self.model(x)\n            loss = self.loss_fnc(logits[0].view(-1), y)\n            # backward pass\n            loss.backward()\n            if (batch_idx+1) % self.cfg.gradient_accumulation_steps == 0 or (batch_idx+1) == len(self.dataloader_train):\n                self.optimizer.step()\n                self.optimizer.zero_grad()\n                self.scheduler.step()\n\n            # update loss meter\n            loss_agg.update(loss.item(), self.cfg.batch_size)\n\n            # save preds/targets for roc computation\n            predictions.append(torch.sigmoid(logits[0].view(-1)).detach().cpu().squeeze().numpy())\n            targets.append(y.detach().cpu().squeeze().numpy())\n\n            # log every cfg.log_steps steps\n            if batch_idx > 0 and batch_idx % self.cfg.log_steps == 0:\n                log_item = {\n                    'step': self.epoch*len(self.dataloader_train) + batch_idx,\n                    'train_loss': loss_agg.avg,\n                    'train_auc': roc_auc_score(np.hstack(targets), np.hstack(predictions)),\n                    'lr': self.optimizer.param_groups[0]['lr']\n                }\n                self.wandb_run.log(log_item)\n            \n        # compute auc for whole epoch\n        auc = roc_auc_score(np.hstack(targets), np.hstack(predictions))\n        return loss_agg.avg, auc\n\n    @torch.no_grad()\n    def step_eval(self, return_predictions=False):\n        self.model.eval()\n        loss_agg = AverageMeter()\n        predictions = []\n        targets = []\n        for batch_idx, (x, y) in enumerate(self.dataloader_eval):\n            x = x.to(self.cfg.device)\n            y = y.to(self.cfg.device)\n            logits = self.model(x)\n#             loss = self.loss_fnc(logits.view(-1), y)\n            loss = self.loss_fnc(logits[0].view(-1), y)\n            # update loss meter\n            loss_agg.update(loss.item(), self.cfg.batch_size)\n\n            # optionally return predictions\n            # save preds/targets for roc computation\n            predictions.append(torch.sigmoid(logits[0].view(-1)).detach().cpu().squeeze().numpy())\n            targets.append(y.detach().cpu().squeeze().numpy())\n\n        # compute auc\n        targets = np.hstack(targets)\n        predictions = np.hstack(predictions)\n        auc = roc_auc_score(targets, predictions)\n\n        # setup output\n        if return_predictions:\n            out = (loss_agg.avg, auc, predictions)\n        else:\n            out = (loss_agg.avg, auc)\n        return out","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:38:36.987096Z","iopub.execute_input":"2021-09-18T19:38:36.987553Z","iopub.status.idle":"2021-09-18T19:38:37.020884Z","shell.execute_reply.started":"2021-09-18T19:38:36.987517Z","shell.execute_reply":"2021-09-18T19:38:37.020022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd ..","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:38:37.023819Z","iopub.execute_input":"2021-09-18T19:38:37.024115Z","iopub.status.idle":"2021-09-18T19:38:37.032542Z","shell.execute_reply.started":"2021-09-18T19:38:37.024081Z","shell.execute_reply":"2021-09-18T19:38:37.031631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#SpatialPad(spatial_size=CFG.img_size),\\\n#                          RandCoarseDropout(holes=5,max_holes=10,prob=0.3,max_spatial_size=(28,28,28),spatial_size=(10,10,10)),\\\n#                          RandGaussianNoise(prob=1),\\\n\n#                          CenterScaleCrop(roi_scale=[0.3,0.4,0.5]),\n\n#                          SpatialPad(spatial_size=CFG.img_size),","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:38:37.035284Z","iopub.execute_input":"2021-09-18T19:38:37.035530Z","iopub.status.idle":"2021-09-18T19:38:37.040962Z","shell.execute_reply.started":"2021-09-18T19:38:37.035505Z","shell.execute_reply":"2021-09-18T19:38:37.040240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transform(phase):\n    if phase in ['train']:\n        #Resize((32, 128, 128))\n        trans = Compose([ScaleIntensity(),  AddChannel(), Resize((64, 128, 128)),\\\n                         SpatialPad(spatial_size=CFG.img_size),\n#                          Flip(),\n#                          RandAdjustContrast(),\n#                          Zoom(zoom=0.5),\n#                          RandShiftIntensity(offsets=10),\\\n                         EnsureType()])\n    else:\n        trans = Compose([ScaleIntensity(),  AddChannel(), Resize((64, 128, 128)), SpatialPad(spatial_size=CFG.img_size), EnsureType()])\n        #CenterScaleCrop(roi_scale=CFG.roi_scale),\n        #CenterScaleCrop(roi_scale=CFG.roi_scale),\n    return trans","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:38:37.043611Z","iopub.execute_input":"2021-09-18T19:38:37.043868Z","iopub.status.idle":"2021-09-18T19:38:37.051189Z","shell.execute_reply.started":"2021-09-18T19:38:37.043837Z","shell.execute_reply":"2021-09-18T19:38:37.050267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train = train_df.loc[train_df.fold != 0, ['BraTS21ID', 'MGMT_value']].reset_index(drop=True)\n# df_eval = train_df.loc[train_df.fold == 0, ['BraTS21ID', 'MGMT_value']].reset_index(drop=True)\n# dataset_train = BTRCDataset(df_train, data_dir='../input/rsna-miccai-png/train', transform = get_transform('train'), cohort=CFG.cohort)\n# dataset_eval = BTRCDataset(df_eval, data_dir='../input/rsna-miccai-png/train', transform = get_transform('valid'), cohort=CFG.cohort)","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:38:37.053821Z","iopub.execute_input":"2021-09-18T19:38:37.054066Z","iopub.status.idle":"2021-09-18T19:38:37.061984Z","shell.execute_reply.started":"2021-09-18T19:38:37.054043Z","shell.execute_reply":"2021-09-18T19:38:37.061194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\ndef show_list_img(img):\n    w = 10\n    h = 10\n    fig = plt.figure(figsize=(15, 15))\n    columns = 2\n    rows = 5\n    for i in range(1, columns*rows +1):\n        image = img[0][i+50]\n        fig.add_subplot(rows, columns, i)\n        plt.imshow(image)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:45:33.542744Z","iopub.execute_input":"2021-09-18T19:45:33.543081Z","iopub.status.idle":"2021-09-18T19:45:33.556193Z","shell.execute_reply.started":"2021-09-18T19:45:33.543049Z","shell.execute_reply":"2021-09-18T19:45:33.552433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize(dataloader_train):\n    dem=0\n    for img, label in dataloader_train:\n        show_list_img(img)\n        if dem==1:\n            break\n        dem+=1","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:45:33.777528Z","iopub.execute_input":"2021-09-18T19:45:33.777854Z","iopub.status.idle":"2021-09-18T19:45:33.790234Z","shell.execute_reply.started":"2021-09-18T19:45:33.777822Z","shell.execute_reply":"2021-09-18T19:45:33.789263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !wandb login --relogin","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:45:33.937331Z","iopub.execute_input":"2021-09-18T19:45:33.937627Z","iopub.status.idle":"2021-09-18T19:45:33.943306Z","shell.execute_reply.started":"2021-09-18T19:45:33.937601Z","shell.execute_reply":"2021-09-18T19:45:33.942393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv')\n\n# drop 3 samples, see https://www.kaggle.com/c/rsna-miccai-brain-tumor-radiogenomic-classification/discussion/262046\ntrain_df = train_df[~train_df.BraTS21ID.isin([109, 123, 709])]\n\n# setup cross validation\ntrain_df = get_folds(train_df, rng=CFG.seed)\ntrain_df.to_csv('folds.csv')\n\n# train each fold\nfor fold_idx in range(int(train_df['fold'].max())+1):\n    # datasets\n    df_train = train_df.loc[train_df.fold != fold_idx, ['BraTS21ID', 'MGMT_value']].reset_index(drop=True)\n    df_eval = train_df.loc[train_df.fold == fold_idx, ['BraTS21ID', 'MGMT_value']].reset_index(drop=True)\n    dataset_train = BTRCDataset(df_train, data_dir='../input/rsna-miccai-png/train', transform = get_transform('train'), cohort=CFG.cohort)\n    dataset_eval = BTRCDataset(df_eval, data_dir='../input/rsna-miccai-png/train', transform = get_transform('valid'), cohort=CFG.cohort)\n    if CFG.visualize:\n        visualize(dataset_train)\n#     model = monai.networks.nets.DenseNet121(spatial_dims=3, in_channels=1, out_channels=1).to(CFG.device)\n    model = monai.networks.nets.ViT(spatial_dims=3,in_channels=1, img_size=CFG.img_size, pos_embed='conv', \\\n                                    patch_size = 32, dropout_rate=CFG.dropout_rate-0.2 , \\\n                                    num_layers=24,num_heads=16,mlp_dim=4096,hidden_size=1024,\n                                    num_classes =1,classification=True).to(CFG.device)\n# #     model = monai.networks.nets.DynUNet(spatial_dims=3,in_channels=1, out_channels=1, img_size=CFG.img_size, pos_embed='conv', \\\n# #                                     patch_size = 32, dropout_rate=CFG.dropout_rate , \\\n# #                                     num_layers=24,num_heads=16,mlp_dim=4096,hidden_size=1024,\n# #                                     num_classes =1,classification=True).to(CFG.device)\n#     # setup wandb\n    wandb_run = wandb.init(project=\"kaggle-BTRC\", config=config_wandb, group=wandb_group_name, name=f\"fold{fold_idx}\", job_type=\"finetuning\")\n    wandb_run.define_metric(\"step\")\n    wandb_run.define_metric(\"*\", step_metric=\"step\", step_sync=True)\n    \n    # setup trainer\n    trainer = Trainer(cfg=TrainerConfig,\n                      model=model,\n                      model_path=f'model_fold{fold_idx}.torch',\n                      dataset_train=dataset_train,\n                      dataset_val=dataset_eval,\n                      wandb_run=wandb_run)\n    train_log = trainer.train()\n    train_log.to_csv(f\"fold{fold_idx}_log.csv\")\n    wandb_run.finish()\n","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:48:54.276219Z","iopub.execute_input":"2021-09-18T19:48:54.276563Z","iopub.status.idle":"2021-09-18T19:49:13.889078Z","shell.execute_reply.started":"2021-09-18T19:48:54.276534Z","shell.execute_reply":"2021-09-18T19:49:13.886592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for img,label in dataset_train:\n#     print(img)\n#     break","metadata":{"execution":{"iopub.status.busy":"2021-09-18T19:47:50.112424Z","iopub.execute_input":"2021-09-18T19:47:50.112767Z","iopub.status.idle":"2021-09-18T19:47:50.217961Z","shell.execute_reply.started":"2021-09-18T19:47:50.112737Z","shell.execute_reply":"2021-09-18T19:47:50.217169Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]}]}