{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":" **This notebook attempts to use Pytorch Lightning to speed up the process to modeling.\n A large part of this code was taken from Roland Leuthy the link to his notebook is [here](https://www.kaggle.com/rluethy/efficientnet3d-with-one-mri-type). It's a great notebook and he has managed to get a great score on the leaderboard.**\n\nThis notebook uses [Pytorch Lightning](https://www.pytorchlightning.ai/) if you know Pytorch you should be able to pick this up quite quickly. I'll attempt explain the best I can in this notebook.\n\n","metadata":{}},{"cell_type":"markdown","source":"![](https://images.unsplash.com/photo-1559757175-5700dde675bc?ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&ixlib=rb-1.2.1&auto=format&fit=crop&w=889&q=80)","metadata":{}},{"cell_type":"markdown","source":"# **Dependencies**","metadata":{}},{"cell_type":"code","source":"import torch \nimport torch.nn as nn\nimport cv2\nimport pytorch_lightning as pl\nfrom pytorch_lightning.core.lightning import LightningModule\n\nfrom torch.utils.data import Dataset,DataLoader \nimport pydicom\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport sys\nimport random\nimport glob\nimport time\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\n\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-02T02:36:02.080181Z","iopub.execute_input":"2021-08-02T02:36:02.080687Z","iopub.status.idle":"2021-08-02T02:36:06.711072Z","shell.execute_reply.started":"2021-08-02T02:36:02.080578Z","shell.execute_reply":"2021-08-02T02:36:06.709925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The goal of this notebook is to build a Functioning model for the **Flair** MRI type. After building a model for each MRI type one should be able to make a separate notebook combining all of them. ","metadata":{}},{"cell_type":"code","source":"if os.path.exists(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification\"):\n    data_directory = '../input/rsna-miccai-brain-tumor-radiogenomic-classification'\n    pytorch3dpath = \"../input/efficientnet3d/EfficientNet-PyTorch-3D-master\"\nelse:\n    data_directory = '/media/roland/data/kaggle/rsna-miccai-brain-tumor-radiogenomic-classification'\n    pytorch3dpath = \"EfficientNet-PyTorch-3D\"\n    \nmri_types = ['FLAIR','T1w','T1wCE','T2w']\nSIZE = 256\nNUM_IMAGES = 64\n","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:36:06.712858Z","iopub.execute_input":"2021-08-02T02:36:06.71319Z","iopub.status.idle":"2021-08-02T02:36:06.720491Z","shell.execute_reply.started":"2021-08-02T02:36:06.713152Z","shell.execute_reply":"2021-08-02T02:36:06.719657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Data**","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv\")\ndf = df.loc[df['BraTS21ID'] != 109]\ndf = df.loc[df['BraTS21ID'] != 709]\ndf = df.reset_index(drop=True)\ndf","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:36:06.721988Z","iopub.execute_input":"2021-08-02T02:36:06.722432Z","iopub.status.idle":"2021-08-02T02:36:06.786472Z","shell.execute_reply.started":"2021-08-02T02:36:06.722386Z","shell.execute_reply":"2021-08-02T02:36:06.785448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = pd.read_csv(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:36:06.788339Z","iopub.execute_input":"2021-08-02T02:36:06.788769Z","iopub.status.idle":"2021-08-02T02:36:06.799294Z","shell.execute_reply.started":"2021-08-02T02:36:06.788727Z","shell.execute_reply":"2021-08-02T02:36:06.798482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = df[[\"BraTS21ID\"]]\ny = df[[\"MGMT_value\"]]\ntrain_x,test_x,train_y,test_y = train_test_split(X,y,test_size = 0.25 , random_state = 42,stratify=y[\"MGMT_value\"])\ntest_x,val_x,test_y,val_y = train_test_split(test_x,test_y,test_size = 0.25 , random_state = 42, stratify=test_y[\"MGMT_value\"])\nprint(len(train_x) , \"\\n\" , len(test_x) , \"\\n\" , len(val_x))","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:36:06.800504Z","iopub.execute_input":"2021-08-02T02:36:06.801107Z","iopub.status.idle":"2021-08-02T02:36:06.819866Z","shell.execute_reply.started":"2021-08-02T02:36:06.801077Z","shell.execute_reply":"2021-08-02T02:36:06.818687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Data Preprocessing**","metadata":{}},{"cell_type":"markdown","source":"The preprocessing will convert each FLAIR dicom file for that patient into a 3D image of dimensions 64,256,256","metadata":{}},{"cell_type":"code","source":"def load_dicom_image(path, img_size=SIZE):\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    if np.min(data)==np.max(data):\n        data = np.zeros((img_size,img_size))\n        return data\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    \n    #data = (data * 255).astype(np.uint8)\n    data = cv2.resize(data, (img_size, img_size))\n    return data\n\ndef load_dicom_images_3d(scan_id, num_imgs=NUM_IMAGES, img_size=SIZE, mri_type=\"FLAIR\", split=\"train\"):\n\n    files = sorted(glob.glob(f\"{data_directory}/{split}/{scan_id}/{mri_type}/*.dcm\"))\n    \n    middle = len(files)//2\n    num_imgs2 = num_imgs//2\n    p1 = max(0, middle - num_imgs2)\n    p2 = min(len(files), middle + num_imgs2)\n    img3d = np.stack([load_dicom_image(f) for f in files[p1:p2]]).T \n    if img3d.shape[-1] < num_imgs:\n        n_zero = np.zeros((img_size, img_size, num_imgs - img3d.shape[-1]))\n        img3d = np.concatenate((img3d,  n_zero), axis = -1)\n            \n    return np.expand_dims(img3d,0)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:36:06.821228Z","iopub.execute_input":"2021-08-02T02:36:06.821778Z","iopub.status.idle":"2021-08-02T02:36:06.831883Z","shell.execute_reply.started":"2021-08-02T02:36:06.821742Z","shell.execute_reply":"2021-08-02T02:36:06.831123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load_dicom_images_3d(\"00002\").shape","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:36:06.833234Z","iopub.execute_input":"2021-08-02T02:36:06.833736Z","iopub.status.idle":"2021-08-02T02:36:07.809569Z","shell.execute_reply.started":"2021-08-02T02:36:06.833704Z","shell.execute_reply":"2021-08-02T02:36:07.808568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Dataset**","metadata":{}},{"cell_type":"code","source":"class RSNA_Dataset(Dataset):\n    def __init__(self, paths, targets=None, mri_type=\"Flair\", label_smoothing=0.0, split=\"train\"):\n        self.paths = paths\n        self.targets = targets\n        self.mri_type = mri_type\n        self.label_smoothing = label_smoothing\n        self.split = split\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        scan_id = self.paths[index]\n        if self.targets is None:\n            data = load_dicom_images_3d(str(scan_id).zfill(5),split = \"test\")\n        else:\n            data = load_dicom_images_3d(str(scan_id).zfill(5))\n\n        if self.targets is None:\n            return torch.tensor(data).float()\n        else:\n            y = torch.tensor(abs(self.targets[index]-self.label_smoothing), dtype=torch.float)\n            return torch.tensor(data).float(),y","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:36:07.812101Z","iopub.execute_input":"2021-08-02T02:36:07.812416Z","iopub.status.idle":"2021-08-02T02:36:07.821064Z","shell.execute_reply.started":"2021-08-02T02:36:07.812386Z","shell.execute_reply":"2021-08-02T02:36:07.820308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = RSNA_Dataset(\n                train_x[\"BraTS21ID\"].values,\n                train_y[\"MGMT_value\"].values\n)\n\ntest_dataset = RSNA_Dataset(\n                    val_x[\"BraTS21ID\"].values,\n                    val_y[\"MGMT_value\"].values\n)\n\nvalidation_dataset = RSNA_Dataset(\n                    test_x[\"BraTS21ID\"].values,\n                    test_y[\"MGMT_value\"].values\n)\n\npredict_dataset = RSNA_Dataset(\n                    sample_df[\"BraTS21ID\"].values,\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:36:07.822955Z","iopub.execute_input":"2021-08-02T02:36:07.82362Z","iopub.status.idle":"2021-08-02T02:36:07.836411Z","shell.execute_reply.started":"2021-08-02T02:36:07.823586Z","shell.execute_reply":"2021-08-02T02:36:07.835435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"you could directly send the code to the trainer but I find using the lightning datamodule to be more visually pleasing and cleaner","metadata":{}},{"cell_type":"code","source":"class RSNA_DataModule(pl.LightningDataModule):\n    def __init__(self):\n        super().__init__()\n        self.train = train_dataset\n        self.val = validation_dataset\n        self.test = test_dataset\n        self.predict = predict_dataset\n        \n    def train_dataloader(self):\n        return DataLoader(self.train,batch_size = 20,shuffle = True,num_workers=1)\n    def val_dataloader(self):  \n        return DataLoader(self.val,batch_size = 20,shuffle = False,num_workers=1)\n    def test_dataloader(self):\n        return DataLoader(self.test,batch_size = 22,shuffle = False,num_workers=1)\n    def predict_dataloader(self):\n        return DataLoader(self.predict,batch_size = 1,shuffle = False,num_workers=1)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:36:07.837778Z","iopub.execute_input":"2021-08-02T02:36:07.838248Z","iopub.status.idle":"2021-08-02T02:36:07.851308Z","shell.execute_reply.started":"2021-08-02T02:36:07.838219Z","shell.execute_reply":"2021-08-02T02:36:07.850536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sanity Check: To check the data we're sending to the model","metadata":{}},{"cell_type":"code","source":"image , label  = next(iter(DataLoader(train_dataset,batch_size = 1,shuffle = True)))\nprint(image,label)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:36:07.852525Z","iopub.execute_input":"2021-08-02T02:36:07.852969Z","iopub.status.idle":"2021-08-02T02:36:08.578155Z","shell.execute_reply.started":"2021-08-02T02:36:07.852928Z","shell.execute_reply":"2021-08-02T02:36:08.577058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"we'll be using a pretrained EfficentNet Model for our model and change the number of classes to 1 since we're going to be using binary classification","metadata":{}},{"cell_type":"code","source":"package_path = \"../input/efficientnet3d/EfficientNet-PyTorch-3D-master\"\nsys.path.append(package_path)\nfrom efficientnet_pytorch_3d import EfficientNet3D\nneural_network = EfficientNet3D.from_name(\"efficientnet-b1\", override_params={'num_classes': 1}, in_channels=1) ","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:36:08.579449Z","iopub.execute_input":"2021-08-02T02:36:08.579729Z","iopub.status.idle":"2021-08-02T02:36:08.721652Z","shell.execute_reply.started":"2021-08-02T02:36:08.579702Z","shell.execute_reply":"2021-08-02T02:36:08.720476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The cell below is a functoin that returns the auc score on the validation. We're Using auc score since it;s the metric thats being used with the competition. The metrics auc and roc have been explained very well in this [video](https://www.youtube.com/watch?v=4jRBRDbJemM&vl=en)","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_curve,auc\n\nprobs = nn.Sigmoid() # Since we're using binary cross entropy we use the Sigmoid function to convert the Logits into probabilities\n\ndef get_score(y_pred,y):\n    probabilities = []\n    for x in y_pred:\n        prob = probs(x)\n        top_p, top_class = prob.topk(1, dim = -1)\n        probabilities.append(float(top_p))\n    y = [float(t) for t in y]\n    logistic_fpr , logistic_tpr,_ = roc_curve(y , probabilities)\n    aoc_score = auc(logistic_fpr , logistic_tpr)\n    return aoc_score","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:36:08.723075Z","iopub.execute_input":"2021-08-02T02:36:08.723407Z","iopub.status.idle":"2021-08-02T02:36:08.730638Z","shell.execute_reply.started":"2021-08-02T02:36:08.72337Z","shell.execute_reply":"2021-08-02T02:36:08.729629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The model, optimizers are all set up in the lightning Module class. Notice the \"auc_score\" being logged in the validation step we're going to be using that to monitor our models performance.","metadata":{}},{"cell_type":"code","source":"class RSNA_Model(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n        self.neural_net = neural_network\n        \n    def forward(self,x):\n        return self.neural_net(x)\n    \n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.parameters() , lr = 1e-4)\n        sch = torch.optim.lr_scheduler.StepLR(optimizer, step_size = 20, gamma=0.5, last_epoch=-1, verbose=False)\n        return {\n        \"optimizer\": optimizer,\n        \"lr_scheduler\": {\n            \"scheduler\": sch,\n          #  \"monitor\": \"\",\n        },\n    }\n    \n    def training_step(self,batch,batch_idx):\n        x,y = batch\n        y_pred = self(x)\n        y = y.unsqueeze(-1)\n        loss = torch.nn.functional.binary_cross_entropy_with_logits(y_pred,y)\n        return loss\n    \n    def validation_step(self,batch,batch_idx):\n        x,y = batch\n        y_pred = self(x)\n        y = y.unsqueeze(-1)\n        loss = torch.nn.functional.binary_cross_entropy_with_logits(y_pred,y)\n        self.log(\"auc_score\" ,get_score(y_pred,y) )\n        return loss\n    def test_step(self,batch,batch_idx):\n        x,y = batch\n        y_pred = self(x)\n        y = y.unsqueeze(-1)\n        loss = torch.nn.functional.binary_cross_entropy_with_logits(y_pred,y)\n        self.log(\"test_loss : \" , loss)\n        return loss","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:36:08.7321Z","iopub.execute_input":"2021-08-02T02:36:08.732691Z","iopub.status.idle":"2021-08-02T02:36:08.745988Z","shell.execute_reply.started":"2021-08-02T02:36:08.732647Z","shell.execute_reply":"2021-08-02T02:36:08.744897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Training**","metadata":{}},{"cell_type":"markdown","source":"the cell below will will passes to the trainer and will return the model with the highest \"auc_score\"","metadata":{}},{"cell_type":"code","source":"from pytorch_lightning.callbacks import ModelCheckpoint\ncheckpoint_callback = ModelCheckpoint(\n                            monitor = \"auc_score\",\n                            mode = \"max\",\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:39:14.592927Z","iopub.execute_input":"2021-08-02T02:39:14.593412Z","iopub.status.idle":"2021-08-02T02:39:14.598861Z","shell.execute_reply.started":"2021-08-02T02:39:14.593371Z","shell.execute_reply":"2021-08-02T02:39:14.597808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I have already used the model and made a checkpoint. You can load checkpoints so you won't have to resume training from scratch. The best part about lightning is that you only have to declare the number of gpus you want to use and Lightning will handle the rest. Note that 15 epochs will take around 45mins on a gpu and 6 hours on a cpu.","metadata":{}},{"cell_type":"code","source":"%%time\nfrom pytorch_lightning import Trainer\nmodel = RSNA_Model()\nmodule = RSNA_DataModule()\nmodel.load_from_checkpoint('../input/d/aristotle609/efficient3d-checkpoint/FLAIR-Best_Checkpoint.ckpt')\ntrainer = Trainer(max_epochs=15,gpus = 1,  callbacks = [checkpoint_callback])\ntrainer.fit(model,module)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T02:39:14.61418Z","iopub.execute_input":"2021-08-02T02:39:14.61501Z","iopub.status.idle":"2021-08-02T03:04:21.051401Z","shell.execute_reply.started":"2021-08-02T02:39:14.614945Z","shell.execute_reply":"2021-08-02T03:04:21.050177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"you can test the trainer below. For more accurate results I would recommend increasing the test size.","metadata":{}},{"cell_type":"code","source":"result = trainer.test()\nprint(result)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T03:04:21.056419Z","iopub.execute_input":"2021-08-02T03:04:21.056779Z","iopub.status.idle":"2021-08-02T03:05:10.385409Z","shell.execute_reply.started":"2021-08-02T03:04:21.056741Z","shell.execute_reply":"2021-08-02T03:05:10.383915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Predictions**","metadata":{}},{"cell_type":"code","source":"predictons = trainer.predict()","metadata":{"execution":{"iopub.status.busy":"2021-08-02T03:05:10.387312Z","iopub.execute_input":"2021-08-02T03:05:10.387892Z","iopub.status.idle":"2021-08-02T03:06:25.181171Z","shell.execute_reply.started":"2021-08-02T03:05:10.38785Z","shell.execute_reply":"2021-08-02T03:06:25.180367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probabilities = []\nfor x in predictons:\n    prob = probs(x)\n    top_p, top_class = prob.topk(1, dim = 1)\n    probabilities.append(float(top_p))","metadata":{"execution":{"iopub.status.busy":"2021-08-02T03:06:25.18309Z","iopub.execute_input":"2021-08-02T03:06:25.183418Z","iopub.status.idle":"2021-08-02T03:06:25.19116Z","shell.execute_reply.started":"2021-08-02T03:06:25.183378Z","shell.execute_reply":"2021-08-02T03:06:25.190106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.save_checkpoint(\"FLAIR-Best_Checkpoint.ckpt\")# this will save the checkpoint","metadata":{"execution":{"iopub.status.busy":"2021-08-02T03:06:25.192323Z","iopub.execute_input":"2021-08-02T03:06:25.192708Z","iopub.status.idle":"2021-08-02T03:06:25.578487Z","shell.execute_reply.started":"2021-08-02T03:06:25.192675Z","shell.execute_reply":"2021-08-02T03:06:25.577396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nshutil.rmtree(\"./lightning_logs\") # you might want to keep the logs if you want to plot them on a graph since I won't be using them I have deleted them","metadata":{"execution":{"iopub.status.busy":"2021-08-02T03:06:25.579806Z","iopub.execute_input":"2021-08-02T03:06:25.580111Z","iopub.status.idle":"2021-08-02T03:06:25.605094Z","shell.execute_reply.started":"2021-08-02T03:06:25.58008Z","shell.execute_reply":"2021-08-02T03:06:25.604314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = {\n    'BraTS21ID' : list(sample_df[\"BraTS21ID\"]),\n    'MGMT_value' : probabilities\n}\nsubmission = pd.DataFrame(data)\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T03:06:25.606389Z","iopub.execute_input":"2021-08-02T03:06:25.606938Z","iopub.status.idle":"2021-08-02T03:06:25.648049Z","shell.execute_reply.started":"2021-08-02T03:06:25.606905Z","shell.execute_reply":"2021-08-02T03:06:25.646652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(pd.read_csv(\"./submission.csv\"))\nsns.displot(submission[\"MGMT_value\"])","metadata":{"execution":{"iopub.status.busy":"2021-08-02T03:06:25.650253Z","iopub.execute_input":"2021-08-02T03:06:25.6506Z","iopub.status.idle":"2021-08-02T03:06:26.105037Z","shell.execute_reply.started":"2021-08-02T03:06:25.65057Z","shell.execute_reply":"2021-08-02T03:06:26.104245Z"},"trusted":true},"execution_count":null,"outputs":[]}]}