{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 2D EfficientNet + InceptionResNet (soon) Benchmark \n\n#### In this notebook, we're going to go through:\n- A brief exploration of the data,\n- What MRI scans are, what [the different scan modes (FLAIR, T1w, T1wCE, T2w) mean](https://case.edu/med/neurology/NR/MRI%20Basics.htm), and how they provide information,\n- A quick 2-hour benchmark model.\n\n##### I hope this exploration helps whoever's reading this - I'm still a beginner myself, so any comments/feedback would be appreciated.\n\n#### if you feel this notebook was helpful, please don't forget to upvote! 😊\n\n#### Credits to Johnathan Basomi for the Brain Tumor dataset in [PNG form.](https://www.kaggle.com/c/rsna-miccai-brain-tumor-radiogenomic-classification/discussion/253000)","metadata":{}},{"cell_type":"markdown","source":"### Load Dataset","metadata":{}},{"cell_type":"code","source":"# Import libraries\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Define data paths\nTRAIN_DATA_PATH = '../input/rsna-miccai-png/train/'\nTEST_DATA_PATH = '../input/rsna-miccai-png/test/'\n\ntrain_labels = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv')\n\n# Mark on the dataframe whether there are missing values for one or more MRI modes\nimport os\n\nimg_modes = set(['T1w','T1wCE','T2w','FLAIR'])\n\nmode_valid = {'T1w':[],\n             'T1wCE':[],\n             'T2w':[],\n             'FLAIR':[]}\n\nfor patient_id in train_labels.iloc:\n    subdirs = set(os.listdir(TRAIN_DATA_PATH + str(patient_id['BraTS21ID']).zfill(5)))\n    for img_mode in img_modes: mode_valid[img_mode].append(int(img_mode in subdirs))\ntrain_labels = train_labels.merge(pd.DataFrame(mode_valid),left_index=True,right_index=True)\n\ntrain_labels","metadata":{"execution":{"iopub.status.busy":"2021-07-27T23:57:36.052159Z","iopub.execute_input":"2021-07-27T23:57:36.052698Z","iopub.status.idle":"2021-07-27T23:57:38.754315Z","shell.execute_reply.started":"2021-07-27T23:57:36.052605Z","shell.execute_reply":"2021-07-27T23:57:38.753342Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Visualize Class Label Distribution:","metadata":{}},{"cell_type":"code","source":"sns.histplot(train_labels['MGMT_value'], bins=2, shrink=.8)\nplt.title(\"Class Distribution: Positive vs Negative\")\nprint(\"# of total data points: \",len(train_labels))","metadata":{"execution":{"iopub.status.busy":"2021-07-27T23:57:38.756053Z","iopub.execute_input":"2021-07-27T23:57:38.756489Z","iopub.status.idle":"2021-07-27T23:57:38.994069Z","shell.execute_reply.started":"2021-07-27T23:57:38.756447Z","shell.execute_reply":"2021-07-27T23:57:38.993127Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Four modes of MRI scans are in our dataset in 3d image format (many images as vertical slices).\n#### According to https://case.edu/med/neurology/NR/MRI%20Basics.htm, the scans mean:\n\n``` \nT1w: T1-Weighted MRI scan. The Cerespinal fluid (fluid outside the brain cavity) will be dark, and brain tissue will be white. Tumors will typically show up as dark patches.\n\nT1wCE: Contrast-material enhanced. Also known as a MRI scan with [Gadolinium infusion](https://www.insideradiology.com.au/gadolinium-contrast-medium/). Blood vessels will show up as white lines.\n\nT2w: T2-Weighted MRI scan. Cerespinal fluid will appear white, and the brain tissue will appear dark.\n\nFLAIR: Both brain matter and cerespinal fluid will appear dark, but abnormalities tend to appear light.\n```\n-----------\n#### Visualizing Depth Distribution:","metadata":{}},{"cell_type":"code","source":"import os\nfrom tqdm import tqdm, trange\n\n# Explore data structure\n\nimg_modes = ['T1w','T1wCE','T2w','FLAIR']\nscans_per_mode = {'T1w':[],\n                     'T1wCE':[],\n                     'T2w':[],\n                     'FLAIR':[]}\n\nfor PATIENT in tqdm(os.listdir(TRAIN_DATA_PATH)):\n    for img_mode in img_modes:\n        try: scans_per_mode[img_mode].append(len(os.listdir(TRAIN_DATA_PATH + PATIENT + '/' + img_mode)))\n        except: scans_per_mode[img_mode].append(0)\n            \nfig, axes = plt.subplots(4, figsize=(10,10), sharex=True)\nfig.suptitle(\"# of vertical slices per MRI category\")\nfor index, img_mode in enumerate(img_modes):\n    sns.distplot(scans_per_mode[img_mode],ax=axes[index], color=['cyan','blue','green','violet'][index])\n    axes[index].set_title(img_mode)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-27T23:57:38.996178Z","iopub.execute_input":"2021-07-27T23:57:38.99659Z","iopub.status.idle":"2021-07-27T23:58:40.564894Z","shell.execute_reply.started":"2021-07-27T23:57:38.996546Z","shell.execute_reply":"2021-07-27T23:58:40.563941Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Load image slices in np array format:\nImage augmentation from [🧠 MRI data augmentation\n](https://www.kaggle.com/furcifer/mri-data-augmentation-pipeline)","metadata":{}},{"cell_type":"code","source":"import imgaug as ia\nimport imgaug.augmenters as iaa\n\ndef image_augment(data):\n    sometimes = lambda aug: iaa.Sometimes(0.2, aug)\n\n    seq = iaa.Sequential(\n        [\n            # apply the following augmenters to most images\n            iaa.Fliplr(0.5), # horizontally flip 50% of all images\n            iaa.Flipud(0.2), # vertically flip 20% of all images\n            # crop images by -5% to 10% of their height/width\n            sometimes(iaa.CropAndPad(\n                percent=(-0.05, 0.05),\n                pad_mode=ia.ALL,\n                pad_cval=(0, 255)\n            )),\n            sometimes(iaa.Affine(\n                scale={\"x\": (0.7, 1.3), \"y\": (0.7, 1.3)}, # scale images to 80-120% of their size, individually per axis\n                translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)\n                rotate=(-45, 45), # rotate by -45 to +45 degrees\n                shear=(-16, 16), # shear by -16 to +16 degrees\n                order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)\n                cval=(0, 255), # if mode is constant, use a cval between 0 and 255\n                mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)\n            )),\n            # execute 0 to 5 of the following (less important) augmenters per image\n            # don't execute all of them, as that would often be way too strong\n            iaa.SomeOf((0, 5),\n                [\n                    sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation\n                    iaa.OneOf([\n                        iaa.GaussianBlur((0, 2.0)), # blur images with a sigma between 0 and 3.0\n                        iaa.AverageBlur(k=(2, 5)), # blur image using local means with kernel sizes between 2 and 7\n                        iaa.MedianBlur(k=(3, 7)), # blur image using local medians with kernel sizes between 2 and 7\n                    ]),\n                    iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images\n                    iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images\n                    # search either for all edges or for directed edges,\n                    # blend the result with the original image using a blobby mask\n                    iaa.SimplexNoiseAlpha(iaa.OneOf([\n                        iaa.EdgeDetect(alpha=(0.5, 1.0)),\n                        iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)),\n                    ])),\n                    iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5), # add gaussian noise to images\n                    iaa.OneOf([\n                        iaa.Dropout((0.01, 0.1), per_channel=0.5), # randomly remove up to 10% of the pixels\n                        iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),\n                    ]),\n                    iaa.Invert(0.05, per_channel=True), # invert color channels\n                    iaa.Add((-10, 10), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)\n\n                    # either change the brightness of the whole image (sometimes\n                    # per channel) or change the brightness of subareas\n                    iaa.OneOf([\n                        iaa.Multiply((0.5, 1.5), per_channel=0.5),\n                        iaa.FrequencyNoiseAlpha(\n                            exponent=(-4, 0),\n                            first=iaa.Multiply((0.5, 1.5), per_channel=True),\n                            second=iaa.LinearContrast((0.5, 2.0))\n                        )\n                    ]),\n                    iaa.LinearContrast((0.5, 2.0), per_channel=0.5), # improve or worsen the contrast\n                    sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)\n                    sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))), # sometimes move parts of the image around\n                    sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))\n                ],\n                random_order=True\n            )\n        ],\n        random_order=True\n    )\n    return seq(images=data)","metadata":{"execution":{"iopub.status.busy":"2021-07-27T23:58:40.566711Z","iopub.execute_input":"2021-07-27T23:58:40.56726Z","iopub.status.idle":"2021-07-27T23:58:41.71026Z","shell.execute_reply.started":"2021-07-27T23:58:40.567216Z","shell.execute_reply":"2021-07-27T23:58:41.709285Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_DIMS = (300,300)\nIMG_CHANNELS = 3","metadata":{"execution":{"iopub.status.busy":"2021-07-27T23:58:41.711808Z","iopub.execute_input":"2021-07-27T23:58:41.712235Z","iopub.status.idle":"2021-07-27T23:58:41.717082Z","shell.execute_reply.started":"2021-07-27T23:58:41.712173Z","shell.execute_reply":"2021-07-27T23:58:41.716005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, PIL\nfrom PIL import Image\nfrom scipy.ndimage import zoom\n    \ndef load_array(path: str, slice_name:str):\n    return np.array(Image.open(path+'/'+slice_name).resize(IMG_DIMS, resample=PIL.Image.BILINEAR))\n\ndef load_array_3d(filename: str, img_mode: str, augment=False):\n    \"\"\"\n    Inputs\n    ----------\n    Filename: Id of scan being loaded (without any further path extensions)\n        etc: \"00000\" -> Patient with ID 00000\n    Img_mode: MRI Scan mode\n        etc: one of ['T1w','T1wCE','T2w','FLAIR']\n        \n    Returns\n    -----------\n    3d array of shape (x, y, num_slices)\n    \"\"\"\n    path = TRAIN_DATA_PATH+filename+'/'+img_mode\n\n    data = []\n    img_paths_in_order = sorted(os.listdir(path), key=lambda x: int(x.split('-')[1].split('.png')[0]))\n    \n    for slice_name in img_paths_in_order:\n        data.append(load_array(path, slice_name))\n            \n    if augment: data = image_augment(data)\n    return np.swapaxes(np.array(data), 0, 2)\n\nimport time\nstarttime = time.time()\nsample = load_array_3d('00000','FLAIR')\nprint(\"Time to load array of %i images:\"%(len(os.listdir('../input/rsna-miccai-png/train/00000/FLAIR'))), time.time()-starttime)","metadata":{"execution":{"iopub.status.busy":"2021-07-27T23:58:41.719075Z","iopub.execute_input":"2021-07-27T23:58:41.719771Z","iopub.status.idle":"2021-07-27T23:58:43.968613Z","shell.execute_reply.started":"2021-07-27T23:58:41.719724Z","shell.execute_reply":"2021-07-27T23:58:43.96763Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize\n\ndef plot_slices(sample, num_rows, num_columns, from_train_gen = False, augment=False):\n    f, axarr = plt.subplots(\n        num_rows,\n        num_columns,\n        figsize=(num_columns*2, num_rows*2)\n    )\n    f.suptitle(\"MRI Slices Visualized\\nAugmentation: %s\"%str(augment))\n    \n    if from_train_gen: num_slices = sample.shape[0]\n    else: num_slices = sample.shape[2]\n    slices_to_increment = num_slices / (num_rows * num_columns)\n    for r in range(num_rows):\n        for c in range(num_columns):\n            slice_index = np.floor((r*num_columns + c) * slices_to_increment)\n            if from_train_gen: axarr[r, c].imshow(sample[slice_index.astype(int)],cmap='gray')\n            else: axarr[r, c].imshow(sample[:,:,slice_index.astype(int)],cmap='gray')\n                \n            axarr[r, c].axis('off')\n            if not from_train_gen: axarr[r, c].set_title(\"Slice %i\"%(slice_index+1))\n    plt.subplots_adjust(wspace=0, hspace=0, left=0, right=1, bottom=0, top=1)\n    plt.tight_layout()\n    plt.show()\n    \nsample = load_array_3d('00000','FLAIR',augment=False)\nplot_slices(sample, 4, 4); del sample\nsample = load_array_3d('00000','FLAIR',augment=True)\nplot_slices(sample, 4, 4, augment=True); del sample","metadata":{"execution":{"iopub.status.busy":"2021-07-27T23:58:43.970077Z","iopub.execute_input":"2021-07-27T23:58:43.970481Z","iopub.status.idle":"2021-07-27T23:58:50.92487Z","shell.execute_reply.started":"2021-07-27T23:58:43.97044Z","shell.execute_reply":"2021-07-27T23:58:50.923956Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data flow\nfrom sklearn.model_selection import train_test_split\nfrom time import sleep\nimport random\nimport gc\n\ntrain_df, val_df = train_test_split(train_labels, test_size=0.2, random_state= 420)\n\n# Multiprocessing\nimport multiprocessing as mp\nMULTIPROCESS = False\nNCORE = 4\n\ndef load_array_and_labels(patient_id, img_mode, augment, testing = False):\n    if testing:\n        \n        path = TEST_DATA_PATH+str(int(patient_id['BraTS21ID'])).zfill(5)+'/'+img_mode\n        img_paths = os.listdir(path)\n        random.shuffle(img_paths)\n\n        batch_data = []\n\n        for img_path in img_paths:\n            batch_data.append(load_array(path, img_path))\n\n        if augment: batch_data = image_augment(batch_data)\n        return batch_data\n    \n    else:\n        path = TRAIN_DATA_PATH+str(int(patient_id['BraTS21ID'])).zfill(5)+'/'+img_mode\n        img_paths = os.listdir(path)\n        random.shuffle(img_paths)\n\n        batch_data = []\n\n        for img_path in img_paths:\n            batch_data.append(load_array(path, img_path))\n\n        if augment: batch_data = image_augment(batch_data)\n        return batch_data, [patient_id['MGMT_value']] * len(batch_data)\n\ndef data_generator(df, img_mode: str, batch_size = 8, db_mult = 20, shuffle_per = 10, augment=True):\n    img_batch, target_batch = [], []\n    while True:\n        # Shuffle dataframe\n        df = df.sample(frac=1).reset_index(drop=True)\n        shuffle_index = 0\n        \n        # Multiprocess\n        if MULTIPROCESS: \n            pool = mp.Pool(NCORE)\n            patient_id_queue = []\n        for patient_id in df.iloc:      \n            if patient_id[img_mode] == 0: continue\n              \n            if MULTIPROCESS: \n                patient_id_queue.append(patient_id)    \n                if len(patient_id_queue) >= NCORE:\n                    jobs = []\n                    for task in patient_id_queue:\n                        jobs.append(pool.apply_async(load_array_and_labels, (task,img_mode,augment)))\n                    for job in jobs:\n                        preprocessed_data = job.get(timeout=30)\n                        img_batch.extend(preprocessed_data[0])\n                        target_batch.extend(preprocessed_data[1])\n\n                    patient_id_queue = []    \n            else:\n                preprocessed_data = load_array_and_labels(patient_id,img_mode, augment)\n                img_batch.extend(preprocessed_data[0])\n                target_batch.extend(preprocessed_data[1])\n                \n            while len(target_batch) > (batch_size * db_mult): \n                if shuffle_index % shuffle_per == 0: \n                    c = list(zip(img_batch, target_batch))\n                    random.shuffle(c)\n                    img_batch, target_batch = zip(*c)\n                    img_batch, target_batch = list(img_batch), list(target_batch)\n                    del c\n                    \n                if IMG_CHANNELS == 1:\n                    yield np.array(img_batch[:batch_size])[:,:,:,np.newaxis], np.array(target_batch[:batch_size])\n                else:\n                    yield np.repeat(np.array(img_batch[:batch_size])[:,:,:,np.newaxis], IMG_CHANNELS, -1), np.array(target_batch[:batch_size])\n\n                img_batch = img_batch[batch_size:]\n                target_batch = target_batch[batch_size:]\n                shuffle_index += 1\n                gc.collect()\n                                \n        pool.close()","metadata":{"execution":{"iopub.status.busy":"2021-07-27T23:58:50.927793Z","iopub.execute_input":"2021-07-27T23:58:50.928348Z","iopub.status.idle":"2021-07-27T23:58:51.113076Z","shell.execute_reply.started":"2021-07-27T23:58:50.928307Z","shell.execute_reply":"2021-07-27T23:58:51.112156Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def timeit():\n    starttime = time.time()\n    datagen = data_generator(train_df,batch_size=64,img_mode='FLAIR')\n    \n    times = []\n    for i in range(6):\n        t = next(iter(datagen)); del t\n        if i % 5 == 0: times.append(\"Batch %i :\"%i +str(round(time.time()-starttime,2)))\n    [print(i) for i in times]\n    return times[-1].split(':')[1]\n\nMULTIPROCESS = False\nprint(\"Without multiprocessing:\",timeit())\nMULTIPROCESS = True\nprint(\"With multiprocessing:\",timeit())","metadata":{"execution":{"iopub.status.busy":"2021-07-27T23:58:51.116321Z","iopub.execute_input":"2021-07-27T23:58:51.116675Z","iopub.status.idle":"2021-07-27T23:59:35.523365Z","shell.execute_reply.started":"2021-07-27T23:58:51.116643Z","shell.execute_reply":"2021-07-27T23:59:35.522397Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Efficientnet Benchmark","metadata":{}},{"cell_type":"code","source":"!pip install -q efficientnet","metadata":{"execution":{"iopub.status.busy":"2021-07-27T23:59:35.525019Z","iopub.execute_input":"2021-07-27T23:59:35.525413Z","iopub.status.idle":"2021-07-27T23:59:44.530708Z","shell.execute_reply.started":"2021-07-27T23:59:35.525375Z","shell.execute_reply":"2021-07-27T23:59:44.529632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import efficientnet.tfkeras as efn\nimport tensorflow as tf\ntf.compat.v1.disable_eager_execution()\nprint(\"Eager execution on:\",tf.executing_eagerly())\nimport tensorflow.keras as keras\nfrom tensorflow.keras import layers, models, optimizers\nfrom keras.utils.vis_utils import plot_model\n\nl_in = layers.Input((IMG_DIMS[0],IMG_DIMS[1],IMG_CHANNELS,))\neffnet_base = efn.EfficientNetB3(weights='noisy-student', input_shape = (IMG_DIMS[0], IMG_DIMS[1], IMG_CHANNELS), include_top=False, drop_connect_rate=0.2)(l_in)  # or weights='imagenet'\nl_pool_1 = layers.GlobalAveragePooling2D()(effnet_base)\nl_dense_1 = layers.Dense(128, activation='relu')(l_pool_1)\nl_out = layers.Dense(1, activation='sigmoid')(l_dense_1)\nmodel = models.Model(l_in, l_out)\n\nmodel.summary()\nplot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-27T23:59:44.534199Z","iopub.execute_input":"2021-07-27T23:59:44.53446Z","iopub.status.idle":"2021-07-28T00:00:08.37667Z","shell.execute_reply.started":"2021-07-27T23:59:44.534433Z","shell.execute_reply":"2021-07-28T00:00:08.375683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 32\nEPOCHS = 20","metadata":{"execution":{"iopub.status.busy":"2021-07-28T00:00:08.378487Z","iopub.execute_input":"2021-07-28T00:00:08.378889Z","iopub.status.idle":"2021-07-28T00:00:08.384488Z","shell.execute_reply.started":"2021-07-28T00:00:08.378846Z","shell.execute_reply":"2021-07-28T00:00:08.382845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow.keras as keras\n\nlr_schedule = keras.optimizers.schedules.ExponentialDecay(\n    1.25e-5*BATCH_SIZE, decay_steps=(25*len(train_df)//BATCH_SIZE), decay_rate=0.8, staircase=True\n)\nplt.title(\"Learning rate over %i epochs:\"%EPOCHS)\nx = range(0,(25*len(train_df)//BATCH_SIZE) * EPOCHS,10)\ndef visualize_lr_schedule(x):\n    return 1.25e-5*BATCH_SIZE * 0.8 ** (x//(25*(len(train_df)//BATCH_SIZE)))\ny = [visualize_lr_schedule(i) for i in x]\nsns.lineplot(x, y)\nprint(\"Starting LR:\",max(y))\nprint(\"Ending LR:\",min(y))","metadata":{"execution":{"iopub.status.busy":"2021-07-28T00:00:08.386216Z","iopub.execute_input":"2021-07-28T00:00:08.386559Z","iopub.status.idle":"2021-07-28T00:00:08.662296Z","shell.execute_reply.started":"2021-07-28T00:00:08.386524Z","shell.execute_reply":"2021-07-28T00:00:08.661344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize processed\ntrain_datagen = data_generator(train_df, img_mode, batch_size=BATCH_SIZE, augment=True)\nsample_batch = next(iter(train_datagen))\nplot_slices(sample_batch[0], 4, 4, from_train_gen=True, augment=True)\nprint(sample_batch[0].shape, sample_batch[1].shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T00:00:08.665384Z","iopub.execute_input":"2021-07-28T00:00:08.665629Z","iopub.status.idle":"2021-07-28T00:00:17.473899Z","shell.execute_reply.started":"2021-07-28T00:00:08.665604Z","shell.execute_reply":"2021-07-28T00:00:17.472824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for weirdness in class distribution in train sample\nsns.histplot(sample_batch[1])","metadata":{"execution":{"iopub.status.busy":"2021-07-28T00:00:17.475989Z","iopub.execute_input":"2021-07-28T00:00:17.476582Z","iopub.status.idle":"2021-07-28T00:00:17.655769Z","shell.execute_reply.started":"2021-07-28T00:00:17.476535Z","shell.execute_reply":"2021-07-28T00:00:17.654684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training:\n#### (Note: You will need to run the block of code below four times to train all the models. Set img_mode to:\n- 'T1w'\n- 'T2w'\n- 'T1wCE'\n- 'FLAIR'\n#### each time.)\n#### Download the models after training is complete, and restart the kernel.\nA memory leak causes RAM to fill up during training and will crash your kernel if you try to train 2 models in a row. I still haven't found a fix.","metadata":{}},{"cell_type":"code","source":"import gc\nimport pickle as pkl\n\n# Select one\nimg_modes = ['T1w','T1wCE','T2w','FLAIR']\nimg_mode = 'FLAIR'\n\nprint(\"Num train samples:\",len(train_df))\nprint(\"Num validation samples:\",len(val_df))\n\n# Clear garbage upon the end of every epoch\nclass gc_callback(tf.keras.callbacks.Callback):\n    def on_epoch_end(self, epoch, logs=None):\n        gc.collect()\ngarbage_callback = gc_callback()\n\n# Load datasets & train model\nprint(\"Now training model on:\", img_mode)\ntrain_datagen = data_generator(train_df, img_mode, batch_size=BATCH_SIZE, augment=True)\nval_datagen = data_generator(val_df, img_mode, batch_size=BATCH_SIZE, augment=False)\n\nmodel.compile(loss='binary_crossentropy', optimizer = optimizers.Adam(learning_rate=lr_schedule), metrics=['AUC','acc','mae'])\nes_callback = tf.keras.callbacks.EarlyStopping(\nmonitor=\"val_loss\",\nmin_delta=0,\npatience=5,\nrestore_best_weights=False\n)\n\ntrain_history = model.fit(train_datagen, epochs=EPOCHS, validation_data=val_datagen, use_multiprocessing=False, workers=1, steps_per_epoch = int(25*len(train_df)//BATCH_SIZE), validation_steps = int(25*len(val_df)//BATCH_SIZE), callbacks = [es_callback, garbage_callback])\nmodel.save(img_mode+'.h5')\n\nsns.lineplot(data=pd.DataFrame(train_history.history)[['loss','val_loss','AUC','val_AUC']])\nplt.title(\"Training History: %s\"%img_mode)\n\ndel train_datagen, val_datagen, train_history # Save memory\ngc.collect()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-28T00:00:17.657476Z","iopub.execute_input":"2021-07-28T00:00:17.657878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluation","metadata":{}},{"cell_type":"code","source":"import efficientnet.tfkeras as efn\nimport tensorflow as tf\ntf.compat.v1.disable_eager_execution()\nprint(\"Eager execution on:\",tf.executing_eagerly())\nimport tensorflow.keras as keras\nfrom tensorflow.keras import layers, models, optimizers\nfrom keras.utils.vis_utils import plot_model\n\n# Load model if you have a pretrained one\n#model = models.load_model('../input/btr-classification-models/FLAIR.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm, trange\n\ndef evaluate(model, val_df, img_mode, augment=False):\n    true, preds = [],[]\n    for patient_id in tqdm(val_df.iloc, total=len(val_df)):\n        if patient_id[img_mode] == 0: preds.append(0.5)\n        else:\n            img_batch = load_array_and_labels(patient_id, img_mode, augment=augment)[0]\n            \n            if IMG_CHANNELS == 1: img_batch = np.array(img_batch)[:,:,:,np.newaxis]\n            else: img_batch = np.repeat(np.array(img_batch)[:,:,:,np.newaxis], IMG_CHANNELS, -1)\n\n            preds.append(np.mean(model.predict(np.array(img_batch))))\n        true.append(patient_id['MGMT_value'])\n    return true, preds\n\ntrue_noaug, preds_noaug = evaluate(model, val_df, img_mode, augment=False)\ntrue_aug, preds_aug = evaluate(model, val_df, img_mode, augment=True)","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc\n\ndef display_roc_curve(true_list, preds_list, name_list):\n    plt.figure(1)\n    plt.plot([0, 1], [0, 1], 'k--')\n    for true, preds, name in zip(true_list, preds_list, name_list):\n        fpr, tpr, thresholds_rf = roc_curve(true, preds, pos_label = 1)\n        auc_rf = auc(fpr, tpr)\n        sns.lineplot(fpr, tpr, label=name+' (area = {:.3f})'.format(auc_rf))\n    plt.xlabel('False positive rate')\n    plt.ylabel('True positive rate')\n    plt.title('ROC curve')\n    plt.legend(loc='best')\n    plt.show()\n    # Zoom in view of the upper left corner.\n    plt.figure(2)\n    plt.xlim(0, 0.5)\n    plt.ylim(0.3, 1)\n    plt.plot([0, 1], [0, 1], 'k--')\n    for true, preds, name in zip(true_list, preds_list, name_list):\n        fpr, tpr, thresholds_rf = roc_curve(true, preds, pos_label = 1)\n        auc_rf = auc(fpr, tpr)\n        sns.lineplot(fpr, tpr, label=name+' (area = {:.3f})'.format(auc_rf))\n    plt.xlabel('False positive rate')\n    plt.ylabel('True positive rate')\n    plt.title('ROC curve (zoomed in at top left)')\n    plt.legend(loc='best')\n    plt.show()","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"true_list = [true_noaug,\n             true_aug]\npreds_list = [preds_noaug,\n             preds_aug]\nname_list = [\"FLAIR - No Augmentation\",\n            \"FLAIR - With Augmentation\"]\ndisplay_roc_curve(true_list, preds_list, name_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Prediction","metadata":{}},{"cell_type":"code","source":"# Import libraries\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Define data paths\nTRAIN_DATA_PATH = '../input/rsna-miccai-png/train/'\nTEST_DATA_PATH = '../input/rsna-miccai-png/test/'\n\nsubmission_df = pd.read_csv('../input/rsna-miccai-brain-tumor-radiogenomic-classification/sample_submission.csv')\n# Mark on the dataframe whether there are missing values for one or more MRI modes\nimport os\n\nimg_modes = set(['T1w','T1wCE','T2w','FLAIR'])\n\nmode_valid = {'T1w':[],\n             'T1wCE':[],\n             'T2w':[],\n             'FLAIR':[]}\n\nfor patient_id in submission_df.iloc:\n    subdirs = set(os.listdir(TEST_DATA_PATH + str(int(patient_id['BraTS21ID'])).zfill(5)))\n    for img_mode in img_modes: mode_valid[img_mode].append(int(img_mode in subdirs))\nsubmission_df = submission_df.merge(pd.DataFrame(mode_valid),left_index=True,right_index=True)\nprint(\"Sample submission:\")\nsubmission_df","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm, trange\n\nIMG_CHANNELS = 3\nimg_mode = 'FLAIR'\ndef make_preds(model, submission_df, img_mode):\n    preds = []\n    for patient_id in tqdm(submission_df.iloc, total=len(submission_df)):\n        if patient_id[img_mode] == 0: preds.append(0.5)\n        else:\n            img_batch = load_array_and_labels(patient_id, img_mode, augment=False, testing = True)\n            \n            if IMG_CHANNELS == 1: img_batch = np.array(img_batch)[:,:,:,np.newaxis]\n            else: img_batch = np.repeat(np.array(img_batch)[:,:,:,np.newaxis], IMG_CHANNELS, -1)\n\n            preds.append(np.mean(model.predict(np.array(img_batch))))\n    return preds\n\npreds = make_preds(model, submission_df, img_mode)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df['MGMT_value'] = preds\nsubmission_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}