{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom glob import glob\n%matplotlib inline\nimport matplotlib.pyplot as plt\n\nimport random\nfrom random import sample\nimport sklearn.model_selection as skl\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom keras_preprocessing.image.dataframe_iterator import DataFrameIterator\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, BatchNormalization, Dense, Dropout, Flatten, Conv2D, MaxPooling2D\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.applications.vgg16 import VGG16\nfrom tensorflow.keras.applications import EfficientNetB3\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\n\nimport pydicom\nimport cv2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-21T02:31:48.566676Z","iopub.execute_input":"2021-09-21T02:31:48.567252Z","iopub.status.idle":"2021-09-21T02:31:48.580030Z","shell.execute_reply.started":"2021-09-21T02:31:48.567214Z","shell.execute_reply":"2021-09-21T02:31:48.577415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_dir = '../input/rsna-miccai-brain-tumor-radiogenomic-classification/'\ndf = pd.read_csv(root_dir+'train_labels.csv')","metadata":{"execution":{"iopub.status.busy":"2021-09-21T02:31:49.216066Z","iopub.execute_input":"2021-09-21T02:31:49.216745Z","iopub.status.idle":"2021-09-21T02:31:49.226363Z","shell.execute_reply.started":"2021-09-21T02:31:49.216694Z","shell.execute_reply":"2021-09-21T02:31:49.225608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add the full paths for each id for different types of sequences to the csv \ndef full_ids(data):\n    zeros = 5 - len(str(data))\n    if zeros > 0:\n        prefix = ''.join(['0' for i in range(zeros)])\n    \n    return prefix+str(data)\n        \n\ndf['BraTS21ID_full'] = df['BraTS21ID'].apply(full_ids)\n\n# Add all the paths to the df for easy access\ndf['flair'] = df['BraTS21ID_full'].apply(lambda file_id : root_dir+'train/'+file_id+'/FLAIR/')\ndf['t1w'] = df['BraTS21ID_full'].apply(lambda file_id : root_dir+'train/'+file_id+'/T1w/')\ndf['t1wce'] = df['BraTS21ID_full'].apply(lambda file_id : root_dir+'train/'+file_id+'/T1wCE/')\ndf['t2w'] = df['BraTS21ID_full'].apply(lambda file_id : root_dir+'train/'+file_id+'/T2w/')","metadata":{"execution":{"iopub.status.busy":"2021-09-21T02:31:49.633146Z","iopub.execute_input":"2021-09-21T02:31:49.634033Z","iopub.status.idle":"2021-09-21T02:31:49.648326Z","shell.execute_reply.started":"2021-09-21T02:31:49.633994Z","shell.execute_reply":"2021-09-21T02:31:49.647450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(root_dir+'sample_submission.csv')\n\ndf_test['BraTS21ID_full'] = df_test['BraTS21ID'].apply(full_ids)\n\n# Add all the paths to the df for easy access\ndf_test['flair'] = df_test['BraTS21ID_full'].apply(lambda file_id : root_dir+'test/'+file_id+'/FLAIR/')\ndf_test['t1w'] = df_test['BraTS21ID_full'].apply(lambda file_id : root_dir+'test/'+file_id+'/T1w/')\ndf_test['t1wce'] = df_test['BraTS21ID_full'].apply(lambda file_id : root_dir+'test/'+file_id+'/T1wCE/')\ndf_test['t2w'] = df_test['BraTS21ID_full'].apply(lambda file_id : root_dir+'test/'+file_id+'/T2w/')","metadata":{"execution":{"iopub.status.busy":"2021-09-21T02:31:50.440215Z","iopub.execute_input":"2021-09-21T02:31:50.440490Z","iopub.status.idle":"2021-09-21T02:31:50.455208Z","shell.execute_reply.started":"2021-09-21T02:31:50.440461Z","shell.execute_reply":"2021-09-21T02:31:50.454397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# load the images\n\n00109 (FLAIR images are blank)\n00123 (T1w images are blank)\n00709 (FLAIR images are blank)","metadata":{}},{"cell_type":"code","source":"def get_train_val_dataframe(mri_type):\n\n    all_img_files = []\n    all_img_labels = []\n    all_img_patient_ids = []\n    for row in df.iterrows():\n        if row[1]['BraTS21ID_full'] == '00109' and mri_type == 'flair':\n            continue\n        if row[1]['BraTS21ID_full'] == '00123' and mri_type == 't1w':\n            continue\n        if row[1]['BraTS21ID_full'] == '00709' and mri_type == 'flair':\n            continue\n        img_dir = row[1][mri_type]\n        img_files = os.listdir(img_dir)\n        img_nums = sorted([int(ele.replace('Image-', '').replace('.dcm', '')) for ele in img_files])\n        mid_point = int(len(img_nums)/2)\n        start_point = mid_point - max(int(mid_point*0.1), 1)\n        end_point = mid_point + max(int(mid_point*0.1), 1)\n        img_names = [f'Image-{img_nums[i]}.dcm' for i in range(start_point, end_point+1)]\n        img_paths = [img_dir+ele for ele in img_names]\n        img_labels = [row[1]['MGMT_value']]*len(img_paths)\n        img_patient_ids = [row[1]['BraTS21ID']]*len(img_paths)\n        all_img_files.extend(img_paths)\n        all_img_labels.extend(img_labels)\n        all_img_patient_ids.extend(img_patient_ids)\n\n    train_val_df = pd.DataFrame({'patient_ids': all_img_patient_ids,\n                  'labels': all_img_labels,\n                  'file_paths': all_img_files})\n\n    train_val_df['labels'] = train_val_df['labels'].map({1: '1', 0: '0'})\n    \n    return train_val_df\n    \ndef get_test_dataframe(mri_type):\n    \n    all_test_img_files = []\n    all_test_img_labels = []\n    all_test_img_patient_ids = []\n    for row in df_test.iterrows():\n        img_dir = row[1][mri_type]\n        img_files = os.listdir(img_dir)\n        img_nums = sorted([int(ele.replace('Image-', '').replace('.dcm', '')) for ele in img_files])\n        mid_point = int(len(img_nums)/2)\n        start_point = mid_point - max(int(mid_point*0.1), 1)\n        end_point = mid_point + max(int(mid_point*0.1), 1)\n        img_names = [f'Image-{img_nums[i]}.dcm' for i in range(start_point, end_point+1)]\n        img_paths = [img_dir+ele for ele in img_names]\n        img_labels = [row[1]['MGMT_value']]*len(img_paths)\n        img_patient_ids = [row[1]['BraTS21ID']]*len(img_paths)\n        all_test_img_files.extend(img_paths)\n        all_test_img_labels.extend(img_labels)\n        all_test_img_patient_ids.extend(img_patient_ids)\n\n    test_df = pd.DataFrame({'patient_ids': all_test_img_patient_ids,\n                  'labels': all_test_img_labels,\n                  'file_paths': all_test_img_files})\n    \n    test_df['labels'] = ['1']*(len(test_df)-1) + ['0'] # workaround for testing data gen\n    \n    return test_df","metadata":{"execution":{"iopub.status.busy":"2021-09-21T02:31:51.839175Z","iopub.execute_input":"2021-09-21T02:31:51.839441Z","iopub.status.idle":"2021-09-21T02:31:51.856509Z","shell.execute_reply.started":"2021-09-21T02:31:51.839413Z","shell.execute_reply":"2021-09-21T02:31:51.855592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DCMDataFrameIterator(DataFrameIterator):\n    def __init__(self, *arg, **kwargs):\n        self.white_list_formats = ('dcm')\n        super(DCMDataFrameIterator, self).__init__(*arg, **kwargs)\n        self.dataframe = kwargs['dataframe']\n        self.x = self.dataframe[kwargs['x_col']]\n        self.y = self.dataframe[kwargs['y_col']]\n        self.color_mode = kwargs['color_mode']\n        self.target_size = kwargs['target_size']\n\n    def _get_batches_of_transformed_samples(self, indices_array):\n        # get batch of images\n        batch_x = np.array([self.read_dcm_as_array(dcm_path, self.target_size, color_mode=self.color_mode)\n                            for dcm_path in self.x.iloc[indices_array]])\n\n        batch_y = np.array(self.y.iloc[indices_array].astype(np.uint8))  # astype because y was passed as str\n\n        # transform images\n        if self.image_data_generator is not None:\n            for i, (x, y) in enumerate(zip(batch_x, batch_y)):\n                transform_params = self.image_data_generator.get_random_transform(x.shape)\n                batch_x[i] = self.image_data_generator.apply_transform(x, transform_params)\n                # you can change y here as well, eg: in semantic segmentation you want to transform masks as well \n                # using the same image_data_generator transformations.\n\n        return batch_x, batch_y\n\n    @staticmethod\n    def read_dcm_as_array(dcm_path, target_size=(224, 224), color_mode='rgb'):\n        image_array = pydicom.dcmread(dcm_path).pixel_array\n        pixels = image_array - np.min(image_array)\n        pixels = pixels / np.max(pixels)\n        image_manual_norm = (pixels * 255).astype(np.uint8)\n        image_array = cv2.resize(image_manual_norm, target_size, interpolation=cv2.INTER_NEAREST)  #this returns a 2d array\n#         image_array = np.expand_dims(image_array, -1)\n        if color_mode == 'rgb':\n            image_array = np.dstack((image_array, np.zeros_like(image_array), np.zeros_like(image_array)))\n        return image_array","metadata":{"execution":{"iopub.status.busy":"2021-09-21T02:31:52.665082Z","iopub.execute_input":"2021-09-21T02:31:52.665336Z","iopub.status.idle":"2021-09-21T02:31:52.676071Z","shell.execute_reply.started":"2021-09-21T02:31:52.665308Z","shell.execute_reply":"2021-09-21T02:31:52.675403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 369\nBATCH_SIZE = 512\nCLASS_MODE = 'binary'\nCOLOR_MODE = 'rgb'\nTARGET_SIZE = (224, 224)","metadata":{"execution":{"iopub.status.busy":"2021-09-21T02:31:53.335566Z","iopub.execute_input":"2021-09-21T02:31:53.336123Z","iopub.status.idle":"2021-09-21T02:31:53.340222Z","shell.execute_reply.started":"2021-09-21T02:31:53.336085Z","shell.execute_reply":"2021-09-21T02:31:53.339409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_data_generators(train_df, test_df):\n    train_augmentation_parameters = dict(\n    #     rescale=1.0/255.0,\n        zoom_range=0.2,\n    #     horizontal_flip=True,\n    #     vertical_flip = True,\n        fill_mode='nearest',\n        height_shift_range= 0.1,\n        width_shift_range=0.1,\n        brightness_range = [0.8, 1.2]\n    )\n\n    test_augmentation_parameters = dict(\n    #     rescale=1.0/255.0\n    )\n\n    train_consts = {\n        'seed': SEED,\n        'batch_size': BATCH_SIZE,\n        'class_mode': CLASS_MODE,\n        'color_mode': COLOR_MODE,\n        'target_size': TARGET_SIZE,  \n    }\n\n    test_consts = {\n        'batch_size': BATCH_SIZE,\n        'class_mode': CLASS_MODE,\n        'color_mode': COLOR_MODE,\n        'target_size': TARGET_SIZE,\n        'shuffle': False\n    }\n\n    train_augmenter = ImageDataGenerator(**train_augmentation_parameters)\n    test_augmenter = ImageDataGenerator(**test_augmentation_parameters)\n\n    train_generator = DCMDataFrameIterator(dataframe=train_df,\n                                 x_col='file_paths',\n                                 y_col='labels',\n                                 image_data_generator=train_augmenter,\n                                 **train_consts)\n\n    test_generator = DCMDataFrameIterator(dataframe=test_df,\n                                 x_col='file_paths',\n                                 y_col='labels',\n                                 image_data_generator=test_augmenter,\n                                 **test_consts)\n    \n    return train_generator, test_generator","metadata":{"execution":{"iopub.status.busy":"2021-09-21T02:32:38.144610Z","iopub.execute_input":"2021-09-21T02:32:38.145294Z","iopub.status.idle":"2021-09-21T02:32:38.154369Z","shell.execute_reply.started":"2021-09-21T02:32:38.145253Z","shell.execute_reply":"2021-09-21T02:32:38.151832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build model and train","metadata":{}},{"cell_type":"code","source":"def build_model(weights_path):\n    model = EfficientNetB3(include_top=False, weights=weights_path)\n\n    # Freeze the pretrained weights\n    model.trainable = False\n\n    # Rebuild top\n    x = GlobalAveragePooling2D(name=\"avg_pool\")(model.output)\n    x = BatchNormalization()(x)\n\n    top_dropout_rate = 0.1\n    x = Dropout(top_dropout_rate, name=\"top_dropout\")(x)\n    dense_1 = Dense(32, activation=\"relu\")(x)\n    outputs = Dense(1, activation=\"sigmoid\", name=\"pred\")(dense_1)\n\n    # Compile\n    model = Model(model.inputs, outputs, name=\"EfficientNet\")\n    optimizer = Adam(learning_rate=1e-3)\n    model.compile(\n        optimizer=optimizer, loss=\"binary_crossentropy\", metrics=[\"binary_accuracy\"]\n    )\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-09-21T02:32:39.214187Z","iopub.execute_input":"2021-09-21T02:32:39.214461Z","iopub.status.idle":"2021-09-21T02:32:39.223527Z","shell.execute_reply.started":"2021-09-21T02:32:39.214432Z","shell.execute_reply":"2021-09-21T02:32:39.222255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model_name, train_generator, epochs):\n    \n    print('training', model_name)\n    \n    model = build_model(\"../input/efficentnet-b0b5-tensorflow-24-notop/efficientnet-b3_tf24_imagenet_1000_notop.h5\")\n    \n    history = model.fit_generator(\n        generator=train_generator,\n        steps_per_epoch=len(train_generator),\n        epochs=epochs,\n        workers=2\n    )\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-09-21T02:32:39.572256Z","iopub.execute_input":"2021-09-21T02:32:39.572801Z","iopub.status.idle":"2021-09-21T02:32:39.577771Z","shell.execute_reply.started":"2021-09-21T02:32:39.572766Z","shell.execute_reply":"2021-09-21T02:32:39.576914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train a model for each of the mri types and then ensemble predictions\nall_test_preds = []\n\nfor mt in ['flair', 't1w', 't1wce', 't2w']:\n    train_df = get_train_val_dataframe(mt)\n    test_df = get_test_dataframe(mt)\n    train_g, test_g = get_data_generators(train_df, test_df)\n    best_model = train_model(mt, train_g, epochs=5)\n    test_pred = best_model.predict(test_g, steps=len(test_g))\n    test_df['pred_y'] = test_pred\n    # aggregate the predictions on all image for each person (take the most confident prediction out of all image predictions)\n    mean_pred = test_pred.mean()\n    test_pred_agg = test_df.groupby('patient_ids').apply(\n        lambda x: x['pred_y'].max()\n        if (x['pred_y'].max() - mean_pred) > (mean_pred - x['pred_y'].min()) \n        else x['pred_y'].min())\n    all_test_preds.append(test_pred_agg.values)","metadata":{"execution":{"iopub.status.busy":"2021-09-21T02:32:39.862101Z","iopub.execute_input":"2021-09-21T02:32:39.862330Z","iopub.status.idle":"2021-09-21T02:36:23.083916Z","shell.execute_reply.started":"2021-09-21T02:32:39.862305Z","shell.execute_reply":"2021-09-21T02:36:23.079376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"all_test_preds = np.array(all_test_preds)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T04:13:43.322892Z","iopub.execute_input":"2021-09-20T04:13:43.323239Z","iopub.status.idle":"2021-09-20T04:13:43.329188Z","shell.execute_reply.started":"2021-09-20T04:13:43.323206Z","shell.execute_reply":"2021-09-20T04:13:43.328409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(all_test_preds.mean(0))","metadata":{"execution":{"iopub.status.busy":"2021-09-20T04:14:41.688742Z","iopub.execute_input":"2021-09-20T04:14:41.689Z","iopub.status.idle":"2021-09-20T04:14:41.903759Z","shell.execute_reply.started":"2021-09-20T04:14:41.688973Z","shell.execute_reply":"2021-09-20T04:14:41.903065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm = pd.read_csv(root_dir+'sample_submission.csv')\nsubm['MGMT_value'] = all_test_preds.mean(0)\nsubm.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T04:14:04.962435Z","iopub.execute_input":"2021-09-20T04:14:04.962699Z","iopub.status.idle":"2021-09-20T04:14:04.987088Z","shell.execute_reply.started":"2021-09-20T04:14:04.962673Z","shell.execute_reply":"2021-09-20T04:14:04.986415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm","metadata":{"execution":{"iopub.status.busy":"2021-09-20T04:14:10.882005Z","iopub.execute_input":"2021-09-20T04:14:10.882262Z","iopub.status.idle":"2021-09-20T04:14:10.894005Z","shell.execute_reply.started":"2021-09-20T04:14:10.882233Z","shell.execute_reply":"2021-09-20T04:14:10.893165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}