{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## RSNA-MICCAI Brain Tumor Radiogenomic Classification - Inference Notebook\n\nIn this notebook we will use different trained models to predict the MGMT_value for the test set using ensemble methods.","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport imageio\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' \n\nimport wandb\nimport re \nimport glob\nimport cv2\nfrom tqdm.notebook import tqdm\nfrom pathlib import Path\n\n# Deep learning packages\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport warnings\nwarnings.filterwarnings('ignore')\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-20T18:01:07.238175Z","iopub.execute_input":"2021-09-20T18:01:07.238504Z","iopub.status.idle":"2021-09-20T18:01:07.252170Z","shell.execute_reply.started":"2021-09-20T18:01:07.238474Z","shell.execute_reply":"2021-09-20T18:01:07.250971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = {\n  'images_source_path' : '../input/rsna-miccai-brain-tumor-radiogenomic-classification/test',\n  'csv_path': '../input/rsna-miccai-brain-tumor-radiogenomic-classification/train_labels.csv',\n  'data_path': '../input/rsna-miccai-brain-tumor-radiogenomic-classification',\n   \"models_path\": \"../input/genetic-biomarker-prediction-with-crnn-train/\", \n  'output_path': './crnn/',\n  'nfolds': 3,\n  'global_seed': 42,\n  'batch_size': 4,\n  'frames_per_seq': 24,\n  'img_size': 224,\n  'learning_rate': 0.0001,\n  'rnn_cells': 16,  \n  'num_epochs': 10,\n  'channels': 3,\n  'scale' : 0.75\n}\n\n\nmri_types = ['T2w'] \n# mri_types = ['FLAIR','T1w','T1wCE','T2w']","metadata":{"execution":{"iopub.status.busy":"2021-09-20T18:01:07.254650Z","iopub.execute_input":"2021-09-20T18:01:07.254973Z","iopub.status.idle":"2021-09-20T18:01:07.269325Z","shell.execute_reply.started":"2021-09-20T18:01:07.254933Z","shell.execute_reply":"2021-09-20T18:01:07.268394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BrainTumor_GeneticSequence():\n    \"\"\"Prepares the train and the validation data pipeline for mri_type, for ex: mri_type = FLAIR\"\"\"\n    mri_type = \"FLAIR\"\n    df_data = None\n    df_train_labels = pd.read_csv(config['csv_path'])\n    \n    def __init__(self, mri_type):\n        self.mri_type = mri_type\n        self.df_data = pd.DataFrame(columns=['BraTS21ID'] + mri_types)\n        for key in mri_types:\n            self.df_data[key] = self.df_data[key].astype(int)\n        self.df_data['BraTS21ID'] = self.df_data['BraTS21ID'].astype(int)\n\n    def prepare_dataframe(self, mode='train'):\n        train_folders = ''\n        if mode == 'test':\n            folders_path = \"test_images_source_path\"\n        else:\n            folders_path = \"images_source_path\"\n        train_folders = config[folders_path] + '/'\n        for f in tqdm(os.listdir(train_folders)):\n            if f in [\"00109\", \"00123\", \"00709\"]: \n                continue\n            BraTS21ID = int(f)\n            self.df_data = self.df_data.append({'BraTS21ID': BraTS21ID, 'FLAIR': 0, 'T1w': 0, 'T1wCE': 0, 'T2w' : 0}, ignore_index=True)\n            BraTS21ID_key_path = f'{config[folders_path]}/{format(BraTS21ID, \"05d\")}/{self.mri_type}/*.dcm'\n            files_len = len(glob.glob(BraTS21ID_key_path))\n            # update file count or remove the patient from the dataset for the mri_type chosen\n            if files_len > 0:\n                self.df_data.loc[self.df_data['BraTS21ID'] == BraTS21ID, self.mri_type] = files_len\n            else:\n                self.df_data = self.df_data.loc[self.df_data.BraTS21ID!=BraTS21ID]\n        self.df_data[\"folder_name\"] = [format(x, '05d') for x in self.df_data[\"BraTS21ID\"]]\n        self.df_data[\"folder_path\"] = [os.path.join(config[folders_path], x) for x in self.df_data[\"folder_name\"]]\n#         self.df_data = self.df_data.head(30) # for testing\n#         print(self.df_data.head())\n        self.df_data = pd.merge(self.df_data, self.df_train_labels,how='left',on='BraTS21ID')\n    \n    def rotate_image(self, image, angle):\n        image_center = tuple(np.array(image.shape[1::-1]) / 2)\n        rot_mat = cv2.getRotationMatrix2D(image_center, angle, 1.0)\n        result = cv2.warpAffine(image, rot_mat, image.shape[1::-1], flags=cv2.INTER_LINEAR)\n        return result\n    \n    def normalize(self, image):\n        result = cv2.normalize(image, None, alpha=0, beta=1, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F)\n        return result\n    \n    def read_mri(self, path, voi_lut = True, fix_monochrome = True):\n        # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n        dicom = pydicom.read_file(path)\n        if voi_lut:\n            data = apply_voi_lut(dicom.pixel_array, dicom)\n        else:\n            data = dicom.pixel_array\n        if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n            data = np.amax(data) - data\n        data = data - np.min(data)\n        data = data / np.max(data)\n        data = (data * 255).astype(np.uint8)\n        data = self.normalize(data)\n        data = self.rotate_image(data, np.random.randint(0,20))\n        data = self.crop_center_square(data)\n        data = cv2.resize(data, (config['img_size'], config['img_size']))\n#         (thresh, im_bw) = cv2.threshold(data, 128, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)\n#         data = cv2.threshold(data, thresh, 255, cv2.THRESH_BINARY)[1]\n        data = np.repeat(data[..., np.newaxis], 3, -1)\n        return data\n    \n    def crop_center_square(self, frame, scale=config['scale']):\n        y, x = frame.shape[0:2]\n        center_x, center_y = x / 2, y / 2\n        width_scaled, height_scaled = x * scale, y * scale\n        left_x, right_x = center_x - width_scaled / 2, center_x + width_scaled / 2\n        top_y, bottom_y = center_y - height_scaled / 2, center_y + height_scaled / 2\n        return frame[int(top_y):int(bottom_y), int(left_x):int(right_x)]\n\n    def get_img_path_3d(self, dir_path):\n        modality_path = os.path.join(dir_path.decode('utf8'), self.mri_type)\n        files = sorted(glob.glob(f\"{modality_path}/*.dcm\"), key=lambda var:[int(x) if x.isdigit() else x for x in re.findall(r'[^0-9]|[0-9]+', var)])\n        total_img_num = len(files) \n        mid_num = total_img_num // 2\n        num_3d2 = config['frames_per_seq'] // 2\n        start_idx = max(0, mid_num - num_3d2)\n        end_idx = min(len(files), mid_num + num_3d2)\n        target_file_paths = tf.convert_to_tensor(files[start_idx:end_idx], dtype=tf.string) \n        \n        def get_frames(path):\n            file_path = path.numpy().decode('UTF-8')\n            image = self.read_mri(file_path)    \n            return image\n    \n        mri_images = tf.nest.map_structure(tf.stop_gradient, tf.map_fn(fn=get_frames, elems=target_file_paths, fn_output_signature=tf.float32))\n\n        # padding null images \n        if mri_images.shape[0] < config['frames_per_seq']:\n            n_zero = tf.zeros((config['frames_per_seq'] - mri_images.shape[0], config['img_size'], config['img_size'], config['channels']), dtype=tf.dtypes.float32)\n            mri_images = np.concatenate((mri_images,  n_zero), axis = 0)\n        return mri_images\n    \n    def load_frame(self, df_dict):\n        dirname = df_dict['folder_path']\n        paths = tf.numpy_function(self.get_img_path_3d, [dirname], tf.float32)\n        label = df_dict['MGMT_value']\n        label = tf.cast(label, tf.float32)\n        return paths, label\n    \n    def get_loader(self):\n        AUTOTUNE = tf.data.AUTOTUNE\n        testloader = tf.data.Dataset.from_tensor_slices(dict(self.df_data))\n        testloader = (testloader\n                    .map(self.load_frame, num_parallel_calls=AUTOTUNE)\n                    .batch(config['batch_size'])\n                    .prefetch(AUTOTUNE)\n                    )\n        return testloader\n    \n    def predict(self, model):\n        AUTOTUNE = tf.data.AUTOTUNE\n        testloader = tf.data.Dataset.from_tensor_slices(dict(self.df_data))\n        testloader = (testloader\n                    .map(self.load_frame, num_parallel_calls=AUTOTUNE)\n                    .batch(config['batch_size'])\n                    .prefetch(AUTOTUNE)\n                    )\n        proba = model.predict(testloader, verbose=1)\n        return proba ","metadata":{"execution":{"iopub.status.busy":"2021-09-20T18:01:07.270815Z","iopub.execute_input":"2021-09-20T18:01:07.271050Z","iopub.status.idle":"2021-09-20T18:01:07.318894Z","shell.execute_reply.started":"2021-09-20T18:01:07.271023Z","shell.execute_reply":"2021-09-20T18:01:07.317966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_path = os.path.join(config['data_path'], 'sample_submission.csv')\nsample_df = pd.read_csv(sample_submission_path); \ntest_df = sample_df.copy(); \ntest_df[\"folder_name\"] = [format(x, \"05d\") for x in test_df.BraTS21ID]\ntest_df[\"folder_path\"] = [os.path.join(config['data_path'], 'test', x) for x in test_df[\"folder_name\"]]\ntest_df['MGMT_pred'] = 0\n\n# def plot_gifs(loader):\n#     os.makedirs('gifs/', exist_ok=True)\n#     frames = next(iter(loader))\n#     for i, frame in enumerate(frames):\n#         imageio.mimsave(f'gifs/out_{i}.gif', (frame).numpy().astype('uint8')) \n    \n# dp = BrainTumor_GeneticSequence(mri_types[0])\n# dp.prepare_dataframe()\n# testloader = dp.get_loader()\n# plot_gifs(testloader)\n\n# model = tf.keras.models.load_model(f'./{model_arch}_1.h5')\n# dp = BrainTumor_GeneticSequence(mri_types[0])\n# dp.prepare_dataframe(mode='test')\n# dp.df_data['MGMT_pred'] = 0\n# proba = dp.predit(model)\n# sample_submission_path = os.path.join(config['data_path'], 'sample_submission.csv')\n# sample_df = pd.read_csv(sample_submission_path); \n# dp.df_data['MGMT_pred'] += proba.squeeze()    \n# dp.df_data[['BraTS21ID','MGMT_pred']].head()","metadata":{"execution":{"iopub.status.busy":"2021-09-20T18:01:07.358054Z","iopub.execute_input":"2021-09-20T18:01:07.358379Z","iopub.status.idle":"2021-09-20T18:01:07.373076Z","shell.execute_reply.started":"2021-09-20T18:01:07.358349Z","shell.execute_reply":"2021-09-20T18:01:07.371943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_arch = 'xception'\nfor m_type in mri_types:\n    print(f'Predicting for....{m_type}')\n    dp = BrainTumor_GeneticSequence(m_type)\n    dp.prepare_dataframe()\n    model_weight_path = f'{config[\"models_path\"]}{model_arch}_3.h5'\n    model = tf.keras.models.load_model(model_weight_path)\n    proba = dp.predict(model)\n    test_df['MGMT_pred'] += proba.squeeze()    \n\ntest_df['MGMT_pred'] /= len(mri_types)\ntest_df['MGMT_pred'] = [round(x,3) for x in test_df['MGMT_pred']]\nsample_df['MGMT_value'] = test_df['MGMT_pred']\nsample_df","metadata":{"execution":{"iopub.status.busy":"2021-09-20T18:01:07.376157Z","iopub.execute_input":"2021-09-20T18:01:07.376639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(sample_df['MGMT_value'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}