{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# FLAIR images","metadata":{}},{"cell_type":"code","source":"import os\nimport zipfile\nimport numpy as np\nimport tensorflow as tf\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.107222Z","iopub.execute_input":"2021-09-28T19:55:55.107652Z","iopub.status.idle":"2021-09-28T19:55:55.113871Z","shell.execute_reply.started":"2021-09-28T19:55:55.107609Z","shell.execute_reply":"2021-09-28T19:55:55.112792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reference: # https://www.kaggle.com/rude009/working-with-dicom-data\nimport pydicom as dicom","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:56:10.950856Z","iopub.execute_input":"2021-09-28T19:56:10.951189Z","iopub.status.idle":"2021-09-28T19:56:10.95689Z","shell.execute_reply.started":"2021-09-28T19:56:10.951142Z","shell.execute_reply":"2021-09-28T19:56:10.956109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# here we create a pd.df with each patient's id and the target (MGMT value)\nimport pandas as pd\ndata_directory = '../input/rsna-miccai-brain-tumor-radiogenomic-classification'\ndf = pd.read_csv(f\"{data_directory}/train_labels.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.127258Z","iopub.status.idle":"2021-09-28T19:55:55.128144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataframe with ID and MGMT value","metadata":{}},{"cell_type":"markdown","source":"As we have to predict the MGMT value, we create a df with these two columns to link the ID of each set of images with the MGMT value","metadata":{}},{"cell_type":"code","source":"def convert_BraTS21ID_to_string(df):\n    \n    patientID = []\n    for ID in df['BraTS21ID']:\n        patientID.append(str(ID))\n    df['BraTS21ID'] = patientID\n    \n    patientID2 = []\n    for ID in df['BraTS21ID']:\n        if len(ID)==1:\n            patientID2.append('0000'+ID)\n        elif len(ID)==2:\n            patientID2.append('000'+ID)\n        elif len(ID)==3:\n            patientID2.append('00'+ID)\n        elif len(ID)==4:\n            patientID2.append('0'+ID)\n        elif len(ID)==5:\n            patientID2.append(ID)\n            \n    df['BraTS21ID'] = patientID2\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.129581Z","iopub.status.idle":"2021-09-28T19:55:55.130369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = convert_BraTS21ID_to_string(df)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.131596Z","iopub.status.idle":"2021-09-28T19:55:55.132431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=df.drop(labels=71 , axis=0)\ndf=df.drop(labels=81 , axis=0)\ndf=df.drop(labels=488 , axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.133825Z","iopub.status.idle":"2021-09-28T19:55:55.134648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we create 2 dfs, one for MGMT=1 and another for MGMT=0\ndf_MGMT_1 = df.loc[df.MGMT_value == 1] #len = \ndf_MGMT_0 = df.loc[df.MGMT_value == 0] #len =","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.135906Z","iopub.status.idle":"2021-09-28T19:55:55.13673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# now, we define functions to load and stack the images\nfrom scipy import ndimage\nimport glob\nimport re\nimport cv2\n\nSIZE=128\ndef load_dicom_image(path, img_size=SIZE, voi_lut=True):\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n        \n    data = cv2.resize(data, (img_size, img_size))\n    return data\n\nNUM_IMAGES = 64\ndef load_dicom_images_3d(scan_id, num_imgs=NUM_IMAGES, img_size=SIZE, mri_type=\"FLAIR\", split=\"train\"):\n\n    files = sorted(glob.glob(f\"{data_directory}/{split}/{scan_id}/{mri_type}/*.dcm\"), \n               key=lambda var:[int(x) if x.isdigit() else x for x in re.findall(r'[^0-9]|[0-9]+', var)])\n\n    middle = len(files)//2\n    num_imgs2 = num_imgs//2\n    p1 = max(0, middle - num_imgs2)\n    p2 = min(len(files), middle + num_imgs2)\n    img3d = np.stack([load_dicom_image(f) for f in files[p1:p2]]).T \n    if img3d.shape[-1] < num_imgs:\n        n_zero = np.zeros((img_size, img_size, num_imgs - img3d.shape[-1]))\n        img3d = np.concatenate((img3d,  n_zero), axis = -1)\n        \n    if np.min(img3d) < np.max(img3d):\n        img3d = img3d - np.min(img3d)\n        img3d = img3d / np.max(img3d)\n            \n    return img3d","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.138011Z","iopub.status.idle":"2021-09-28T19:55:55.138788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# now we specify the geral directory and load one image\ndata_directory = '../input/rsna-miccai-brain-tumor-radiogenomic-classification'","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.139956Z","iopub.status.idle":"2021-09-28T19:55:55.140735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transforming the Data","metadata":{}},{"cell_type":"markdown","source":"We now define some functions to apply transformations to the images: normalization, resizing, cropping and dicom processing","metadata":{}},{"cell_type":"code","source":"# now, we define functions to apply transformations to the images (stacked images)\n\n# values of the array are between 0 and aprox. 2000\n\ndef normalize(volume):\n    \"\"\"Normalize the volume\"\"\"\n    minv = np.min(volume)\n    maxv = np.max(volume)\n    volume[volume < minv] = minv\n    volume[volume > maxv] = maxv\n    volume = (volume - minv) / (maxv - minv)\n    volume = volume.astype(\"float32\")\n    return volume\n\n\ndef resize_volume(img):\n    \"\"\"Resize across z-axis\"\"\"\n    # Set the desired depth\n    desired_depth = 64\n    desired_width = 128\n    desired_height = 128\n    # Get current depth\n    current_depth = img.shape[-1]\n    current_width = img.shape[0]\n    current_height = img.shape[1]\n    # Compute depth factor\n    depth = current_depth / desired_depth\n    width = current_width / desired_width\n    height = current_height / desired_height\n    depth_factor = 1 / depth\n    width_factor = 1 / width\n    height_factor = 1 / height\n    # Rotate\n    img = ndimage.rotate(img, 90, reshape=False)\n    # Resize across z-axis\n    img = ndimage.zoom(img, (width_factor, height_factor, depth_factor), order=1)\n    return img\n\ndef cropped_images(images):\n    min=np.array(np.nonzero(images)).min(axis=1)\n    max=np.array(np.nonzero(images)).max(axis=1)\n    return images[min[0]:max[0],min[1]:max[1],min[2]:max[2]]\n    \n# path= patient's ID\ndef process_scan(path):\n    \"\"\"Read and resize volume\"\"\"\n    # Read scan\n    volume = load_dicom_images_3d(path)\n    # Normalize\n    volume = normalize(volume)\n    # Resize width, height and depth\n    volume = cropped_images(volume)\n    \n    volume = resize_volume(volume)\n    \n    return volume","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.141952Z","iopub.status.idle":"2021-09-28T19:55:55.142718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"process_scan('00000').shape","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.1439Z","iopub.status.idle":"2021-09-28T19:55:55.144669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since we have a big amount of images, we reduce the dataset, so it can fit to 16GB ram","metadata":{}},{"cell_type":"code","source":"# reducing the amount of data to 200 scans per category\n\n# Read and process the scans.\n# Each scan is resized across height, width, and depth and rescaled.\nMGMT_scans = np.array([process_scan(path) for path in df_MGMT_1['BraTS21ID'][:200]])\nno_MGMT_scans = np.array([process_scan(path) for path in df_MGMT_0['BraTS21ID'][:200]])\n\n# For the MRI scans having presence of metylation\n# assign 1, for the normal ones assign 0.\nMGMT_labels = np.array([1 for _ in range(len(df_MGMT_1['BraTS21ID'][:200]))])\nno_MGMT_labels = np.array([0 for _ in range(len(df_MGMT_0['BraTS21ID'][:200]))])","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.171526Z","iopub.execute_input":"2021-09-28T19:55:55.171769Z","iopub.status.idle":"2021-09-28T19:55:55.458271Z","shell.execute_reply.started":"2021-09-28T19:55:55.171746Z","shell.execute_reply":"2021-09-28T19:55:55.456492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We now split the data to be then processed by CNN","metadata":{}},{"cell_type":"code","source":"# Split data in the ratio 70-30 for training and validation.\nx_train = np.concatenate((MGMT_scans[:140], no_MGMT_scans[:140]), axis=0)\ny_train = np.concatenate((MGMT_labels[:140], no_MGMT_labels[:140]), axis=0)\nx_val = np.concatenate((MGMT_scans[140:], no_MGMT_scans[140:]), axis=0)\ny_val = np.concatenate((MGMT_labels[140:], no_MGMT_labels[140:]), axis=0)\nprint(\n    \"Number of samples in train and validation are %d and %d.\"\n    % (x_train.shape[0], x_val.shape[0])\n)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.459695Z","iopub.status.idle":"2021-09-28T19:55:55.460408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating the dataset","metadata":{}},{"cell_type":"markdown","source":"Now that we have store the transformed data into numpy tensors, wue proceed to save it to then create the tensor's dataset","metadata":{}},{"cell_type":"code","source":"x_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.461937Z","iopub.status.idle":"2021-09-28T19:55:55.462528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs(\"Dataset\")","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.463753Z","iopub.status.idle":"2021-09-28T19:55:55.46433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs(\"x_train_dataset\")\nos.makedirs(\"y_train_dataset\")\nos.makedirs(\"x_val_dataset\")\nos.makedirs(\"y_val_dataset\")","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.465595Z","iopub.status.idle":"2021-09-28T19:55:55.466139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_path='/kaggle/working/x_train_dataset'\nnp.save(x_train_path, x_train)\n\ny_train_path='/kaggle/working/y_train_dataset'\nnp.save(y_train_path, y_train)\n\nx_val_path='/kaggle/working/x_val_dataset'\nnp.save(x_val_path, x_val)\n\ny_val_path='/kaggle/working/y_val_dataset'\nnp.save(y_val_path, y_val)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:55:55.467442Z","iopub.status.idle":"2021-09-28T19:55:55.467994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The FLAIR dataset is stored in the following kaggle Dataset:\nhttps://www.kaggle.com/hugovallejo/numpy-for-rsna-compet-flair","metadata":{}}]}