{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This notebook helps you prepare the 100+gb data into smaller dataset. Convert raw data into tensorflow recordset (TfRecords)","metadata":{}},{"cell_type":"markdown","source":"# UP Vote if this notebook is helpful and comment if you have any doubt.","metadata":{}},{"cell_type":"code","source":"import os, math, glob, re\nimport numpy as np\nimport pandas as pd\nimport cv2\n\nimport matplotlib.pyplot as plt\nimport pydicom\n\nfrom kaggle_datasets import KaggleDatasets\n\nfrom sklearn.model_selection import train_test_split\n\n\nfrom random import shuffle\nimport tensorflow as tf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hyperparameters for your final images","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE  = 256   # consider this as hyper parameter for your data prepration\nIMAGE_DEPTH = 128 # consider this as hyper parameter for your data prepration\n\nmri_types = ['FLAIR','T1w','T1wCE','T2w']\nCHANNELS  = len(mri_types)\n\nlocal_directory = \"../input/rsna-miccai-brain-tumor-radiogenomic-classification\"\nlocal_label_path = local_directory+\"/train_labels.csv\"\nlocal_submission_path = local_directory+\"/sample_submission.csv\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Data","metadata":{}},{"cell_type":"code","source":"# test data\ndf_test = pd.read_csv(local_submission_path)\ndf_test['BraTS21ID'] = [format(x, '05d') for x in df_test.BraTS21ID]\n\n# train data\ntrain_df = pd.read_csv(local_label_path)\nEXCLUDE = [109, 123, 709]\ntrain_df = train_df[~train_df.BraTS21ID.isin(EXCLUDE)]\ntrain_df['BraTS21ID'] = [format(x, '05d') for x in train_df.BraTS21ID]\ntrain_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:23.008213Z","iopub.execute_input":"2021-09-22T16:12:23.00841Z","iopub.status.idle":"2021-09-22T16:12:23.031954Z","shell.execute_reply.started":"2021-09-22T16:12:23.008389Z","shell.execute_reply":"2021-09-22T16:12:23.031314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:23.034495Z","iopub.execute_input":"2021-09-22T16:12:23.034685Z","iopub.status.idle":"2021-09-22T16:12:23.042083Z","shell.execute_reply.started":"2021-09-22T16:12:23.034665Z","shell.execute_reply":"2021-09-22T16:12:23.041145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split Data","metadata":{}},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(train_df.BraTS21ID, \n                                                  train_df.MGMT_value, \n                                                  test_size=0.2, \n                                                  random_state=42,\n                                                  stratify=train_df.MGMT_value)\nprint(X_train.shape)\nprint(X_val.shape)\nprint(y_train.shape)\nprint(y_val.shape)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:23.044714Z","iopub.execute_input":"2021-09-22T16:12:23.044979Z","iopub.status.idle":"2021-09-22T16:12:23.055512Z","shell.execute_reply.started":"2021-09-22T16:12:23.044948Z","shell.execute_reply":"2021-09-22T16:12:23.054667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing the data\nThere are few things to note about the data.\n\n* Data has been preprocessed via this notebook\n* For one patient e.g. '00002' a TFRecord will look this IMAGE_SIZE x IMAGE_SIZE x IMAGE_DEPTH x 4\n* IMAGE_SIZE x IMAGE_SIZE (128 x 128) is the black and white image provided to us (resized, original was 512x512 I guess)\n* IMAGE_DEPTH (16) means, there are a total of IMAGE_DEPTH such images of IMAGE_SIZE x IMAGE_SIZE size (there are multiple images in each folder. E.g. FLAIR folder has 148 images for a patient, I picked 16 (IMAGE_DEPTH)\n* 4 means there are IMAGE_SIZE x IMAGE_SIZE x IMAGE_DEPTH images of each MRI Image type. \n* In a nutshell, each TFRecord has 16 images of 256x256 size for each MRI SCAN/IMAGE type","metadata":{}},{"cell_type":"code","source":"# load 1 dicom img, e.g. image-1.dcm\ndef read_one_dicom_image(path, img_size=256):\n\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    data = cv2.resize(data, (img_size, img_size))\n    return data\n","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:23.056896Z","iopub.execute_input":"2021-09-22T16:12:23.057318Z","iopub.status.idle":"2021-09-22T16:12:23.062185Z","shell.execute_reply.started":"2021-09-22T16:12:23.057283Z","shell.execute_reply":"2021-09-22T16:12:23.061369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p = \"../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/00000/FLAIR/Image-103.dcm\"\nimg = read_one_dicom_image(p, IMAGE_SIZE)\nimg.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:23.063797Z","iopub.execute_input":"2021-09-22T16:12:23.064301Z","iopub.status.idle":"2021-09-22T16:12:23.118484Z","shell.execute_reply.started":"2021-09-22T16:12:23.064267Z","shell.execute_reply":"2021-09-22T16:12:23.117865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# load all dicoms in a modality folder, e.g. FLAIR/*.dcm\ndef load_dicom_modality(mri_type, scan_id, img_depth, img_size, split):\n    files_path = f\"{local_directory}/{split}/{scan_id}/{mri_type}/*.dcm\"\n    files = sorted(tf.io.gfile.glob(files_path), \n               key=lambda var:[int(x) if x.isdigit() else x for x in re.findall(r'[^0-9]|[0-9]+', var)])\n    \n   \n    num_files = len(files)\n    \n#     print(f\"total files in path {files_path}: = {num_files}\")\n    \n    num_files_middle = num_files//2\n    img_depth_middle = img_depth//2\n    \n    start_depth = max(0, num_files_middle - img_depth_middle)\n    end_depth   = min(num_files, num_files_middle + img_depth_middle)\n    \n    lst_imgs = []\n\n    \n    # picking up only the middle files because\n    for dicom_img in files[start_depth:end_depth]:\n        \n#     for dicom_img in files:        \n        resized_img = read_one_dicom_image(dicom_img, img_size=img_size)\n        lst_imgs.append(resized_img)\n        \n#     convert list of nested list into one big nested list\n    img3d = np.stack(lst_imgs, axis=-1)\n    \n    # print(f\"Old Shape of img3d = : {img3d.shape}\")\n    # basically put 0s to make sure shape is same\n    # effectively - add more images a x b x num_of_images\n    if img3d.shape[-1] < img_depth:\n        # if this patient does not have enough images to make a depth of 32 (img_depth)\n        # we will add additional 3d images with zero values\n        \n        for n in range(img_depth - img3d.shape[-1]):\n            n_zero = np.zeros_like(img3d[:, :, 0])\n            n_zero = tf.expand_dims(n_zero, axis=-1)\n            img3d  = np.concatenate((img3d, n_zero), axis=-1)        \n    \n#     print(f\"New Shape of img3d = : {img3d.shape}\")\n    return img3d\n\n# load all modality for a single sample, e.g. 00010/*/*.dcm\ndef load_dicom_3D(scan_id, img_depth=128, img_size=256, split=\"train\"):\n#     print(scan_id, end=\" \")\n    dicom_channels = [load_dicom_modality(scan_id=scan_id,\n                                         img_depth=img_depth, \n                                         img_size=img_size, \n                                         split=split,\n                                         mri_type=mtype) \n                      for mtype in mri_types]\n    \n#     stacking all type images for one patient into one huge image\n#     for 0001->flair, T1w, T1wCE and T2w\n#     print(dicom_channels)\n    img = np.stack(dicom_channels, axis=-1)\n    \n    # Normalize\n    if np.min(img) < np.max(img):\n        img = img - np.min(img)\n        img = img / np.max(img)\n        \n#     so we need image which looks something like this\n#     img_row x img_col x num_images x 4_class_of_image\n#     print(f\"Image for patient {scan_id} = {img.shape}\")\n    return img","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:23.128157Z","iopub.execute_input":"2021-09-22T16:12:23.128384Z","iopub.status.idle":"2021-09-22T16:12:23.143442Z","shell.execute_reply.started":"2021-09-22T16:12:23.12836Z","shell.execute_reply":"2021-09-22T16:12:23.142668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = load_dicom_3D(\"00002\", img_size=IMAGE_SIZE, img_depth=64, split=\"train\")\nimg.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:23.144804Z","iopub.execute_input":"2021-09-22T16:12:23.145525Z","iopub.status.idle":"2021-09-22T16:12:27.554572Z","shell.execute_reply.started":"2021-09-22T16:12:23.145489Z","shell.execute_reply":"2021-09-22T16:12:27.553844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img.size, img.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:27.555949Z","iopub.execute_input":"2021-09-22T16:12:27.556211Z","iopub.status.idle":"2021-09-22T16:12:27.56248Z","shell.execute_reply.started":"2021-09-22T16:12:27.556179Z","shell.execute_reply":"2021-09-22T16:12:27.561518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Convert to TFRecords","metadata":{}},{"cell_type":"code","source":"def _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() \n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:27.566302Z","iopub.execute_input":"2021-09-22T16:12:27.566626Z","iopub.status.idle":"2021-09-22T16:12:27.574472Z","shell.execute_reply.started":"2021-09-22T16:12:27.56659Z","shell.execute_reply":"2021-09-22T16:12:27.5737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def serialize_example(image, label):\n    feature = {\n        'image': _bytes_feature(image.tobytes()),\n        'MGMT_value': _float_feature(label)\n    }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:27.575546Z","iopub.execute_input":"2021-09-22T16:12:27.576362Z","iopub.status.idle":"2021-09-22T16:12:27.583334Z","shell.execute_reply.started":"2021-09-22T16:12:27.576327Z","shell.execute_reply":"2021-09-22T16:12:27.582589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_all_images_as_tensor_records(dest_path, writer_options, folder_name, X, y, n_sample=32):\n    with tf.io.TFRecordWriter(dest_path, options=writer_options) as writer:    \n    \n        # patient id and label e.g. (00002, 1)\n        cnt = 1\n        for patient_id, label in zip(X, y):\n            \n            img3d_for_one_patient = load_dicom_3D(patient_id, \n                                                  img_size=IMAGE_SIZE, \n                                                  img_depth=IMAGE_DEPTH, \n                                                  split=folder_name) # to look in test or train folder\n\n            example = serialize_example(img3d_for_one_patient, label)\n            writer.write(example)\n            if cnt > n_sample:\n                break\n\n            print(f\"{cnt}: Completed for patient id : {patient_id}\")\n            cnt +=1\n            \n        print(\"Finished\")","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:27.58529Z","iopub.execute_input":"2021-09-22T16:12:27.58548Z","iopub.status.idle":"2021-09-22T16:12:27.594421Z","shell.execute_reply.started":"2021-09-22T16:12:27.585459Z","shell.execute_reply":"2021-09-22T16:12:27.593558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating folders and uploading input images as Tensorflow Records","metadata":{}},{"cell_type":"code","source":"!rm -r tfrecords\n\n# creating trian data folder\n! mkdir -p ./tfrecords/train/\n                \n# creating validation data folder\n! mkdir -p ./tfrecords/valid/\n\n# creating real test data folder\n! mkdir -p ./tfrecords/test/","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:27.595535Z","iopub.execute_input":"2021-09-22T16:12:27.59597Z","iopub.status.idle":"2021-09-22T16:12:30.240871Z","shell.execute_reply.started":"2021-09-22T16:12:27.595936Z","shell.execute_reply":"2021-09-22T16:12:30.239519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_writer_options = tf.io.TFRecordOptions(compression_type=\"GZIP\")","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:30.245803Z","iopub.execute_input":"2021-09-22T16:12:30.246378Z","iopub.status.idle":"2021-09-22T16:12:30.255748Z","shell.execute_reply.started":"2021-09-22T16:12:30.246337Z","shell.execute_reply":"2021-09-22T16:12:30.255008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outpath_train = \"./tfrecords/train\"\ndest_path = str(outpath_train) + os.sep + \"brain_train.tfrec\"\n\n# creating tensor records for the train data\nsave_all_images_as_tensor_records(dest_path, \n                                  my_writer_options, \n                                  \"train\", \n                                  X_train,\n                                 y_train,\n                                 n_sample=32000) \n# use n_sample to save 10 samples to test your Model \n# then later you can put a big value to get all the data into the dataset\n                \n","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:30.260515Z","iopub.execute_input":"2021-09-22T16:12:30.262441Z","iopub.status.idle":"2021-09-22T16:12:47.688858Z","shell.execute_reply.started":"2021-09-22T16:12:30.2624Z","shell.execute_reply":"2021-09-22T16:12:47.687455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\noutpath_valid = \"./tfrecords/valid\"\ndest_path = str(outpath_valid) + os.sep + \"brain_valid.tfrec\"\n\n# creating tensor records for the test data\nsave_all_images_as_tensor_records(dest_path, \n                                  my_writer_options, \n                                  \"train\", \n                                  X_val,\n                                 y_val,\n                                 n_sample=32000)\n","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:12:47.690125Z","iopub.execute_input":"2021-09-22T16:12:47.690601Z","iopub.status.idle":"2021-09-22T16:13:04.189338Z","shell.execute_reply.started":"2021-09-22T16:12:47.690564Z","shell.execute_reply":"2021-09-22T16:13:04.188579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reading the ground test data\ngt_df = pd.read_csv(local_submission_path)\n\ngt_df['BraTS21ID'] = [format(x, '05d') for x in gt_df[\"BraTS21ID\"]]\n\n# gt_df","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:13:04.190518Z","iopub.execute_input":"2021-09-22T16:13:04.190968Z","iopub.status.idle":"2021-09-22T16:13:04.201463Z","shell.execute_reply.started":"2021-09-22T16:13:04.190929Z","shell.execute_reply":"2021-09-22T16:13:04.200603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outpath_valid = \"./tfrecords/test\"\ndest_path = str(outpath_valid) + os.sep + \"brain_test.tfrec\"\n\n# creating tensor records for the test data\nsave_all_images_as_tensor_records(dest_path, \n                                  my_writer_options, \n                                  \"test\", \n                                  gt_df[\"BraTS21ID\"].values,\n                                 gt_df[\"MGMT_value\"].values,\n                                 n_sample=32000)","metadata":{"execution":{"iopub.status.busy":"2021-09-22T16:13:04.202633Z","iopub.execute_input":"2021-09-22T16:13:04.203003Z","iopub.status.idle":"2021-09-22T16:13:23.788305Z","shell.execute_reply.started":"2021-09-22T16:13:04.202968Z","shell.execute_reply":"2021-09-22T16:13:23.786731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}