{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, math, glob, re\nimport numpy as np\nimport pandas as pd\nimport cv2\n\nimport matplotlib.pyplot as plt\nimport pydicom\n\nfrom kaggle_datasets import KaggleDatasets\n\nfrom sklearn.model_selection import train_test_split\n\n\nfrom random import shuffle\nimport tensorflow as tf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE  = 256\nIMAGE_DEPTH = 64\n\nmri_types = ['FLAIR','T1w','T1wCE','T2w']\nCHANNELS  = len(mri_types)\n\nlocal_directory = \"../input/rsna-miccai-brain-tumor-radiogenomic-classification\"\nlocal_label_path = local_directory+\"/train_labels.csv\"\nlocal_submission_path = local_directory+\"/sample_submission.csv\"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Data","metadata":{}},{"cell_type":"code","source":"# test data\ndf_test = pd.read_csv(local_submission_path)\ndf_test['BraTS21ID'] = [format(x, '05d') for x in df_test.BraTS21ID]\n\n# train data\ntrain_df = pd.read_csv(local_label_path)\nEXCLUDE = [109, 123, 709]\ntrain_df = train_df[~train_df.BraTS21ID.isin(EXCLUDE)]\ntrain_df['BraTS21ID'] = [format(x, '05d') for x in train_df.BraTS21ID]\ntrain_df.head(5)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split Data","metadata":{}},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(train_df.BraTS21ID, \n                                                  train_df.MGMT_value, \n                                                  test_size=0.2, \n                                                  random_state=42,\n                                                  stratify=train_df.MGMT_value)\nprint(X_train.shape)\nprint(X_val.shape)\nprint(y_train.shape)\nprint(y_val.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load DICOM","metadata":{}},{"cell_type":"code","source":"# load 1 dicom img, e.g. image-1.dcm\ndef load_dicom_slice(path, img_size=256):\n\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    data = cv2.resize(data, (img_size, img_size))\n    \n    return data\n\n# load all dicoms in a modality folder, e.g. FLAIR/*.dcm\ndef load_dicom_modality(mri_type, scan_id, img_depth, img_size, split):\n\n    files = sorted(tf.io.gfile.glob(f\"{local_directory}/{split}/{scan_id}/{mri_type}/*.dcm\"), \n               key=lambda var:[int(x) if x.isdigit() else x for x in re.findall(r'[^0-9]|[0-9]+', var)])\n    num_files = len(files)\n    num_files_middle = num_files//2\n    img_depth_middle = img_depth//2\n    \n    start_depth = max(0, num_files_middle - img_depth_middle)\n    end_depth   = min(num_files, num_files_middle + img_depth_middle)\n    img3d = np.stack([load_dicom_slice(dicom_img, img_size=img_size) \n                      for dicom_img in files[start_depth:end_depth]]).T\n    \n    if img3d.shape[-1] < img_depth:\n        n_zero = np.zeros((img_size, img_size, img_depth - img3d.shape[-1]))\n        img3d  = np.concatenate((img3d, n_zero), axis=-1)        \n\n    return img3d\n\n# load all modality for a single sample, e.g. 00010/*/*.dcm\ndef load_dicom_3D(scan_id, img_depth=128, img_size=256, split=\"train\"):\n    print(scan_id, end=\" \")\n    dicom_channels = [load_dicom_modality(scan_id=scan_id,\n                                         img_depth=img_depth, \n                                         img_size=img_size, \n                                         split=split,\n                                         mri_type=mtype) \n                      for mtype in mri_types]\n        \n    img = np.stack(dicom_channels, axis=-1)\n    # Normalize\n    if np.min(img) < np.max(img):\n        img = img - np.min(img)\n        img = img / np.max(img)\n    return img","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Convert to TFRecords","metadata":{}},{"cell_type":"code","source":"def _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() \n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def serialize_example(image, label):\n    feature = {\n        'image': _bytes_feature(image.tobytes()),\n        'MGMT_value': _float_feature(label)\n    }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! mkdir -p ./tfrecords/train/\noutpath_train = \"./tfrecords/train\"\nwith tf.io.TFRecordWriter(str(outpath_train + os.sep + 'brain_train.tfrec'),\n                          options=tf.io.TFRecordOptions(compression_type=\"GZIP\")) as writer:\n    for x,y in zip(X_train,y_train):\n        img = load_dicom_3D(x, img_size=IMAGE_SIZE, img_depth=IMAGE_DEPTH, split=\"train\")\n        example = serialize_example(img, y)\n        writer.write(example)\n\n! mkdir -p ./tfrecords/valid/\noutpath_valid = \"./tfrecords/valid\"\nwith tf.io.TFRecordWriter(str(outpath_valid + os.sep + 'brain_val.tfrec'),\n                          options=tf.io.TFRecordOptions(compression_type=\"GZIP\")) as writer:\n    for x,y in zip(X_val,y_val):\n        img = load_dicom_3D(x, img_size=IMAGE_SIZE, img_depth=IMAGE_DEPTH, split=\"train\")\n        example = serialize_example(img, y)\n        writer.write(example)","metadata":{},"execution_count":null,"outputs":[]}]}