{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This kernel uses NiBabel to read the NifTi files in the \"processed/train\" folder and split them into a Training and Validation dataset.\n\nThis is part of a larger solution found at: https://www.kaggle.com/ohbewise/a-rsna-mri-solution-from-dicom-to-submission","metadata":{}},{"cell_type":"code","source":"import os\nimport pickle\nimport numpy as np\nimport pandas as pd\nimport nibabel as nib\nimport tensorflow as tf\nfrom pathlib import Path","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir  = '/kaggle/input/rsna-miccai-brain-tumor-radiogenomic-classification/'\nout_dir   = '../input/dicom-to-normalized-nifti-with-torchio/processed'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# build datasets\n\n# dataset processing functions\ndef read_nifti_file(filepath):\n    \"\"\"Read and load volume\"\"\"\n    # Read file\n    scan = nib.load(filepath)\n    # Get raw data\n    scan = scan.get_fdata()\n    return scan\n\ndef add_batch_channel(volume):\n    \"\"\"Process validation data by adding a channel.\"\"\"\n    volume = tf.expand_dims(volume, axis=-1)\n    volume = tf.expand_dims(volume, axis=0)\n    return volume\n\ndef process_scan(filepath):\n    scan = read_nifti_file(filepath)\n    volume = add_batch_channel(scan)\n    return volume\n\n# get labels\nlabels_df = pd.read_csv(data_dir+'train_labels.csv', index_col=0)\n\n# split patients\npatients = os.listdir(f'{out_dir}/train')\nfrom sklearn.model_selection import train_test_split\ntrain, validation = train_test_split(patients, test_size=0.3, random_state=42)\nprint(f'{len(patients)} total patients.\\n   {len(train)} in the train split.\\n   {len(validation)} in the validation split')\n\nscan_types  = ['FLAIR','T1w','T1wCE','T2w']\nsplits_dict = {'train':train, 'validation':validation}\n\nfor scan_type in scan_types:\n    print(f'{scan_type} start')\n    for split_name, split_list in splits_dict.items():\n        print(f'   {split_name} start')\n        label_list = []\n        filepaths = []\n        for patient in split_list:\n            label = labels_df._get_value(int(patient), 'MGMT_value')\n            label = add_batch_channel(label)\n            label_list.append(label)\n            filepath  = f'{out_dir}/train/{patient}/{scan_type}/{scan_type}.nii.gz'\n            filepaths.append(filepath)\n\n        features = np.array([process_scan(filepath) for filepath in filepaths if filepath])\n        labels = np.array(label_list, dtype=np.uint8)\n        dataset = tf.data.Dataset.from_tensor_slices((features, labels))\n        \n        # save dataset   \n        tf_data_path = f'./datasets/{scan_type}_{split_name}_dataset'\n        tf.data.experimental.save(dataset, tf_data_path, compression='GZIP')\n        with open(tf_data_path + '/element_spec', 'wb') as out_:  # also save the element_spec to disk for future loading\n            pickle.dump(dataset.element_spec, out_)\n        print(f'   {split_name} done')\n    print(f'{scan_type} done')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}